From 2676ba9e8ed0a349ee240f0ba3afdd873019f345 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Tue, 9 Feb 2021 18:21:29 +0000 Subject: [PATCH 01/30] refactor: Begin refactor of... everything --- .devcontainer.json | 8 +- docker-compose.dev.yml | 26 ++ package-lock.json | 353 ++++++++++++++++-- package.json | 2 + src/Library/Fastify.test.ts | 2 +- src/Library/Fastify.ts | 3 +- .../Fastify/Routes1/Route1/Fastifty1Route.ts | 1 + src/Modules/Ques/Que.ts | 29 ++ src/Modules/Redis/RedisController.ts | 15 + src/Modules/Redis/RedisOptions.ts | 11 + src/Modules/SSR/SSRController.tsx | 6 + src/Modules/SSR/SSROptions.ts | 27 ++ src/Modules/SSR/SSRRoute.ts | 2 + src/Modules/Server/SSRServer.tsx | 3 +- src/Modules/Server/ServerController.ts | 53 +++ src/Modules/Server/ServerOptions.ts | 22 ++ .../TypeScript/TypeScriptController.ts | 20 + src/Modules/WebModule/WebModuleResolver.ts | 14 +- src/index.tsx | 21 +- 19 files changed, 573 insertions(+), 45 deletions(-) create mode 100644 docker-compose.dev.yml create mode 100644 src/Modules/Ques/Que.ts create mode 100644 src/Modules/Redis/RedisController.ts create mode 100644 src/Modules/Redis/RedisOptions.ts create mode 100644 src/Modules/SSR/SSRController.tsx create mode 100644 src/Modules/SSR/SSROptions.ts create mode 100644 src/Modules/Server/ServerController.ts create mode 100644 src/Modules/Server/ServerOptions.ts create mode 100644 src/Modules/TypeScript/TypeScriptController.ts diff --git a/.devcontainer.json b/.devcontainer.json index 7de21a4..73ca2c0 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -1,6 +1,8 @@ { "name": "TS-ESWeb Container", - "dockerFile": "./Dockerfile.dev", + "dockerComposeFile": "./docker-compose.dev.yml", + "service": "TSESWeb", + "runServices": ["TSESWeb", "Redis"], "settings": { // This dev container does include /bin/bash if you prefer to use it instead of ash. "terminal.integrated.shell.linux": "/bin/bash" @@ -18,10 +20,6 @@ "redhat.vscode-yaml" ], "remoteUser": "node", - "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached", "workspaceFolder": "/workspace", - "mounts": [ - "source=ts-esweb-modules,target=/workspace/node_modules,type=volume" - ], "postCreateCommand": "sudo chown node node_modules && npm i" } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..76caaf9 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,26 @@ +version: '3.8' + +volumes: + modules: + +networks: + internalRedis: + +services: + TSESWeb: + image: ts-esweb:dev + command: /bin/sh -c "while sleep 1000; do :; done" + build: + context: ./ + dockerfile: Dockerfile.dev + volumes: + # API Folders + - ./:/workspace:cached + - modules:/workspace/node_modules + networks: + - internalRedis + + Redis: + image: redis:alpine3.12 + networks: + - internalRedis diff --git a/package-lock.json b/package-lock.json index b119566..9358c12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,11 +11,13 @@ "dependencies": { "@k-foss/ts-worker": "^1.0.5", "apollo-server-fastify": "^3.0.0-alpha.3", + "bullmq": "^1.14.4", "cjstoesm": "^0.1.3", "fastify": "^3.11.0", "fastify-websocket": "^3.0.0", "get-port": "^5.1.1", "graphql": "^15.4.0", + "hyperid": "^2.1.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3" @@ -337,6 +339,14 @@ "@types/node": "*" } }, + "node_modules/@types/ioredis": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.19.3.tgz", + "integrity": "sha512-r9rXLYeZVmvTf0nKDmFYrpBeVzK5B6jgE2ncfkzPOAuRM5RTzKnYdW2wVn+ZONMnFxV6H4R3NrCdRdC3CL4nFA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/json-schema": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", @@ -1394,6 +1404,21 @@ "node": ">=8" } }, + "node_modules/bullmq": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.14.4.tgz", + "integrity": "sha512-Gx5NX+wA+5E1IReSWuW/ybeq0+JUKUk5BvkiO6QWKowrnJaGJLi8r1rl/VSE7h4reErslLmcXqEvBrpg0yMOwg==", + "dependencies": { + "@types/ioredis": "^4.16.3", + "cron-parser": "^2.7.3", + "get-port": "^5.0.0", + "ioredis": "^4.17.3", + "lodash": "^4.17.11", + "semver": "^6.3.0", + "tslib": "^1.10.0", + "uuid": "^8.2.0" + } + }, "node_modules/cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -1437,7 +1462,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.0.tgz", "integrity": "sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.0" @@ -1620,6 +1644,14 @@ "mimic-response": "^1.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", @@ -1738,6 +1770,18 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "node_modules/cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "dependencies": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -1840,6 +1884,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/deprecated-decorator": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/deprecated-decorator/-/deprecated-decorator-0.1.6.tgz", @@ -2800,7 +2852,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", "integrity": "sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -3055,6 +3106,23 @@ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", "dev": true }, + "node_modules/hyperid": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-2.1.0.tgz", + "integrity": "sha512-cSakhxbUsaIuqjfvvcUuvl/Fl342J65xgLLYrYxSSr9qmJ/EJK+S8crS6mIlQd/a7i+Pe4D0MgSrtZPLze+aCw==", + "dependencies": { + "uuid": "^3.4.0", + "uuid-parse": "^1.1.0" + } + }, + "node_modules/hyperid/node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "bin": { + "uuid": "bin/uuid" + } + }, "node_modules/ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -3119,13 +3187,10 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", - "dev": true, - "engines": { - "node": "*" - } + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true }, "node_modules/internal-slot": { "version": "1.0.2", @@ -3141,6 +3206,30 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.22.0.tgz", + "integrity": "sha512-mtC+jNFMPRxReWx0HodDbcwj34Gj5pK/P4+aE6Nh0pdqgtZKvxUh4z2lVtLjqnRIvMhKaBnIgMYFR8qH/xtttA==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -3249,6 +3338,21 @@ "node": ">=8" } }, + "node_modules/is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-negative-zero": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", @@ -3508,8 +3612,17 @@ "node_modules/lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", - "dev": true + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, "node_modules/lodash.get": { "version": "4.4.2", @@ -3661,6 +3774,25 @@ "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true }, + "node_modules/moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone": { + "version": "0.5.33", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", + "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", + "dependencies": { + "moment": ">= 2.9.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -3972,6 +4104,14 @@ "node": ">=4" } }, + "node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "engines": { + "node": ">=6" + } + }, "node_modules/p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -4365,6 +4505,30 @@ "node": ">=8.10.0" } }, + "node_modules/redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -4570,8 +4734,7 @@ "node_modules/semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" }, "node_modules/semver-diff": { "version": "3.1.1", @@ -4791,6 +4954,11 @@ "node": "*" } }, + "node_modules/standard-as-callback": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -5396,6 +5564,11 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uuid-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz", + "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==" + }, "node_modules/v8-compile-cache": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", @@ -5848,6 +6021,14 @@ "@types/node": "*" } }, + "@types/ioredis": { + "version": "4.19.3", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.19.3.tgz", + "integrity": "sha512-r9rXLYeZVmvTf0nKDmFYrpBeVzK5B6jgE2ncfkzPOAuRM5RTzKnYdW2wVn+ZONMnFxV6H4R3NrCdRdC3CL4nFA==", + "requires": { + "@types/node": "*" + } + }, "@types/json-schema": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", @@ -6669,6 +6850,21 @@ "fill-range": "^7.0.1" } }, + "bullmq": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.14.4.tgz", + "integrity": "sha512-Gx5NX+wA+5E1IReSWuW/ybeq0+JUKUk5BvkiO6QWKowrnJaGJLi8r1rl/VSE7h4reErslLmcXqEvBrpg0yMOwg==", + "requires": { + "@types/ioredis": "^4.16.3", + "cron-parser": "^2.7.3", + "get-port": "^5.0.0", + "ioredis": "^4.17.3", + "lodash": "^4.17.11", + "semver": "^6.3.0", + "tslib": "^1.10.0", + "uuid": "^8.2.0" + } + }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -6705,7 +6901,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.0.tgz", "integrity": "sha512-AEXsYIyyDY3MCzbwdhzG3Jx1R0J2wetQyUynn6dYHAO+bg8l1k7jwZtRv4ryryFs7EP+NDlikJlVe59jr0cM2w==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.0" @@ -6847,6 +7042,11 @@ "mimic-response": "^1.0.0" } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "color": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", @@ -6945,6 +7145,15 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "requires": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -7026,6 +7235,11 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + }, "deprecated-decorator": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/deprecated-decorator/-/deprecated-decorator-0.1.6.tgz", @@ -7792,7 +8006,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", "integrity": "sha512-ZnWP+AmS1VUaLgTRy47+zKtjTxz+0xMpx3I52i+aalBK1QP19ggLF3Db89KJX7kjfOfP2eoa01qc++GwPgufPg==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -7983,6 +8196,22 @@ "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", "dev": true }, + "hyperid": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hyperid/-/hyperid-2.1.0.tgz", + "integrity": "sha512-cSakhxbUsaIuqjfvvcUuvl/Fl342J65xgLLYrYxSSr9qmJ/EJK+S8crS6mIlQd/a7i+Pe4D0MgSrtZPLze+aCw==", + "requires": { + "uuid": "^3.4.0", + "uuid-parse": "^1.1.0" + }, + "dependencies": { + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + } + } + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -8032,9 +8261,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "dev": true }, "internal-slot": { @@ -8048,6 +8277,23 @@ "side-channel": "^1.0.2" } }, + "ioredis": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.22.0.tgz", + "integrity": "sha512-mtC+jNFMPRxReWx0HodDbcwj34Gj5pK/P4+aE6Nh0pdqgtZKvxUh4z2lVtLjqnRIvMhKaBnIgMYFR8qH/xtttA==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.1.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.0.1" + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8126,6 +8372,15 @@ "is-path-inside": "^3.0.1" } }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, "is-negative-zero": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", @@ -8337,8 +8592,17 @@ "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", - "dev": true + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" }, "lodash.get": { "version": "4.4.2", @@ -8453,6 +8717,19 @@ "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" + }, + "moment-timezone": { + "version": "0.5.33", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.33.tgz", + "integrity": "sha512-PTc2vcT8K9J5/9rDEPe5czSIKgLoGsH8UNpA4qZTVw0Vd/Uz19geE9abbIOQKaAQFcnQ3v5YEXrbSc5BpshH+w==", + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -8696,6 +8973,11 @@ "p-limit": "^1.1.0" } }, + "p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" + }, "p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -8993,6 +9275,24 @@ "picomatch": "^2.2.1" } }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "reflect-metadata": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", @@ -9133,8 +9433,7 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" }, "semver-diff": { "version": "3.1.1", @@ -9327,6 +9626,11 @@ "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, + "standard-as-callback": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.0.1.tgz", + "integrity": "sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==" + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -9822,6 +10126,11 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" }, + "uuid-parse": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/uuid-parse/-/uuid-parse-1.1.0.tgz", + "integrity": "sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A==" + }, "v8-compile-cache": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", diff --git a/package.json b/package.json index c8e628d..900976d 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,13 @@ "dependencies": { "@k-foss/ts-worker": "^1.0.5", "apollo-server-fastify": "^3.0.0-alpha.3", + "bullmq": "^1.14.4", "cjstoesm": "^0.1.3", "fastify": "^3.11.0", "fastify-websocket": "^3.0.0", "get-port": "^5.1.1", "graphql": "^15.4.0", + "hyperid": "^2.1.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3" diff --git a/src/Library/Fastify.test.ts b/src/Library/Fastify.test.ts index e99db4f..c5a43fa 100644 --- a/src/Library/Fastify.test.ts +++ b/src/Library/Fastify.test.ts @@ -45,7 +45,7 @@ export class FastifyTestSuite extends TestSuite { return webServer.route({ ...route.options, - handler: route.handler, + handler: async (...params) => route.handler(...params), }); }); diff --git a/src/Library/Fastify.ts b/src/Library/Fastify.ts index c9c256c..570e7cb 100644 --- a/src/Library/Fastify.ts +++ b/src/Library/Fastify.ts @@ -3,7 +3,6 @@ import fastify, { FastifyInstance, FastifyReply, FastifyRequest, - RouteHandlerMethod, RouteOptions, } from 'fastify'; import fastifyWS from 'fastify-websocket'; @@ -86,7 +85,7 @@ export async function createFastifyServer( routes.map((route) => { return webServer.route({ ...route.options, - handler: async (...params: unknown[]) => route.handler(...params), + handler: async (...params) => route.handler(...params), }); }); diff --git a/src/Library/fixtures/Fastify/Routes1/Route1/Fastifty1Route.ts b/src/Library/fixtures/Fastify/Routes1/Route1/Fastifty1Route.ts index 5cc7b95..93c2d0c 100644 --- a/src/Library/fixtures/Fastify/Routes1/Route1/Fastifty1Route.ts +++ b/src/Library/fixtures/Fastify/Routes1/Route1/Fastifty1Route.ts @@ -15,6 +15,7 @@ export default class FastifyRoute1 implements Route { url: '/Route1', }; + // eslint-disable-next-line @typescript-eslint/require-await public async handler( this: FastifyInstance, request: FastifyRequest, diff --git a/src/Modules/Ques/Que.ts b/src/Modules/Ques/Que.ts new file mode 100644 index 0000000..fa1bcb9 --- /dev/null +++ b/src/Modules/Ques/Que.ts @@ -0,0 +1,29 @@ +// src/Modules/Ques/Que.ts + +import { ContainerInstance, Inject, Service, Token, Container } from 'typedi'; +import { Queue as BullMQQueue } from 'bullmq'; +import { logger } from '../../Library/Logger'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; + +@Service() +export class Queue { + public container: ContainerInstance; + + public constructor( + @Inject(serverOptionsToken) private options: ServerOptions, + ) { + logger.info(`Queue.constructor() ${JSON.stringify(options)}`); + + this.container = Container.of(options.serverId); + } + + public async createQueue(queName: string): Promise { + logger.info(`Que.createQueue(${queName})`); + + return new BullMQQueue(queName, { + connection: { + host: this.options.redis.hostname, + }, + }); + } +} diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts new file mode 100644 index 0000000..c61064a --- /dev/null +++ b/src/Modules/Redis/RedisController.ts @@ -0,0 +1,15 @@ +// src/Modules/Redis/RedisController.ts +import { Inject } from 'typedi'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; +import { Queue } from 'bullmq' + +const queue = new Queue(''); + +queue.add('cars', { color: 'blue' }); + +export class RedisController { + @Inject(serverOptionsToken) + public options: ServerOptions; + + public redis = +} \ No newline at end of file diff --git a/src/Modules/Redis/RedisOptions.ts b/src/Modules/Redis/RedisOptions.ts new file mode 100644 index 0000000..c154dd2 --- /dev/null +++ b/src/Modules/Redis/RedisOptions.ts @@ -0,0 +1,11 @@ +// src/Modules/Redis/RedisOptions.ts +import { IsDefined, IsString } from 'class-validator'; +import { Token } from 'typedi'; + +export class RedisOptions { + @IsString() + @IsDefined() + public hostname: string; +} + +export const redisOptionsToken = new Token('redisOptions'); diff --git a/src/Modules/SSR/SSRController.tsx b/src/Modules/SSR/SSRController.tsx new file mode 100644 index 0000000..157f062 --- /dev/null +++ b/src/Modules/SSR/SSRController.tsx @@ -0,0 +1,6 @@ +// src/Modules/SSR/SSRController.tsx + +import { Service } from 'typedi'; + +@Service() +export class SSRController {} diff --git a/src/Modules/SSR/SSROptions.ts b/src/Modules/SSR/SSROptions.ts new file mode 100644 index 0000000..e57248f --- /dev/null +++ b/src/Modules/SSR/SSROptions.ts @@ -0,0 +1,27 @@ +// src/Modules/SSR/SSROptions.ts +import { Exclude } from 'class-transformer'; +import { IsDefined, IsString } from 'class-validator'; + +export class SSROptions { + /** + * Root Directory for Web/Client + * + * @example + * ```ts + * webRoot: path.resolve('./Web') + * ``` + */ + @IsString() + @IsDefined() + public webRoot: string; + + /** + * Entrypoint Path relative to @see webRoot + */ + @IsString() + @IsDefined() + public entrypoint: string; + + @Exclude() + public appComponent: () => React.ReactElement; +} diff --git a/src/Modules/SSR/SSRRoute.ts b/src/Modules/SSR/SSRRoute.ts index 4713244..187afd8 100644 --- a/src/Modules/SSR/SSRRoute.ts +++ b/src/Modules/SSR/SSRRoute.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ /* eslint-disable @typescript-eslint/ban-types */ // src/Modules/SSR/SSRRoute.ts import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; @@ -15,6 +16,7 @@ export default class SSRRoute implements Route { url: '/Test', }; + // eslint-disable-next-line @typescript-eslint/require-await public async handler( this: FastifyInstance, request: FastifyRequest, diff --git a/src/Modules/Server/SSRServer.tsx b/src/Modules/Server/SSRServer.tsx index bb4a1d2..cfec77b 100644 --- a/src/Modules/Server/SSRServer.tsx +++ b/src/Modules/Server/SSRServer.tsx @@ -3,7 +3,7 @@ import fastify, { FastifyInstance } from 'fastify'; import { resolve } from 'path'; import React from 'react'; import { renderToString } from 'react-dom/server'; -import Container from 'typedi'; +import Container, { Service } from 'typedi'; import { createApolloServer } from '../../Library/Apollo'; import { getRoutes, Route } from '../../Library/Fastify'; import { logger } from '../../Library/Logger'; @@ -37,6 +37,7 @@ export interface ClientOptions { /** * TS-ESWeb SSR Server Controller */ +@Service() export class SSRServer { /** * Client Provided Options diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts new file mode 100644 index 0000000..4445b82 --- /dev/null +++ b/src/Modules/Server/ServerController.ts @@ -0,0 +1,53 @@ +// src/Modules/Server/ServerController.ts +import { plainToClass } from 'class-transformer'; +import { validateOrReject } from 'class-validator'; +import hyperid from 'hyperid'; +import { ContainerInstance, Service, Container, Inject } from 'typedi'; +import { logger } from '../../Library/Logger'; +import { TypeScriptController } from '../TypeScript/TypeScriptController'; +import { ServerOptions, serverOptionsToken } from './ServerOptions'; + +@Service() +export class ServerController { + @Inject(serverOptionsToken) + public options: ServerOptions; + + public constructor(private typescriptController: TypeScriptController) {} + + /** + * Create a new ServerController with the provided Configuration within the TypeDi Container + * @param options Server Configuration + * @param container Optional TypeDi Container defaults to `Container.of()` + * @returns Promise resolving to newly configured ServerController class + */ + public static async createServer( + options: ServerOptions, + ): Promise { + logger.info(`ServerController.createServer()`); + + const serverId = hyperid().uuid; + + const container = Container.of(serverId); + + const serverOptions = plainToClass(ServerOptions, { + ...options, + serverId, + }); + + await validateOrReject(serverOptions); + + container.set(serverOptionsToken, serverOptions); + + return container.get(ServerController); + } + + public async startTypeScript(): Promise { + logger.info( + `ServerController.startTypeScript() starting TypeScript Workers`, + ); + + const workers = await this.typescriptController.createWorkers(); + + logger.debug(`ServerController.startTypeScript() workers: `, workers); + } +} diff --git a/src/Modules/Server/ServerOptions.ts b/src/Modules/Server/ServerOptions.ts new file mode 100644 index 0000000..588fbbe --- /dev/null +++ b/src/Modules/Server/ServerOptions.ts @@ -0,0 +1,22 @@ +// src/Modules/Server/ServerOptions.ts +import { Transform, Type } from 'class-transformer'; +import { IsDefined, IsUUID, ValidateNested } from 'class-validator'; +import { Token } from 'typedi'; +import { RedisOptions } from '../Redis/RedisOptions'; +import { SSROptions } from '../SSR/SSROptions'; + +export class ServerOptions { + @ValidateNested() + @Type(() => RedisOptions) + public redis: RedisOptions; + + @ValidateNested() + @Type(() => SSROptions) + public ssr: SSROptions; + + @IsUUID() + @IsDefined() + public serverId: string; +} + +export const serverOptionsToken = new Token('serverOptions'); diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts new file mode 100644 index 0000000..48c70a4 --- /dev/null +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -0,0 +1,20 @@ +// src/Modules/TypeScript/TypeScriptController.ts +import { Inject, Service } from 'typedi'; +import { logger } from '../../Library/Logger'; +import { timeout } from '../../Utils/timeout'; +import { Queue } from '../Ques/Que'; + +@Service() +export class TypeScriptController { + public constructor(private typescriptQue: Queue) { + logger.info(`TypeScriptController.constructor()`, typescriptQue); + + typescriptQue.createQueue('typescriptTranspile') + } + + public async createWorkers(): Promise { + logger.info(`TypeScriptController.createWorkers()`); + + await timeout(50); + } +} diff --git a/src/Modules/WebModule/WebModuleResolver.ts b/src/Modules/WebModule/WebModuleResolver.ts index de3cd55..a09edc1 100644 --- a/src/Modules/WebModule/WebModuleResolver.ts +++ b/src/Modules/WebModule/WebModuleResolver.ts @@ -8,6 +8,7 @@ import { Resolver, Root, } from 'type-graphql'; +import { Service } from 'typedi'; import { logger } from '../../Library/Logger'; import { WebModule } from './WebModule'; import { webModuleController } from './WebModuleController'; @@ -22,6 +23,7 @@ class WebModuleFilter { } @Resolver(() => WebModule) +@Service() export class WebModuleResolver { @Query(() => [WebModule]) public webModules(): WebModule[] { @@ -67,14 +69,14 @@ export class WebModuleResolver { @Query(() => Boolean) public helloTest(): boolean { logger.debug( - `WebModuleResolver.helloTest() Array.from(webModuleController.modules): ${Array.from( - webModuleController.modules, + `WebModuleResolver.helloTest() Array.from(webModuleController.modules): ${JSON.stringify( + Array.from(webModuleController.modules), )} - Array.from(webModuleController.specifierMap): ${Array.from( - webModuleController.specifierMap, + Array.from(webModuleController.specifierMap): ${JSON.stringify( + Array.from(webModuleController.specifierMap), )} - Array.from(webModuleController.specifierTest): ${Array.from( - webModuleController.specifierTest, + Array.from(webModuleController.specifierTest): ${JSON.stringify( + Array.from(webModuleController.specifierTest), )}`, ); diff --git a/src/index.tsx b/src/index.tsx index 1e41f66..dd74fcc 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -3,6 +3,7 @@ import { resolve } from 'path'; import 'reflect-metadata'; import { logger } from './Library/Logger'; import { createSSRServer } from './Modules/Server'; +import { ServerController } from './Modules/Server/ServerController'; import { WebAppManifest } from './Modules/WebAppManifest/WebAppManifest'; import { WebAppManfiestController } from './Modules/WebAppManifest/WebAppManifestController'; import { App } from './Web_Test/App'; @@ -30,25 +31,29 @@ const manifest: WebAppManifest = { await WebAppManfiestController.loadManifest(manifest); -export const ssrServer = await createSSRServer({ - options: { - appComponent: App, +export const server = await ServerController.createServer({ + redis: { + hostname: 'Redis', + }, + ssr: { webRoot: resolve('./src/Web_Test'), entrypoint: 'Imports.ts', - serverRoutes: [], + appComponent: App, }, }); logger.debug(`Starting the TS-ESWeb SSR Server Transpiler`); -await ssrServer.startTranspiler(); +await server.startTypeScript(); + +// await ssrServer.startTranspiler(); logger.debug(`Creating the Fastify Server`); -const fastifyServer = await ssrServer.createFastifyServer(); +// const fastifyServer = await ssrServer.createFastifyServer(); -const serverString = await fastifyServer.listen(8082, '0.0.0.0'); +// const serverString = await fastifyServer.listen(8082, '0.0.0.0'); -logger.debug(`Fastify server is listening ${serverString}`); +// logger.debug(`Fastify server is listening ${serverString}`); export {}; From 043f143e82b7098fa63f28e2ef932caa1eba7a15 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Tue, 9 Feb 2021 21:08:39 +0000 Subject: [PATCH 02/30] feat: Begin work on new Que system --- .vscode/launch.json | 4 ++ src/Modules/Ques/Que.ts | 39 +++++++++++++--- src/Modules/Redis/RedisController.ts | 2 +- src/Modules/Server/ServerController.ts | 6 ++- .../TypeScript/TypeScriptController.ts | 45 ++++++++++++++++++- src/Modules/TypeScript/TypeScriptWorker.ts | 33 ++++++++++++++ 6 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 src/Modules/TypeScript/TypeScriptWorker.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 105d3ef..0bd5dd4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,6 +8,10 @@ "protocol": "inspector", "port": 9229, "restart": true, + "outputCapture": "console", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "autoAttachChildProcesses": true, "cwd": "${workspaceFolder}", "runtimeExecutable": "npx", diff --git a/src/Modules/Ques/Que.ts b/src/Modules/Ques/Que.ts index fa1bcb9..db1be75 100644 --- a/src/Modules/Ques/Que.ts +++ b/src/Modules/Ques/Que.ts @@ -1,14 +1,23 @@ // src/Modules/Ques/Que.ts - import { ContainerInstance, Inject, Service, Token, Container } from 'typedi'; -import { Queue as BullMQQueue } from 'bullmq'; +import { Queue as BullMQQueue, Worker, QueueEvents } from 'bullmq'; import { logger } from '../../Library/Logger'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; @Service() -export class Queue { +export class Queue { public container: ContainerInstance; + public queueToken: Token>; + + public get queueEvents(): QueueEvents { + return new QueueEvents(this.queue.name, { + connection: { + host: this.options.redis.hostname, + }, + }); + } + public constructor( @Inject(serverOptionsToken) private options: ServerOptions, ) { @@ -17,13 +26,33 @@ export class Queue { this.container = Container.of(options.serverId); } - public async createQueue(queName: string): Promise { + public get queue(): BullMQQueue { + return this.container.get(this.queueToken); + } + + public createQueue(queName: string): void { logger.info(`Que.createQueue(${queName})`); - return new BullMQQueue(queName, { + this.queueToken = new Token>(`bullMQQue-${queName}`); + + const queue = new BullMQQueue(queName, { connection: { host: this.options.redis.hostname, }, }); + + this.container.set(this.queueToken, queue); + } + + public async createWorker(workerFile: string): Promise> { + const queue = this.queue; + + logger.info(`Queue.createWorker(${workerFile})`); + + worker.on('data', (...args) => { + console.log('Worker Data: ', ...args); + }); + + return worker; } } diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index c61064a..99b2929 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -1,7 +1,7 @@ // src/Modules/Redis/RedisController.ts import { Inject } from 'typedi'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; -import { Queue } from 'bullmq' +import { Queue } from 'bullmq/src' const queue = new Queue(''); diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 4445b82..52d1ac8 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -21,7 +21,7 @@ export class ServerController { * @returns Promise resolving to newly configured ServerController class */ public static async createServer( - options: ServerOptions, + options: Partial, ): Promise { logger.info(`ServerController.createServer()`); @@ -49,5 +49,9 @@ export class ServerController { const workers = await this.typescriptController.createWorkers(); logger.debug(`ServerController.startTypeScript() workers: `, workers); + + await this.typescriptController.createTask('helloFucker'); + + logger.info('Done'); } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 48c70a4..0cf3499 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,20 +1,61 @@ // src/Modules/TypeScript/TypeScriptController.ts +import { spawnWorker } from '@k-foss/ts-worker'; +import { Worker } from 'bullmq/src'; +import { resolve } from 'path'; import { Inject, Service } from 'typedi'; +import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; import { timeout } from '../../Utils/timeout'; import { Queue } from '../Ques/Que'; +import { serverOptionsToken, ServerOptions } from '../Server/ServerOptions'; + +type TypeScriptQueType = 'helloFucker'; @Service() export class TypeScriptController { - public constructor(private typescriptQue: Queue) { + public worker: Worker; + + @Inject(serverOptionsToken) + public options: ServerOptions; + + public constructor(private typescriptQue: Queue) { logger.info(`TypeScriptController.constructor()`, typescriptQue); - typescriptQue.createQueue('typescriptTranspile') + typescriptQue.createQueue('typescriptTranspile'); } public async createWorkers(): Promise { logger.info(`TypeScriptController.createWorkers()`); + const workerPath = resolve( + fileURLToPath(import.meta.url), + '../TypeScriptWorker.ts', + ); + + logger.debug( + `TypeScriptController.createWorkers() path: ${workerPath} containerId: ${this.options.serverId}`, + ); + + const worker = spawnWorker(workerPath, { + redisOptions: JSON.stringify(this.options.redis), + }); + + console.log('Worker: ', worker); + + logger.info(`workerPath: ${workerPath}`); + + await timeout(50); + } + + public async createTask(taskInput: TypeScriptQueType): Promise { + const job = this.typescriptQue.queue.add( + this.typescriptQue.queue.name, + 'helloFucker', + ); + + logger.info(`TypeScriptController.createTask()`); await timeout(50); + + return 'test'; } } diff --git a/src/Modules/TypeScript/TypeScriptWorker.ts b/src/Modules/TypeScript/TypeScriptWorker.ts new file mode 100644 index 0000000..ea24978 --- /dev/null +++ b/src/Modules/TypeScript/TypeScriptWorker.ts @@ -0,0 +1,33 @@ +// src/Modules/TypeScript/TypeScriptWorker.ts +import { Worker } from 'bullmq'; +import { logger } from '../../Library/Logger'; +import { workerData } from 'worker_threads'; +import { getWorkerData } from '@k-foss/ts-worker'; +import Container from 'typedi'; +import { RedisOptions } from '../Redis/RedisOptions'; + +const data = getWorkerData(import.meta.url); + +const redisOptions = JSON.parse(data.redisOptions); + +function isRedisOptions(input: RedisOptions): input is RedisOptions { + return 'hostname' in input; +} + +if (!isRedisOptions(redisOptions)) { + throw new Error('Invalid Redis Options provided to worker'); +} + +const TSWorker = new Worker( + 'typescriptTranspile', + async (job) => { + logger.info(`TypeScriptWorker.TSWorker job ${JSON.stringify(job)}`); + + return 'helloWorld'; + }, + { + connection: { + host: redisOptions.hostname, + }, + }, +); From 80e9721798edfff7723ad73a4e485073c531af77 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Tue, 9 Feb 2021 21:14:26 +0000 Subject: [PATCH 03/30] fix: Fix Codespace issue? --- docker-compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 76caaf9..954ab8f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,6 +10,7 @@ services: TSESWeb: image: ts-esweb:dev command: /bin/sh -c "while sleep 1000; do :; done" + user: node build: context: ./ dockerfile: Dockerfile.dev From c86b997beb00cc5a31cc49095841f2ae35d07cdb Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Tue, 9 Feb 2021 23:23:03 +0000 Subject: [PATCH 04/30] feat(debugging): Setup debugging and source maps for breakpoints and inspector development --- .vscode/launch.json | 11 ++++++----- nodemon.json | 5 ++++- package.json | 2 +- src/Library/Logger.ts | 4 +++- src/Modules/TypeScript/TypeScriptWorker.ts | 2 ++ src/index.tsx | 2 ++ tsconfig.json | 3 ++- 7 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0bd5dd4..3654830 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,14 +8,15 @@ "protocol": "inspector", "port": 9229, "restart": true, - "outputCapture": "console", - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen", + "showAsyncStacks": true, "autoAttachChildProcesses": true, + "console": "integratedTerminal", + "outputCapture": "std", + "sourceMaps": true, "cwd": "${workspaceFolder}", - "runtimeExecutable": "npx", - "runtimeArgs": ["nodemon"] + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "start:debug"] } ] } diff --git a/nodemon.json b/nodemon.json index 452c43f..3446adb 100644 --- a/nodemon.json +++ b/nodemon.json @@ -1,6 +1,9 @@ { + "$schema": "https://raw.githubusercontent.com/SchemaStore/schemastore/master/src/schemas/json/nodemon.json", "watch": ["src"], "ext": "ts,tsx", "ignore": ["src/**/*.test.ts"], - "exec": "npm run start:debug" + "exec": "npm run start:debug", + "spawn": true, + "stdin": false } diff --git a/package.json b/package.json index 900976d..c5f7e66 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "test": "ts-estest ./src", "prepublishOnly": "npm run build", "build": "tsc -p ./tsconfig.build.json", - "start:debug": "node --loader @k-foss/ts-esnode --experimental-modules --experimental-import-meta-resolve --experimental-specifier-resolution=node --harmony-top-level-await ./", + "start:debug": "node --loader @k-foss/ts-esnode --experimental-modules --experimental-import-meta-resolve --experimental-specifier-resolution=node --enable-source-maps --harmony-top-level-await ./", "dev": "nodemon", "prettier": "prettier --config .prettierrc --check \"src/**/*ts\"", "lint": "eslint ./src --ext .js,.ts" diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index 49829f8..e648a2a 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -10,6 +10,8 @@ export const logger = winston.createLogger({ // - Write all logs with level `error` and below to `error.log` // - Write all logs with level `info` and below to `combined.log` // - new winston.transports.Console({}), + new winston.transports.Console({ + debugStdout: true, + }), ], }); diff --git a/src/Modules/TypeScript/TypeScriptWorker.ts b/src/Modules/TypeScript/TypeScriptWorker.ts index ea24978..51a0ffe 100644 --- a/src/Modules/TypeScript/TypeScriptWorker.ts +++ b/src/Modules/TypeScript/TypeScriptWorker.ts @@ -18,6 +18,8 @@ if (!isRedisOptions(redisOptions)) { throw new Error('Invalid Redis Options provided to worker'); } +console.log('Fucker'); + const TSWorker = new Worker( 'typescriptTranspile', async (job) => { diff --git a/src/index.tsx b/src/index.tsx index dd74fcc..a1ae1c7 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -31,6 +31,8 @@ const manifest: WebAppManifest = { await WebAppManfiestController.loadManifest(manifest); +console.log('Hello'); + export const server = await ServerController.createServer({ redis: { hostname: 'Redis', diff --git a/tsconfig.json b/tsconfig.json index 1da78ff..178bd4a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,8 @@ "outDir": "dist", // Source Maps - "inlineSourceMap": false, + "inlineSourceMap": true, + "lib": ["ESNext", "DOM", "WebWorker"], // Features From 050ab5db87c453977e6e70bd048fcb17c4cd7f3d Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Wed, 10 Feb 2021 03:53:01 +0000 Subject: [PATCH 05/30] feat: Continue working on Transpiler Worker --- src/Modules/Server/ServerController.ts | 10 +++- .../TypeScript/TranspilerWorkerInput.ts | 16 +++++++ .../TypeScript/TranspilerWorkerJobInput.ts | 2 + src/Modules/TypeScript/TypeScriptConsts.ts | 3 ++ .../TypeScript/TypeScriptController.ts | 48 +++++++++---------- .../TypeScript/TypeScriptTranspilerWorker.ts | 47 ++++++++++++++++++ src/Modules/TypeScript/TypeScriptWorker.ts | 35 -------------- src/index.tsx | 2 - 8 files changed, 100 insertions(+), 63 deletions(-) create mode 100644 src/Modules/TypeScript/TranspilerWorkerInput.ts create mode 100644 src/Modules/TypeScript/TranspilerWorkerJobInput.ts create mode 100644 src/Modules/TypeScript/TypeScriptConsts.ts create mode 100644 src/Modules/TypeScript/TypeScriptTranspilerWorker.ts delete mode 100644 src/Modules/TypeScript/TypeScriptWorker.ts diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 52d1ac8..76c4c42 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -50,7 +50,15 @@ export class ServerController { logger.debug(`ServerController.startTypeScript() workers: `, workers); - await this.typescriptController.createTask('helloFucker'); + const randomArray = [...Array(10).fill(0)]; + + await Promise.all( + randomArray.map(() => { + logger.info(`Adding a task to workers`); + + return this.typescriptController.createTask('helloFucker'); + }), + ); logger.info('Done'); } diff --git a/src/Modules/TypeScript/TranspilerWorkerInput.ts b/src/Modules/TypeScript/TranspilerWorkerInput.ts new file mode 100644 index 0000000..652bc25 --- /dev/null +++ b/src/Modules/TypeScript/TranspilerWorkerInput.ts @@ -0,0 +1,16 @@ +// src/Modules/TypeScript/TranspilerWorkerInput.ts +import { Type } from 'class-transformer'; +import { ValidateNested, IsString } from 'class-validator'; +import { RedisOptions } from '../Redis/RedisOptions'; + +export class TranspilerWorkerInput { + @ValidateNested() + @Type(() => RedisOptions) + public redisOptions: RedisOptions; + + /** + * Name of the TypeScript Transpiler Que for this worker + */ + @IsString() + public queName: string; +} diff --git a/src/Modules/TypeScript/TranspilerWorkerJobInput.ts b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts new file mode 100644 index 0000000..84a6735 --- /dev/null +++ b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts @@ -0,0 +1,2 @@ +// src/Modules/TypeScript/TranspilerWorkerJobInput.ts +import {} from 'class-validator' \ No newline at end of file diff --git a/src/Modules/TypeScript/TypeScriptConsts.ts b/src/Modules/TypeScript/TypeScriptConsts.ts new file mode 100644 index 0000000..019df35 --- /dev/null +++ b/src/Modules/TypeScript/TypeScriptConsts.ts @@ -0,0 +1,3 @@ +// src/Modules/TypeScript/TypeScriptConsts.ts + +export const transpilerQueName = 'typescriptTranspile'; diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 0cf3499..fb81827 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,61 +1,59 @@ // src/Modules/TypeScript/TypeScriptController.ts import { spawnWorker } from '@k-foss/ts-worker'; -import { Worker } from 'bullmq/src'; -import { resolve } from 'path'; +import { Job } from 'bullmq'; +import { cpus } from 'os'; import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; -import { timeout } from '../../Utils/timeout'; import { Queue } from '../Ques/Que'; -import { serverOptionsToken, ServerOptions } from '../Server/ServerOptions'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; -type TypeScriptQueType = 'helloFucker'; +type filePath = string; @Service() export class TypeScriptController { - public worker: Worker; - @Inject(serverOptionsToken) public options: ServerOptions; - public constructor(private typescriptQue: Queue) { - logger.info(`TypeScriptController.constructor()`, typescriptQue); + public constructor(private transpilerQue: Queue) { + logger.debug(`TypeScriptController.constructor()`); - typescriptQue.createQueue('typescriptTranspile'); + transpilerQue.createQueue('typescriptTranspiler'); } public async createWorkers(): Promise { logger.info(`TypeScriptController.createWorkers()`); - const workerPath = resolve( - fileURLToPath(import.meta.url), - '../TypeScriptWorker.ts', + const workerPath = await import.meta.resolve( + './TypeScriptTranspilerWorker', ); logger.debug( `TypeScriptController.createWorkers() path: ${workerPath} containerId: ${this.options.serverId}`, ); - const worker = spawnWorker(workerPath, { - redisOptions: JSON.stringify(this.options.redis), - }); + for (const _workerThread of Array(cpus().length - 1).fill(0)) { + logger.info('Spawning worker'); - console.log('Worker: ', worker); + const worker = spawnWorker(fileURLToPath(workerPath), { + redisOptions: JSON.stringify(this.options.redis), + queName: this.transpilerQue.queue.name, + }); - logger.info(`workerPath: ${workerPath}`); + logger.debug(`worker: `, worker.threadId); + } - await timeout(50); + logger.info(`workerPath: ${workerPath}`); } - public async createTask(taskInput: TypeScriptQueType): Promise { - const job = this.typescriptQue.queue.add( - this.typescriptQue.queue.name, - 'helloFucker', + public createTask(filePath: string): Promise { + const job = this.transpilerQue.queue.add( + this.transpilerQue.queue.name, + filePath, ); logger.info(`TypeScriptController.createTask()`); - await timeout(50); - return 'test'; + return job; } } diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts new file mode 100644 index 0000000..cea5361 --- /dev/null +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -0,0 +1,47 @@ +// src/Modules/TypeScript/TypeScriptWorker.ts +import { getWorkerData } from '@k-foss/ts-worker'; +import '../../Utils/Setup'; +import { plainToClass } from 'class-transformer'; +import { Worker } from 'bullmq'; +import { logger } from '../../Library/Logger'; +import { TranspilerWorkerInput } from './TranspilerWorkerInput'; +import { validateOrReject } from 'class-validator'; +import { RedisOptions } from '../Redis/RedisOptions'; + +const data = getWorkerData(import.meta.url); + +logger.debug(`TypeScriptWorker`, data); + +const workerInput = plainToClass(TranspilerWorkerInput, { + redisOptions: JSON.parse(data.redisOptions) as RedisOptions, + queName: data?.queName as string, +}); + +logger.debug(`TypeScriptTranspilerWorker: `, workerInput); + +await validateOrReject(workerInput); + +async function transformFile(filePath: string): Promise { + logger.info(`Transforming ${filePath}`); + + return `console.log('helloWorld')`; +} + +const transpilerWorker = new Worker( + workerInput.queName, + async (job) => { + logger.info(`TypeScript Transpiler Worker: filePath: ${job.data}`); + + const transformedModule = await transformFile(job.data); + + logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); + }, + { + connection: { + host: workerInput.redisOptions.hostname, + }, + concurrency: 2, + }, +); + +logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker); diff --git a/src/Modules/TypeScript/TypeScriptWorker.ts b/src/Modules/TypeScript/TypeScriptWorker.ts deleted file mode 100644 index 51a0ffe..0000000 --- a/src/Modules/TypeScript/TypeScriptWorker.ts +++ /dev/null @@ -1,35 +0,0 @@ -// src/Modules/TypeScript/TypeScriptWorker.ts -import { Worker } from 'bullmq'; -import { logger } from '../../Library/Logger'; -import { workerData } from 'worker_threads'; -import { getWorkerData } from '@k-foss/ts-worker'; -import Container from 'typedi'; -import { RedisOptions } from '../Redis/RedisOptions'; - -const data = getWorkerData(import.meta.url); - -const redisOptions = JSON.parse(data.redisOptions); - -function isRedisOptions(input: RedisOptions): input is RedisOptions { - return 'hostname' in input; -} - -if (!isRedisOptions(redisOptions)) { - throw new Error('Invalid Redis Options provided to worker'); -} - -console.log('Fucker'); - -const TSWorker = new Worker( - 'typescriptTranspile', - async (job) => { - logger.info(`TypeScriptWorker.TSWorker job ${JSON.stringify(job)}`); - - return 'helloWorld'; - }, - { - connection: { - host: redisOptions.hostname, - }, - }, -); diff --git a/src/index.tsx b/src/index.tsx index a1ae1c7..dd74fcc 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -31,8 +31,6 @@ const manifest: WebAppManifest = { await WebAppManfiestController.loadManifest(manifest); -console.log('Hello'); - export const server = await ServerController.createServer({ redis: { hostname: 'Redis', From fbed0cbadfd204866e1c5ebdce51d994a990f859 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Wed, 10 Feb 2021 04:23:37 +0000 Subject: [PATCH 06/30] feat(ModuleMaps): Begin work on first discovering module map without transpiling --- src/Modules/Server/ServerController.ts | 37 ++++++-- .../TypeScript/ModuleMapWorkerJobInput.ts | 11 +++ .../TypeScript/TranspilerWorkerJobInput.ts | 7 +- .../TypeScript/TypeScriptController.ts | 66 +++++++++++--- .../TypeScript/TypeScriptModuleMapWorker.ts | 90 +++++++++++++++++++ .../TypeScript/TypeScriptTranspilerWorker.ts | 15 ++-- ...ranspilerWorkerInput.ts => WorkerInput.ts} | 4 +- 7 files changed, 201 insertions(+), 29 deletions(-) create mode 100644 src/Modules/TypeScript/ModuleMapWorkerJobInput.ts create mode 100644 src/Modules/TypeScript/TypeScriptModuleMapWorker.ts rename src/Modules/TypeScript/{TranspilerWorkerInput.ts => WorkerInput.ts} (80%) diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 76c4c42..b47c5d9 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -2,7 +2,8 @@ import { plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; import hyperid from 'hyperid'; -import { ContainerInstance, Service, Container, Inject } from 'typedi'; +import { Service, Container, Inject } from 'typedi'; +import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @@ -43,23 +44,43 @@ export class ServerController { public async startTypeScript(): Promise { logger.info( - `ServerController.startTypeScript() starting TypeScript Workers`, + `ServerController.startTypeScript() starting TypeScript Module Map`, ); - const workers = await this.typescriptController.createWorkers(); - - logger.debug(`ServerController.startTypeScript() workers: `, workers); + const workers = await this.typescriptController.createModuleMapWorkers(); const randomArray = [...Array(10).fill(0)]; await Promise.all( - randomArray.map(() => { + randomArray.map(async () => { logger.info(`Adding a task to workers`); - return this.typescriptController.createTask('helloFucker'); + return this.typescriptController.createModuleMapTask({ + filePath: fileURLToPath( + await import.meta.resolve('../../Web_Test/Imports.ts'), + ), + }); }), ); - logger.info('Done'); + // const workers = await this.typescriptController.createTranspilerWorkers(); + + // logger.debug(`ServerController.startTypeScript() workers: `, workers); + + // const randomArray = [...Array(10).fill(0)]; + + // await Promise.all( + // randomArray.map(async () => { + // logger.info(`Adding a task to workers`); + + // return this.typescriptController.createTranspilerTask({ + // filePath: fileURLToPath( + // await import.meta.resolve('../../Web_Test/Imports.ts'), + // ), + // }); + // }), + // ); + + // logger.info('Done'); } } diff --git a/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts b/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts new file mode 100644 index 0000000..a0e2706 --- /dev/null +++ b/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts @@ -0,0 +1,11 @@ +// src/Modules/TypeScript/ModuleMapWorkerJobInput.ts +import { IsOptional, IsString } from 'class-validator'; + +export class ModuleMapWorkerJobInput { + @IsString() + public filePath: string; + + @IsString() + @IsOptional() + public specifier?: string; +} diff --git a/src/Modules/TypeScript/TranspilerWorkerJobInput.ts b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts index 84a6735..173fd76 100644 --- a/src/Modules/TypeScript/TranspilerWorkerJobInput.ts +++ b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts @@ -1,2 +1,7 @@ // src/Modules/TypeScript/TranspilerWorkerJobInput.ts -import {} from 'class-validator' \ No newline at end of file +import { IsString } from 'class-validator'; + +export class TranspilerWorkerJobInput { + @IsString() + public filePath: string; +} diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index fb81827..7cca71d 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -7,53 +7,93 @@ import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; import { Queue } from '../Ques/Que'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; - -type filePath = string; +import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; +import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; @Service() export class TypeScriptController { @Inject(serverOptionsToken) public options: ServerOptions; - public constructor(private transpilerQue: Queue) { + public constructor( + private transpilerQue: Queue, + private moduleMapQue: Queue, + ) { logger.debug(`TypeScriptController.constructor()`); transpilerQue.createQueue('typescriptTranspiler'); + moduleMapQue.createQueue('typescriptModuleMap'); } - public async createWorkers(): Promise { - logger.info(`TypeScriptController.createWorkers()`); + public async createModuleMapWorkers(): Promise { + logger.info(`TypeScriptController.createModuleMapWorkers()`); - const workerPath = await import.meta.resolve( - './TypeScriptTranspilerWorker', + const workerPathURI = await import.meta.resolve( + './TypeScriptModuleMapWorker', ); logger.debug( - `TypeScriptController.createWorkers() path: ${workerPath} containerId: ${this.options.serverId}`, + `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, ); for (const _workerThread of Array(cpus().length - 1).fill(0)) { logger.info('Spawning worker'); - const worker = spawnWorker(fileURLToPath(workerPath), { + const worker = spawnWorker(fileURLToPath(workerPathURI), { redisOptions: JSON.stringify(this.options.redis), - queName: this.transpilerQue.queue.name, + queName: this.moduleMapQue.queue.name, }); logger.debug(`worker: `, worker.threadId); } + } + + public async createTranspilerWorkers(): Promise { + logger.info(`TypeScriptController.createTranspilerWorkers()`); + + const workerPathURI = await import.meta.resolve( + './TypeScriptTranspilerWorker', + ); + + logger.debug( + `TypeScriptController.createTranspilerWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, + ); + + for (const _workerThread of Array(cpus().length - 1).fill(0)) { + logger.info('Spawning Transpiler worker'); - logger.info(`workerPath: ${workerPath}`); + const worker = spawnWorker(fileURLToPath(workerPathURI), { + redisOptions: JSON.stringify(this.options.redis), + queName: this.transpilerQue.queue.name, + }); + + logger.debug(`worker: `, worker.threadId); + } } - public createTask(filePath: string): Promise { + public createTranspilerTask( + jobInput: TranspilerWorkerJobInput, + ): Promise { const job = this.transpilerQue.queue.add( this.transpilerQue.queue.name, - filePath, + jobInput, ); logger.info(`TypeScriptController.createTask()`); return job; } + + public createModuleMapTask(jobInput: ModuleMapWorkerJobInput): Promise { + const job = this.moduleMapQue.queue.add( + this.moduleMapQue.queue.name, + jobInput, + ); + + logger.info( + `TypeScriptController.createModuleMapTask(${JSON.stringify(jobInput)})`, + ); + + return job; + } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts new file mode 100644 index 0000000..e2be104 --- /dev/null +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -0,0 +1,90 @@ +// src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +import { getWorkerData } from '@k-foss/ts-worker'; +import '../../Utils/Setup'; +import { plainToClass } from 'class-transformer'; +import { Worker } from 'bullmq'; +import { logger } from '../../Library/Logger'; +import { WorkerInput } from './WorkerInput'; +import { validateOrReject } from 'class-validator'; +import { RedisOptions } from '../Redis/RedisOptions'; +import * as ts from 'typescript'; +import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; +import { dirname } from 'path'; +import { getTSConfig } from './TSConfig'; + +const data = getWorkerData(import.meta.url); + +logger.debug(`TypeScriptWorker`, data); + +const workerInput = plainToClass(WorkerInput, { + redisOptions: JSON.parse(data.redisOptions) as RedisOptions, + queName: data?.queName as string, +}); + +logger.debug(`TypeScriptModuleMapWorker: `, workerInput); + +await validateOrReject(workerInput); + +async function discoverModuleMap( + moduleInput: ModuleMapWorkerJobInput, +): Promise { + logger.debug(`discoverModuleMap(${JSON.stringify(moduleInput)})`); + + const rootDir = dirname(moduleInput.filePath); + + const tsConfig = getTSConfig(moduleInput.filePath); + const defaultOptions = ts.getDefaultCompilerOptions(); + const options = { ...defaultOptions, ...tsConfig }; + + const compilierHost = ts.createCompilerHost({ + ...options, + rootDir, + }); + + const compilerProgram = ts.createProgram({ + rootNames: [moduleInput.filePath], + options: { + ...options, + jsxFragmentFactory: 'Fragment', + }, + host: compilierHost, + }); + + const sourceFile = compilerProgram.getSourceFile(moduleInput.filePath); + + if (!sourceFile) { + throw new Error('Invalid Source File'); + } + + for (const [specifier, resovledModule] of sourceFile.resolvedModules) { + console.log( + `Resolved Module: specifier: ${specifier} resolved:`, + resovledModule, + ); + } + + return `console.log('helloWorld')`; +} + +const transpilerWorker = new Worker( + workerInput.queName, + async (job) => { + const jobInput = plainToClass(ModuleMapWorkerJobInput, job.data); + + await validateOrReject(jobInput); + + logger.info(`TypeScript Module Map Worker: filePath: ${jobInput.filePath}`); + + const transformedModule = await discoverModuleMap(jobInput); + + logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); + }, + { + connection: { + host: workerInput.redisOptions.hostname, + }, + concurrency: 2, + }, +); + +logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index cea5361..92d08a5 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -4,15 +4,16 @@ import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; import { Worker } from 'bullmq'; import { logger } from '../../Library/Logger'; -import { TranspilerWorkerInput } from './TranspilerWorkerInput'; +import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; import { RedisOptions } from '../Redis/RedisOptions'; +import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; const data = getWorkerData(import.meta.url); logger.debug(`TypeScriptWorker`, data); -const workerInput = plainToClass(TranspilerWorkerInput, { +const workerInput = plainToClass(WorkerInput, { redisOptions: JSON.parse(data.redisOptions) as RedisOptions, queName: data?.queName as string, }); @@ -27,12 +28,16 @@ async function transformFile(filePath: string): Promise { return `console.log('helloWorld')`; } -const transpilerWorker = new Worker( +const transpilerWorker = new Worker( workerInput.queName, async (job) => { - logger.info(`TypeScript Transpiler Worker: filePath: ${job.data}`); + const jobInput = plainToClass(TranspilerWorkerJobInput, job.data); - const transformedModule = await transformFile(job.data); + await validateOrReject(jobInput); + + logger.info(`TypeScript Transpiler Worker: filePath: ${jobInput.filePath}`); + + const transformedModule = await transformFile(jobInput.filePath); logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); }, diff --git a/src/Modules/TypeScript/TranspilerWorkerInput.ts b/src/Modules/TypeScript/WorkerInput.ts similarity index 80% rename from src/Modules/TypeScript/TranspilerWorkerInput.ts rename to src/Modules/TypeScript/WorkerInput.ts index 652bc25..f3732a7 100644 --- a/src/Modules/TypeScript/TranspilerWorkerInput.ts +++ b/src/Modules/TypeScript/WorkerInput.ts @@ -1,9 +1,9 @@ -// src/Modules/TypeScript/TranspilerWorkerInput.ts +// src/Modules/TypeScript/WorkerInput.ts import { Type } from 'class-transformer'; import { ValidateNested, IsString } from 'class-validator'; import { RedisOptions } from '../Redis/RedisOptions'; -export class TranspilerWorkerInput { +export class WorkerInput { @ValidateNested() @Type(() => RedisOptions) public redisOptions: RedisOptions; From 939d039d2867f8cf4ebd690fb81a5d2a7fb3186b Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Wed, 10 Feb 2021 04:53:00 +0000 Subject: [PATCH 07/30] feat(ModuleMap): Working Module Map Generation --- src/Modules/Server/ServerController.ts | 20 ++---- .../TypeScript/TypeScriptController.ts | 10 +-- .../TypeScript/TypeScriptModuleMapWorker.ts | 64 +++++++++++++------ 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index b47c5d9..dbb7cc3 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -47,21 +47,13 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - const workers = await this.typescriptController.createModuleMapWorkers(); + await this.typescriptController.createModuleMapWorkers(); - const randomArray = [...Array(10).fill(0)]; - - await Promise.all( - randomArray.map(async () => { - logger.info(`Adding a task to workers`); - - return this.typescriptController.createModuleMapTask({ - filePath: fileURLToPath( - await import.meta.resolve('../../Web_Test/Imports.ts'), - ), - }); - }), - ); + await this.typescriptController.createModuleMapTask({ + filePath: fileURLToPath( + await import.meta.resolve('../../Web_Test/Imports.ts'), + ), + }); // const workers = await this.typescriptController.createTranspilerWorkers(); diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 7cca71d..9b740da 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -36,6 +36,12 @@ export class TypeScriptController { `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, ); + await this.moduleMapQue.queue.clean(0, 1000); + + this.moduleMapQue.queueEvents.on('completed', async (msg) => { + console.log('Module Map Que Recieved', msg); + }); + for (const _workerThread of Array(cpus().length - 1).fill(0)) { logger.info('Spawning worker'); @@ -90,10 +96,6 @@ export class TypeScriptController { jobInput, ); - logger.info( - `TypeScriptController.createModuleMapTask(${JSON.stringify(jobInput)})`, - ); - return job; } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index e2be104..afbf515 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -2,7 +2,7 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; -import { Worker } from 'bullmq'; +import { Queue, Worker } from 'bullmq'; import { logger } from '../../Library/Logger'; import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; @@ -11,23 +11,32 @@ import * as ts from 'typescript'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { dirname } from 'path'; import { getTSConfig } from './TSConfig'; +import { fileURLToPath, pathToFileURL } from 'url'; const data = getWorkerData(import.meta.url); -logger.debug(`TypeScriptWorker`, data); - const workerInput = plainToClass(WorkerInput, { redisOptions: JSON.parse(data.redisOptions) as RedisOptions, queName: data?.queName as string, }); -logger.debug(`TypeScriptModuleMapWorker: `, workerInput); - await validateOrReject(workerInput); +const moduleMapQue = new Queue(workerInput.queName, { + connection: { + host: workerInput.redisOptions.hostname, + }, +}); + +interface ModuleMap { + filePath: string; + + importedModules: string[]; +} + async function discoverModuleMap( moduleInput: ModuleMapWorkerJobInput, -): Promise { +): Promise { logger.debug(`discoverModuleMap(${JSON.stringify(moduleInput)})`); const rootDir = dirname(moduleInput.filePath); @@ -56,14 +65,35 @@ async function discoverModuleMap( throw new Error('Invalid Source File'); } - for (const [specifier, resovledModule] of sourceFile.resolvedModules) { - console.log( - `Resolved Module: specifier: ${specifier} resolved:`, - resovledModule, - ); - } - - return `console.log('helloWorld')`; + const resolvedArray = Array.from(sourceFile.resolvedModules); + + const importedModules = await Promise.all( + resolvedArray.map(async ([specifier]) => { + const resolvePathURI = await import.meta.resolve( + specifier, + pathToFileURL(moduleInput.filePath).href, + ); + + const filePath = fileURLToPath(resolvePathURI); + + await moduleMapQue.add( + workerInput.queName, + plainToClass(ModuleMapWorkerJobInput, { + filePath, + }), + { + jobId: filePath, + }, + ); + + return filePath; + }), + ); + + return { + filePath: moduleInput.filePath, + importedModules, + }; } const transpilerWorker = new Worker( @@ -75,9 +105,7 @@ const transpilerWorker = new Worker( logger.info(`TypeScript Module Map Worker: filePath: ${jobInput.filePath}`); - const transformedModule = await discoverModuleMap(jobInput); - - logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); + return discoverModuleMap(jobInput); }, { connection: { @@ -87,4 +115,4 @@ const transpilerWorker = new Worker( }, ); -logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker); +logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker.name); From 12944b59096e9650e98239a4281b47a3dc237ca6 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Wed, 10 Feb 2021 05:21:30 +0000 Subject: [PATCH 08/30] feat: Getting closer to functional --- .../TypeScript/TypeScriptController.ts | 50 +++++++++++++++++-- .../TypeScript/TypeScriptModuleMapWorker.ts | 8 +++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 9b740da..e4c94ff 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,6 +1,7 @@ // src/Modules/TypeScript/TypeScriptController.ts import { spawnWorker } from '@k-foss/ts-worker'; import { Job } from 'bullmq'; +import { Worker } from 'worker_threads'; import { cpus } from 'os'; import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; @@ -28,6 +29,8 @@ export class TypeScriptController { public async createModuleMapWorkers(): Promise { logger.info(`TypeScriptController.createModuleMapWorkers()`); + const moduleMapQue = this.moduleMapQue.queue; + const workerPathURI = await import.meta.resolve( './TypeScriptModuleMapWorker', ); @@ -36,21 +39,60 @@ export class TypeScriptController { `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, ); - await this.moduleMapQue.queue.clean(0, 1000); + await moduleMapQue.clean(0, 1000); + // eslint-disable-next-line @typescript-eslint/no-misused-promises this.moduleMapQue.queueEvents.on('completed', async (msg) => { - console.log('Module Map Que Recieved', msg); + const job = await moduleMapQue.getJob(msg.jobId); + + const specifier = job?.data?.specifier as string; + + console.log( + `${specifier} resovled ${JSON.stringify( + msg.returnvalue.importedModules, + )}`, + ); }); + const workers: Worker[] = []; + for (const _workerThread of Array(cpus().length - 1).fill(0)) { logger.info('Spawning worker'); const worker = spawnWorker(fileURLToPath(workerPathURI), { redisOptions: JSON.stringify(this.options.redis), - queName: this.moduleMapQue.queue.name, + queName: moduleMapQue.name, }); - logger.debug(`worker: `, worker.threadId); + workers.push(worker); + } + + const interval = setInterval(() => { + checkActiveJobs(); + }, 500); + + let hasRun: boolean = false; + + async function checkActiveJobs(): Promise { + const activeJobCount = await moduleMapQue.getActiveCount(); + + if (hasRun === false && activeJobCount > 0) { + hasRun = true; + } + + if (hasRun === false) { + return; + } + + console.log(activeJobCount); + + if (activeJobCount === 0) { + clearInterval(interval); + + console.log('Shutting down workers'); + + return Promise.all(workers.map((worker) => worker.terminate())); + } } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index afbf515..8d117bd 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -76,10 +76,18 @@ async function discoverModuleMap( const filePath = fileURLToPath(resolvePathURI); + let moduleSpecifier: string; + if (ts.isExternalModuleNameRelative(specifier)) { + moduleSpecifier = filePath; + } else { + moduleSpecifier = specifier; + } + await moduleMapQue.add( workerInput.queName, plainToClass(ModuleMapWorkerJobInput, { filePath, + specifier: moduleSpecifier, }), { jobId: filePath, From f6364740fd3c9b2e594b5a45557ffe428d5df7f4 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Thu, 11 Feb 2021 14:49:59 +0000 Subject: [PATCH 09/30] feat: More work --- src/Modules/Ques/Que.ts | 58 ------ src/Modules/Queues/Queue.ts | 170 +++++++++++++++++ src/Modules/Queues/QueueController.ts | 42 +++++ src/Modules/Queues/QueueOptions.ts | 15 ++ src/Modules/Redis/RedisController.ts | 9 +- src/Modules/Redis/RedisOptions.ts | 2 +- src/Modules/Server/ServerController.ts | 25 +-- src/Modules/TypeScript/ResolvedModuleMap.ts | 10 + .../TypeScript/TranspiledModuleOutput.ts | 7 + .../TypeScript/TypeScriptController.ts | 174 ++++++++++-------- .../TypeScript/TypeScriptModuleMapWorker.ts | 11 +- .../TypeScript/TypeScriptTranspilerWorker.ts | 10 +- src/Modules/TypeScript/WorkerInput.ts | 17 +- src/index.tsx | 3 +- 14 files changed, 372 insertions(+), 181 deletions(-) delete mode 100644 src/Modules/Ques/Que.ts create mode 100644 src/Modules/Queues/Queue.ts create mode 100644 src/Modules/Queues/QueueController.ts create mode 100644 src/Modules/Queues/QueueOptions.ts create mode 100644 src/Modules/TypeScript/ResolvedModuleMap.ts create mode 100644 src/Modules/TypeScript/TranspiledModuleOutput.ts diff --git a/src/Modules/Ques/Que.ts b/src/Modules/Ques/Que.ts deleted file mode 100644 index db1be75..0000000 --- a/src/Modules/Ques/Que.ts +++ /dev/null @@ -1,58 +0,0 @@ -// src/Modules/Ques/Que.ts -import { ContainerInstance, Inject, Service, Token, Container } from 'typedi'; -import { Queue as BullMQQueue, Worker, QueueEvents } from 'bullmq'; -import { logger } from '../../Library/Logger'; -import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; - -@Service() -export class Queue { - public container: ContainerInstance; - - public queueToken: Token>; - - public get queueEvents(): QueueEvents { - return new QueueEvents(this.queue.name, { - connection: { - host: this.options.redis.hostname, - }, - }); - } - - public constructor( - @Inject(serverOptionsToken) private options: ServerOptions, - ) { - logger.info(`Queue.constructor() ${JSON.stringify(options)}`); - - this.container = Container.of(options.serverId); - } - - public get queue(): BullMQQueue { - return this.container.get(this.queueToken); - } - - public createQueue(queName: string): void { - logger.info(`Que.createQueue(${queName})`); - - this.queueToken = new Token>(`bullMQQue-${queName}`); - - const queue = new BullMQQueue(queName, { - connection: { - host: this.options.redis.hostname, - }, - }); - - this.container.set(this.queueToken, queue); - } - - public async createWorker(workerFile: string): Promise> { - const queue = this.queue; - - logger.info(`Queue.createWorker(${workerFile})`); - - worker.on('data', (...args) => { - console.log('Worker Data: ', ...args); - }); - - return worker; - } -} diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts new file mode 100644 index 0000000..0cf3fa9 --- /dev/null +++ b/src/Modules/Queues/Queue.ts @@ -0,0 +1,170 @@ +// src/Modules/Queue/Queue.ts +import { + Job, + Queue as BullMQQueue, + QueueEvents as BullMQQueueEvents, +} from 'bullmq'; +import { Worker } from 'worker_threads'; +import { spawnWorker } from '@k-foss/ts-worker'; +import { cpus } from 'os'; +import { logger } from '../../Library/Logger'; +import { QueueOptions } from './QueueOptions'; +import { ClassConstructor } from 'class-transformer'; + +/** + * Task Queue handled by dedicated `worker_threads` created by + * [`@K-FOSS/TS-ESWorkers`](https://www.npmjs.com/package/@k-foss/ts-worker) + * + * The Queue is handled by [`bullmq`](https://www.npmjs.com/package/bullmq) with + * a Redis backend + */ +export class Queue { + public queue: BullMQQueue; + public queueEvents: BullMQQueueEvents; + + public workers: Worker[] = []; + + public hasRun = false; + + public checkInterval: NodeJS.Timeout; + + public options: QueueOptions< + QueueName, + ClassConstructor, + ClassConstructor + >; + + public constructor( + options: QueueOptions< + QueueName, + ClassConstructor, + ClassConstructor + >, + ) { + this.queue = new BullMQQueue(options.name, options.bullOptions); + this.queueEvents = new BullMQQueueEvents(options.name, options.bullOptions); + + this.options = options; + } + + private terminateWorkers(): Promise { + return Promise.all(this.workers.map((worker) => worker.terminate())); + } + + private async checkActiveJobs(): Promise { + const activeJobCount = await this.queue.getActiveCount(); + + if (this.hasRun === false) { + if (activeJobCount > 0) { + this.hasRun = true; + } + + return; + } + + if (activeJobCount === 0) { + clearInterval(this.checkInterval); + + logger.info(`Shutting down workers`); + + return this.terminateWorkers(); + } + } + + /** + * Start watching workers for active tasks and kill workers upon empty Queue + */ + private startWatching(): void { + this.checkInterval = setInterval(() => { + this.checkActiveJobs().catch((err) => { + logger.error( + `Queue({${this.queue.name}) startWatching error: ${JSON.stringify( + err, + )}`, + ); + + return this.terminateWorkers(); + }); + }, 500); + } + + /** + * + * @param workerPath Absolute path to the Worker File + * + * @example + * ```ts + * const moduleMapQueue = new Queue('randomQueue'); + * + * const workerPathURI = await import.meta.resolve( + * './TypeScriptModuleMapWorker', + * ); + * const workerPath = fileURLToPath(workerPathURI); + * + * await moduleMapQueue.createWorkers(workerPath); + * + * ``` + * + * @returns Promise resolving once the workers threads have all been created + */ + public async createWorkers(workerPath: string): Promise { + await this.queue.clean(0, 1000); + + for (const _workerThread of Array(cpus().length - 1).fill(0)) { + logger.info('Spawning worker'); + + const worker = spawnWorker(workerPath, { + redisOptions: JSON.stringify(this.queue.opts), + queName: this.queue.name, + }); + + this.workers.push(worker); + } + + return this.startWatching(); + } + + /** + * Create a new Job/Task for the queue + * @param input Job Input data + * + * @example + * ```ts + * interface ModuleMapJobInput { + * redisHost: string; + * filePath: string; + * } + * + * const moduleMapQueue = new Queue('randomQueue'); + * + * const workerPathURI = await import.meta.resolve( + * './ModuleMapWorker', + * ); + * const workerPath = fileURLToPath(workerPathURI); + * + * await moduleMapQueue.createWorkers(workerPath); + * + * const jobInput: ModuleMapJobInput = { + * redisHost: 'redis', + * filePath: '/workspace/src/index.ts' + * } + * + * await moduleMapQueue.addTask(jobInput) + * ``` + * + * @returns BullMQ Job object + */ + public addTask(input: JobInput): Promise { + const job = this.queue.add(this.options.name, input); + + logger.debug(`Queue.addTask(${JSON.stringify(input)})`); + + return job; + } + + public async waitForTask(task: Job): Promise { + const job = await task.waitUntilFinished(this.queueEvents); + + logger.debug(`Queue.waitForTask(${task.name}) ${JSON.stringify(job)}`); + } +} diff --git a/src/Modules/Queues/QueueController.ts b/src/Modules/Queues/QueueController.ts new file mode 100644 index 0000000..62b9def --- /dev/null +++ b/src/Modules/Queues/QueueController.ts @@ -0,0 +1,42 @@ +// src/Modules/Ques/QueueController.ts +import { QueueOptions } from 'bullmq'; +import { ClassConstructor } from 'class-transformer'; +import { Inject, Service } from 'typedi'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; +import { Queue } from './Queue'; + +@Service() +export class QueueController { + @Inject(serverOptionsToken) + public options: ServerOptions; + + /** + * Create the IORedis Options + */ + private createBullOptions(): QueueOptions { + return { + connection: { + ...this.options.redis, + }, + }; + } + + /** + * Create a new Queue + * @param queKey Queue key + */ + public createQueue( + name: QueueName, + inputClass: ClassConstructor, + outputClass: ClassConstructor, + ): Queue { + const bullOptions = this.createBullOptions(); + + return new Queue({ + name, + bullOptions: bullOptions, + outputClass, + in + }); + } +} diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts new file mode 100644 index 0000000..3857f9b --- /dev/null +++ b/src/Modules/Queues/QueueOptions.ts @@ -0,0 +1,15 @@ +// src/Modules/Queues/QueueOptions.ts +import { IsDefined, IsString } from 'class-validator'; +import { QueueOptions as BullMQOptions } from 'bullmq'; + +export class QueueOptions { + @IsString() + @IsDefined() + public name: QueueName; + + public bullOptions: BullMQOptions; + + public outputClass: JobOutput; + + public inputClass: JobInput; +} diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index 99b2929..1b0c419 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -1,15 +1,8 @@ // src/Modules/Redis/RedisController.ts import { Inject } from 'typedi'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; -import { Queue } from 'bullmq/src' - -const queue = new Queue(''); - -queue.add('cars', { color: 'blue' }); export class RedisController { @Inject(serverOptionsToken) public options: ServerOptions; - - public redis = -} \ No newline at end of file +} diff --git a/src/Modules/Redis/RedisOptions.ts b/src/Modules/Redis/RedisOptions.ts index c154dd2..43355f0 100644 --- a/src/Modules/Redis/RedisOptions.ts +++ b/src/Modules/Redis/RedisOptions.ts @@ -5,7 +5,7 @@ import { Token } from 'typedi'; export class RedisOptions { @IsString() @IsDefined() - public hostname: string; + public host: string; } export const redisOptionsToken = new Token('redisOptions'); diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index dbb7cc3..37cedcd 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -47,32 +47,15 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - await this.typescriptController.createModuleMapWorkers(); + await Promise.all([ + this.typescriptController.createModuleMapWorkers(), + this.typescriptController.createTranspilerWorkers(), + ]); await this.typescriptController.createModuleMapTask({ filePath: fileURLToPath( await import.meta.resolve('../../Web_Test/Imports.ts'), ), }); - - // const workers = await this.typescriptController.createTranspilerWorkers(); - - // logger.debug(`ServerController.startTypeScript() workers: `, workers); - - // const randomArray = [...Array(10).fill(0)]; - - // await Promise.all( - // randomArray.map(async () => { - // logger.info(`Adding a task to workers`); - - // return this.typescriptController.createTranspilerTask({ - // filePath: fileURLToPath( - // await import.meta.resolve('../../Web_Test/Imports.ts'), - // ), - // }); - // }), - // ); - - // logger.info('Done'); } } diff --git a/src/Modules/TypeScript/ResolvedModuleMap.ts b/src/Modules/TypeScript/ResolvedModuleMap.ts new file mode 100644 index 0000000..dcbc01a --- /dev/null +++ b/src/Modules/TypeScript/ResolvedModuleMap.ts @@ -0,0 +1,10 @@ +// src/Modules/TypeScript/ResolvedModuleMap.ts +import { IsDefined, IsString } from 'class-validator'; + +export class ResolvedModuleMap { + @IsString() + @IsDefined() + public filePath: string; + + public importedModules: string[]; +} diff --git a/src/Modules/TypeScript/TranspiledModuleOutput.ts b/src/Modules/TypeScript/TranspiledModuleOutput.ts new file mode 100644 index 0000000..ce86757 --- /dev/null +++ b/src/Modules/TypeScript/TranspiledModuleOutput.ts @@ -0,0 +1,7 @@ +// src/Modules/TypeScript/TranspiledModuleOutput.ts +import { IsString } from 'class-validator'; + +export class TranspiledModuleOutput { + @IsString() + public filePath: string; +} diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index e4c94ff..0776a04 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,142 +1,156 @@ // src/Modules/TypeScript/TypeScriptController.ts -import { spawnWorker } from '@k-foss/ts-worker'; import { Job } from 'bullmq'; -import { Worker } from 'worker_threads'; -import { cpus } from 'os'; import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; -import { Queue } from '../Ques/Que'; +import { Queue } from '../Queues/Queue'; +import { QueueController } from '../Queues/QueueController'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; +import { ResolvedModuleMap } from './ResolvedModuleMap'; +import { TranspiledModuleOutput } from './TranspiledModuleOutput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; @Service() export class TypeScriptController { - @Inject(serverOptionsToken) - public options: ServerOptions; + /** + * Transpiler Que and Workers for transpiling Files to ESModules + */ + private transpilerQueue: Queue< + 'typescriptTranspiler', + TranspilerWorkerJobInput, + TranspiledModuleOutput + >; + + /** + * Module Map Que and Workers for determining all imported and related modules and files + */ + private moduleMapQueue: Queue< + 'typescriptModuleMap', + ModuleMapWorkerJobInput, + ResolvedModuleMap + >; public constructor( - private transpilerQue: Queue, - private moduleMapQue: Queue, + @Inject(serverOptionsToken) + public options: ServerOptions, + @Inject(() => QueueController) + private queueController: QueueController, ) { logger.debug(`TypeScriptController.constructor()`); - transpilerQue.createQueue('typescriptTranspiler'); - moduleMapQue.createQueue('typescriptModuleMap'); + const typescriptTranspilerKey = 'typescriptTranspiler'; + const moduleMapQueKey = 'typescriptModuleMap'; + + this.transpilerQueue = this.queueController.createQueue( + typescriptTranspilerKey, + TranspilerWorkerJobInput, + TranspiledModuleOutput, + ); + + this.moduleMapQueue = this.queueController.createQueue( + moduleMapQueKey, + ModuleMapWorkerJobInput, + ResolvedModuleMap, + ); } + /** + * Spawn the Module Map Workers + */ public async createModuleMapWorkers(): Promise { logger.info(`TypeScriptController.createModuleMapWorkers()`); - const moduleMapQue = this.moduleMapQue.queue; - const workerPathURI = await import.meta.resolve( './TypeScriptModuleMapWorker', ); + const workerPath = fileURLToPath(workerPathURI); + + /** + * Create the @K-FOSS/TS-ESWorkers. + */ + await this.moduleMapQueue.createWorkers(workerPath); logger.debug( - `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, + `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, ); - await moduleMapQue.clean(0, 1000); - // eslint-disable-next-line @typescript-eslint/no-misused-promises - this.moduleMapQue.queueEvents.on('completed', async (msg) => { - const job = await moduleMapQue.getJob(msg.jobId); - - const specifier = job?.data?.specifier as string; - - console.log( - `${specifier} resovled ${JSON.stringify( - msg.returnvalue.importedModules, - )}`, - ); - }); - - const workers: Worker[] = []; - - for (const _workerThread of Array(cpus().length - 1).fill(0)) { - logger.info('Spawning worker'); + // this.moduleMapQueue.queueEvents.on('completed', async (msg) => { + // const job = await this.moduleMapQueue.queue.getJob(msg.jobId); - const worker = spawnWorker(fileURLToPath(workerPathURI), { - redisOptions: JSON.stringify(this.options.redis), - queName: moduleMapQue.name, - }); + // console.log('JobInput: ', job.data); - workers.push(worker); - } + // const jobOutput = plainToClass(ResolvedModuleMap, msg.returnvalue); + // const jobInput = plainToClass(ModuleMapWorkerJobInput, job?.data); - const interval = setInterval(() => { - checkActiveJobs(); - }, 500); + // console.log('MotherFucker', msg.returnvalue); - let hasRun: boolean = false; - - async function checkActiveJobs(): Promise { - const activeJobCount = await moduleMapQue.getActiveCount(); - - if (hasRun === false && activeJobCount > 0) { - hasRun = true; - } - - if (hasRun === false) { - return; - } - - console.log(activeJobCount); - - if (activeJobCount === 0) { - clearInterval(interval); - - console.log('Shutting down workers'); - - return Promise.all(workers.map((worker) => worker.terminate())); - } - } + // console.log( + // `${jobInput.specifier || jobInput.filePath} resovled ${JSON.stringify( + // jobOutput?.importedModules, + // )}`, + // ); + // }); } + /** + * Spawn the TypeScript Transpiler Workers + */ public async createTranspilerWorkers(): Promise { logger.info(`TypeScriptController.createTranspilerWorkers()`); const workerPathURI = await import.meta.resolve( './TypeScriptTranspilerWorker', ); + const workerPath = fileURLToPath(workerPathURI); + + await this.moduleMapQueue.createWorkers(workerPath); logger.debug( - `TypeScriptController.createTranspilerWorkers() workerPathURI: ${workerPathURI} containerId: ${this.options.serverId}`, + `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, ); - for (const _workerThread of Array(cpus().length - 1).fill(0)) { - logger.info('Spawning Transpiler worker'); + // for (const _workerThread of Array(cpus().length - 1).fill(0)) { + // logger.info('Spawning Transpiler worker'); + + // const { spawnWorker } = await import('@k-foss/ts-worker'); - const worker = spawnWorker(fileURLToPath(workerPathURI), { - redisOptions: JSON.stringify(this.options.redis), - queName: this.transpilerQue.queue.name, - }); + // console.log('TranspileQueFucker: ', this.transpilerQueue.name); - logger.debug(`worker: `, worker.threadId); - } + // const worker = spawnWorker(fileURLToPath(workerPathURI), { + // redisOptions: JSON.stringify(this.options.redis), + // queName: this.transpilerQueue.name, + // }); + // } } + /** + * Transpile a new FilePath + * + * @param jobInput Input to pass to the Transpiler Worker + * + * @returns Promise resolving to the created BullMQ Job + */ public createTranspilerTask( jobInput: TranspilerWorkerJobInput, ): Promise { - const job = this.transpilerQue.queue.add( - this.transpilerQue.queue.name, - jobInput, - ); + const job = this.transpilerQueue.addTask(jobInput); logger.info(`TypeScriptController.createTask()`); return job; } + /** + * Create a new "job" to create a module map for a specified input module + * + * @param jobInput Input to pass to the Module Map Worker + * + * @returns Promise resolving to the created BullMQ Job + */ public createModuleMapTask(jobInput: ModuleMapWorkerJobInput): Promise { - const job = this.moduleMapQue.queue.add( - this.moduleMapQue.queue.name, - jobInput, - ); + const job = this.moduleMapQueue.addTask(jobInput); return job; } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 8d117bd..05d98a7 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -12,6 +12,7 @@ import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { dirname } from 'path'; import { getTSConfig } from './TSConfig'; import { fileURLToPath, pathToFileURL } from 'url'; +import { ResolvedModuleMap } from './ResolvedModuleMap'; const data = getWorkerData(import.meta.url); @@ -23,9 +24,7 @@ const workerInput = plainToClass(WorkerInput, { await validateOrReject(workerInput); const moduleMapQue = new Queue(workerInput.queName, { - connection: { - host: workerInput.redisOptions.hostname, - }, + connection: workerInput.redisOptions, }); interface ModuleMap { @@ -104,9 +103,11 @@ async function discoverModuleMap( }; } -const transpilerWorker = new Worker( +const moduleWorker = new Worker( workerInput.queName, async (job) => { + logger.info(`Recieved a task for moduleWorker`); + const jobInput = plainToClass(ModuleMapWorkerJobInput, job.data); await validateOrReject(jobInput); @@ -123,4 +124,4 @@ const transpilerWorker = new Worker( }, ); -logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker.name); +logger.debug(`TypeScriptModuleMapWorker.ts worker: `, moduleWorker.name); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 92d08a5..ab7b44b 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -11,15 +11,13 @@ import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; const data = getWorkerData(import.meta.url); -logger.debug(`TypeScriptWorker`, data); +console.log(`TranspilerWorker:`, data, import.meta.url); const workerInput = plainToClass(WorkerInput, { redisOptions: JSON.parse(data.redisOptions) as RedisOptions, queName: data?.queName as string, }); -logger.debug(`TypeScriptTranspilerWorker: `, workerInput); - await validateOrReject(workerInput); async function transformFile(filePath: string): Promise { @@ -40,6 +38,10 @@ const transpilerWorker = new Worker( const transformedModule = await transformFile(jobInput.filePath); logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); + + return { + test: 'shti', + }; }, { connection: { @@ -48,5 +50,3 @@ const transpilerWorker = new Worker( concurrency: 2, }, ); - -logger.debug(`TypeScriptTranspilerWorker.ts worker: `, transpilerWorker); diff --git a/src/Modules/TypeScript/WorkerInput.ts b/src/Modules/TypeScript/WorkerInput.ts index f3732a7..13dbc76 100644 --- a/src/Modules/TypeScript/WorkerInput.ts +++ b/src/Modules/TypeScript/WorkerInput.ts @@ -1,11 +1,26 @@ // src/Modules/TypeScript/WorkerInput.ts -import { Type } from 'class-transformer'; +import { Transform, Type } from 'class-transformer'; import { ValidateNested, IsString } from 'class-validator'; +import { logger } from '../../Library/Logger'; import { RedisOptions } from '../Redis/RedisOptions'; export class WorkerInput { @ValidateNested() @Type(() => RedisOptions) + @Transform((input) => { + if (typeof input === 'string') { + const parsedJSON = JSON.parse(input) as RedisOptions; + + logger.debug( + `new WorkerInput().redisOptions typeof redisOptions === 'string'`, + parsedJSON, + ); + + return parsedJSON; + } + + return input; + }) public redisOptions: RedisOptions; /** diff --git a/src/index.tsx b/src/index.tsx index dd74fcc..eaca2f8 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -2,7 +2,6 @@ import { resolve } from 'path'; import 'reflect-metadata'; import { logger } from './Library/Logger'; -import { createSSRServer } from './Modules/Server'; import { ServerController } from './Modules/Server/ServerController'; import { WebAppManifest } from './Modules/WebAppManifest/WebAppManifest'; import { WebAppManfiestController } from './Modules/WebAppManifest/WebAppManifestController'; @@ -33,7 +32,7 @@ await WebAppManfiestController.loadManifest(manifest); export const server = await ServerController.createServer({ redis: { - hostname: 'Redis', + host: 'Redis', }, ssr: { webRoot: resolve('./src/Web_Test'), From 83183db05827756dbd9641981d77ac7a709d5631 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Thu, 11 Feb 2021 16:17:06 +0000 Subject: [PATCH 10/30] feat: More work --- src/Modules/Queues/QueueController.ts | 4 ++-- src/Modules/Queues/QueueOptions.ts | 2 +- src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Modules/Queues/QueueController.ts b/src/Modules/Queues/QueueController.ts index 62b9def..3db3a30 100644 --- a/src/Modules/Queues/QueueController.ts +++ b/src/Modules/Queues/QueueController.ts @@ -34,9 +34,9 @@ export class QueueController { return new Queue({ name, - bullOptions: bullOptions, + bullOptions, outputClass, - in + inputClass, }); } } diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts index 3857f9b..e239bde 100644 --- a/src/Modules/Queues/QueueOptions.ts +++ b/src/Modules/Queues/QueueOptions.ts @@ -2,7 +2,7 @@ import { IsDefined, IsString } from 'class-validator'; import { QueueOptions as BullMQOptions } from 'bullmq'; -export class QueueOptions { +export class QueueOptions { @IsString() @IsDefined() public name: QueueName; diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 05d98a7..949c849 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -16,11 +16,15 @@ import { ResolvedModuleMap } from './ResolvedModuleMap'; const data = getWorkerData(import.meta.url); +console.log('ModuleMapWorkerData: ', data); + const workerInput = plainToClass(WorkerInput, { redisOptions: JSON.parse(data.redisOptions) as RedisOptions, queName: data?.queName as string, }); +console.log('ModuleMapWorker: ', workerInput); + await validateOrReject(workerInput); const moduleMapQue = new Queue(workerInput.queName, { From f546c3806e2ca1b0c91962ace484945d1b2e6fd2 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Thu, 11 Feb 2021 16:22:39 +0000 Subject: [PATCH 11/30] style(lint): Fix ESLint issues --- src/Modules/Queues/Queue.ts | 8 ++++++-- src/Modules/Server/ServerController.ts | 1 + src/Modules/Server/ServerOptions.ts | 2 +- src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 4 +--- src/Modules/TypeScript/TypeScriptTranspilerWorker.ts | 9 +++++---- src/Modules/TypeScript/transpileWorker.ts | 8 ++++++++ 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 0cf3fa9..700d39d 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -163,8 +163,12 @@ export class Queue { } public async waitForTask(task: Job): Promise { - const job = await task.waitUntilFinished(this.queueEvents); + const jobOutput = (await task.waitUntilFinished( + this.queueEvents, + )) as JobOutput; - logger.debug(`Queue.waitForTask(${task.name}) ${JSON.stringify(job)}`); + logger.debug( + `Queue.waitForTask(${task.name}) ${JSON.stringify(jobOutput)}`, + ); } } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 37cedcd..397fe71 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -13,6 +13,7 @@ export class ServerController { @Inject(serverOptionsToken) public options: ServerOptions; + // eslint-disable-next-line no-useless-constructor public constructor(private typescriptController: TypeScriptController) {} /** diff --git a/src/Modules/Server/ServerOptions.ts b/src/Modules/Server/ServerOptions.ts index 588fbbe..7a9b525 100644 --- a/src/Modules/Server/ServerOptions.ts +++ b/src/Modules/Server/ServerOptions.ts @@ -1,5 +1,5 @@ // src/Modules/Server/ServerOptions.ts -import { Transform, Type } from 'class-transformer'; +import { Type } from 'class-transformer'; import { IsDefined, IsUUID, ValidateNested } from 'class-validator'; import { Token } from 'typedi'; import { RedisOptions } from '../Redis/RedisOptions'; diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 949c849..8edc2d7 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -121,9 +121,7 @@ const moduleWorker = new Worker( return discoverModuleMap(jobInput); }, { - connection: { - host: workerInput.redisOptions.hostname, - }, + ...workerInput.redisOptions, concurrency: 2, }, ); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index ab7b44b..55874cf 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -8,6 +8,7 @@ import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; import { RedisOptions } from '../Redis/RedisOptions'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; +import { timeout } from '../../Utils/timeout'; const data = getWorkerData(import.meta.url); @@ -23,10 +24,12 @@ await validateOrReject(workerInput); async function transformFile(filePath: string): Promise { logger.info(`Transforming ${filePath}`); + await timeout(50); + return `console.log('helloWorld')`; } -const transpilerWorker = new Worker( +const _transpilerWorker = new Worker( workerInput.queName, async (job) => { const jobInput = plainToClass(TranspilerWorkerJobInput, job.data); @@ -44,9 +47,7 @@ const transpilerWorker = new Worker( }; }, { - connection: { - host: workerInput.redisOptions.hostname, - }, + ...workerInput.redisOptions, concurrency: 2, }, ); diff --git a/src/Modules/TypeScript/transpileWorker.ts b/src/Modules/TypeScript/transpileWorker.ts index 3e8df87..b96a385 100644 --- a/src/Modules/TypeScript/transpileWorker.ts +++ b/src/Modules/TypeScript/transpileWorker.ts @@ -1,4 +1,12 @@ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/restrict-template-expressions */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ // src/Server/Utils/Workers/transpileWorker.ts +/** + * eslint-disable + */ import { readFileSync } from 'fs'; import { dirname } from 'path'; import ts from 'typescript'; From c80aedb8c332429cde960482c8e2aa9a22c2daf7 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Thu, 11 Feb 2021 16:42:47 +0000 Subject: [PATCH 12/30] feat: Getting closer --- .../TypeScript/TypeScriptController.ts | 2 +- .../TypeScript/TypeScriptModuleMapWorker.ts | 4 ++-- .../TypeScript/TypeScriptTranspilerWorker.ts | 6 ++++- src/Modules/TypeScript/WorkerInput.ts | 23 ++++--------------- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 0776a04..4eb8930 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -105,7 +105,7 @@ export class TypeScriptController { ); const workerPath = fileURLToPath(workerPathURI); - await this.moduleMapQueue.createWorkers(workerPath); + await this.transpilerQueue.createWorkers(workerPath); logger.debug( `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 8edc2d7..b537fbd 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -28,7 +28,7 @@ console.log('ModuleMapWorker: ', workerInput); await validateOrReject(workerInput); const moduleMapQue = new Queue(workerInput.queName, { - connection: workerInput.redisOptions, + ...workerInput.redisOptions, }); interface ModuleMap { @@ -121,7 +121,7 @@ const moduleWorker = new Worker( return discoverModuleMap(jobInput); }, { - ...workerInput.redisOptions, + connection: workerInput.redisOptions.connection, concurrency: 2, }, ); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 55874cf..8ec6bf7 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -19,8 +19,12 @@ const workerInput = plainToClass(WorkerInput, { queName: data?.queName as string, }); +console.log('TranspilerWorkerInput: ', workerInput); + await validateOrReject(workerInput); +console.log(workerInput.redisOptions); + async function transformFile(filePath: string): Promise { logger.info(`Transforming ${filePath}`); @@ -47,7 +51,7 @@ const _transpilerWorker = new Worker( }; }, { - ...workerInput.redisOptions, + connection: workerInput.redisOptions.connection, concurrency: 2, }, ); diff --git a/src/Modules/TypeScript/WorkerInput.ts b/src/Modules/TypeScript/WorkerInput.ts index 13dbc76..e1a81ba 100644 --- a/src/Modules/TypeScript/WorkerInput.ts +++ b/src/Modules/TypeScript/WorkerInput.ts @@ -1,27 +1,12 @@ // src/Modules/TypeScript/WorkerInput.ts -import { Transform, Type } from 'class-transformer'; -import { ValidateNested, IsString } from 'class-validator'; -import { logger } from '../../Library/Logger'; +import type { QueueOptions } from 'bullmq'; +import { Type } from 'class-transformer'; +import { IsString } from 'class-validator'; import { RedisOptions } from '../Redis/RedisOptions'; export class WorkerInput { - @ValidateNested() @Type(() => RedisOptions) - @Transform((input) => { - if (typeof input === 'string') { - const parsedJSON = JSON.parse(input) as RedisOptions; - - logger.debug( - `new WorkerInput().redisOptions typeof redisOptions === 'string'`, - parsedJSON, - ); - - return parsedJSON; - } - - return input; - }) - public redisOptions: RedisOptions; + public redisOptions: QueueOptions; /** * Name of the TypeScript Transpiler Que for this worker From 03bf6af92733c4a60bd78d26de3b37f66a54d6c5 Mon Sep 17 00:00:00 2001 From: Kristian Jones Date: Thu, 11 Feb 2021 16:53:46 +0000 Subject: [PATCH 13/30] fix: Fix que --- src/Modules/Queues/Queue.ts | 4 ++++ src/Modules/Server/ServerController.ts | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 700d39d..768018b 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -54,6 +54,8 @@ export class Queue { private async checkActiveJobs(): Promise { const activeJobCount = await this.queue.getActiveCount(); + console.log('Active Jobs: ', activeJobCount); + if (this.hasRun === false) { if (activeJobCount > 0) { this.hasRun = true; @@ -110,6 +112,8 @@ export class Queue { public async createWorkers(workerPath: string): Promise { await this.queue.clean(0, 1000); + console.log('Creating worker: ', workerPath); + for (const _workerThread of Array(cpus().length - 1).fill(0)) { logger.info('Spawning worker'); diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 397fe71..cf6a6b3 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -48,10 +48,10 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - await Promise.all([ - this.typescriptController.createModuleMapWorkers(), - this.typescriptController.createTranspilerWorkers(), - ]); + await this.typescriptController.createModuleMapWorkers(); + await this.typescriptController.createTranspilerWorkers(); + + console.log('Starting Module Task'); await this.typescriptController.createModuleMapTask({ filePath: fileURLToPath( From c325033eec47d7954979139c56ff05d96bfc86f2 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Thu, 11 Feb 2021 18:39:30 +0000 Subject: [PATCH 14/30] feat: Clean up debugging and general worker cleanup --- src/@types/global.d.ts | 1 + src/Library/Logger.ts | 27 +++++- src/Modules/Queues/Queue.ts | 85 +++++++++++++--- src/Modules/Server/ServerController.ts | 8 +- .../TypeScript/TypeScriptController.ts | 2 +- .../TypeScript/TypeScriptModuleMapWorker.ts | 97 ++++++++++++++++--- .../TypeScript/TypeScriptTranspilerWorker.ts | 70 ++++++++++--- src/Modules/TypeScript/WorkerInput.ts | 18 +++- 8 files changed, 250 insertions(+), 58 deletions(-) diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index 07f969f..b5c9988 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -1,6 +1,7 @@ declare namespace NodeJS { interface ProcessEnv { NODE_ENV: import('../Utils/Environment').Environment; + LOG_LEVEL: keyof typeof import('../Library/Logger').LogLevel; } } diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index e648a2a..b53defc 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -1,10 +1,33 @@ // src/Library/Logger.ts import * as winston from 'winston'; +export enum LogLevel { + SILLY = 'silly', + DEBUG = 'debug', + INFO = 'info', + WARN = 'warn', + ERROR = 'error', +} + +function isKeyOf( + obj: Obj, + key: K | string, +): key is K { + return Object.keys(obj).includes(key as string); +} + +function getLevel(value: string): LogLevel { + if (isKeyOf(LogLevel, value)) { + return LogLevel[value]; + } + + return process.env.NODE_ENV === 'production' ? LogLevel.INFO : LogLevel.DEBUG; +} + export const logger = winston.createLogger({ - level: 'debug', + level: getLevel(process.env.LOG_LEVEL), format: winston.format.prettyPrint(), - defaultMeta: { service: 'user-service' }, + defaultMeta: { appName: 'TS-ESWeb' }, transports: [ // // - Write all logs with level `error` and below to `error.log` diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 768018b..75191e2 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -9,7 +9,9 @@ import { spawnWorker } from '@k-foss/ts-worker'; import { cpus } from 'os'; import { logger } from '../../Library/Logger'; import { QueueOptions } from './QueueOptions'; -import { ClassConstructor } from 'class-transformer'; +import { ClassConstructor, plainToClass } from 'class-transformer'; +import { Logger } from 'winston'; +import { WorkerInput } from '../TypeScript/WorkerInput'; /** * Task Queue handled by dedicated `worker_threads` created by @@ -19,13 +21,26 @@ import { ClassConstructor } from 'class-transformer'; * a Redis backend */ export class Queue { + /** + * BullMQ Queue and QueueEvents object + */ public queue: BullMQQueue; public queueEvents: BullMQQueueEvents; + /** + * Array of Child Workers + */ public workers: Worker[] = []; + /** + * Boolean value if a single job has been created. + */ public hasRun = false; + /** + * Node.JS Interval to check if there are any active jobs if not, + * and a job has already been created, the child workers are exited. + */ public checkInterval: NodeJS.Timeout; public options: QueueOptions< @@ -34,6 +49,11 @@ export class Queue { ClassConstructor >; + /** + * Local logger object adding additional metadata + */ + private logger: Logger; + public constructor( options: QueueOptions< QueueName, @@ -45,16 +65,40 @@ export class Queue { this.queueEvents = new BullMQQueueEvents(options.name, options.bullOptions); this.options = options; + + this.logger = logger.child({ + queName: this.queue.name, + }); } + /** + * Terminate all worker threads. + */ private terminateWorkers(): Promise { return Promise.all(this.workers.map((worker) => worker.terminate())); } + /** + * Clean all jobs from Redis/BullMQ + */ + private async cleanQueue(): Promise { + this.logger.debug(`Queue.cleanQueue()`); + + return Promise.all([ + this.queue.clean(0, 1000, 'active'), + this.queue.clean(0, 1000), + ]); + } + + /** + * Check if there are any active jobs/tasks + */ private async checkActiveJobs(): Promise { const activeJobCount = await this.queue.getActiveCount(); - console.log('Active Jobs: ', activeJobCount); + this.logger.debug(`Queue.checkActiveJobs()`, { + activeJobCount, + }); if (this.hasRun === false) { if (activeJobCount > 0) { @@ -67,7 +111,7 @@ export class Queue { if (activeJobCount === 0) { clearInterval(this.checkInterval); - logger.info(`Shutting down workers`); + this.logger.info(`Queue.checkActiveJobs() Shutting down workers`); return this.terminateWorkers(); } @@ -79,10 +123,8 @@ export class Queue { private startWatching(): void { this.checkInterval = setInterval(() => { this.checkActiveJobs().catch((err) => { - logger.error( - `Queue({${this.queue.name}) startWatching error: ${JSON.stringify( - err, - )}`, + this.logger.error( + `Queue.startWatching() startWatching error: ${JSON.stringify(err)}`, ); return this.terminateWorkers(); @@ -110,17 +152,24 @@ export class Queue { * @returns Promise resolving once the workers threads have all been created */ public async createWorkers(workerPath: string): Promise { - await this.queue.clean(0, 1000); + this.logger.debug(`Queue.createWorkers('${workerPath}')`); + + this.logger.debug(`Queue.createWorkers() cleaning old jobs`); + + await this.cleanQueue(); - console.log('Creating worker: ', workerPath); + const workerInput: WorkerInput = { + queueOptions: this.queue.opts, + queName: this.queue.name, + }; for (const _workerThread of Array(cpus().length - 1).fill(0)) { - logger.info('Spawning worker'); + this.logger.info(`Queue.createWorkers() spawning worker`); - const worker = spawnWorker(workerPath, { - redisOptions: JSON.stringify(this.queue.opts), - queName: this.queue.name, - }); + const worker = spawnWorker( + workerPath, + plainToClass(WorkerInput, workerInput), + ); this.workers.push(worker); } @@ -161,17 +210,21 @@ export class Queue { public addTask(input: JobInput): Promise { const job = this.queue.add(this.options.name, input); - logger.debug(`Queue.addTask(${JSON.stringify(input)})`); + this.logger.debug(`Queue.addTask(${JSON.stringify(input)})`); return job; } + /** + * Wait for the result of the specified task + * @param task BullMQ Job entity + */ public async waitForTask(task: Job): Promise { const jobOutput = (await task.waitUntilFinished( this.queueEvents, )) as JobOutput; - logger.debug( + this.logger.debug( `Queue.waitForTask(${task.name}) ${JSON.stringify(jobOutput)}`, ); } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index cf6a6b3..397fe71 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -48,10 +48,10 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - await this.typescriptController.createModuleMapWorkers(); - await this.typescriptController.createTranspilerWorkers(); - - console.log('Starting Module Task'); + await Promise.all([ + this.typescriptController.createModuleMapWorkers(), + this.typescriptController.createTranspilerWorkers(), + ]); await this.typescriptController.createModuleMapTask({ filePath: fileURLToPath( diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 4eb8930..d440003 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -108,7 +108,7 @@ export class TypeScriptController { await this.transpilerQueue.createWorkers(workerPath); logger.debug( - `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, + `TypeScriptController.createTranspilerWorkers() workerPathURI: ${workerPathURI}`, ); // for (const _workerThread of Array(cpus().length - 1).fill(0)) { diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index b537fbd..8740823 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -3,10 +3,10 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; import { Queue, Worker } from 'bullmq'; -import { logger } from '../../Library/Logger'; +import { logger as coreLogger } from '../../Library/Logger'; import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; -import { RedisOptions } from '../Redis/RedisOptions'; +import { threadId } from 'worker_threads'; import * as ts from 'typescript'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { dirname } from 'path'; @@ -14,21 +14,31 @@ import { getTSConfig } from './TSConfig'; import { fileURLToPath, pathToFileURL } from 'url'; import { ResolvedModuleMap } from './ResolvedModuleMap'; -const data = getWorkerData(import.meta.url); +const logger = coreLogger.child({ + workerFile: 'TypeScriptModuleMapWorker.ts', + workerId: threadId, +}); -console.log('ModuleMapWorkerData: ', data); +logger.info(`Worker starting`); -const workerInput = plainToClass(WorkerInput, { - redisOptions: JSON.parse(data.redisOptions) as RedisOptions, - queName: data?.queName as string, +const data = getWorkerData(import.meta.url); + +logger.debug(`Retrieved workerData:`, { + objectName: 'data', + data, }); -console.log('ModuleMapWorker: ', workerInput); +const workerInput = plainToClass(WorkerInput, data); + +logger.debug(`Transformed to class`, { + objectName: 'workerInput', + workerInput, +}); await validateOrReject(workerInput); const moduleMapQue = new Queue(workerInput.queName, { - ...workerInput.redisOptions, + ...workerInput.queueOptions, }); interface ModuleMap { @@ -40,7 +50,9 @@ interface ModuleMap { async function discoverModuleMap( moduleInput: ModuleMapWorkerJobInput, ): Promise { - logger.debug(`discoverModuleMap(${JSON.stringify(moduleInput)})`); + logger.debug(`discoverModuleMap()`, { + params: moduleInput, + }); const rootDir = dirname(moduleInput.filePath); @@ -72,20 +84,50 @@ async function discoverModuleMap( const importedModules = await Promise.all( resolvedArray.map(async ([specifier]) => { + const parentURI = pathToFileURL(moduleInput.filePath); + + logger.silly(`Resolving Module`, { + specifier, + parentURI, + }); + const resolvePathURI = await import.meta.resolve( specifier, - pathToFileURL(moduleInput.filePath).href, + parentURI.href, ); + logger.silly(`Resolved moduleURI`, { + specifier, + parentURI, + resolvePathURI, + }); + const filePath = fileURLToPath(resolvePathURI); let moduleSpecifier: string; if (ts.isExternalModuleNameRelative(specifier)) { + logger.silly(`Specifier is relative`, { + specifier, + parentURI, + resolvePathURI, + }); + moduleSpecifier = filePath; } else { + logger.silly(`Specifier isn't relative`, { + specifier, + parentURI, + resolvePathURI, + }); + moduleSpecifier = specifier; } + logger.silly(`Adding module to Que`, { + specifier, + parentURI, + resolvePathURI, + }); await moduleMapQue.add( workerInput.queName, plainToClass(ModuleMapWorkerJobInput, { @@ -110,20 +152,43 @@ async function discoverModuleMap( const moduleWorker = new Worker( workerInput.queName, async (job) => { - logger.info(`Recieved a task for moduleWorker`); + logger.info(`Recieved a task for moduleWorker`, { + worker: 'moduleWorker', + }); + + logger.debug(`Task input`, { + worker: 'moduleWorker', + jobInput: job.data, + }); const jobInput = plainToClass(ModuleMapWorkerJobInput, job.data); + logger.debug(`Transformed job.data to Class`, { + worker: 'moduleWorker', + objectName: 'jobInput', + jobInput, + }); + await validateOrReject(jobInput); - logger.info(`TypeScript Module Map Worker: filePath: ${jobInput.filePath}`); + logger.debug(`Validated jobInput`, { + worker: 'moduleWorker', + jobInput, + }); + + logger.debug(`Returning promise of discoverModuleMap`, { + worker: 'moduleWorker', + }); return discoverModuleMap(jobInput); }, { - connection: workerInput.redisOptions.connection, - concurrency: 2, + connection: workerInput.queueOptions.connection, + concurrency: 1, }, ); -logger.debug(`TypeScriptModuleMapWorker.ts worker: `, moduleWorker.name); +logger.debug(`Created moduleWorker`, { + objectName: 'moduleWorker', + moduleWorker, +}); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 8ec6bf7..5549913 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -3,44 +3,79 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; import { Worker } from 'bullmq'; -import { logger } from '../../Library/Logger'; +import { logger as coreLogger } from '../../Library/Logger'; import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; -import { RedisOptions } from '../Redis/RedisOptions'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; -import { timeout } from '../../Utils/timeout'; +import { readFile } from 'fs/promises'; +import { threadId } from 'worker_threads'; -const data = getWorkerData(import.meta.url); +const logger = coreLogger.child({ + workerFile: 'TypeScriptTranspilerWorker.ts', + workerId: threadId, +}); -console.log(`TranspilerWorker:`, data, import.meta.url); +logger.info(`Worker starting`); + +const data = getWorkerData(import.meta.url); -const workerInput = plainToClass(WorkerInput, { - redisOptions: JSON.parse(data.redisOptions) as RedisOptions, - queName: data?.queName as string, +logger.debug(`Retrieved workerData:`, { + objectName: 'data', + data, }); -console.log('TranspilerWorkerInput: ', workerInput); +const workerInput = plainToClass(WorkerInput, data); -await validateOrReject(workerInput); +logger.debug(`Transformed to class`, { + objectName: 'workerInput', + workerInput, +}); -console.log(workerInput.redisOptions); +await validateOrReject(workerInput); async function transformFile(filePath: string): Promise { logger.info(`Transforming ${filePath}`); - await timeout(50); + logger.silly(`Loading File`, { + filePath, + }); + + const file = await readFile(filePath); + + logger.debug(`Opened file`, { + filePath, + file, + }); return `console.log('helloWorld')`; } -const _transpilerWorker = new Worker( +const transpilerWorker = new Worker( workerInput.queName, async (job) => { + logger.info(`Recieved a task for transpilerWorker`, { + worker: 'transpilerWorker', + }); + + logger.debug(`Task input`, { + worker: 'transpilerWorker', + jobInput: job.data, + }); + const jobInput = plainToClass(TranspilerWorkerJobInput, job.data); + logger.debug(`Transformed job.data to Class`, { + worker: 'transpilerWorker', + objectName: 'jobInput', + jobInput, + }); + await validateOrReject(jobInput); - logger.info(`TypeScript Transpiler Worker: filePath: ${jobInput.filePath}`); + logger.debug(`Validated jobInput`, { + worker: 'transpilerWorker', + jobInput, + }); const transformedModule = await transformFile(jobInput.filePath); @@ -51,7 +86,12 @@ const _transpilerWorker = new Worker( }; }, { - connection: workerInput.redisOptions.connection, + connection: workerInput.queueOptions.connection, concurrency: 2, }, ); + +logger.debug(`Created transpilerWorker`, { + objectName: 'transpilerWorker', + transpilerWorker, +}); diff --git a/src/Modules/TypeScript/WorkerInput.ts b/src/Modules/TypeScript/WorkerInput.ts index e1a81ba..3d59d1f 100644 --- a/src/Modules/TypeScript/WorkerInput.ts +++ b/src/Modules/TypeScript/WorkerInput.ts @@ -1,12 +1,22 @@ // src/Modules/TypeScript/WorkerInput.ts import type { QueueOptions } from 'bullmq'; -import { Type } from 'class-transformer'; +import { Transform } from 'class-transformer'; import { IsString } from 'class-validator'; -import { RedisOptions } from '../Redis/RedisOptions'; +import { logger } from '../../Library/Logger'; export class WorkerInput { - @Type(() => RedisOptions) - public redisOptions: QueueOptions; + @Transform(({ value }) => { + if (typeof value === 'string') { + logger.debug(`WorkerInput.queueOptions is string`, { + value, + }); + + return JSON.parse(value) as QueueOptions; + } + + return JSON.stringify(value); + }) + public queueOptions: QueueOptions; /** * Name of the TypeScript Transpiler Que for this worker From ef817715083d1ee2b51b97e14c6344d777e9b1a5 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Thu, 11 Feb 2021 19:24:03 +0000 Subject: [PATCH 15/30] chore(cleanup): Remove legacy code --- src/Modules/HMR/HMR.ts | 78 ------- src/Modules/HMR/index.ts | 2 - src/Modules/Server/SSRServer.tsx | 13 -- src/Modules/TypeScript/TSTranspiler.ts | 32 --- src/Modules/TypeScript/TypeScriptConsts.ts | 3 - .../TypeScript/TypeScriptController.ts | 31 --- src/Modules/TypeScript/WorkerController.ts | 211 ------------------ src/Modules/TypeScript/WorkerMessages.ts | 39 ---- src/Modules/TypeScript/index.ts | 2 - src/Modules/TypeScript/transpileWorker.ts | 168 -------------- 10 files changed, 579 deletions(-) delete mode 100644 src/Modules/HMR/HMR.ts delete mode 100644 src/Modules/HMR/index.ts delete mode 100644 src/Modules/TypeScript/TSTranspiler.ts delete mode 100644 src/Modules/TypeScript/TypeScriptConsts.ts delete mode 100644 src/Modules/TypeScript/WorkerController.ts delete mode 100644 src/Modules/TypeScript/WorkerMessages.ts delete mode 100644 src/Modules/TypeScript/index.ts delete mode 100644 src/Modules/TypeScript/transpileWorker.ts diff --git a/src/Modules/HMR/HMR.ts b/src/Modules/HMR/HMR.ts deleted file mode 100644 index 15cdefe..0000000 --- a/src/Modules/HMR/HMR.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ -// src/Server/Modules/HMR/HMR.ts -import type { PathLike } from 'fs'; -import { logger } from '../../Library/Logger'; -import { debounce } from '../../Utils/debounce'; -import { BaseEventEmitter } from '../../Utils/Events'; -import { WorkerController } from '../TypeScript/WorkerController'; - -/** - * HMR Class Event Map - */ -interface HMREventsMap { - fileChanged: { - filePath: PathLike; - }; - - moduleUpdated: string; -} - -type WatchedFileEvents = 'change' | string; - -/** - * Hot Module Reload Controller Class - * this controls the HMR features of this system - */ -class HMRController extends BaseEventEmitter { - /** - * Files to be watched for changes, on file change event it is send to a - * transpiler thread to be transpled and the module graph's entry is updated with the returned output code. - */ - public watchedFiles = new Set(); - - /** - * Creates the fs watcher on all HMRed files - */ - public async createWatcher(): Promise { - const fs = await import('fs'); - - const workerController = await WorkerController.spawnWorkers(2, { - cache: false, - }); - workerController.startPolling(); - - Array.from(this.watchedFiles).map((filePath) => { - const watcher = fs.watch(filePath); - - logger.debug( - `HMRController.createWatcher() watching ${filePath.toString()}`, - ); - - watcher.on( - 'change', - debounce((eventType: WatchedFileEvents) => { - switch (eventType) { - case 'change': - this.emit('fileChanged', { - filePath, - }); - } - }, 1500), - ); - - return watcher; - }); - - this.on('fileChanged', async ({ filePath }) => { - logger.info( - `HMRController Spawned threads. Starting jobs with ${filePath.toString()} as the entrypoint`, - ); - - return workerController - .forceAddJob(filePath.toString()) - .then(() => this.emit('moduleUpdated', filePath.toString())); - }); - } -} - -export const HMR = new HMRController(); diff --git a/src/Modules/HMR/index.ts b/src/Modules/HMR/index.ts deleted file mode 100644 index 75634b2..0000000 --- a/src/Modules/HMR/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -// src/Server/Modules/HMR/index.ts -export { HMR } from './HMR'; diff --git a/src/Modules/Server/SSRServer.tsx b/src/Modules/Server/SSRServer.tsx index cfec77b..69e5465 100644 --- a/src/Modules/Server/SSRServer.tsx +++ b/src/Modules/Server/SSRServer.tsx @@ -71,19 +71,6 @@ export class SSRServer { this.options = opt; } - /** - * Start transpiling Modules from Imports - */ - public async startTranspiler(): Promise { - logger.info(`Starting transpiler for ${this.entrypoint}`); - - logger.debug( - `SSRServer.startTranspiler() entrypoint: ${this.entrypoint} this.options.webRoot: ${this.options.webRoot}`, - ); - - await startWebTranspiler(this.entrypoint); - } - public async createFastifyServer(): Promise { /** * Get All Route Modules. diff --git a/src/Modules/TypeScript/TSTranspiler.ts b/src/Modules/TypeScript/TSTranspiler.ts deleted file mode 100644 index b3b5d64..0000000 --- a/src/Modules/TypeScript/TSTranspiler.ts +++ /dev/null @@ -1,32 +0,0 @@ -// src/Server/Modules/TypeScript/WebCompiler.ts -import { cpus } from 'os'; -import { logger } from '../../Library/Logger'; -import { WorkerController } from './WorkerController'; - -interface TranspilerOptions { - /** - * Number of worker threads to spin up. - */ - threadCount: number; -} - -/** - * Starts a Web Transpiler starting with the specified entrypoint - * @param entryPath Client Entrypoint to App - */ -export async function startWebTranspiler( - filePath: string, - opts: TranspilerOptions = { - threadCount: cpus().length - 1, - }, -): Promise { - const workerController = await WorkerController.spawnWorkers( - opts.threadCount, - ); - - logger.info( - `startWebTranspiler() Spawned threads. Starting jobs with ${filePath} as the entrypoint`, - ); - - await workerController.start(filePath); -} diff --git a/src/Modules/TypeScript/TypeScriptConsts.ts b/src/Modules/TypeScript/TypeScriptConsts.ts deleted file mode 100644 index 019df35..0000000 --- a/src/Modules/TypeScript/TypeScriptConsts.ts +++ /dev/null @@ -1,3 +0,0 @@ -// src/Modules/TypeScript/TypeScriptConsts.ts - -export const transpilerQueName = 'typescriptTranspile'; diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index d440003..201312a 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -74,24 +74,6 @@ export class TypeScriptController { logger.debug( `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, ); - - // eslint-disable-next-line @typescript-eslint/no-misused-promises - // this.moduleMapQueue.queueEvents.on('completed', async (msg) => { - // const job = await this.moduleMapQueue.queue.getJob(msg.jobId); - - // console.log('JobInput: ', job.data); - - // const jobOutput = plainToClass(ResolvedModuleMap, msg.returnvalue); - // const jobInput = plainToClass(ModuleMapWorkerJobInput, job?.data); - - // console.log('MotherFucker', msg.returnvalue); - - // console.log( - // `${jobInput.specifier || jobInput.filePath} resovled ${JSON.stringify( - // jobOutput?.importedModules, - // )}`, - // ); - // }); } /** @@ -110,19 +92,6 @@ export class TypeScriptController { logger.debug( `TypeScriptController.createTranspilerWorkers() workerPathURI: ${workerPathURI}`, ); - - // for (const _workerThread of Array(cpus().length - 1).fill(0)) { - // logger.info('Spawning Transpiler worker'); - - // const { spawnWorker } = await import('@k-foss/ts-worker'); - - // console.log('TranspileQueFucker: ', this.transpilerQueue.name); - - // const worker = spawnWorker(fileURLToPath(workerPathURI), { - // redisOptions: JSON.stringify(this.options.redis), - // queName: this.transpilerQueue.name, - // }); - // } } /** diff --git a/src/Modules/TypeScript/WorkerController.ts b/src/Modules/TypeScript/WorkerController.ts deleted file mode 100644 index edb4330..0000000 --- a/src/Modules/TypeScript/WorkerController.ts +++ /dev/null @@ -1,211 +0,0 @@ -// src/Server/Modules/TypeScript/WorkerController.ts -import { spawnWorker } from '@k-foss/ts-worker'; -import { fileURLToPath } from 'url'; -import { Worker } from 'worker_threads'; -import { logger } from '../../Library/Logger'; -import { BaseEventEmitter } from '../../Utils/Events'; -import { HMR } from '../HMR'; -import { WebModule } from '../WebModule'; -import { webModuleController } from '../WebModule/WebModuleController'; -import { - TranspileWorkerMessage, - TranspileWorkerMessageType, -} from './WorkerMessages'; - -interface WorkerControllerEventMap { - fileTranspiled: unknown; - done: boolean; -} - -interface WorkerThread { - workerThread: Worker; - - ready: boolean; - - online: boolean; -} - -interface SpawnWorkersOptions { - cache: boolean; -} - -export class WorkerController extends BaseEventEmitter { - public workers: WorkerThread[] = []; - public started = false; - public cache = true; - - public get threads(): number { - return this.workers.length; - } - - public get lazyWorkers(): WorkerThread[] { - return this.workers.filter( - ({ ready, online }) => ready === true && online === true, - ); - } - - public get lazyThreads(): number { - return this.lazyWorkers.length; - } - - private pathHistory = new Set(); - private jobQue = new Set(); - - public static async spawnWorkers( - threadCount: number, - { cache } = { cache: true } as SpawnWorkersOptions, - ): Promise { - const controller = new WorkerController(); - controller.cache = cache; - - const workerModulePath = await import.meta.resolve( - './transpileWorker', - import.meta.url, - ); - - for (const _workerThread of Array(threadCount).fill(0)) { - const worker = spawnWorker(fileURLToPath(workerModulePath), {}); - controller.workers.push({ - workerThread: worker, - online: false, - ready: false, - }); - - worker.on('online', () => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const workerThread = controller.workers.find( - ({ workerThread }) => workerThread.threadId === worker.threadId, - )!; - workerThread.ready = true; - workerThread.online = true; - }); - - worker.on('message', controller.handleWorkerMessage(worker)); - - controller.on('done', () => { - // eslint-disable-next-line @typescript-eslint/no-floating-promises - if (controller.cache) worker.terminate(); - }); - } - - return controller; - } - - public addJob(filePath: string): void { - if (this.pathHistory.has(filePath)) { - return; - } else if (this.jobQue.has(filePath)) { - return; - } else if (webModuleController.getModule(filePath)) { - return; - } - - this.jobQue.add(filePath); - this.pathHistory.add(filePath); - } - - public handleWorkerMessage( - worker: Worker, - ): (msg: TranspileWorkerMessage) => void { - return (msg): void => { - switch (msg.type) { - case TranspileWorkerMessageType.PUSH_DEPENDENCY: - if (this.started === false) this.started = true; - - logger.debug( - `WorkerController handleWorkerMessage(): added ${msg.filePath}`, - ); - - this.addJob(msg.filePath); - - break; - case TranspileWorkerMessageType.PUSH_HMR: - HMR.watchedFiles.add(msg.filePath); - - break; - case TranspileWorkerMessageType.PUSH_OUTPUT: - webModuleController.pushModule( - msg.filePath, - new WebModule({ - code: msg.outputCode, - dependencies: new Set(), - filePath: msg.filePath, - }), - ); - - break; - case TranspileWorkerMessageType.READY: - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.workers.find( - ({ workerThread }) => workerThread.threadId === worker.threadId, - )!.ready = true; - - break; - } - }; - } - - public removeJob(filePath: string): void { - this.jobQue.delete(filePath); - } - - public forceAddJob(filePath: string): Promise { - this.jobQue.add(filePath); - - return new Promise((resolve) => { - this.on('done', resolve); - }); - } - - public startPolling(): void { - webModuleController.on('newModule', (msg) => { - if (this.jobQue.has(msg.filePath)) this.removeJob(msg.filePath); - }); - - const poll = setInterval(() => { - const jobQueArray = Array.from(this.jobQue); - - if (this.lazyThreads > 0 && jobQueArray.length > 0) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const lazyWorker = this.lazyWorkers.pop()!; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const filePath = jobQueArray.pop()!; - - if (webModuleController.getModule(filePath) && this.cache) { - this.removeJob(filePath); - return; - } - - this.removeJob(filePath); - lazyWorker.workerThread.postMessage(filePath); - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.workers.find( - ({ workerThread }) => - workerThread.threadId === lazyWorker.workerThread.threadId, - )!.ready = false; - } - - if (this.lazyThreads === this.threads && this.started === true) { - this.emit('done', true); - if (this.cache) { - clearInterval(poll); - } - } - }, 500); - } - - /** - * Starts transpiling a module graph starting with the entry - * @param entrypoint Path to entryfile - */ - public async start(filePath: string): Promise { - this.addJob(filePath); - - this.startPolling(); - - return new Promise((resolve) => { - if (this.cache) this.on('done', resolve); - }); - } -} diff --git a/src/Modules/TypeScript/WorkerMessages.ts b/src/Modules/TypeScript/WorkerMessages.ts deleted file mode 100644 index 0890b0c..0000000 --- a/src/Modules/TypeScript/WorkerMessages.ts +++ /dev/null @@ -1,39 +0,0 @@ -// src/Modules/TypeScript/WorkerMessages.ts -export enum TranspileWorkerMessageType { - READY, - PUSH_OUTPUT, - PUSH_DEPENDENCY, - PUSH_HMR, -} - -interface TranspileWorkerPushOutputMessage { - type: TranspileWorkerMessageType.PUSH_OUTPUT; - - filePath: string; - - outputCode: string; -} - -interface TranspileWorkerPushDependencyMessage { - type: TranspileWorkerMessageType.PUSH_DEPENDENCY; - - specifier: string; - - filePath: string; -} - -interface TranspileWorkerPushHMRMessage { - type: TranspileWorkerMessageType.PUSH_HMR; - - filePath: string; -} - -interface TranspileWorkerReadyMessage { - type: TranspileWorkerMessageType.READY; -} - -export type TranspileWorkerMessage = - | TranspileWorkerPushOutputMessage - | TranspileWorkerReadyMessage - | TranspileWorkerPushHMRMessage - | TranspileWorkerPushDependencyMessage; diff --git a/src/Modules/TypeScript/index.ts b/src/Modules/TypeScript/index.ts deleted file mode 100644 index c4d3586..0000000 --- a/src/Modules/TypeScript/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -// src/Modules/TypeScript/index.ts -export { startWebTranspiler } from './TSTranspiler'; diff --git a/src/Modules/TypeScript/transpileWorker.ts b/src/Modules/TypeScript/transpileWorker.ts deleted file mode 100644 index b96a385..0000000 --- a/src/Modules/TypeScript/transpileWorker.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unsafe-call */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unsafe-return */ -/* eslint-disable @typescript-eslint/restrict-template-expressions */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment */ -// src/Server/Utils/Workers/transpileWorker.ts -/** - * eslint-disable - */ -import { readFileSync } from 'fs'; -import { dirname } from 'path'; -import ts from 'typescript'; -import { fileURLToPath, pathToFileURL } from 'url'; -import { parentPort } from 'worker_threads'; -import { logger } from '../../Library/Logger'; -import { getTransformers } from '../../Library/Transformers'; -import { exportsHandler, objectExport, processNodeReplacement } from './Regex'; -import { getTSConfig } from './TSConfig'; -import { - TranspileWorkerMessage, - TranspileWorkerMessageType, -} from './WorkerMessages'; - -if (parentPort === null) { - throw new Error(`Worker does not have parentPort open`); -} - -/** - * Send the Worker ready for new file message to the Worker Controller - */ -function sendReady(): void { - if (parentPort === null) { - throw new Error(`Worker does not have parentPort open`); - } - - return parentPort.postMessage({ - type: TranspileWorkerMessageType.READY, - } as TranspileWorkerMessage); -} - -/** - * Transpile filePath to brower compatible code - * @param filePath Path to file to Transpile - */ -async function transpilePath(filePath: string): Promise { - const rootDir = dirname(filePath); - - const tsConfig = getTSConfig(filePath); - const defaultOptions = ts.getDefaultCompilerOptions(); - const options = { ...defaultOptions, ...tsConfig }; - - const compilierHost = ts.createCompilerHost({ - ...options, - rootDir, - }); - - const webModulePromises: Promise[] = []; - - compilierHost.readFile = (fileName: string): string => { - const fileContents = readFileSync(fileName); - - const exportVars: string[] = []; - - /** - * https://regex101.com/r/uwAq1N/1 - */ - - let moduleContents = fileContents - ?.toString() - .replace(processNodeReplacement, '$') - .replaceAll(exportsHandler, (test, todo, varName) => { - exportVars.push(varName); - - return `var ${varName}`; - }) - .replace(objectExport, (...args) => { - const { coreCode } = args[args.length - 1]; - - return coreCode.replaceAll('exports.', ''); - }); - - exportVars.map( - (exportVar) => - (moduleContents += `exports.${exportVar} = ${exportVar}\n`), - ); - - return moduleContents; - }; - - /** - * Overwrite the TypeScript write file function so we can intercept the module data - */ - compilierHost.writeFile = ( - _filePath, - contents, - _writeByteOrderMark, - _onError, - sourceFiles, - ): void => { - if (sourceFiles) { - if (!sourceFiles[0]) { - throw new Error('No inital source file. Issue with TS Compile'); - } - - webModulePromises.push( - ...sourceFiles.map(async (sourceFile) => { - parentPort?.postMessage({ - type: TranspileWorkerMessageType.PUSH_OUTPUT, - filePath: sourceFile.fileName, - outputCode: contents.replaceAll('exports.', ''), - } as TranspileWorkerMessage); - - if (sourceFile.resolvedModules) { - for (const [moduleName] of sourceFile.resolvedModules) { - const moduleURLPath = await import.meta.resolve( - moduleName, - pathToFileURL(sourceFile.fileName).href, - ); - - logger.debug( - `transpilerWorker.ts resolvedModuleURL: ${moduleURLPath}`, - ); - - if (moduleURLPath.startsWith('node:')) { - continue; - } - const modulePath = fileURLToPath(moduleURLPath); - - parentPort?.postMessage({ - type: TranspileWorkerMessageType.PUSH_DEPENDENCY, - filePath: modulePath, - specifier: moduleName, - } as TranspileWorkerMessage); - } - } - }), - ); - } - }; - - const compilerProgram = ts.createProgram({ - rootNames: [filePath], - options: { - ...options, - jsxFragmentFactory: 'Fragment', - }, - host: compilierHost, - }); - compilerProgram.emit( - undefined, - undefined, - undefined, - undefined, - await getTransformers(compilerProgram), - ); - - return Promise.all(webModulePromises); -} - -parentPort.on('message', (filePath: string) => { - transpilePath(filePath) - .then(sendReady) - .catch((err) => { - console.error('Transpile has errorer', err); - }); -}); - -sendReady(); From 550e287d640c65a6cee2a9054be1cf5764f9a5b3 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Thu, 11 Feb 2021 20:29:10 +0000 Subject: [PATCH 16/30] feat: Some minor work --- package-lock.json | 55 +++++++++++++++++++ package.json | 1 + src/Modules/Queues/Queue.ts | 39 ++++++++++--- src/Modules/Server/ServerController.ts | 10 ++++ .../TypeScript/TypeScriptController.ts | 16 ++++-- .../TypeScript/TypeScriptModuleMapWorker.ts | 10 +++- 6 files changed, 118 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9358c12..b2df3b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "get-port": "^5.1.1", "graphql": "^15.4.0", "hyperid": "^2.1.0", + "p-event": "^4.2.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3" @@ -4080,6 +4081,28 @@ "node": ">=6" } }, + "node_modules/p-event": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz", + "integrity": "sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ==", + "dependencies": { + "p-timeout": "^3.1.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", @@ -4112,6 +4135,17 @@ "node": ">=6" } }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -8955,6 +8989,19 @@ "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", "dev": true }, + "p-event": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.2.0.tgz", + "integrity": "sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ==", + "requires": { + "p-timeout": "^3.1.0" + } + }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" + }, "p-limit": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", @@ -8978,6 +9025,14 @@ "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + }, "p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", diff --git a/package.json b/package.json index c5f7e66..6484b37 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "get-port": "^5.1.1", "graphql": "^15.4.0", "hyperid": "^2.1.0", + "p-event": "^4.2.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3" diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 75191e2..0c5d5bc 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -54,6 +54,8 @@ export class Queue { */ private logger: Logger; + public startDate = Date.now(); + public constructor( options: QueueOptions< QueueName, @@ -74,8 +76,20 @@ export class Queue { /** * Terminate all worker threads. */ - private terminateWorkers(): Promise { - return Promise.all(this.workers.map((worker) => worker.terminate())); + private terminateWorkers(): Promise< + [ + ...PromiseSettledResult[], + PromiseSettledResult, + PromiseSettledResult, + ] + > { + const workerTeminiations = this.workers.map((worker) => worker.terminate()); + + return Promise.allSettled([ + ...workerTeminiations, + this.queue.close(), + this.queueEvents.close(), + ]); } /** @@ -93,7 +107,14 @@ export class Queue { /** * Check if there are any active jobs/tasks */ - private async checkActiveJobs(): Promise { + private async checkActiveJobs(): Promise< + | [ + ...PromiseSettledResult[], + PromiseSettledResult, + PromiseSettledResult, + ] + | undefined + > { const activeJobCount = await this.queue.getActiveCount(); this.logger.debug(`Queue.checkActiveJobs()`, { @@ -111,7 +132,11 @@ export class Queue { if (activeJobCount === 0) { clearInterval(this.checkInterval); - this.logger.info(`Queue.checkActiveJobs() Shutting down workers`); + this.logger.info( + `Queue.checkActiveJobs() Shutting down workers ${ + (Date.now() - this.startDate) / 1000 + }`, + ); return this.terminateWorkers(); } @@ -207,12 +232,12 @@ export class Queue { * * @returns BullMQ Job object */ - public addTask(input: JobInput): Promise { - const job = this.queue.add(this.options.name, input); + public async addTask(input: JobInput): Promise { + const job = await this.queue.add(this.options.name, input); this.logger.debug(`Queue.addTask(${JSON.stringify(input)})`); - return job; + return job.waitUntilFinished(this.queueEvents); } /** diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 397fe71..4e96afa 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -48,6 +48,8 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); + const timeStart = Date.now(); + await Promise.all([ this.typescriptController.createModuleMapWorkers(), this.typescriptController.createTranspilerWorkers(), @@ -58,5 +60,13 @@ export class ServerController { await import.meta.resolve('../../Web_Test/Imports.ts'), ), }); + await this.typescriptController.waitForModuleMap(); + + const timeEnd = Date.now(); + + const timeMs = timeEnd - timeStart; + const timeS = timeMs / 1000; + + console.log('Done Fuckers', timeS); } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 201312a..be54e99 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,5 +1,4 @@ // src/Modules/TypeScript/TypeScriptController.ts -import { Job } from 'bullmq'; import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; @@ -103,7 +102,7 @@ export class TypeScriptController { */ public createTranspilerTask( jobInput: TranspilerWorkerJobInput, - ): Promise { + ): Promise { const job = this.transpilerQueue.addTask(jobInput); logger.info(`TypeScriptController.createTask()`); @@ -118,9 +117,16 @@ export class TypeScriptController { * * @returns Promise resolving to the created BullMQ Job */ - public createModuleMapTask(jobInput: ModuleMapWorkerJobInput): Promise { - const job = this.moduleMapQueue.addTask(jobInput); + public async createModuleMapTask( + jobInput: ModuleMapWorkerJobInput, + ): Promise { + return this.moduleMapQueue.addTask(jobInput); + } - return job; + public async waitForModuleMap(): Promise { + return Promise.all([ + this.moduleMapQueue.queue.closing, + this.moduleMapQueue.queueEvents.closing, + ]); } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 8740823..68b89ae 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -2,7 +2,7 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; -import { Queue, Worker } from 'bullmq'; +import { Queue, QueueEvents, Worker } from 'bullmq'; import { logger as coreLogger } from '../../Library/Logger'; import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; @@ -41,6 +41,14 @@ const moduleMapQue = new Queue(workerInput.queName, { ...workerInput.queueOptions, }); +const moduleMapQueEvents = new QueueEvents(workerInput.queName, { + ...workerInput.queueOptions, +}); + +logger.silly(`Testing`, { + moduleMapQueEvents, +}); + interface ModuleMap { filePath: string; From 186f89af72e74419c2e230a756ec8e04812f9189 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Thu, 11 Feb 2021 20:35:44 +0000 Subject: [PATCH 17/30] feat: ...IDK --- src/Modules/Queues/Queue.ts | 7 ------- src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 0c5d5bc..dcf5a42 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -131,13 +131,6 @@ export class Queue { if (activeJobCount === 0) { clearInterval(this.checkInterval); - - this.logger.info( - `Queue.checkActiveJobs() Shutting down workers ${ - (Date.now() - this.startDate) / 1000 - }`, - ); - return this.terminateWorkers(); } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 68b89ae..28e037d 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -192,7 +192,7 @@ const moduleWorker = new Worker( }, { connection: workerInput.queueOptions.connection, - concurrency: 1, + concurrency: 2, }, ); From 2456a48796dbceef04f8915f291229232d3c3b57 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Thu, 11 Feb 2021 21:21:28 +0000 Subject: [PATCH 18/30] feat: Work towards perf improvements --- src/Modules/Queues/Queue.ts | 42 +++++++++---------- src/Modules/Server/ServerController.ts | 10 ----- .../TypeScript/TypeScriptController.ts | 7 ---- .../TypeScript/TypeScriptModuleMapWorker.ts | 2 +- .../TypeScript/TypeScriptTranspilerWorker.ts | 2 +- 5 files changed, 22 insertions(+), 41 deletions(-) diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index dcf5a42..d01274f 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -76,20 +76,10 @@ export class Queue { /** * Terminate all worker threads. */ - private terminateWorkers(): Promise< - [ - ...PromiseSettledResult[], - PromiseSettledResult, - PromiseSettledResult, - ] - > { + private async terminateWorkers(): Promise { const workerTeminiations = this.workers.map((worker) => worker.terminate()); - return Promise.allSettled([ - ...workerTeminiations, - this.queue.close(), - this.queueEvents.close(), - ]); + return Promise.all(workerTeminiations); } /** @@ -107,23 +97,23 @@ export class Queue { /** * Check if there are any active jobs/tasks */ - private async checkActiveJobs(): Promise< - | [ - ...PromiseSettledResult[], - PromiseSettledResult, - PromiseSettledResult, - ] - | undefined - > { + private async checkActiveJobs(): Promise { const activeJobCount = await this.queue.getActiveCount(); this.logger.debug(`Queue.checkActiveJobs()`, { activeJobCount, + hasRun: this.hasRun, }); if (this.hasRun === false) { if (activeJobCount > 0) { + this.logger.debug(`Queue.checkActiveJobs() activeJobs more than 0`); + this.hasRun = true; + + this.queueEvents.on('drained', () => { + this.logger.debug(`Queue has been drained?`); + }); } return; @@ -131,6 +121,14 @@ export class Queue { if (activeJobCount === 0) { clearInterval(this.checkInterval); + + const endDate = Date.now(); + const runTime = endDate - this.startDate; + + const endS = runTime / 1000; + + this.logger.debug(`endDate: ${endDate.toString()} ${endS} ${runTime}`); + return this.terminateWorkers(); } } @@ -225,12 +223,12 @@ export class Queue { * * @returns BullMQ Job object */ - public async addTask(input: JobInput): Promise { + public async addTask(input: JobInput): Promise { const job = await this.queue.add(this.options.name, input); this.logger.debug(`Queue.addTask(${JSON.stringify(input)})`); - return job.waitUntilFinished(this.queueEvents); + return job; } /** diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 4e96afa..397fe71 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -48,8 +48,6 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - const timeStart = Date.now(); - await Promise.all([ this.typescriptController.createModuleMapWorkers(), this.typescriptController.createTranspilerWorkers(), @@ -60,13 +58,5 @@ export class ServerController { await import.meta.resolve('../../Web_Test/Imports.ts'), ), }); - await this.typescriptController.waitForModuleMap(); - - const timeEnd = Date.now(); - - const timeMs = timeEnd - timeStart; - const timeS = timeMs / 1000; - - console.log('Done Fuckers', timeS); } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index be54e99..fbcf76c 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -122,11 +122,4 @@ export class TypeScriptController { ): Promise { return this.moduleMapQueue.addTask(jobInput); } - - public async waitForModuleMap(): Promise { - return Promise.all([ - this.moduleMapQueue.queue.closing, - this.moduleMapQueue.queueEvents.closing, - ]); - } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 28e037d..cf6f072 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -196,7 +196,7 @@ const moduleWorker = new Worker( }, ); -logger.debug(`Created moduleWorker`, { +logger.silly(`Created moduleWorker`, { objectName: 'moduleWorker', moduleWorker, }); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 5549913..fdc5ec4 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -91,7 +91,7 @@ const transpilerWorker = new Worker( }, ); -logger.debug(`Created transpilerWorker`, { +logger.silly(`Created transpilerWorker`, { objectName: 'transpilerWorker', transpilerWorker, }); From f706bc38b87b3b6aa7c1e2102980396ec16c315c Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Sun, 14 Feb 2021 19:50:21 +0000 Subject: [PATCH 19/30] feat: Cleanup ModuleMap handler and setup proper logging and tracing --- .devcontainer.json | 2 +- .vscode/launch.json | 5 +- docker-compose.dev.yml | 15 + package-lock.json | 1247 ++++++++++++++++- package.json | 4 +- src/Library/Logger.ts | 20 +- src/Modules/Queues/Queue.ts | 100 +- src/Modules/Redis/RedisController.ts | 13 +- src/Modules/Server/ServerController.ts | 49 +- .../TypeScript/TypeScriptController.ts | 37 +- .../TypeScript/TypeScriptModuleMapWorker.ts | 82 +- src/Utils/Types.ts | 1 + 12 files changed, 1472 insertions(+), 103 deletions(-) create mode 100644 src/Utils/Types.ts diff --git a/.devcontainer.json b/.devcontainer.json index 73ca2c0..81782e8 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -2,7 +2,7 @@ "name": "TS-ESWeb Container", "dockerComposeFile": "./docker-compose.dev.yml", "service": "TSESWeb", - "runServices": ["TSESWeb", "Redis"], + "runServices": ["TSESWeb", "Redis", "Loki", "Grafana"], "settings": { // This dev container does include /bin/bash if you prefer to use it instead of ash. "terminal.integrated.shell.linux": "/bin/bash" diff --git a/.vscode/launch.json b/.vscode/launch.json index 3654830..99cb82e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,7 +16,10 @@ "cwd": "${workspaceFolder}", "runtimeExecutable": "npm", - "runtimeArgs": ["run", "start:debug"] + "runtimeArgs": ["run", "start:debug"], + "env": { + "LOG_LEVEL": "SILLY" + } } ] } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 954ab8f..acc8d8a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -5,6 +5,7 @@ volumes: networks: internalRedis: + internalLoki: services: TSESWeb: @@ -20,8 +21,22 @@ services: - modules:/workspace/node_modules networks: - internalRedis + - internalLoki Redis: image: redis:alpine3.12 networks: - internalRedis + + Loki: + image: grafana/loki:latest + command: -config.file=/etc/loki/local-config.yaml + networks: + - internalLoki + + Grafana: + image: grafana/grafana:latest + ports: + - 3000:3000 + networks: + - internalLoki \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index b2df3b6..6f65cb6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,10 +18,12 @@ "get-port": "^5.1.1", "graphql": "^15.4.0", "hyperid": "^2.1.0", + "ioredis": "^4.22.0", "p-event": "^4.2.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", - "winston": "^3.3.3" + "winston": "^3.3.3", + "winston-loki": "^6.0.0-rc.9" }, "devDependencies": { "@k-foss/ts-esnode": "^2.0.0", @@ -1146,11 +1148,57 @@ "graphql": "^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0" } }, + "node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "optional": true + }, "node_modules/archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=" }, + "node_modules/are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "node_modules/are-we-there-yet/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/are-we-there-yet/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "node_modules/are-we-there-yet/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -1254,6 +1302,14 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" }, + "node_modules/async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/async-retry": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", @@ -1300,6 +1356,55 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "optional": true, + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/bl/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "node_modules/bl/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/boxen": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", @@ -1405,6 +1510,39 @@ "node": ">=8" } }, + "node_modules/btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==", + "bin": { + "btoa": "bin/btoa.js" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "optional": true, + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "optional": true + }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", + "optional": true + }, "node_modules/bullmq": { "version": "1.14.4", "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.14.4.tgz", @@ -1569,6 +1707,12 @@ "fsevents": "~2.1.2" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "optional": true + }, "node_modules/ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -1653,6 +1797,15 @@ "node": ">=0.10.0" } }, + "node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/color": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", @@ -1739,6 +1892,12 @@ "node": ">=8" } }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "optional": true + }, "node_modules/contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -1829,7 +1988,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "dev": true, + "devOptional": true, "dependencies": { "mimic-response": "^1.0.0" }, @@ -1841,7 +2000,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true, + "devOptional": true, "engines": { "node": ">=4.0.0" } @@ -1885,6 +2044,12 @@ "node": ">=0.4.0" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "optional": true + }, "node_modules/denque": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", @@ -1898,6 +2063,18 @@ "resolved": "https://registry.npmjs.org/deprecated-decorator/-/deprecated-decorator-0.1.6.tgz", "integrity": "sha1-AJZjF7ehL+kvPMgx91g68ym4bDc=" }, + "node_modules/detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "optional": true, + "bin": { + "detect-libc": "bin/detect-libc.js" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -1964,7 +2141,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, + "devOptional": true, "dependencies": { "once": "^1.4.0" } @@ -2512,6 +2689,15 @@ "node": ">=0.10.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -2733,6 +2919,12 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -2820,6 +3012,12 @@ "node": ">= 0.6" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "optional": true + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2849,6 +3047,69 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "node_modules/gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "optional": true, + "dependencies": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "node_modules/gauge/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gauge/node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "optional": true, + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gauge/node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "optional": true, + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gauge/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "optional": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/get-intrinsic": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", @@ -2885,6 +3146,12 @@ "node": ">=6" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "optional": true + }, "node_modules/glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -3086,6 +3353,12 @@ "node": ">= 0.4" } }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "optional": true + }, "node_modules/has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -3191,7 +3464,7 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "devOptional": true }, "node_modules/internal-slot": { "version": "1.0.2", @@ -3753,7 +4026,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "dev": true, + "devOptional": true, "engines": { "node": ">=4" } @@ -3773,7 +4046,19 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "devOptional": true + }, + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "optional": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } }, "node_modules/moment": { "version": "2.29.1", @@ -3799,6 +4084,18 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", + "optional": true + }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "optional": true + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -3813,6 +4110,24 @@ "node": ">= 0.6" } }, + "node_modules/node-abi": { + "version": "2.19.3", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz", + "integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==", + "optional": true, + "dependencies": { + "semver": "^5.4.1" + } + }, + "node_modules/node-abi/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "optional": true, + "bin": { + "semver": "bin/semver" + } + }, "node_modules/node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -3868,6 +4183,12 @@ "semver": "bin/semver" } }, + "node_modules/noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=", + "optional": true + }, "node_modules/nopt": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", @@ -3919,11 +4240,32 @@ "node": ">=8" } }, + "node_modules/npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "optional": true, + "dependencies": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -4072,6 +4414,15 @@ "node": ">= 0.8.0" } }, + "node_modules/os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/p-cancelable": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", @@ -4288,6 +4639,46 @@ "node": ">=4" } }, + "node_modules/prebuild-install": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", + "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", + "optional": true, + "dependencies": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.7.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "os-homedir": "^1.0.1", + "pump": "^2.0.1", + "rc": "^1.2.7", + "simple-get": "^2.7.0", + "tar-fs": "^1.13.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/prebuild-install/node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "optional": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -4352,6 +4743,36 @@ "react-is": "^16.8.1" } }, + "node_modules/protobufjs": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", + "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, + "node_modules/protobufjs/node_modules/@types/node": { + "version": "13.13.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.41.tgz", + "integrity": "sha512-qLT9IvHiXJfdrje9VmsLzun7cQ65obsBTmtU3EOnCSLFOoSHx1hpiRHoBnpdbyFqnzqdUUIv81JcEJQCB8un9g==" + }, "node_modules/proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -4428,7 +4849,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, + "devOptional": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -4443,13 +4864,13 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "devOptional": true }, "node_modules/rc/node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -4784,6 +5205,12 @@ "resolved": "https://registry.npmjs.org/semver-store/-/semver-store-0.3.0.tgz", "integrity": "sha512-TcZvGMMy9vodEFSse30lWinkj+JgOBvPn8wRItpQRSayhc+4ssDs335uklkfvQQJgL/WvmHLVj4Ycv2s7QCQMg==" }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "optional": true + }, "node_modules/set-cookie-parser": { "version": "2.4.6", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.4.6.tgz", @@ -4859,7 +5286,38 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", - "dev": true + "devOptional": true + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/simple-get": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", + "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", + "optional": true, + "dependencies": { + "decompress-response": "^3.3.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } }, "node_modules/simple-swizzle": { "version": "0.2.2", @@ -4933,6 +5391,18 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/snappy": { + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", + "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "bindings": "^1.3.1", + "nan": "^2.14.1", + "prebuild-install": "5.3.0" + } + }, "node_modules/sonic-boom": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", @@ -5229,6 +5699,76 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true }, + "node_modules/tar-fs": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", + "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", + "optional": true, + "dependencies": { + "chownr": "^1.0.1", + "mkdirp": "^0.5.1", + "pump": "^1.0.0", + "tar-stream": "^1.1.2" + } + }, + "node_modules/tar-fs/node_modules/pump": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", + "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", + "optional": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "optional": true, + "dependencies": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/tar-stream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "node_modules/tar-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/term-size": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz", @@ -5257,6 +5797,12 @@ "node": ">=6" } }, + "node_modules/to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", + "optional": true + }, "node_modules/to-readable-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", @@ -5335,6 +5881,18 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "optional": true, + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -5650,6 +6208,64 @@ "node": ">= 8" } }, + "node_modules/which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", + "optional": true + }, + "node_modules/wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "optional": true, + "dependencies": { + "string-width": "^1.0.2 || 2" + } + }, + "node_modules/wide-align/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/wide-align/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/wide-align/node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "optional": true, + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/wide-align/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "optional": true, + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/widest-line": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", @@ -5681,6 +6297,21 @@ "node": ">= 6.4.0" } }, + "node_modules/winston-loki": { + "version": "6.0.0-rc.9", + "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.0.0-rc.9.tgz", + "integrity": "sha512-xH0wO8fJDC9kcjfWBLBYgHSbL2ecIYVRa0oIQ4rYlK8+PygfHcmwDOUWdoS5k8sm6oVOZGnpdSyEclZUhhwwlA==", + "dependencies": { + "async-exit-hook": "2.0.1", + "btoa": "^1.2.1", + "protobufjs": "^6.8.8", + "snappy": "^6.1.1", + "winston-transport": "^4.3.0" + }, + "optionalDependencies": { + "snappy": "^6.1.1" + } + }, "node_modules/winston-transport": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", @@ -5790,6 +6421,15 @@ "node": ">= 0.10.0" } }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "optional": true, + "engines": { + "node": ">=0.4" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -6672,11 +7312,59 @@ "tslib": "^1.10.0" } }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "optional": true + }, "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=" }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "optional": true, + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -6758,6 +7446,11 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" }, + "async-exit-hook": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/async-exit-hook/-/async-exit-hook-2.0.1.tgz", + "integrity": "sha512-NW2cX8m1Q7KPA7a5M2ULQeZ2wR5qI5PAbw5L0UOMxdioVk9PMZ0h1TmyZEkPYrCvYjDlFICusOu1dlEKAAeXBw==" + }, "async-retry": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", @@ -6798,6 +7491,57 @@ "integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, + "bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "optional": true, + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "boxen": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", @@ -6884,6 +7628,33 @@ "fill-range": "^7.0.1" } }, + "btoa": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz", + "integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==" + }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "optional": true, + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "optional": true + }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", + "optional": true + }, "bullmq": { "version": "1.14.4", "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.14.4.tgz", @@ -7013,6 +7784,12 @@ "readdirp": "~3.4.0" } }, + "chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "optional": true + }, "ci-info": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", @@ -7081,6 +7858,12 @@ "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "optional": true + }, "color": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", @@ -7158,6 +7941,12 @@ "xdg-basedir": "^4.0.0" } }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "optional": true + }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", @@ -7228,7 +8017,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "dev": true, + "devOptional": true, "requires": { "mimic-response": "^1.0.0" } @@ -7237,7 +8026,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "dev": true + "devOptional": true }, "deep-is": { "version": "0.1.3", @@ -7269,6 +8058,12 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "optional": true + }, "denque": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", @@ -7279,6 +8074,12 @@ "resolved": "https://registry.npmjs.org/deprecated-decorator/-/deprecated-decorator-0.1.6.tgz", "integrity": "sha1-AJZjF7ehL+kvPMgx91g68ym4bDc=" }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "optional": true + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -7335,7 +8136,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, + "devOptional": true, "requires": { "once": "^1.4.0" } @@ -7753,6 +8554,12 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "optional": true + }, "fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -7944,6 +8751,12 @@ "flat-cache": "^3.0.4" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "optional": true + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -8013,6 +8826,12 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "optional": true + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -8036,6 +8855,59 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "optional": true, + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "optional": true + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "optional": true, + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "optional": true, + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "optional": true, + "requires": { + "ansi-regex": "^2.0.0" + } + } + } + }, "get-intrinsic": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.0.1.tgz", @@ -8060,6 +8932,12 @@ "pump": "^3.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", + "optional": true + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -8212,6 +9090,12 @@ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "optional": true + }, "has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -8298,7 +9182,7 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "devOptional": true }, "internal-slot": { "version": "1.0.2", @@ -8735,7 +9619,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "dev": true + "devOptional": true }, "minimatch": { "version": "3.0.4", @@ -8749,7 +9633,16 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "devOptional": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "optional": true, + "requires": { + "minimist": "^1.2.5" + } }, "moment": { "version": "2.29.1", @@ -8769,6 +9662,18 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==", + "optional": true + }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", + "optional": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -8780,6 +9685,23 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "node-abi": { + "version": "2.19.3", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.19.3.tgz", + "integrity": "sha512-9xZrlyfvKhWme2EXFKQhZRp1yNWT/uI1luYPr3sFl+H4keYY4xR+1jO7mvTTijIsHf1M+QDe9uWuKeEpLInIlg==", + "optional": true, + "requires": { + "semver": "^5.4.1" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "optional": true + } + } + }, "node-fetch": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", @@ -8820,6 +9742,12 @@ } } }, + "noop-logger": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", + "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=", + "optional": true + }, "nopt": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", @@ -8861,11 +9789,29 @@ "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", "dev": true }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "optional": true, + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "optional": true + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true + "devOptional": true }, "object-inspect": { "version": "1.8.0", @@ -8983,6 +9929,12 @@ "word-wrap": "^1.2.3" } }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "optional": true + }, "p-cancelable": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", @@ -9139,6 +10091,42 @@ "find-up": "^2.1.0" } }, + "prebuild-install": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", + "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", + "optional": true, + "requires": { + "detect-libc": "^1.0.3", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.0", + "mkdirp": "^0.5.1", + "napi-build-utils": "^1.0.1", + "node-abi": "^2.7.0", + "noop-logger": "^0.1.1", + "npmlog": "^4.0.1", + "os-homedir": "^1.0.1", + "pump": "^2.0.1", + "rc": "^1.2.7", + "simple-get": "^2.7.0", + "tar-fs": "^1.13.0", + "tunnel-agent": "^0.6.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "optional": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -9188,6 +10176,33 @@ "react-is": "^16.8.1" } }, + "protobufjs": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.10.2.tgz", + "integrity": "sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": "^13.7.0", + "long": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "13.13.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.41.tgz", + "integrity": "sha512-qLT9IvHiXJfdrje9VmsLzun7cQ65obsBTmtU3EOnCSLFOoSHx1hpiRHoBnpdbyFqnzqdUUIv81JcEJQCB8un9g==" + } + } + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -9241,7 +10256,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dev": true, + "devOptional": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -9253,13 +10268,13 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "devOptional": true }, "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", - "dev": true + "devOptional": true } } }, @@ -9504,6 +10519,12 @@ "resolved": "https://registry.npmjs.org/semver-store/-/semver-store-0.3.0.tgz", "integrity": "sha512-TcZvGMMy9vodEFSse30lWinkj+JgOBvPn8wRItpQRSayhc+4ssDs335uklkfvQQJgL/WvmHLVj4Ycv2s7QCQMg==" }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "optional": true + }, "set-cookie-parser": { "version": "2.4.6", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.4.6.tgz", @@ -9569,7 +10590,24 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", - "dev": true + "devOptional": true + }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "optional": true + }, + "simple-get": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", + "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", + "optional": true, + "requires": { + "decompress-response": "^3.3.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } }, "simple-swizzle": { "version": "0.2.2", @@ -9629,6 +10667,17 @@ } } }, + "snappy": { + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", + "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", + "optional": true, + "requires": { + "bindings": "^1.3.1", + "nan": "^2.14.1", + "prebuild-install": "5.3.0" + } + }, "sonic-boom": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.3.0.tgz", @@ -9888,6 +10937,77 @@ } } }, + "tar-fs": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", + "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", + "optional": true, + "requires": { + "chownr": "^1.0.1", + "mkdirp": "^0.5.1", + "pump": "^1.0.0", + "tar-stream": "^1.1.2" + }, + "dependencies": { + "pump": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", + "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", + "optional": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, + "tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "optional": true, + "requires": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "optional": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "optional": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "optional": true, + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "term-size": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz", @@ -9910,6 +11030,12 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", "integrity": "sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow==" }, + "to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", + "optional": true + }, "to-readable-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", @@ -9973,6 +11099,15 @@ "tslib": "^1.8.1" } }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "optional": true, + "requires": { + "safe-buffer": "^5.0.1" + } + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -10221,6 +11356,54 @@ "isexe": "^2.0.0" } }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", + "optional": true + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "optional": true, + "requires": { + "string-width": "^1.0.2 || 2" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "optional": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "optional": true + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "optional": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "optional": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, "widest-line": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", @@ -10246,6 +11429,18 @@ "winston-transport": "^4.4.0" } }, + "winston-loki": { + "version": "6.0.0-rc.9", + "resolved": "https://registry.npmjs.org/winston-loki/-/winston-loki-6.0.0-rc.9.tgz", + "integrity": "sha512-xH0wO8fJDC9kcjfWBLBYgHSbL2ecIYVRa0oIQ4rYlK8+PygfHcmwDOUWdoS5k8sm6oVOZGnpdSyEclZUhhwwlA==", + "requires": { + "async-exit-hook": "2.0.1", + "btoa": "^1.2.1", + "protobufjs": "^6.8.8", + "snappy": "^6.1.1", + "winston-transport": "^4.3.0" + } + }, "winston-transport": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", @@ -10328,6 +11523,12 @@ "cssfilter": "0.0.10" } }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "optional": true + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", diff --git a/package.json b/package.json index 6484b37..74b4613 100644 --- a/package.json +++ b/package.json @@ -57,9 +57,11 @@ "get-port": "^5.1.1", "graphql": "^15.4.0", "hyperid": "^2.1.0", + "ioredis": "^4.22.0", "p-event": "^4.2.0", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", - "winston": "^3.3.3" + "winston": "^3.3.3", + "winston-loki": "^6.0.0-rc.9" } } diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index b53defc..58e9028 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -1,5 +1,6 @@ // src/Library/Logger.ts import * as winston from 'winston'; +import LokiTransport from 'winston-loki'; export enum LogLevel { SILLY = 'silly', @@ -26,15 +27,28 @@ function getLevel(value: string): LogLevel { export const logger = winston.createLogger({ level: getLevel(process.env.LOG_LEVEL), - format: winston.format.prettyPrint(), - defaultMeta: { appName: 'TS-ESWeb' }, + defaultMeta: { + appName: 'TS-ESWeb', + get date(): string { + return new Date().toLocaleTimeString(); + }, + }, transports: [ // // - Write all logs with level `error` and below to `error.log` // - Write all logs with level `info` and below to `combined.log` // new winston.transports.Console({ - debugStdout: true, + level: 'silly', + format: winston.format.prettyPrint({ + colorize: true, + }), + }), + new LokiTransport({ + host: 'http://Loki:3100', + format: winston.format.json({}), + labels: { appName: 'TS-ESWeb' }, + json: false, }), ], }); diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index d01274f..355d4c0 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -1,12 +1,12 @@ // src/Modules/Queue/Queue.ts import { Job, + JobsOptions, Queue as BullMQQueue, QueueEvents as BullMQQueueEvents, } from 'bullmq'; import { Worker } from 'worker_threads'; import { spawnWorker } from '@k-foss/ts-worker'; -import { cpus } from 'os'; import { logger } from '../../Library/Logger'; import { QueueOptions } from './QueueOptions'; import { ClassConstructor, plainToClass } from 'class-transformer'; @@ -71,14 +71,33 @@ export class Queue { this.logger = logger.child({ queName: this.queue.name, }); + + this.handleError = (args): void => { + this.logger.info(`Que has errored`, { + ...args, + }); + }; + + this.queueEvents.on('failed', this.handleError); } + private handleError: ( + args: { + jobId: string; + failedReason: string; + prev?: string; + }, + id: string, + ) => void; + /** * Terminate all worker threads. */ private async terminateWorkers(): Promise { const workerTeminiations = this.workers.map((worker) => worker.terminate()); + this.queueEvents.removeListener('drained', this.handleDrained); + return Promise.all(workerTeminiations); } @@ -90,6 +109,10 @@ export class Queue { return Promise.all([ this.queue.clean(0, 1000, 'active'), + this.queue.clean(0, 1000, 'failed'), + this.queue.clean(0, 1000, 'completed'), + this.queue.clean(0, 1000, 'wait'), + this.queue.clean(0, 1000, 'paused'), this.queue.clean(0, 1000), ]); } @@ -97,55 +120,46 @@ export class Queue { /** * Check if there are any active jobs/tasks */ - private async checkActiveJobs(): Promise { - const activeJobCount = await this.queue.getActiveCount(); + private async isRunningJobs(): Promise { + const jobCounts = await Promise.all([ + this.queue.getDelayedCount(), + this.queue.getActiveCount(), + this.queue.getFailedCount(), + this.queue.getFailed(), + ]); - this.logger.debug(`Queue.checkActiveJobs()`, { - activeJobCount, - hasRun: this.hasRun, + this.logger.silly(`jobCounts: `, { + jobCounts, }); - if (this.hasRun === false) { - if (activeJobCount > 0) { - this.logger.debug(`Queue.checkActiveJobs() activeJobs more than 0`); + const anyJobs = jobCounts.some((jobCount) => jobCount > 0); + if (this.hasRun === false) { + if (anyJobs === true) { this.hasRun = true; - - this.queueEvents.on('drained', () => { - this.logger.debug(`Queue has been drained?`); - }); } return; } - if (activeJobCount === 0) { - clearInterval(this.checkInterval); - - const endDate = Date.now(); - const runTime = endDate - this.startDate; - - const endS = runTime / 1000; - - this.logger.debug(`endDate: ${endDate.toString()} ${endS} ${runTime}`); - + if (anyJobs === false) { return this.terminateWorkers(); } } + private handleDrained: () => void; + /** * Start watching workers for active tasks and kill workers upon empty Queue */ private startWatching(): void { - this.checkInterval = setInterval(() => { - this.checkActiveJobs().catch((err) => { - this.logger.error( - `Queue.startWatching() startWatching error: ${JSON.stringify(err)}`, - ); - + this.handleDrained = (): void => { + this.isRunningJobs().catch(() => { return this.terminateWorkers(); }); - }, 500); + }; + + this.queueEvents.on('drained', this.handleDrained); } /** @@ -179,7 +193,7 @@ export class Queue { queName: this.queue.name, }; - for (const _workerThread of Array(cpus().length - 1).fill(0)) { + for (const _workerThread of Array(4).fill(0)) { this.logger.info(`Queue.createWorkers() spawning worker`); const worker = spawnWorker( @@ -187,6 +201,14 @@ export class Queue { plainToClass(WorkerInput, workerInput), ); + worker.on('error', (err) => { + this.logger.error(`Worker thread has errored`, { + err, + }); + + this.logger.info(`TODO: Proper worker error handling`); + }); + this.workers.push(worker); } @@ -223,8 +245,8 @@ export class Queue { * * @returns BullMQ Job object */ - public async addTask(input: JobInput): Promise { - const job = await this.queue.add(this.options.name, input); + public async addTask(input: JobInput, opts?: JobsOptions): Promise { + const job = await this.queue.add(this.options.name, input, opts); this.logger.debug(`Queue.addTask(${JSON.stringify(input)})`); @@ -235,13 +257,9 @@ export class Queue { * Wait for the result of the specified task * @param task BullMQ Job entity */ - public async waitForTask(task: Job): Promise { - const jobOutput = (await task.waitUntilFinished( - this.queueEvents, - )) as JobOutput; - - this.logger.debug( - `Queue.waitForTask(${task.name}) ${JSON.stringify(jobOutput)}`, - ); + public async waitForTask(task: Job): Promise { + this.logger.silly(`waitForTask()`); + + return task.waitUntilFinished(this.queueEvents) as Promise; } } diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index 1b0c419..ce10533 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -1,8 +1,19 @@ // src/Modules/Redis/RedisController.ts -import { Inject } from 'typedi'; +import IORedis from 'ioredis'; +import { Inject, Service } from 'typedi'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; +@Service() export class RedisController { @Inject(serverOptionsToken) public options: ServerOptions; + + /** + * Redis Instance + */ + public IORedis: IORedis.Redis; + + public constructor(options: IORedis.RedisOptions) { + this.IORedis = new IORedis(options); + } } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 397fe71..3cca76c 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -2,9 +2,10 @@ import { plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; import hyperid from 'hyperid'; -import { Service, Container, Inject } from 'typedi'; +import { Container, Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; +import { timeout } from '../../Utils/timeout'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @@ -35,10 +36,13 @@ export class ServerController { ...options, serverId, }); - await validateOrReject(serverOptions); - container.set(serverOptionsToken, serverOptions); + container.set({ + id: serverOptionsToken, + global: true, + value: serverOptions, + }); return container.get(ServerController); } @@ -48,15 +52,44 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - await Promise.all([ - this.typescriptController.createModuleMapWorkers(), - this.typescriptController.createTranspilerWorkers(), - ]); + await this.typescriptController.createModuleMapWorkers(); - await this.typescriptController.createModuleMapTask({ + const entrypointJob = await this.typescriptController.createModuleMapTask({ filePath: fileURLToPath( await import.meta.resolve('../../Web_Test/Imports.ts'), ), }); + + await timeout(60000); + + const jobOutput = await this.typescriptController.waitForJob(entrypointJob); + + const typescriptController = this.typescriptController; + + async function getAllChildModules(filePath: string): Promise { + const job = await typescriptController.getModuleMap(filePath); + + const fullModules = [...job.importedModules]; + + const childModules = job.importedModules ?? []; + + for (const importedModule of childModules) { + if (importedModule === null) { + continue; + } + + fullModules.push(...(await getAllChildModules(importedModule))); + } + + return [filePath, ...fullModules]; + } + + const moduleFiles = await getAllChildModules(jobOutput.filePath); + + logger.silly('Output', moduleFiles); + + logger.info(`ServerController.startTypeScript() jobOutput: `, { + jobOutput, + }); } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index fbcf76c..e55e276 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -1,4 +1,5 @@ // src/Modules/TypeScript/TypeScriptController.ts +import { Job } from 'bullmq'; import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; @@ -103,7 +104,13 @@ export class TypeScriptController { public createTranspilerTask( jobInput: TranspilerWorkerJobInput, ): Promise { - const job = this.transpilerQueue.addTask(jobInput); + logger.silly(`TypeScriptController.createTranspilerTask()`, { + jobInput, + }); + + const job = this.transpilerQueue.addTask(jobInput, { + jobId: jobInput.filePath, + }); logger.info(`TypeScriptController.createTask()`); @@ -119,7 +126,31 @@ export class TypeScriptController { */ public async createModuleMapTask( jobInput: ModuleMapWorkerJobInput, - ): Promise { - return this.moduleMapQueue.addTask(jobInput); + ): Promise { + return this.moduleMapQueue.addTask(jobInput, { + jobId: jobInput.filePath, + }); + } + + public async waitForJob(job: Job): Promise { + return this.moduleMapQueue.waitForTask(job); + } + + public async getModuleMap(filePath: string): Promise { + const job = await this.moduleMapQueue.queue.getJob(filePath); + + if (job?.returnvalue) { + logger.silly(`job?.returnValue: `, { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + returnValue: job.returnvalue, + }); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return job?.returnvalue; + } else { + logger.warn(`getModuleMap(${filePath}) failed`); + } + + throw new Error('Invalid file path'); } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index cf6f072..a9dd024 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -10,7 +10,6 @@ import { threadId } from 'worker_threads'; import * as ts from 'typescript'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { dirname } from 'path'; -import { getTSConfig } from './TSConfig'; import { fileURLToPath, pathToFileURL } from 'url'; import { ResolvedModuleMap } from './ResolvedModuleMap'; @@ -45,10 +44,6 @@ const moduleMapQueEvents = new QueueEvents(workerInput.queName, { ...workerInput.queueOptions, }); -logger.silly(`Testing`, { - moduleMapQueEvents, -}); - interface ModuleMap { filePath: string; @@ -63,10 +58,20 @@ async function discoverModuleMap( }); const rootDir = dirname(moduleInput.filePath); - - const tsConfig = getTSConfig(moduleInput.filePath); const defaultOptions = ts.getDefaultCompilerOptions(); - const options = { ...defaultOptions, ...tsConfig }; + const options: ts.CompilerOptions = { + ...defaultOptions, + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, + }; const compilierHost = ts.createCompilerHost({ ...options, @@ -77,7 +82,6 @@ async function discoverModuleMap( rootNames: [moduleInput.filePath], options: { ...options, - jsxFragmentFactory: 'Fragment', }, host: compilierHost, }); @@ -88,7 +92,19 @@ async function discoverModuleMap( throw new Error('Invalid Source File'); } - const resolvedArray = Array.from(sourceFile.resolvedModules); + logger.silly(`sourceFile: `, { + objectName: 'sourceFile', + sourceFile: sourceFile.fileName, + }); + + const resolvedArray = Array.from( + sourceFile.resolvedModules ?? new Map([]), + ); + + logger.silly(`Resolved modules array`, { + objectName: 'resolvedArray', + resolvedArray, + }); const importedModules = await Promise.all( resolvedArray.map(async ([specifier]) => { @@ -104,6 +120,14 @@ async function discoverModuleMap( parentURI.href, ); + if (resolvePathURI.startsWith('node:')) { + logger.debug(`resolvePathURI is a node: path`, { + objectName: 'resolvePathURI', + resolvePathURI, + }); + return; + } + logger.silly(`Resolved moduleURI`, { specifier, parentURI, @@ -136,24 +160,40 @@ async function discoverModuleMap( parentURI, resolvePathURI, }); - await moduleMapQue.add( - workerInput.queName, - plainToClass(ModuleMapWorkerJobInput, { - filePath, - specifier: moduleSpecifier, - }), - { - jobId: filePath, - }, - ); + + const jobData = plainToClass(ModuleMapWorkerJobInput, { + filePath, + specifier: moduleSpecifier, + }); + + logger.debug(`newJobData: `, { + jobData, + jobId: filePath, + }); + + const job = await moduleMapQue.add(workerInput.queName, jobData, { + jobId: filePath, + lifo: true, + }); + + logger.silly(`Testing123...`, { + test: job.asJSON(), + }); + + // const output = await job.waitUntilFinished(moduleMapQueEvents); + // logger.info(`output shit: `, { + // output, + // }); return filePath; }), ); + logger.silly(`discoverModuleMap(${JSON.stringify(moduleInput)})`); + return { filePath: moduleInput.filePath, - importedModules, + importedModules: importedModules.filter(Boolean) as string[], }; } diff --git a/src/Utils/Types.ts b/src/Utils/Types.ts new file mode 100644 index 0000000..71a58c0 --- /dev/null +++ b/src/Utils/Types.ts @@ -0,0 +1 @@ +// src/Utils/Types.ts From e9f0f3a0e43a3d5119e6e9cf51222058503e8806 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Sun, 14 Feb 2021 19:52:18 +0000 Subject: [PATCH 20/30] style(lint): Fix ESLint issue --- src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index a9dd024..e5b13b3 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -2,7 +2,7 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; -import { Queue, QueueEvents, Worker } from 'bullmq'; +import { Queue, Worker } from 'bullmq'; import { logger as coreLogger } from '../../Library/Logger'; import { WorkerInput } from './WorkerInput'; import { validateOrReject } from 'class-validator'; @@ -40,9 +40,9 @@ const moduleMapQue = new Queue(workerInput.queName, { ...workerInput.queueOptions, }); -const moduleMapQueEvents = new QueueEvents(workerInput.queName, { - ...workerInput.queueOptions, -}); +// const moduleMapQueEvents = new QueueEvents(workerInput.queName, { +// ...workerInput.queueOptions, +// }); interface ModuleMap { filePath: string; From 9468af1b60989f1e27a15e0013280922b1ec5ea6 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Sun, 14 Feb 2021 20:04:26 +0000 Subject: [PATCH 21/30] feat(Queue-EventEmitters): Setup waitForDone --- src/Library/Logger.ts | 4 +--- src/Modules/Queues/Queue.ts | 10 ++++++++++ src/Modules/Server/ServerController.ts | 3 +-- src/Modules/TypeScript/TypeScriptController.ts | 4 ++++ src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 9 ++------- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index 58e9028..0e6a394 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -29,9 +29,6 @@ export const logger = winston.createLogger({ level: getLevel(process.env.LOG_LEVEL), defaultMeta: { appName: 'TS-ESWeb', - get date(): string { - return new Date().toLocaleTimeString(); - }, }, transports: [ // @@ -49,6 +46,7 @@ export const logger = winston.createLogger({ format: winston.format.json({}), labels: { appName: 'TS-ESWeb' }, json: false, + replaceTimestamp: true, }), ], }); diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 355d4c0..540e3a8 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -12,6 +12,8 @@ import { QueueOptions } from './QueueOptions'; import { ClassConstructor, plainToClass } from 'class-transformer'; import { Logger } from 'winston'; import { WorkerInput } from '../TypeScript/WorkerInput'; +import EventEmitter from 'events'; +import pEvent from 'p-event'; /** * Task Queue handled by dedicated `worker_threads` created by @@ -27,6 +29,8 @@ export class Queue { public queue: BullMQQueue; public queueEvents: BullMQQueueEvents; + private events = new EventEmitter(); + /** * Array of Child Workers */ @@ -98,6 +102,8 @@ export class Queue { this.queueEvents.removeListener('drained', this.handleDrained); + this.events.emit('done'); + return Promise.all(workerTeminiations); } @@ -262,4 +268,8 @@ export class Queue { return task.waitUntilFinished(this.queueEvents) as Promise; } + + public async waitForDone(): Promise { + return pEvent(this.events, 'done'); + } } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 3cca76c..8224a5c 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -5,7 +5,6 @@ import hyperid from 'hyperid'; import { Container, Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; -import { timeout } from '../../Utils/timeout'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @@ -60,7 +59,7 @@ export class ServerController { ), }); - await timeout(60000); + await this.typescriptController.waitForModuleMapDone(); const jobOutput = await this.typescriptController.waitForJob(entrypointJob); diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index e55e276..df2a33b 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -153,4 +153,8 @@ export class TypeScriptController { throw new Error('Invalid file path'); } + + public waitForModuleMapDone(): Promise { + return this.moduleMapQueue.waitForDone(); + } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index e5b13b3..7c611d1 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -101,11 +101,6 @@ async function discoverModuleMap( sourceFile.resolvedModules ?? new Map([]), ); - logger.silly(`Resolved modules array`, { - objectName: 'resolvedArray', - resolvedArray, - }); - const importedModules = await Promise.all( resolvedArray.map(async ([specifier]) => { const parentURI = pathToFileURL(moduleInput.filePath); @@ -177,7 +172,7 @@ async function discoverModuleMap( }); logger.silly(`Testing123...`, { - test: job.asJSON(), + test: job.id, }); // const output = await job.waitUntilFinished(moduleMapQueEvents); @@ -232,7 +227,7 @@ const moduleWorker = new Worker( }, { connection: workerInput.queueOptions.connection, - concurrency: 2, + concurrency: 3, }, ); From 304360cafb7a32e7fef67f1ede1a45cab04834c8 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Sun, 14 Feb 2021 20:24:45 +0000 Subject: [PATCH 22/30] chore: Improve threads usage. --- src/Modules/Queues/Queue.ts | 6 ++++-- src/Modules/TypeScript/TypeScriptModuleMapWorker.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 540e3a8..934de3e 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -100,7 +100,7 @@ export class Queue { private async terminateWorkers(): Promise { const workerTeminiations = this.workers.map((worker) => worker.terminate()); - this.queueEvents.removeListener('drained', this.handleDrained); + // this.queueEvents.removeListener('drained', this.handleDrained); this.events.emit('done'); @@ -149,6 +149,8 @@ export class Queue { } if (anyJobs === false) { + logger.silly('Terminating workers'); + return this.terminateWorkers(); } } @@ -199,7 +201,7 @@ export class Queue { queName: this.queue.name, }; - for (const _workerThread of Array(4).fill(0)) { + for (const _workerThread of Array(6).fill(0)) { this.logger.info(`Queue.createWorkers() spawning worker`); const worker = spawnWorker( diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 7c611d1..a788c67 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -227,7 +227,7 @@ const moduleWorker = new Worker( }, { connection: workerInput.queueOptions.connection, - concurrency: 3, + concurrency: 6, }, ); From 71466866cf369ecf37da91e2f39b88dd768e2e74 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Mon, 22 Feb 2021 14:58:21 +0000 Subject: [PATCH 23/30] feat: Continue development of prod ready module --- src/Library/Fastify.test.ts | 6 - src/Library/Logger.ts | 3 +- src/Modules/Queues/Queue.ts | 63 ++-- src/Modules/Queues/QueueController.ts | 6 +- src/Modules/Queues/QueueOptions.ts | 4 +- src/Modules/Queues/WorkerInput.ts | 48 ++++ src/Modules/Server/ServerController.ts | 48 ++-- .../TypeScript/ModuleMapWorkerJobInput.ts | 17 +- .../TypeScript/TranspilerWorkerJobInput.ts | 12 +- .../TypeScript/TypeScriptController.ts | 22 +- .../TypeScript/TypeScriptModuleMapWorker.ts | 270 +++++++++++------- .../TypeScript/TypeScriptTranspilerWorker.ts | 42 ++- src/Modules/TypeScript/Utils.test.ts | 57 ++++ src/Modules/TypeScript/Utils.ts | 80 ++++++ src/Modules/TypeScript/WorkerInput.ts | 26 -- .../WebAppManifest/WebAppManifest.test.ts | 8 +- src/Modules/WebModule/WebModuleController.ts | 155 ++-------- src/Modules/WebModule/WebModuleJobInput.ts | 26 ++ src/Modules/WebModule/WebModuleRoute.ts | 51 ---- src/Modules/WebModule/WebModuleWorker.ts | 59 ++++ 20 files changed, 597 insertions(+), 406 deletions(-) create mode 100644 src/Modules/Queues/WorkerInput.ts create mode 100644 src/Modules/TypeScript/Utils.test.ts create mode 100644 src/Modules/TypeScript/Utils.ts delete mode 100644 src/Modules/TypeScript/WorkerInput.ts create mode 100644 src/Modules/WebModule/WebModuleJobInput.ts delete mode 100644 src/Modules/WebModule/WebModuleRoute.ts create mode 100644 src/Modules/WebModule/WebModuleWorker.ts diff --git a/src/Library/Fastify.test.ts b/src/Library/Fastify.test.ts index c5a43fa..a04fcd0 100644 --- a/src/Library/Fastify.test.ts +++ b/src/Library/Fastify.test.ts @@ -34,15 +34,9 @@ export class FastifyTestSuite extends TestSuite { 'fastifyRoutes[0].options.method === GET', ); - logger.debug(`HelloTest`); - const webServer = fastify(); - logger.debug(`HelloTest`, webServer); - fastifyRoutes.map((route) => { - logger.debug(`Fastify Route: `, route); - return webServer.route({ ...route.options, handler: async (...params) => route.handler(...params), diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index 0e6a394..c30e8ef 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -44,8 +44,7 @@ export const logger = winston.createLogger({ new LokiTransport({ host: 'http://Loki:3100', format: winston.format.json({}), - labels: { appName: 'TS-ESWeb' }, - json: false, + json: true, replaceTimestamp: true, }), ], diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index 934de3e..c34c505 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -11,7 +11,7 @@ import { logger } from '../../Library/Logger'; import { QueueOptions } from './QueueOptions'; import { ClassConstructor, plainToClass } from 'class-transformer'; import { Logger } from 'winston'; -import { WorkerInput } from '../TypeScript/WorkerInput'; +import { WorkerInput } from './WorkerInput'; import EventEmitter from 'events'; import pEvent from 'p-event'; @@ -22,7 +22,7 @@ import pEvent from 'p-event'; * The Queue is handled by [`bullmq`](https://www.npmjs.com/package/bullmq) with * a Redis backend */ -export class Queue { +export class Queue { /** * BullMQ Queue and QueueEvents object */ @@ -34,12 +34,12 @@ export class Queue { /** * Array of Child Workers */ - public workers: Worker[] = []; + private workers: Worker[] = []; /** * Boolean value if a single job has been created. */ - public hasRun = false; + private hasRun = false; /** * Node.JS Interval to check if there are any active jobs if not, @@ -47,25 +47,15 @@ export class Queue { */ public checkInterval: NodeJS.Timeout; - public options: QueueOptions< - QueueName, - ClassConstructor, - ClassConstructor - >; + public options: QueueOptions>; /** * Local logger object adding additional metadata */ private logger: Logger; - public startDate = Date.now(); - public constructor( - options: QueueOptions< - QueueName, - ClassConstructor, - ClassConstructor - >, + options: QueueOptions>, ) { this.queue = new BullMQQueue(options.name, options.bullOptions); this.queueEvents = new BullMQQueueEvents(options.name, options.bullOptions); @@ -76,7 +66,7 @@ export class Queue { queName: this.queue.name, }); - this.handleError = (args): void => { + this.handleError = (...args): void => { this.logger.info(`Que has errored`, { ...args, }); @@ -123,16 +113,19 @@ export class Queue { ]); } - /** - * Check if there are any active jobs/tasks - */ - private async isRunningJobs(): Promise { - const jobCounts = await Promise.all([ + private async getCounts(): Promise { + return Promise.all([ this.queue.getDelayedCount(), this.queue.getActiveCount(), this.queue.getFailedCount(), - this.queue.getFailed(), ]); + } + + /** + * Check if there are any active jobs/tasks + */ + private async isRunningJobs(): Promise { + const jobCounts = await this.getCounts(); this.logger.silly(`jobCounts: `, { jobCounts, @@ -163,6 +156,8 @@ export class Queue { private startWatching(): void { this.handleDrained = (): void => { this.isRunningJobs().catch(() => { + this.logger.silly('Error isRunningJobs, terminating workers'); + return this.terminateWorkers(); }); }; @@ -184,30 +179,36 @@ export class Queue { * const workerPath = fileURLToPath(workerPathURI); * * await moduleMapQueue.createWorkers(workerPath); - * * ``` * * @returns Promise resolving once the workers threads have all been created */ - public async createWorkers(workerPath: string): Promise { + public async createWorkers( + workerPath: string, + workerCount = 3, + ): Promise { this.logger.debug(`Queue.createWorkers('${workerPath}')`); this.logger.debug(`Queue.createWorkers() cleaning old jobs`); await this.cleanQueue(); - const workerInput: WorkerInput = { + const workerInputParams: WorkerInput = { queueOptions: this.queue.opts, queName: this.queue.name, + workerCount, }; - for (const _workerThread of Array(6).fill(0)) { + this.logger.silly(`workerInputParams: `, { + workerInputParams, + }); + + const workerInput = plainToClass(WorkerInput, workerInputParams); + + for (const _workerThread of Array(workerInputParams.workerCount).fill(0)) { this.logger.info(`Queue.createWorkers() spawning worker`); - const worker = spawnWorker( - workerPath, - plainToClass(WorkerInput, workerInput), - ); + const worker = spawnWorker(workerPath, workerInput); worker.on('error', (err) => { this.logger.error(`Worker thread has errored`, { diff --git a/src/Modules/Queues/QueueController.ts b/src/Modules/Queues/QueueController.ts index 3db3a30..536c7f5 100644 --- a/src/Modules/Queues/QueueController.ts +++ b/src/Modules/Queues/QueueController.ts @@ -25,17 +25,15 @@ export class QueueController { * Create a new Queue * @param queKey Queue key */ - public createQueue( + public createQueue( name: QueueName, inputClass: ClassConstructor, - outputClass: ClassConstructor, - ): Queue { + ): Queue { const bullOptions = this.createBullOptions(); return new Queue({ name, bullOptions, - outputClass, inputClass, }); } diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts index e239bde..d0d5a5a 100644 --- a/src/Modules/Queues/QueueOptions.ts +++ b/src/Modules/Queues/QueueOptions.ts @@ -2,14 +2,12 @@ import { IsDefined, IsString } from 'class-validator'; import { QueueOptions as BullMQOptions } from 'bullmq'; -export class QueueOptions { +export class QueueOptions { @IsString() @IsDefined() public name: QueueName; public bullOptions: BullMQOptions; - public outputClass: JobOutput; - public inputClass: JobInput; } diff --git a/src/Modules/Queues/WorkerInput.ts b/src/Modules/Queues/WorkerInput.ts new file mode 100644 index 0000000..3fa751b --- /dev/null +++ b/src/Modules/Queues/WorkerInput.ts @@ -0,0 +1,48 @@ +// src/Modules/TypeScript/WorkerInput.ts +import type { QueueOptions } from 'bullmq'; +import { plainToClass, Transform } from 'class-transformer'; +import { IsInt, IsString, validateOrReject } from 'class-validator'; +import { logger } from '../../Library/Logger'; + +export class WorkerInput { + @Transform(({ value }) => { + if (typeof value === 'string') { + logger.debug(`WorkerInput.queueOptions is string`, { + value, + }); + + return JSON.parse(value) as QueueOptions; + } + + return JSON.stringify(value); + }) + public queueOptions: QueueOptions; + + /** + * Name of the TypeScript Transpiler Que for this worker + */ + @IsString() + public queName: string; + + /** + * Number of workers spawned + */ + @IsInt() + @Transform(({ value }) => { + if (typeof value === 'number') { + return `${value}`; + } + + return parseInt(value); + }) + public workerCount: number; + + public static async createWorkerInput( + inputParams: Partial, + ): Promise { + const jobInput = plainToClass(WorkerInput, inputParams); + await validateOrReject(jobInput); + + return jobInput; + } +} diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 8224a5c..ed49e0d 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -6,6 +6,7 @@ import { Container, Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; +import { WebModuleController } from '../WebModule/WebModuleController'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @Service() @@ -14,7 +15,10 @@ export class ServerController { public options: ServerOptions; // eslint-disable-next-line no-useless-constructor - public constructor(private typescriptController: TypeScriptController) {} + public constructor( + private typescriptController: TypeScriptController, + private webModuleController: WebModuleController, + ) {} /** * Create a new ServerController with the provided Configuration within the TypeDi Container @@ -51,7 +55,11 @@ export class ServerController { `ServerController.startTypeScript() starting TypeScript Module Map`, ); - await this.typescriptController.createModuleMapWorkers(); + await Promise.all([ + this.typescriptController.createModuleMapWorkers(), + this.webModuleController.spawnWebModuleWorkers(), + this.typescriptController.createTranspilerWorkers(), + ]); const entrypointJob = await this.typescriptController.createModuleMapTask({ filePath: fileURLToPath( @@ -61,34 +69,28 @@ export class ServerController { await this.typescriptController.waitForModuleMapDone(); - const jobOutput = await this.typescriptController.waitForJob(entrypointJob); + await this.typescriptController.waitForJob(entrypointJob); - const typescriptController = this.typescriptController; + // const typescriptController = this.typescriptController; - async function getAllChildModules(filePath: string): Promise { - const job = await typescriptController.getModuleMap(filePath); + // async function getAllChildModules(filePath: string): Promise { + // const job = await typescriptController.getModuleMap(filePath); - const fullModules = [...job.importedModules]; + // const fullModules = [...job.importedModules]; - const childModules = job.importedModules ?? []; + // const childModules = job.importedModules ?? []; - for (const importedModule of childModules) { - if (importedModule === null) { - continue; - } + // for (const importedModule of childModules) { + // if (importedModule === null) { + // continue; + // } - fullModules.push(...(await getAllChildModules(importedModule))); - } + // fullModules.push(...(await getAllChildModules(importedModule))); + // } - return [filePath, ...fullModules]; - } + // return [filePath, ...fullModules]; + // } - const moduleFiles = await getAllChildModules(jobOutput.filePath); - - logger.silly('Output', moduleFiles); - - logger.info(`ServerController.startTypeScript() jobOutput: `, { - jobOutput, - }); + // const moduleFiles = await getAllChildModules(jobOutput.filePath); } } diff --git a/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts b/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts index a0e2706..06a1b2a 100644 --- a/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts +++ b/src/Modules/TypeScript/ModuleMapWorkerJobInput.ts @@ -1,5 +1,7 @@ // src/Modules/TypeScript/ModuleMapWorkerJobInput.ts -import { IsOptional, IsString } from 'class-validator'; +import { plainToClass } from 'class-transformer'; +import { IsOptional, IsString, validateOrReject } from 'class-validator'; +import { logger } from '../../Library/Logger'; export class ModuleMapWorkerJobInput { @IsString() @@ -8,4 +10,17 @@ export class ModuleMapWorkerJobInput { @IsString() @IsOptional() public specifier?: string; + + public static async createModuleMapJobInput( + inputParams: Partial, + ): Promise { + logger.silly(`ModuleMapWorkerJobInput.createModuleMapJobInput()`, { + inputParams, + }); + + const jobInput = plainToClass(ModuleMapWorkerJobInput, inputParams); + await validateOrReject(jobInput); + + return jobInput; + } } diff --git a/src/Modules/TypeScript/TranspilerWorkerJobInput.ts b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts index 173fd76..75dc83b 100644 --- a/src/Modules/TypeScript/TranspilerWorkerJobInput.ts +++ b/src/Modules/TypeScript/TranspilerWorkerJobInput.ts @@ -1,7 +1,17 @@ // src/Modules/TypeScript/TranspilerWorkerJobInput.ts -import { IsString } from 'class-validator'; +import { plainToClass } from 'class-transformer'; +import { IsString, validateOrReject } from 'class-validator'; export class TranspilerWorkerJobInput { @IsString() public filePath: string; + + public static async createTranspilerWorkerJobInput( + inputParams: Partial, + ): Promise { + const jobInput = plainToClass(TranspilerWorkerJobInput, inputParams); + await validateOrReject(jobInput); + + return jobInput; + } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index df2a33b..545a51f 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -8,7 +8,6 @@ import { QueueController } from '../Queues/QueueController'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { ResolvedModuleMap } from './ResolvedModuleMap'; -import { TranspiledModuleOutput } from './TranspiledModuleOutput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; @Service() @@ -18,18 +17,13 @@ export class TypeScriptController { */ private transpilerQueue: Queue< 'typescriptTranspiler', - TranspilerWorkerJobInput, - TranspiledModuleOutput + TranspilerWorkerJobInput >; /** * Module Map Que and Workers for determining all imported and related modules and files */ - private moduleMapQueue: Queue< - 'typescriptModuleMap', - ModuleMapWorkerJobInput, - ResolvedModuleMap - >; + private moduleMapQueue: Queue<'typescriptModuleMap', ModuleMapWorkerJobInput>; public constructor( @Inject(serverOptionsToken) @@ -45,13 +39,21 @@ export class TypeScriptController { this.transpilerQueue = this.queueController.createQueue( typescriptTranspilerKey, TranspilerWorkerJobInput, - TranspiledModuleOutput, ); + this.transpilerQueue.queueEvents.on('completed', (job) => { + logger.silly(`Transpiler Que Responds with`, { + jobId: job.jobId, + labels: { + queue: 'transpilerQueue', + appName: 'TS-ESWeb', + }, + }); + }); + this.moduleMapQueue = this.queueController.createQueue( moduleMapQueKey, ModuleMapWorkerJobInput, - ResolvedModuleMap, ); } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index a788c67..898881f 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -1,89 +1,99 @@ // src/Modules/TypeScript/TypeScriptModuleMapWorker.ts import { getWorkerData } from '@k-foss/ts-worker'; -import '../../Utils/Setup'; -import { plainToClass } from 'class-transformer'; import { Queue, Worker } from 'bullmq'; -import { logger as coreLogger } from '../../Library/Logger'; -import { WorkerInput } from './WorkerInput'; -import { validateOrReject } from 'class-validator'; -import { threadId } from 'worker_threads'; -import * as ts from 'typescript'; -import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { dirname } from 'path'; +import * as ts from 'typescript'; import { fileURLToPath, pathToFileURL } from 'url'; -import { ResolvedModuleMap } from './ResolvedModuleMap'; +import { threadId } from 'worker_threads'; +import { logger as coreLogger } from '../../Library/Logger'; +import { envMode } from '../../Utils/Environment'; +import '../../Utils/Setup'; +import { WorkerInput } from '../Queues/WorkerInput'; +import { WebModuleJobInput } from '../WebModule/WebModuleJobInput'; +import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; +import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; +import { createTypeScriptProgram, isCommonJSImportSplit } from './Utils'; const logger = coreLogger.child({ - workerFile: 'TypeScriptModuleMapWorker.ts', - workerId: threadId, + labels: { worker: 'TypeScriptModuleMapWorker.ts', workerId: threadId }, }); logger.info(`Worker starting`); -const data = getWorkerData(import.meta.url); +const workerInput = await WorkerInput.createWorkerInput( + getWorkerData(import.meta.url), +); -logger.debug(`Retrieved workerData:`, { - objectName: 'data', - data, +logger.silly(`workerInput`, { + workerInput, }); -const workerInput = plainToClass(WorkerInput, data); - -logger.debug(`Transformed to class`, { - objectName: 'workerInput', - workerInput, +const webModuleQue = new Queue('webModuleQueue', { + ...workerInput.queueOptions, }); -await validateOrReject(workerInput); +const transpilerQue = new Queue('typescriptTranspiler', { + ...workerInput.queueOptions, +}); const moduleMapQue = new Queue(workerInput.queName, { ...workerInput.queueOptions, }); -// const moduleMapQueEvents = new QueueEvents(workerInput.queName, { -// ...workerInput.queueOptions, -// }); - -interface ModuleMap { - filePath: string; - - importedModules: string[]; -} - +/** + * Discover all imported modules and add to the TypeScript Module Map + * @param moduleInput Input Params + * @returns Promise resolving to void once completed + */ async function discoverModuleMap( moduleInput: ModuleMapWorkerJobInput, -): Promise { - logger.debug(`discoverModuleMap()`, { +): Promise { + const moduleMapLogger = logger.child({ + labels: { + filePath: moduleInput.filePath, + worker: 'TypeScriptModuleMapWorker.ts', + workerId: threadId, + }, + }); + + moduleMapLogger.debug(`discoverModuleMap()`, { params: moduleInput, }); const rootDir = dirname(moduleInput.filePath); - const defaultOptions = ts.getDefaultCompilerOptions(); - const options: ts.CompilerOptions = { - ...defaultOptions, - jsxFragmentFactory: 'Fragment', - target: ts.ScriptTarget.ESNext, - module: ts.ModuleKind.ESNext, - moduleResolution: ts.ModuleResolutionKind.NodeJs, - allowJs: true, - checkJs: false, - noEmit: true, - noEmitHelpers: true, - sourceMap: false, - inlineSourceMap: false, - }; - const compilierHost = ts.createCompilerHost({ - ...options, + const compilerProgram = await createTypeScriptProgram({ rootDir, + rootNames: [moduleInput.filePath], + compilerOptions: { + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, + }, }); - const compilerProgram = ts.createProgram({ + moduleMapLogger.silly(`compilerProgram created`, { + rootDir, rootNames: [moduleInput.filePath], - options: { - ...options, + compilerOptions: { + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, }, - host: compilierHost, }); const sourceFile = compilerProgram.getSourceFile(moduleInput.filePath); @@ -92,7 +102,7 @@ async function discoverModuleMap( throw new Error('Invalid Source File'); } - logger.silly(`sourceFile: `, { + moduleMapLogger.silly(`sourceFile: `, { objectName: 'sourceFile', sourceFile: sourceFile.fileName, }); @@ -101,11 +111,64 @@ async function discoverModuleMap( sourceFile.resolvedModules ?? new Map([]), ); + moduleMapLogger.silly(`resolvedArray`, { + resolvedArray, + }); + + if (isCommonJSImportSplit(resolvedArray) && moduleInput.specifier) { + moduleMapLogger.silly(`Module has two resolvedModules`, { + resolvedArray, + moduleInput, + specifier: moduleInput.specifier, + }); + + try { + const [, outputModule] = resolvedArray.reduce( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-expect-error + // eslint-disable-next-line array-callback-return + (_test, [_filePath, resolvedModule]) => { + if (resolvedModule.packageId?.subModuleName.includes(envMode)) { + return [_filePath, resolvedModule]; + } + }, + ); + + moduleMapLogger.silly(`HelloWorld`, { + outputModule, + }); + + const jobData = await ModuleMapWorkerJobInput.createModuleMapJobInput({ + filePath: outputModule.resolvedFileName, + specifier: moduleInput.specifier, + }); + + moduleMapLogger.silly('Output real module: ', { + outputModule, + jobData, + specifier: moduleInput.specifier, + }); + + await moduleMapQue.add(workerInput.queName, jobData, { + jobId: outputModule.resolvedFileName, + }); + + return; + } catch (err) { + moduleMapLogger.error('Error during reduce', { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + err, + }); + } + } + + moduleMapLogger.silly('Fucker'); + const importedModules = await Promise.all( resolvedArray.map(async ([specifier]) => { const parentURI = pathToFileURL(moduleInput.filePath); - logger.silly(`Resolving Module`, { + moduleMapLogger.silly(`Resolving Module`, { specifier, parentURI, }); @@ -116,14 +179,14 @@ async function discoverModuleMap( ); if (resolvePathURI.startsWith('node:')) { - logger.debug(`resolvePathURI is a node: path`, { + moduleMapLogger.debug(`resolvePathURI is a node: path`, { objectName: 'resolvePathURI', resolvePathURI, }); return; } - logger.silly(`Resolved moduleURI`, { + moduleMapLogger.silly(`Resolved moduleURI`, { specifier, parentURI, resolvePathURI, @@ -133,7 +196,7 @@ async function discoverModuleMap( let moduleSpecifier: string; if (ts.isExternalModuleNameRelative(specifier)) { - logger.silly(`Specifier is relative`, { + moduleMapLogger.silly(`Specifier is relative`, { specifier, parentURI, resolvePathURI, @@ -141,7 +204,7 @@ async function discoverModuleMap( moduleSpecifier = filePath; } else { - logger.silly(`Specifier isn't relative`, { + moduleMapLogger.silly(`Specifier isn't relative`, { specifier, parentURI, resolvePathURI, @@ -150,84 +213,89 @@ async function discoverModuleMap( moduleSpecifier = specifier; } - logger.silly(`Adding module to Que`, { + moduleMapLogger.silly(`Adding module to Que`, { specifier, parentURI, resolvePathURI, }); - const jobData = plainToClass(ModuleMapWorkerJobInput, { + const jobData = await ModuleMapWorkerJobInput.createModuleMapJobInput({ filePath, specifier: moduleSpecifier, }); - logger.debug(`newJobData: `, { + moduleMapLogger.debug(`newJobData: `, { jobData, jobId: filePath, }); + /** + * TODO: Add dep to moduleMap + */ + const job = await moduleMapQue.add(workerInput.queName, jobData, { jobId: filePath, - lifo: true, - }); - - logger.silly(`Testing123...`, { - test: job.id, }); - // const output = await job.waitUntilFinished(moduleMapQueEvents); - // logger.info(`output shit: `, { - // output, - // }); - - return filePath; + return job.id; }), ); - logger.silly(`discoverModuleMap(${JSON.stringify(moduleInput)})`); - - return { + const webModuleJobInputParams: WebModuleJobInput = { filePath: moduleInput.filePath, + specifier: moduleInput.specifier, importedModules: importedModules.filter(Boolean) as string[], }; + + moduleMapLogger.silly(`webModuleJobInputParams`, { + webModuleJobInputParams, + }); + + const [webModuleJobInput, transpilerJobInput] = await Promise.all([ + WebModuleJobInput.createWebModuleJobInput(webModuleJobInputParams), + TranspilerWorkerJobInput.createTranspilerWorkerJobInput({ + filePath: moduleInput.filePath, + }), + ]); + + moduleMapLogger.silly(`discoverModuleMap(${JSON.stringify(moduleInput)})`, { + webModuleJobInput, + }); + + await Promise.all([ + webModuleQue.add('webModuleQueue', webModuleJobInput), + transpilerQue.add('typescriptTranspiler', transpilerJobInput), + ]); } -const moduleWorker = new Worker( +const moduleWorkerLogger = logger.child({ + worker: 'moduleWorker', +}); + +const moduleWorker = new Worker( workerInput.queName, async (job) => { - logger.info(`Recieved a task for moduleWorker`, { - worker: 'moduleWorker', - }); - - logger.debug(`Task input`, { - worker: 'moduleWorker', - jobInput: job.data, - }); + if (moduleWorkerLogger.isSillyEnabled()) { + moduleWorkerLogger.silly(`Task recieved`, { + jobInput: job.data, + }); + } else { + moduleWorkerLogger.info(`Recieved a task for moduleWorker`); + } - const jobInput = plainToClass(ModuleMapWorkerJobInput, job.data); + const jobInput = await ModuleMapWorkerJobInput.createModuleMapJobInput( + job.data, + ); - logger.debug(`Transformed job.data to Class`, { - worker: 'moduleWorker', + moduleWorkerLogger.silly(`Transformed job.data to Class`, { objectName: 'jobInput', jobInput, }); - await validateOrReject(jobInput); - - logger.debug(`Validated jobInput`, { - worker: 'moduleWorker', - jobInput, - }); - - logger.debug(`Returning promise of discoverModuleMap`, { - worker: 'moduleWorker', - }); - - return discoverModuleMap(jobInput); + await discoverModuleMap(jobInput); }, { connection: workerInput.queueOptions.connection, - concurrency: 6, }, ); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index fdc5ec4..872f583 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -1,18 +1,24 @@ -// src/Modules/TypeScript/TypeScriptWorker.ts +// src/Modules/TypeScript/TypeScriptTranspilerWorker.ts import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; import { Worker } from 'bullmq'; import { logger as coreLogger } from '../../Library/Logger'; -import { WorkerInput } from './WorkerInput'; +import { + transpileModule, + ModuleKind, + ScriptTarget, + ScriptKind, + ModuleResolutionKind, +} from 'typescript'; +import { WorkerInput } from '../Queues/WorkerInput'; import { validateOrReject } from 'class-validator'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { readFile } from 'fs/promises'; import { threadId } from 'worker_threads'; const logger = coreLogger.child({ - workerFile: 'TypeScriptTranspilerWorker.ts', - workerId: threadId, + labels: { worker: 'TypeScriptTranspilerWorker.ts', workeId: threadId }, }); logger.info(`Worker starting`); @@ -42,12 +48,24 @@ async function transformFile(filePath: string): Promise { const file = await readFile(filePath); - logger.debug(`Opened file`, { + const transpiledModule = transpileModule(file.toString(), { + compilerOptions: { + allowJs: true, + checkJs: false, + module: ModuleKind.ESNext, + moduleResolution: ModuleResolutionKind.NodeJs, + target: ScriptTarget.ESNext, + isolatedModules: true, + inlineSourceMap: true, + }, + }); + + logger.silly(`Transpiled Module`, { filePath, - file, + transpiledModule: transpiledModule.outputText, }); - return `console.log('helloWorld')`; + return transpiledModule.outputText; } const transpilerWorker = new Worker( @@ -77,17 +95,11 @@ const transpilerWorker = new Worker( jobInput, }); - const transformedModule = await transformFile(jobInput.filePath); - - logger.debug(`transpilerWorker transformedModule: ${transformedModule}`); - - return { - test: 'shti', - }; + return transformFile(jobInput.filePath); }, { connection: workerInput.queueOptions.connection, - concurrency: 2, + concurrency: workerInput.workerCount, }, ); diff --git a/src/Modules/TypeScript/Utils.test.ts b/src/Modules/TypeScript/Utils.test.ts new file mode 100644 index 0000000..0325f5b --- /dev/null +++ b/src/Modules/TypeScript/Utils.test.ts @@ -0,0 +1,57 @@ +// src/Modules/TypeScript/Utils.test.ts +import { TestSuite } from '@k-foss/ts-estests'; +import { dirname } from 'path'; +import * as ts from 'typescript'; +import { fileURLToPath } from 'url'; +import { createTypeScriptProgram } from './Utils'; + +export class TypeScriptUtilsSuite extends TestSuite { + public testName = 'TypeScript Utils Test Suite'; + + public async relativeFile(): Promsie { + const filePathURI = await import.meta.resolve('./TSConfig.ts'); + const filePath = fileURLToPath(filePathURI); + + const program1 = await createTypeScriptProgram({ + rootDir: filePath, + compilerOptions: { + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, + }, + }); + } + + public async relativePath(): Promsie { + const filePathURI = await import.meta.resolve('./TSConfig.ts'); + const filePath = fileURLToPath(filePathURI); + const rootDir = dirname(filePath); + + const program1 = await createTypeScriptProgram({ + rootDir, + compilerOptions: { + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, + }, + }); + } + + public async test(): Promise { + await Promise.all([this.relativePath(), this.relativeFile()]); + } +} diff --git a/src/Modules/TypeScript/Utils.ts b/src/Modules/TypeScript/Utils.ts new file mode 100644 index 0000000..122b0f2 --- /dev/null +++ b/src/Modules/TypeScript/Utils.ts @@ -0,0 +1,80 @@ +// src/Modules/TypeScript/Utils.ts +import { plainToClass, Transform } from 'class-transformer'; +import { + IsObject, + isString, + IsString, + validateOrReject, +} from 'class-validator'; +import { parse } from 'path'; +import * as ts from 'typescript'; + +class CreateTypeScriptProgramInput { + @IsString() + @Transform(({ value }) => { + if (isString(value)) { + const parsedPath = parse(value); + + if (parsedPath.ext.length > 0) { + return parsedPath.dir; + } else { + return value; + } + } + + throw new Error('Invalid string'); + }) + public rootDir: string; + + @IsString({ + each: true, + }) + public rootNames: string[]; + + @IsObject() + public compilerOptions: ts.CompilerOptions; +} + +export async function createTypeScriptProgram( + inputParams: CreateTypeScriptProgramInput, +): Promise { + const input = plainToClass(CreateTypeScriptProgramInput, inputParams); + await validateOrReject(input); + + const defaultOptions = ts.getDefaultCompilerOptions(); + const options: ts.CompilerOptions = { + ...defaultOptions, + ...inputParams.compilerOptions, + }; + + const compilierHost = ts.createCompilerHost({ + ...options, + rootDir: inputParams.rootDir, + }); + + return ts.createProgram({ + rootNames: inputParams.rootNames, + options, + host: compilierHost, + }); +} + +type NonPartial = { [P in keyof T]: T[P] }; + +export function isCommonJSImportSplit( + input: [string, ts.ResolvedModuleFull][], +): input is [ + [string, NonPartial], + [string, NonPartial], +] { + if ( + input.length === 2 && + input.every(([_path, resolvedModule]) => + resolvedModule.packageId?.subModuleName.includes('cjs'), + ) + ) { + return true; + } + + return false; +} diff --git a/src/Modules/TypeScript/WorkerInput.ts b/src/Modules/TypeScript/WorkerInput.ts deleted file mode 100644 index 3d59d1f..0000000 --- a/src/Modules/TypeScript/WorkerInput.ts +++ /dev/null @@ -1,26 +0,0 @@ -// src/Modules/TypeScript/WorkerInput.ts -import type { QueueOptions } from 'bullmq'; -import { Transform } from 'class-transformer'; -import { IsString } from 'class-validator'; -import { logger } from '../../Library/Logger'; - -export class WorkerInput { - @Transform(({ value }) => { - if (typeof value === 'string') { - logger.debug(`WorkerInput.queueOptions is string`, { - value, - }); - - return JSON.parse(value) as QueueOptions; - } - - return JSON.stringify(value); - }) - public queueOptions: QueueOptions; - - /** - * Name of the TypeScript Transpiler Que for this worker - */ - @IsString() - public queName: string; -} diff --git a/src/Modules/WebAppManifest/WebAppManifest.test.ts b/src/Modules/WebAppManifest/WebAppManifest.test.ts index b1f1c95..413bfaa 100644 --- a/src/Modules/WebAppManifest/WebAppManifest.test.ts +++ b/src/Modules/WebAppManifest/WebAppManifest.test.ts @@ -1,9 +1,9 @@ // src/Modules/WebAppManifest/WebAppManifest.test.ts import { TestSuite } from '@k-foss/ts-estests'; import { deepStrictEqual } from 'assert'; -import { classToPlainFromExist, plainToClass } from 'class-transformer'; +import { plainToClass } from 'class-transformer'; import { createFastifyTestServer } from '../../Library/Fastify'; -import { logger } from '../../Library/Logger'; +// import { logger } from '../../Library/Logger'; import '../../Utils/Setup'; import { WebAppManifest } from './WebAppManifest'; import { WebAppManfiestController } from './WebAppManifestController'; @@ -33,7 +33,7 @@ export class WebAppManifestTestSuite extends TestSuite { const webServer = await createFastifyTestServer(); - console.log('Created test server?'); + // console.log('Created test server?'); const manifestRequest = await webServer.inject({ path: '/WebManifest.json', @@ -46,7 +46,7 @@ export class WebAppManifestTestSuite extends TestSuite { `GET Request to /WebManifest returns status code 200 with valid loaded Manifest`, ); - logger.debug(classToPlainFromExist(manifest, manifestRequest)); + // logger.debug(classToPlainFromExist(manifest, manifestRequest)); deepStrictEqual( plainToClass(WebAppManifest, manifestRequest.json()), diff --git a/src/Modules/WebModule/WebModuleController.ts b/src/Modules/WebModule/WebModuleController.ts index 87f86bd..103c47f 100644 --- a/src/Modules/WebModule/WebModuleController.ts +++ b/src/Modules/WebModule/WebModuleController.ts @@ -1,137 +1,36 @@ // src/Modules/WebModule/WebModuleController.ts - -import { WebModule } from './WebModule'; -import { envMode } from '../../Utils/Environment'; -import { BaseEventEmitter } from '../../Utils/Events'; - -interface WebModuleEventModule { - filePath: string; - - webModule: WebModule; -} - -interface WebModuleEventMap { - newModule: WebModuleEventModule; - - updatedModule: string; -} - -/** - * Controller for managing WebModules - */ -class WebModuleController extends BaseEventEmitter { - public modules = new Map(); - public specifierTest = new Map(); - - public relativePathMatcher = /^\.{0,2}[/]/gm; - - /** - * Matcher to extract submodules under /cjs/MODULE_NAME.${NODE_ENV}.js to fix issues with the CJS to ESM transformer and stuff that has an - * entrypoint with an if block with just require - */ - - /** - * Maps specifiers to filePaths - */ - public specifierMap = new Map>(); - - private createEventModule(filePath: string): WebModuleEventModule { - // eslint-disable-next-line @typescript-eslint/unbound-method - const getModule = this.getModule; - - class EventModule implements WebModuleEventModule { - public filePath = filePath; - - public get webModule(): WebModule { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return getModule(this.filePath)!; - } - } - - return new EventModule(); - } - - public pushModule(filePath: string, webModule: WebModule): void { - this.emit('newModule', this.createEventModule(filePath)); - - this.createSpecifiers(filePath); - - this.modules.set(filePath, webModule); - } - - public getModule(filePath: string): WebModule | undefined { - return this.modules.get(filePath); - } - - public createSpecifiers(filePath: string): void { - let specifier: string | undefined; - - const cjsMatch = this.testCJS(filePath); - if (cjsMatch) { - specifier = cjsMatch; - } - - const indexMatch = this.testIndex(filePath); - if (indexMatch) { - specifier = indexMatch; - } - - if (!specifier) { - specifier = filePath; - - const moduleSubPath = /\S+\/(node_modules)\/(?(?!\/)\S+)\.((ts|js)x?)$/gm; - const execResult = moduleSubPath.exec(specifier); - - if (execResult?.groups?.module) { - this.setSpecifier(execResult.groups.module, filePath); - } - - this.setSpecifier(specifier.replace(/\.((ts|js)x?)$/gim, ''), filePath); - } - - this.setSpecifier(specifier, filePath); - } - - public testCJS(filePath: string): string | undefined { - const cjsMatcher = new RegExp( - `(?(?!/)\\S+)/cjs/(\\k)-?(?\\w+)?.${envMode}.js`, - 'gm', +import { Inject, Service } from 'typedi'; +import { fileURLToPath } from 'url'; +import { logger } from '../../Library/Logger'; +import { Queue } from '../Queues/Queue'; +import { QueueController } from '../Queues/QueueController'; +import { WebModuleJobInput } from './WebModuleJobInput'; + +@Service() +export class WebModuleController { + private webModuleQueue: Queue<'webModuleQueue', WebModuleJobInput>; + + public constructor( + @Inject(() => QueueController) + private queueController: QueueController, + ) { + this.webModuleQueue = queueController.createQueue( + 'webModuleQueue', + WebModuleJobInput, ); - - const cjsExec = cjsMatcher.exec(filePath); - - if (cjsExec?.groups?.moduleSub) { - return `${cjsExec?.groups?.module}/${cjsExec?.groups?.moduleSub}`; - } - - return cjsExec?.groups?.module; } - public testIndex(filePath: string): string | undefined { - // eslint-disable-next-line prefer-regex-literals - const indexMatcher = new RegExp(`(?(?!/)[\\w,-]+)/index.js`, 'gm'); - const indexExec = indexMatcher.exec(filePath); + public async spawnWebModuleWorkers(): Promise { + const workerPathURI = await import.meta.resolve('./WebModuleWorker'); + const workerPath = fileURLToPath(workerPathURI); - return indexExec?.groups?.module; - } - - public setSpecifier(specifier: string, filePath: string): void { - // const specifierSet = this.specifierMap.get(filePath); - // if (specifierSet) { - // specifierSet.add(specifier); - // } else { - // this.specifierMap.set(filePath, new Set([filePath, specifier])); - // } + /** + * Create the @K-FOSS/TS-ESWorkers. + */ + await this.webModuleQueue.createWorkers(workerPath, 1); - this.specifierTest.set(specifier, filePath); - } - - public getSpecifiers(filePath: string): string { - const helloWorld = Array.from(this.specifierTest).find( - ([, specifierFilePath]) => specifierFilePath === filePath, + logger.debug( + `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, ); - return helloWorld?.[0] || filePath; } } - -export const webModuleController = new WebModuleController(); diff --git a/src/Modules/WebModule/WebModuleJobInput.ts b/src/Modules/WebModule/WebModuleJobInput.ts new file mode 100644 index 0000000..a669b11 --- /dev/null +++ b/src/Modules/WebModule/WebModuleJobInput.ts @@ -0,0 +1,26 @@ +// src/Modules/WebModules/WebModuleJobInput.ts +import { plainToClass } from 'class-transformer'; +import { IsOptional, IsString, validateOrReject } from 'class-validator'; + +export class WebModuleJobInput { + @IsString() + public filePath: string; + + @IsString() + @IsOptional() + public specifier?: string; + + @IsString({ + each: true, + }) + public importedModules?: string[]; + + public static async createWebModuleJobInput( + inputParams: Partial, + ): Promise { + const jobInput = plainToClass(WebModuleJobInput, inputParams); + await validateOrReject(jobInput); + + return jobInput; + } +} diff --git a/src/Modules/WebModule/WebModuleRoute.ts b/src/Modules/WebModule/WebModuleRoute.ts deleted file mode 100644 index b6adf21..0000000 --- a/src/Modules/WebModule/WebModuleRoute.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ -// src/Server/Modules/WebModule/ModuleRoute.ts -import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; -import { Service } from 'typedi'; -import { Route } from '../../Library/Fastify'; -import { webModuleController } from './WebModuleController'; - -@Service() -export default class WebModuleRoute implements Route { - public options: Route['options'] = { - method: 'GET', - url: '/Static/*', - }; - - // eslint-disable-next-line @typescript-eslint/require-await - public async handler( - this: FastifyInstance, - request: FastifyRequest<{ Params: { '*': string } }>, - reply: FastifyReply, - ): Promise { - const filePath = request.params['*']; - if (!filePath) { - const err = (new Error() as unknown) as { - statusCode: number; - message: string; - }; - err.statusCode = 400; - err.message = 'Invalid file path'; - throw err; - } - - const moduleFilePath = filePath.startsWith('/') ? filePath : `/${filePath}`; - - const webModule = webModuleController.getModule(moduleFilePath); - - if (!webModule) { - const err = (new Error() as unknown) as { - statusCode: number; - message: string; - }; - err.statusCode = 404; - err.message = 'Invalid file'; - throw err; - } - - reply.type('text/javascript'); - reply.header('Service-Worker-Allowed', '/'); - - return webModule.code; - } -} diff --git a/src/Modules/WebModule/WebModuleWorker.ts b/src/Modules/WebModule/WebModuleWorker.ts new file mode 100644 index 0000000..2588844 --- /dev/null +++ b/src/Modules/WebModule/WebModuleWorker.ts @@ -0,0 +1,59 @@ +// src/Modules/WebModule/WebModuleWorker.ts +import { getWorkerData } from '@k-foss/ts-worker'; +import '../../Utils/Setup'; +import { threadId } from 'worker_threads'; +import { logger as coreLogger } from '../../Library/Logger'; +import { plainToClass } from 'class-transformer'; +import { validateOrReject } from 'class-validator'; +import { Queue, Worker } from 'bullmq'; +import { WorkerInput } from '../Queues/WorkerInput'; +import { WebModuleJobInput } from './WebModuleJobInput'; +import { timeout } from '../../Utils/timeout'; + +const logger = coreLogger.child({ + labels: { worker: 'WebModuleWorker.ts', workeId: threadId }, +}); + +logger.info(`Worker starting`); + +const data = getWorkerData(import.meta.url); + +logger.debug(`Retrieved workerData:`, { + objectName: 'data', + data, +}); + +const workerInput = plainToClass(WorkerInput, data); + +logger.debug(`Transformed to class`, { + objectName: 'workerInput', + workerInput, +}); + +await validateOrReject(workerInput); + +const webModuleQue = new Queue(workerInput.queName, { + ...workerInput.queueOptions, +}); + +logger.silly(`webModuleQue has been created`, { + webModuleQue, +}); + +const moduleWorker = new Worker( + workerInput.queName, + async (job) => { + logger.silly(`New Web Module Job:`, { + data: job.data, + }); + + await timeout(60); + }, + { + connection: workerInput.queueOptions.connection, + }, +); + +logger.silly(`moduleWorker`, { + moduleWorker, +}); From 206e827126a34f5b422c168467809c79db3f39e8 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Tue, 23 Feb 2021 02:48:28 +0000 Subject: [PATCH 24/30] feat: Improve flow, store module code in Redis, properly clear redis on clear --- package-lock.json | 14 +- package.json | 2 +- src/Library/Logger.ts | 4 + src/Modules/Queues/Queue.ts | 18 +-- src/Modules/Queues/QueueController.ts | 2 + src/Modules/Queues/QueueOptions.ts | 6 +- src/Modules/Redis/RedisController.ts | 133 +++++++++++++++++- src/Modules/Redis/RedisTypes.ts | 5 + src/Modules/Server/ServerController.ts | 46 ++++-- .../TypeScript/TypeScriptController.ts | 41 +----- .../TypeScript/TypeScriptModuleMapWorker.ts | 81 ++++------- .../TypeScript/TypeScriptTranspilerWorker.ts | 42 +++--- src/Modules/WebModule/WebModule.ts | 5 - src/Modules/WebModule/WebModuleController.ts | 22 ++- src/Modules/WebModule/WebModuleJobInput.ts | 12 +- src/Modules/WebModule/WebModuleMapJobInput.ts | 26 ++++ src/Modules/WebModule/WebModuleWorker.ts | 53 ++++--- 17 files changed, 338 insertions(+), 174 deletions(-) create mode 100644 src/Modules/Redis/RedisTypes.ts create mode 100644 src/Modules/WebModule/WebModuleMapJobInput.ts diff --git a/package-lock.json b/package-lock.json index 6f65cb6..6cff348 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "devDependencies": { "@k-foss/ts-esnode": "^2.0.0", "@k-foss/ts-estests": "^1.0.6", - "@types/node": "^14.14.25", + "@types/node": "^14.14.31", "@types/react": "^17.0.0", "@types/react-dom": "^17.0.0", "@types/workbox-sw": "^4.3.1", @@ -373,9 +373,9 @@ "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" }, "node_modules/@types/node": { - "version": "14.14.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.25.tgz", - "integrity": "sha512-EPpXLOVqDvisVxtlbvzfyqSsFeQxltFbluZNRndIb8tr9KiBnYNLzrc1N3pyKUCww2RNrfHDViqDWWE1LCJQtQ==" + "version": "14.14.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.31.tgz", + "integrity": "sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==" }, "node_modules/@types/node-fetch": { "version": "2.5.7", @@ -6726,9 +6726,9 @@ "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==" }, "@types/node": { - "version": "14.14.25", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.25.tgz", - "integrity": "sha512-EPpXLOVqDvisVxtlbvzfyqSsFeQxltFbluZNRndIb8tr9KiBnYNLzrc1N3pyKUCww2RNrfHDViqDWWE1LCJQtQ==" + "version": "14.14.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.31.tgz", + "integrity": "sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==" }, "@types/node-fetch": { "version": "2.5.7", diff --git a/package.json b/package.json index 74b4613..385f870 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "devDependencies": { "@k-foss/ts-esnode": "^2.0.0", "@k-foss/ts-estests": "^1.0.6", - "@types/node": "^14.14.25", + "@types/node": "^14.14.31", "@types/react": "^17.0.0", "@types/react-dom": "^17.0.0", "@types/workbox-sw": "^4.3.1", diff --git a/src/Library/Logger.ts b/src/Library/Logger.ts index c30e8ef..6334c16 100644 --- a/src/Library/Logger.ts +++ b/src/Library/Logger.ts @@ -29,6 +29,9 @@ export const logger = winston.createLogger({ level: getLevel(process.env.LOG_LEVEL), defaultMeta: { appName: 'TS-ESWeb', + labels: { + appName: 'TS-ESWeb', + }, }, transports: [ // @@ -45,6 +48,7 @@ export const logger = winston.createLogger({ host: 'http://Loki:3100', format: winston.format.json({}), json: true, + level: 'silly', replaceTimestamp: true, }), ], diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index c34c505..ebebe12 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -142,7 +142,7 @@ export class Queue { } if (anyJobs === false) { - logger.silly('Terminating workers'); + this.logger.silly('Terminating workers'); return this.terminateWorkers(); } @@ -154,15 +154,17 @@ export class Queue { * Start watching workers for active tasks and kill workers upon empty Queue */ private startWatching(): void { - this.handleDrained = (): void => { - this.isRunningJobs().catch(() => { - this.logger.silly('Error isRunningJobs, terminating workers'); + if (this.options.disableTermination !== true) { + this.handleDrained = (): void => { + this.isRunningJobs().catch(() => { + this.logger.silly('Error isRunningJobs, terminating workers'); - return this.terminateWorkers(); - }); - }; + return this.terminateWorkers(); + }); + }; - this.queueEvents.on('drained', this.handleDrained); + this.queueEvents.on('drained', this.handleDrained); + } } /** diff --git a/src/Modules/Queues/QueueController.ts b/src/Modules/Queues/QueueController.ts index 536c7f5..ab1f38e 100644 --- a/src/Modules/Queues/QueueController.ts +++ b/src/Modules/Queues/QueueController.ts @@ -28,6 +28,7 @@ export class QueueController { public createQueue( name: QueueName, inputClass: ClassConstructor, + disableTermination?: boolean, ): Queue { const bullOptions = this.createBullOptions(); @@ -35,6 +36,7 @@ export class QueueController { name, bullOptions, inputClass, + disableTermination, }); } } diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts index d0d5a5a..cf3523e 100644 --- a/src/Modules/Queues/QueueOptions.ts +++ b/src/Modules/Queues/QueueOptions.ts @@ -1,5 +1,5 @@ // src/Modules/Queues/QueueOptions.ts -import { IsDefined, IsString } from 'class-validator'; +import { IsBoolean, IsDefined, IsOptional, IsString } from 'class-validator'; import { QueueOptions as BullMQOptions } from 'bullmq'; export class QueueOptions { @@ -10,4 +10,8 @@ export class QueueOptions { public bullOptions: BullMQOptions; public inputClass: JobInput; + + @IsBoolean() + @IsOptional() + public disableTermination?: boolean; } diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index ce10533..e0a862a 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -1,19 +1,140 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ // src/Modules/Redis/RedisController.ts +import { plainToClass } from 'class-transformer'; +import { + isNumberString, + isString, + IsString, + validateOrReject, +} from 'class-validator'; import IORedis from 'ioredis'; -import { Inject, Service } from 'typedi'; -import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; +import { logger } from '../../Library/Logger'; +import { RedisType } from './RedisTypes'; -@Service() -export class RedisController { - @Inject(serverOptionsToken) - public options: ServerOptions; +export class SetValueInput { + @IsString() + public key: string; + + @IsString() + public value: string; +} +export class RedisController { /** * Redis Instance */ public IORedis: IORedis.Redis; + private async clearRedis(): Promise> { + logger.debug('Cleaning up redis'); + + const pipeline = this.IORedis.pipeline(); + + for (const redisTypeKey in RedisType) { + if (isNumberString(redisTypeKey) === false) { + const redisKeyPrefix = this.getRedisKeyPrefix( + RedisType[redisTypeKey as keyof typeof RedisType], + ); + + logger.debug(`redisKeyPrefix: `, { + redisKeyPrefix, + }); + + const redisKeys = await this.IORedis.keys(`${redisKeyPrefix}-*`); + + for (const redisKey of redisKeys) { + pipeline.del(redisKey); + } + } + } + + return pipeline.exec(); + } + public constructor(options: IORedis.RedisOptions) { this.IORedis = new IORedis(options); + + process.on('exit', () => { + // Do some cleanup such as close db + this.clearRedis(); + }); + + // catching signals and do something before exit + [ + 'SIGHUP', + 'SIGINT', + 'SIGQUIT', + 'SIGILL', + 'SIGTRAP', + 'SIGABRT', + 'SIGBUS', + 'SIGFPE', + 'SIGUSR1', + 'SIGSEGV', + 'SIGUSR2', + 'SIGTERM', + ].forEach(function (sig) { + process.on(sig, function () { + terminator(sig); + console.log('signal: ' + sig); + }); + }); + + const terminator = async (sig: string | number): Promise => { + if (typeof sig === 'string') { + // call your async task here and then call process.exit() after async task is done + await this.clearRedis(); + } + }; + } + + private getRedisKeyPrefix(type: RedisType): string { + switch (type) { + case RedisType.MODULE: + return 'module'; + case RedisType.MODULE_MAP: + return 'moduleMap'; + } + } + + private getRedisKey(type: RedisType, key: string): string { + return `${this.getRedisKeyPrefix(type)}-${key}`; + } + + /** + * Set a value in Redis + * @param type Redis DB Type + * @param input Input + */ + public async setValue( + type: RedisType, + inputObj: SetValueInput, + ): Promise { + const input = plainToClass(SetValueInput, inputObj); + await validateOrReject(input); + + await this.IORedis.set(this.getRedisKey(type, input.key), input.value); + } + + /** + * Get a value for a type and key in Redis + */ + public async getValue(type: RedisType, key: string): Promise { + const value = await this.IORedis.get(this.getRedisKey(type, key)); + + return value; + } + + public async getValueOrThrow(type: RedisType, key: string): Promise { + const value = await this.getValue(type, key); + + if (isString(value)) { + return value; + } + + logger.error( + `RedisController.getValue(${type}, ${key}) returned value is invalid`, + ); + throw new Error(`Invalid value from Redis for key ${key}`); } } diff --git a/src/Modules/Redis/RedisTypes.ts b/src/Modules/Redis/RedisTypes.ts new file mode 100644 index 0000000..7f22b16 --- /dev/null +++ b/src/Modules/Redis/RedisTypes.ts @@ -0,0 +1,5 @@ +// src/Modules/Redis/RedisTypes.ts +export enum RedisType { + MODULE_MAP, + MODULE, +} diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index ed49e0d..a29004d 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -5,20 +5,27 @@ import hyperid from 'hyperid'; import { Container, Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { WebModuleController } from '../WebModule/WebModuleController'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @Service() export class ServerController { - @Inject(serverOptionsToken) - public options: ServerOptions; + public redisController: RedisController; // eslint-disable-next-line no-useless-constructor public constructor( private typescriptController: TypeScriptController, private webModuleController: WebModuleController, - ) {} + @Inject(serverOptionsToken) + public options: ServerOptions, + ) { + this.redisController = new RedisController({ + host: options.redis.host, + }); + } /** * Create a new ServerController with the provided Configuration within the TypeDi Container @@ -50,6 +57,19 @@ export class ServerController { return container.get(ServerController); } + public async getPath(filePath: string): Promise { + logger.silly('HelloWorld'); + + const result = await this.redisController.getValue( + RedisType.MODULE, + filePath, + ); + + logger.silly(`getPath(${filePath})`, { + result, + }); + } + public async startTypeScript(): Promise { logger.info( `ServerController.startTypeScript() starting TypeScript Module Map`, @@ -61,15 +81,25 @@ export class ServerController { this.typescriptController.createTranspilerWorkers(), ]); + const filePath = fileURLToPath( + await import.meta.resolve('../../Web_Test/Imports.ts'), + ); + const entrypointJob = await this.typescriptController.createModuleMapTask({ - filePath: fileURLToPath( - await import.meta.resolve('../../Web_Test/Imports.ts'), - ), + filePath, }); + logger.silly(`entrypointJob`, { + entrypointJob, + }); + + await this.typescriptController.waitForTranspileDone(); - await this.typescriptController.waitForModuleMapDone(); + logger.silly('Done'); - await this.typescriptController.waitForJob(entrypointJob); + await this.getPath(filePath); + await this.getPath( + '/workspace/node_modules/react-dom/cjs/react-dom.development.js', + ); // const typescriptController = this.typescriptController; diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index 545a51f..f208d6a 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -7,7 +7,6 @@ import { Queue } from '../Queues/Queue'; import { QueueController } from '../Queues/QueueController'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; -import { ResolvedModuleMap } from './ResolvedModuleMap'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; @Service() @@ -41,16 +40,6 @@ export class TypeScriptController { TranspilerWorkerJobInput, ); - this.transpilerQueue.queueEvents.on('completed', (job) => { - logger.silly(`Transpiler Que Responds with`, { - jobId: job.jobId, - labels: { - queue: 'transpilerQueue', - appName: 'TS-ESWeb', - }, - }); - }); - this.moduleMapQueue = this.queueController.createQueue( moduleMapQueKey, ModuleMapWorkerJobInput, @@ -82,7 +71,7 @@ export class TypeScriptController { * Spawn the TypeScript Transpiler Workers */ public async createTranspilerWorkers(): Promise { - logger.info(`TypeScriptController.createTranspilerWorkers()`); + logger.debug(`TypeScriptController.createTranspilerWorkers()`); const workerPathURI = await import.meta.resolve( './TypeScriptTranspilerWorker', @@ -91,7 +80,7 @@ export class TypeScriptController { await this.transpilerQueue.createWorkers(workerPath); - logger.debug( + logger.silly( `TypeScriptController.createTranspilerWorkers() workerPathURI: ${workerPathURI}`, ); } @@ -134,29 +123,11 @@ export class TypeScriptController { }); } - public async waitForJob(job: Job): Promise { - return this.moduleMapQueue.waitForTask(job); - } - - public async getModuleMap(filePath: string): Promise { - const job = await this.moduleMapQueue.queue.getJob(filePath); - - if (job?.returnvalue) { - logger.silly(`job?.returnValue: `, { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - returnValue: job.returnvalue, - }); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return job?.returnvalue; - } else { - logger.warn(`getModuleMap(${filePath}) failed`); - } - - throw new Error('Invalid file path'); - } - public waitForModuleMapDone(): Promise { return this.moduleMapQueue.waitForDone(); } + + public waitForTranspileDone(): Promise { + return this.transpilerQueue.waitForDone(); + } } diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 898881f..2534f25 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -9,7 +9,7 @@ import { logger as coreLogger } from '../../Library/Logger'; import { envMode } from '../../Utils/Environment'; import '../../Utils/Setup'; import { WorkerInput } from '../Queues/WorkerInput'; -import { WebModuleJobInput } from '../WebModule/WebModuleJobInput'; +import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { createTypeScriptProgram, isCommonJSImportSplit } from './Utils'; @@ -28,7 +28,7 @@ logger.silly(`workerInput`, { workerInput, }); -const webModuleQue = new Queue('webModuleQueue', { +const webModuleMapQue = new Queue('webModuleMapQueue', { ...workerInput.queueOptions, }); @@ -62,38 +62,29 @@ async function discoverModuleMap( const rootDir = dirname(moduleInput.filePath); + const compilerOptions: ts.CompilerOptions = { + jsxFragmentFactory: 'Fragment', + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + allowJs: true, + checkJs: false, + noEmit: true, + noEmitHelpers: true, + sourceMap: false, + inlineSourceMap: false, + }; + const compilerProgram = await createTypeScriptProgram({ rootDir, rootNames: [moduleInput.filePath], - compilerOptions: { - jsxFragmentFactory: 'Fragment', - target: ts.ScriptTarget.ESNext, - module: ts.ModuleKind.ESNext, - moduleResolution: ts.ModuleResolutionKind.NodeJs, - allowJs: true, - checkJs: false, - noEmit: true, - noEmitHelpers: true, - sourceMap: false, - inlineSourceMap: false, - }, + compilerOptions, }); moduleMapLogger.silly(`compilerProgram created`, { rootDir, rootNames: [moduleInput.filePath], - compilerOptions: { - jsxFragmentFactory: 'Fragment', - target: ts.ScriptTarget.ESNext, - module: ts.ModuleKind.ESNext, - moduleResolution: ts.ModuleResolutionKind.NodeJs, - allowJs: true, - checkJs: false, - noEmit: true, - noEmitHelpers: true, - sourceMap: false, - inlineSourceMap: false, - }, + compilerOptions, }); const sourceFile = compilerProgram.getSourceFile(moduleInput.filePath); @@ -134,10 +125,6 @@ async function discoverModuleMap( }, ); - moduleMapLogger.silly(`HelloWorld`, { - outputModule, - }); - const jobData = await ModuleMapWorkerJobInput.createModuleMapJobInput({ filePath: outputModule.resolvedFileName, specifier: moduleInput.specifier, @@ -162,8 +149,6 @@ async function discoverModuleMap( } } - moduleMapLogger.silly('Fucker'); - const importedModules = await Promise.all( resolvedArray.map(async ([specifier]) => { const parentURI = pathToFileURL(moduleInput.filePath); @@ -179,10 +164,6 @@ async function discoverModuleMap( ); if (resolvePathURI.startsWith('node:')) { - moduleMapLogger.debug(`resolvePathURI is a node: path`, { - objectName: 'resolvePathURI', - resolvePathURI, - }); return; } @@ -196,20 +177,8 @@ async function discoverModuleMap( let moduleSpecifier: string; if (ts.isExternalModuleNameRelative(specifier)) { - moduleMapLogger.silly(`Specifier is relative`, { - specifier, - parentURI, - resolvePathURI, - }); - moduleSpecifier = filePath; } else { - moduleMapLogger.silly(`Specifier isn't relative`, { - specifier, - parentURI, - resolvePathURI, - }); - moduleSpecifier = specifier; } @@ -224,7 +193,7 @@ async function discoverModuleMap( specifier: moduleSpecifier, }); - moduleMapLogger.debug(`newJobData: `, { + moduleMapLogger.silly(`newJobData: `, { jobData, jobId: filePath, }); @@ -241,7 +210,7 @@ async function discoverModuleMap( }), ); - const webModuleJobInputParams: WebModuleJobInput = { + const webModuleJobInputParams: WebModuleMapJobInput = { filePath: moduleInput.filePath, specifier: moduleInput.specifier, importedModules: importedModules.filter(Boolean) as string[], @@ -252,7 +221,7 @@ async function discoverModuleMap( }); const [webModuleJobInput, transpilerJobInput] = await Promise.all([ - WebModuleJobInput.createWebModuleJobInput(webModuleJobInputParams), + WebModuleMapJobInput.createWebModuleJobInput(webModuleJobInputParams), TranspilerWorkerJobInput.createTranspilerWorkerJobInput({ filePath: moduleInput.filePath, }), @@ -262,10 +231,12 @@ async function discoverModuleMap( webModuleJobInput, }); - await Promise.all([ - webModuleQue.add('webModuleQueue', webModuleJobInput), - transpilerQue.add('typescriptTranspiler', transpilerJobInput), - ]); + await webModuleMapQue.add('webModuleMapQueue', webModuleJobInput, { + jobId: webModuleJobInput.filePath, + }); + await transpilerQue.add('typescriptTranspiler', transpilerJobInput, { + jobId: transpilerJobInput.filePath, + }); } const moduleWorkerLogger = logger.child({ diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 872f583..84eac56 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -2,13 +2,12 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; -import { Worker } from 'bullmq'; +import { Queue, Worker } from 'bullmq'; import { logger as coreLogger } from '../../Library/Logger'; import { transpileModule, ModuleKind, ScriptTarget, - ScriptKind, ModuleResolutionKind, } from 'typescript'; import { WorkerInput } from '../Queues/WorkerInput'; @@ -16,6 +15,8 @@ import { validateOrReject } from 'class-validator'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { readFile } from 'fs/promises'; import { threadId } from 'worker_threads'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; const logger = coreLogger.child({ labels: { worker: 'TypeScriptTranspilerWorker.ts', workeId: threadId }, @@ -23,22 +24,24 @@ const logger = coreLogger.child({ logger.info(`Worker starting`); -const data = getWorkerData(import.meta.url); +const workerInput = await WorkerInput.createWorkerInput( + getWorkerData(import.meta.url), +); -logger.debug(`Retrieved workerData:`, { - objectName: 'data', - data, +const redisController = new RedisController({ + ...(workerInput.queueOptions.connection as { host: string }), }); -const workerInput = plainToClass(WorkerInput, data); - -logger.debug(`Transformed to class`, { +logger.debug(`Retrieved workerData:`, { objectName: 'workerInput', workerInput, }); -await validateOrReject(workerInput); - +/** + * Transform provided filepath to ESM with TypeScript Compiler API + * @param filePath Path to file to load, transform, and transpile to ESM + * @returns Promise resolving to string of tranformed file. + */ async function transformFile(filePath: string): Promise { logger.info(`Transforming ${filePath}`); @@ -60,11 +63,6 @@ async function transformFile(filePath: string): Promise { }, }); - logger.silly(`Transpiled Module`, { - filePath, - transpiledModule: transpiledModule.outputText, - }); - return transpiledModule.outputText; } @@ -95,11 +93,19 @@ const transpilerWorker = new Worker( jobInput, }); - return transformFile(jobInput.filePath); + const sourceText = await transformFile(jobInput.filePath); + + await redisController.setValue(RedisType.MODULE, { + key: jobInput.filePath, + value: sourceText, + }); + + logger.silly('Done', { + filePath: jobInput.filePath, + }); }, { connection: workerInput.queueOptions.connection, - concurrency: workerInput.workerCount, }, ); diff --git a/src/Modules/WebModule/WebModule.ts b/src/Modules/WebModule/WebModule.ts index 123d530..9e4cbcf 100644 --- a/src/Modules/WebModule/WebModule.ts +++ b/src/Modules/WebModule/WebModule.ts @@ -9,11 +9,6 @@ export class WebModule { @Field() public filePath: string; - @Field() - public code: string; - - public dependencies = new Set(); - public constructor(opts: Partial = {}) { Object.assign(this, opts); } diff --git a/src/Modules/WebModule/WebModuleController.ts b/src/Modules/WebModule/WebModuleController.ts index 103c47f..d850800 100644 --- a/src/Modules/WebModule/WebModuleController.ts +++ b/src/Modules/WebModule/WebModuleController.ts @@ -4,22 +4,34 @@ import { fileURLToPath } from 'url'; import { logger } from '../../Library/Logger'; import { Queue } from '../Queues/Queue'; import { QueueController } from '../Queues/QueueController'; -import { WebModuleJobInput } from './WebModuleJobInput'; +import { WebModuleMapJobInput } from './WebModuleMapJobInput'; @Service() export class WebModuleController { - private webModuleQueue: Queue<'webModuleQueue', WebModuleJobInput>; + private webModuleMapQueue: Queue<'webModuleMapQueue', WebModuleMapJobInput>; + + private webModuleQueue: Queue<'webModuleQueue', WebModuleMapJobInput>; public constructor( @Inject(() => QueueController) private queueController: QueueController, ) { + this.webModuleMapQueue = queueController.createQueue( + 'webModuleMapQueue', + WebModuleMapJobInput, + true, + ); + this.webModuleQueue = queueController.createQueue( 'webModuleQueue', - WebModuleJobInput, + WebModuleMapJobInput, + true, ); } + /** + * Spawn Web Module Map Workers to handle the dep map + */ public async spawnWebModuleWorkers(): Promise { const workerPathURI = await import.meta.resolve('./WebModuleWorker'); const workerPath = fileURLToPath(workerPathURI); @@ -27,10 +39,10 @@ export class WebModuleController { /** * Create the @K-FOSS/TS-ESWorkers. */ - await this.webModuleQueue.createWorkers(workerPath, 1); + await this.webModuleMapQueue.createWorkers(workerPath, 1); logger.debug( - `TypeScriptController.createModuleMapWorkers() workerPathURI: ${workerPathURI}`, + `TypeScriptController.spawnWebModuleMapWorkers() workerPathURI: ${workerPathURI}`, ); } } diff --git a/src/Modules/WebModule/WebModuleJobInput.ts b/src/Modules/WebModule/WebModuleJobInput.ts index a669b11..453a991 100644 --- a/src/Modules/WebModule/WebModuleJobInput.ts +++ b/src/Modules/WebModule/WebModuleJobInput.ts @@ -1,19 +1,13 @@ -// src/Modules/WebModules/WebModuleJobInput.ts +// src/Modules/WebModule/WebModuleJobInput.ts import { plainToClass } from 'class-transformer'; -import { IsOptional, IsString, validateOrReject } from 'class-validator'; +import { IsString, validateOrReject } from 'class-validator'; export class WebModuleJobInput { @IsString() public filePath: string; @IsString() - @IsOptional() - public specifier?: string; - - @IsString({ - each: true, - }) - public importedModules?: string[]; + public sourceText: string; public static async createWebModuleJobInput( inputParams: Partial, diff --git a/src/Modules/WebModule/WebModuleMapJobInput.ts b/src/Modules/WebModule/WebModuleMapJobInput.ts new file mode 100644 index 0000000..e14a991 --- /dev/null +++ b/src/Modules/WebModule/WebModuleMapJobInput.ts @@ -0,0 +1,26 @@ +// src/Modules/WebModules/WebModuleJobInput.ts +import { plainToClass } from 'class-transformer'; +import { IsOptional, IsString, validateOrReject } from 'class-validator'; + +export class WebModuleMapJobInput { + @IsString() + public filePath: string; + + @IsString() + @IsOptional() + public specifier?: string; + + @IsString({ + each: true, + }) + public importedModules?: string[]; + + public static async createWebModuleJobInput( + inputParams: Partial, + ): Promise { + const jobInput = plainToClass(WebModuleMapJobInput, inputParams); + await validateOrReject(jobInput); + + return jobInput; + } +} diff --git a/src/Modules/WebModule/WebModuleWorker.ts b/src/Modules/WebModule/WebModuleWorker.ts index 2588844..e28343a 100644 --- a/src/Modules/WebModule/WebModuleWorker.ts +++ b/src/Modules/WebModule/WebModuleWorker.ts @@ -3,12 +3,12 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; -import { plainToClass } from 'class-transformer'; -import { validateOrReject } from 'class-validator'; import { Queue, Worker } from 'bullmq'; import { WorkerInput } from '../Queues/WorkerInput'; -import { WebModuleJobInput } from './WebModuleJobInput'; +import { WebModuleMapJobInput } from './WebModuleMapJobInput'; import { timeout } from '../../Utils/timeout'; +import { RedisController } from '../Redis/RedisController'; +import { WebModuleJobInput } from './WebModuleJobInput'; const logger = coreLogger.child({ labels: { worker: 'WebModuleWorker.ts', workeId: threadId }, @@ -16,34 +16,35 @@ const logger = coreLogger.child({ logger.info(`Worker starting`); -const data = getWorkerData(import.meta.url); +const workerInput = await WorkerInput.createWorkerInput( + getWorkerData(import.meta.url), +); logger.debug(`Retrieved workerData:`, { - objectName: 'data', - data, -}); - -const workerInput = plainToClass(WorkerInput, data); - -logger.debug(`Transformed to class`, { objectName: 'workerInput', workerInput, }); -await validateOrReject(workerInput); +const redisController = new RedisController({ + ...(workerInput.queueOptions.connection as { host: string }), +}); + +const webModuleMapQue = new Queue(workerInput.queName, { + ...workerInput.queueOptions, +}); -const webModuleQue = new Queue(workerInput.queName, { +const webModuleQue = new Queue('webModuleQueue', { ...workerInput.queueOptions, }); logger.silly(`webModuleQue has been created`, { - webModuleQue, + webModuleMapQue, }); -const moduleWorker = new Worker( +const moduleMapWorker = new Worker( workerInput.queName, async (job) => { - logger.silly(`New Web Module Job:`, { + logger.silly(`New Web Module Map Job:`, { data: job.data, }); @@ -54,6 +55,26 @@ const moduleWorker = new Worker( }, ); +logger.silly(`moduleMapWorker`, { + moduleMapWorker, +}); + +const moduleWorker = new Worker( + 'webModuleQueue', + async (job) => { + logger.silly(`New Web Module Job:`, { + data: job.data.filePath, + }); + + const input = await WebModuleJobInput.createWebModuleJobInput(job.data); + + await redisController.IORedis.set(input.filePath, input.sourceText); + }, + { + connection: workerInput.queueOptions.connection, + }, +); + logger.silly(`moduleWorker`, { moduleWorker, }); From 0d690bb4e38d99ab77ee5f2d0774c8f9f880ec3f Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Tue, 23 Feb 2021 13:44:36 +0000 Subject: [PATCH 25/30] feat: Clean up Redis Controller and ModuleMap in Redis --- src/Modules/Redis/RedisController.ts | 8 ++++-- src/Modules/Server/ServerController.ts | 28 +++++++++++++++++-- .../TypeScript/TypeScriptModuleMapWorker.ts | 11 ++++++++ .../TypeScript/TypeScriptTranspilerWorker.ts | 2 +- src/Modules/WebModule/WebModule.ts | 19 ------------- src/Modules/WebModule/WebModuleController.ts | 9 +----- src/index.tsx | 2 +- 7 files changed, 44 insertions(+), 35 deletions(-) diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index e0a862a..4af35f6 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -15,8 +15,10 @@ export class SetValueInput { @IsString() public key: string; - @IsString() - public value: string; + @IsString({ + each: true, + }) + public value: string | string[]; } export class RedisController { @@ -97,7 +99,7 @@ export class RedisController { } } - private getRedisKey(type: RedisType, key: string): string { + public getRedisKey(type: RedisType, key: string): string { return `${this.getRedisKeyPrefix(type)}-${key}`; } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index a29004d..c21b08f 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -9,6 +9,8 @@ import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { WebModuleController } from '../WebModule/WebModuleController'; +import { WebModuleJobInput } from '../WebModule/WebModuleJobInput'; +import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; import { ServerOptions, serverOptionsToken } from './ServerOptions'; @Service() @@ -57,7 +59,20 @@ export class ServerController { return container.get(ServerController); } - public async getPath(filePath: string): Promise { + public async getModuleMap(filePath: string): Promise { + const result = await this.redisController.getValue( + RedisType.MODULE_MAP, + filePath, + ); + + if (typeof result === 'string') { + return plainToClass(WebModuleJobInput, JSON.parse(result)); + } + + throw new Error('Invalid result from Redis'); + } + + public async getPathModule(filePath: string): Promise { logger.silly('HelloWorld'); const result = await this.redisController.getValue( @@ -96,11 +111,18 @@ export class ServerController { logger.silly('Done'); - await this.getPath(filePath); - await this.getPath( + await this.getPathModule(filePath); + await this.getPathModule( '/workspace/node_modules/react-dom/cjs/react-dom.development.js', ); + const value = await this.getModuleMap( + '/workspace/node_modules/react-dom/server.js', + ); + logger.debug('React-DOM Server', { + value, + }); + // const typescriptController = this.typescriptController; // async function getAllChildModules(filePath: string): Promise { diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 2534f25..a8e8a65 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -13,6 +13,8 @@ import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { createTypeScriptProgram, isCommonJSImportSplit } from './Utils'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; const logger = coreLogger.child({ labels: { worker: 'TypeScriptModuleMapWorker.ts', workerId: threadId }, @@ -40,6 +42,10 @@ const moduleMapQue = new Queue(workerInput.queName, { ...workerInput.queueOptions, }); +const redisController = new RedisController({ + ...(workerInput.queueOptions.connection as { host: string }), +}); + /** * Discover all imported modules and add to the TypeScript Module Map * @param moduleInput Input Params @@ -231,6 +237,11 @@ async function discoverModuleMap( webModuleJobInput, }); + await redisController.IORedis.set( + redisController.getRedisKey(RedisType.MODULE_MAP, moduleInput.filePath), + JSON.stringify(webModuleJobInputParams), + ); + await webModuleMapQue.add('webModuleMapQueue', webModuleJobInput, { jobId: webModuleJobInput.filePath, }); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 84eac56..96f4a7f 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -2,7 +2,7 @@ import { getWorkerData } from '@k-foss/ts-worker'; import '../../Utils/Setup'; import { plainToClass } from 'class-transformer'; -import { Queue, Worker } from 'bullmq'; +import { Worker } from 'bullmq'; import { logger as coreLogger } from '../../Library/Logger'; import { transpileModule, diff --git a/src/Modules/WebModule/WebModule.ts b/src/Modules/WebModule/WebModule.ts index 9e4cbcf..9451923 100644 --- a/src/Modules/WebModule/WebModule.ts +++ b/src/Modules/WebModule/WebModule.ts @@ -1,9 +1,6 @@ // src/Modules/WebModule/WebModule.ts import { ObjectType, Field } from 'type-graphql'; -// const distDir = resolvePath('dist'); -// const mapPath = resolvePath(distDir, 'moduleMap.json'); - @ObjectType() export class WebModule { @Field() @@ -13,19 +10,3 @@ export class WebModule { Object.assign(this, opts); } } - -// let mapData: any; - -// if (process.env.NODE_ENV === 'production') { -// try { -// const { promises: fs } = await import('fs'); - -// const mapFile = await fs.readFile(mapPath); - -// mapData = JSON.parse(mapFile.toString()); -// } catch {} -// } - -// type Specifier = string; - -// export const moduleMap = new Map(mapData); diff --git a/src/Modules/WebModule/WebModuleController.ts b/src/Modules/WebModule/WebModuleController.ts index d850800..7746aed 100644 --- a/src/Modules/WebModule/WebModuleController.ts +++ b/src/Modules/WebModule/WebModuleController.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars-experimental */ // src/Modules/WebModule/WebModuleController.ts import { Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; @@ -10,8 +11,6 @@ import { WebModuleMapJobInput } from './WebModuleMapJobInput'; export class WebModuleController { private webModuleMapQueue: Queue<'webModuleMapQueue', WebModuleMapJobInput>; - private webModuleQueue: Queue<'webModuleQueue', WebModuleMapJobInput>; - public constructor( @Inject(() => QueueController) private queueController: QueueController, @@ -21,12 +20,6 @@ export class WebModuleController { WebModuleMapJobInput, true, ); - - this.webModuleQueue = queueController.createQueue( - 'webModuleQueue', - WebModuleMapJobInput, - true, - ); } /** diff --git a/src/index.tsx b/src/index.tsx index eaca2f8..0c57241 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -47,7 +47,7 @@ await server.startTypeScript(); // await ssrServer.startTranspiler(); -logger.debug(`Creating the Fastify Server`); +// logger.debug(`Creating the Fastify Server`); // const fastifyServer = await ssrServer.createFastifyServer(); From c39b04649ece76ae49e090b29cb514fd032f556a Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Tue, 23 Feb 2021 17:10:59 +0000 Subject: [PATCH 26/30] feat: Refactor workers, and it's working!!! --- .gitignore | 3 +- src/Library/Resolvers.ts | 3 + src/Modules/Files/processModule.ts | 45 +++++++ src/Modules/Queues/Queue.ts | 14 ++- src/Modules/Queues/QueueController.ts | 17 +-- src/Modules/Queues/QueueOptions.ts | 5 +- src/Modules/Queues/QueueToken.ts | 4 + src/Modules/Queues/Worker.ts | 25 ++++ src/Modules/Queues/WorkerController.ts | 51 ++++++++ src/Modules/Queues/WorkerInput.ts | 22 +++- src/Modules/Redis/RedisController.ts | 7 +- src/Modules/SSR/SSRController.tsx | 17 ++- src/Modules/SSR/SSROptions.ts | 7 +- src/Modules/Server/ServerController.ts | 117 ++++++++++++++---- .../TypeScript/TypeScriptController.ts | 16 +-- .../TypeScript/TypeScriptModuleMapWorker.ts | 40 ++++-- .../TypeScript/TypeScriptTranspilerWorker.ts | 60 ++++++--- src/Modules/WebModule/ImportTransformer.ts | 12 +- src/Modules/WebModule/WebModule.ts | 5 +- src/Modules/WebModule/WebModuleController.ts | 13 +- src/Modules/WebModule/WebModuleJobInput.ts | 5 + src/Modules/WebModule/WebModuleMap.ts | 30 +++++ .../WebModule/WebModuleMapController.ts | 57 +++++++++ src/Modules/WebModule/WebModuleResolver.ts | 103 ++++++--------- src/Web_Test/App.tsx | 6 +- src/Web_Test/ServiceWorker.ts | 2 +- src/index.tsx | 38 ++++++ 27 files changed, 561 insertions(+), 163 deletions(-) create mode 100644 src/Modules/Files/processModule.ts create mode 100644 src/Modules/Queues/QueueToken.ts create mode 100644 src/Modules/Queues/Worker.ts create mode 100644 src/Modules/Queues/WorkerController.ts create mode 100644 src/Modules/WebModule/WebModuleMap.ts create mode 100644 src/Modules/WebModule/WebModuleMapController.ts diff --git a/.gitignore b/.gitignore index 602dcb4..46acda9 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,5 @@ temp/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) -dist \ No newline at end of file +dist +Archive \ No newline at end of file diff --git a/src/Library/Resolvers.ts b/src/Library/Resolvers.ts index c695909..df932e2 100644 --- a/src/Library/Resolvers.ts +++ b/src/Library/Resolvers.ts @@ -3,6 +3,7 @@ import { buildSchema, NonEmptyArray } from 'type-graphql'; import { findModuleFiles } from '../Utils/moduleFileFinder'; import { GraphQLSchema } from 'graphql'; +import { ContainerInstance } from 'typedi'; type ResolverModule = { [key: string]: Function }; @@ -18,8 +19,10 @@ export async function getResolvers(): Promise { export async function buildGQLSchema( resolvers: Function[], + container: ContainerInstance, ): Promise { return buildSchema({ resolvers: resolvers as NonEmptyArray, + container, }); } diff --git a/src/Modules/Files/processModule.ts b/src/Modules/Files/processModule.ts new file mode 100644 index 0000000..7fc1912 --- /dev/null +++ b/src/Modules/Files/processModule.ts @@ -0,0 +1,45 @@ +// src/Modules/Files/processModule.ts + +import { logger } from '../../Library/Logger'; +import { + exportsHandler, + objectExport, + processNodeReplacement, +} from '../TypeScript/Regex'; + +export function processModule(fileContents: string): string { + const exportVars: string[] = []; + + logger.silly(`processModule`, { + fileContents, + }); + + /** + * https://regex101.com/r/uwAq1N/1 + */ + + let moduleContents = fileContents + .replace(processNodeReplacement, '$') + .replaceAll(exportsHandler, (test, todo, varName) => { + exportVars.push(varName); + logger.silly(`exportVars:`, { varName }); + + return `var ${varName as string}`; + }) + .replace(objectExport, (...args) => { + const { coreCode } = args[args.length - 1]; + + return coreCode.replaceAll('exports.', ''); + }); + + exportVars.map((exportVar) => { + moduleContents = moduleContents.replaceAll( + `exports.${exportVar}`, + `${exportVar}`, + ); + + moduleContents += `exports.${exportVar} = ${exportVar}\n`; + }); + + return moduleContents; +} diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index ebebe12..c5d746d 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -14,6 +14,7 @@ import { Logger } from 'winston'; import { WorkerInput } from './WorkerInput'; import EventEmitter from 'events'; import pEvent from 'p-event'; +import { fileURLToPath } from 'node:url'; /** * Task Queue handled by dedicated `worker_threads` created by @@ -186,19 +187,23 @@ export class Queue { * @returns Promise resolving once the workers threads have all been created */ public async createWorkers( - workerPath: string, - workerCount = 3, + subWorkerPath: string, + workerCount = 2, ): Promise { - this.logger.debug(`Queue.createWorkers('${workerPath}')`); + this.logger.debug( + `Queue.createWorkers('${subWorkerPath}', ${workerCount})`, + ); this.logger.debug(`Queue.createWorkers() cleaning old jobs`); await this.cleanQueue(); const workerInputParams: WorkerInput = { + serverOptions: this.options.serverOptions, queueOptions: this.queue.opts, queName: this.queue.name, workerCount, + workerPath: subWorkerPath, }; this.logger.silly(`workerInputParams: `, { @@ -207,6 +212,9 @@ export class Queue { const workerInput = plainToClass(WorkerInput, workerInputParams); + const workerPathURI = await import.meta.resolve('./Worker'); + const workerPath = fileURLToPath(workerPathURI); + for (const _workerThread of Array(workerInputParams.workerCount).fill(0)) { this.logger.info(`Queue.createWorkers() spawning worker`); diff --git a/src/Modules/Queues/QueueController.ts b/src/Modules/Queues/QueueController.ts index ab1f38e..b9657ab 100644 --- a/src/Modules/Queues/QueueController.ts +++ b/src/Modules/Queues/QueueController.ts @@ -1,9 +1,10 @@ // src/Modules/Ques/QueueController.ts -import { QueueOptions } from 'bullmq'; +import type { QueueOptions as BullMQQueueOptions } from 'bullmq'; import { ClassConstructor } from 'class-transformer'; import { Inject, Service } from 'typedi'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { Queue } from './Queue'; +import { QueueOptions } from './QueueOptions'; @Service() export class QueueController { @@ -13,7 +14,7 @@ export class QueueController { /** * Create the IORedis Options */ - private createBullOptions(): QueueOptions { + private createBullOptions(): BullMQQueueOptions { return { connection: { ...this.options.redis, @@ -26,17 +27,17 @@ export class QueueController { * @param queKey Queue key */ public createQueue( - name: QueueName, - inputClass: ClassConstructor, - disableTermination?: boolean, + input: Omit< + QueueOptions>, + 'bullOptions' | 'serverOptions' + >, ): Queue { const bullOptions = this.createBullOptions(); return new Queue({ - name, + ...input, bullOptions, - inputClass, - disableTermination, + serverOptions: this.options, }); } } diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts index cf3523e..10c4f32 100644 --- a/src/Modules/Queues/QueueOptions.ts +++ b/src/Modules/Queues/QueueOptions.ts @@ -1,12 +1,15 @@ // src/Modules/Queues/QueueOptions.ts import { IsBoolean, IsDefined, IsOptional, IsString } from 'class-validator'; -import { QueueOptions as BullMQOptions } from 'bullmq'; +import type { QueueOptions as BullMQOptions } from 'bullmq'; +import { ServerOptions } from '../Server/ServerOptions'; export class QueueOptions { @IsString() @IsDefined() public name: QueueName; + public serverOptions: ServerOptions; + public bullOptions: BullMQOptions; public inputClass: JobInput; diff --git a/src/Modules/Queues/QueueToken.ts b/src/Modules/Queues/QueueToken.ts new file mode 100644 index 0000000..669bdf0 --- /dev/null +++ b/src/Modules/Queues/QueueToken.ts @@ -0,0 +1,4 @@ +// src/Modules/Queues/QueueToken.ts +import { Token } from 'typedi'; + +export const queueToken = new Token('queueToken'); diff --git a/src/Modules/Queues/Worker.ts b/src/Modules/Queues/Worker.ts new file mode 100644 index 0000000..0d3d99d --- /dev/null +++ b/src/Modules/Queues/Worker.ts @@ -0,0 +1,25 @@ +// src/Modules/Queues/Worker.ts +import { getWorkerData } from '@k-foss/ts-worker'; +import '../../Utils/Setup'; +import {} from 'worker_threads'; +import { logger } from '../../Library/Logger'; +import { WorkerInput } from './WorkerInput'; +import { WorkerController } from './WorkerController'; + +const workerInput = await WorkerInput.createWorkerInput( + getWorkerData(import.meta.url), +); + +logger.silly(`workerInput`, { + workerInput, +}); + +const workerController = await WorkerController.createWorkerController( + workerInput, +); + +logger.silly('WorkerSpawned: ', { + workerController, +}); + +await import(workerInput.workerPath); diff --git a/src/Modules/Queues/WorkerController.ts b/src/Modules/Queues/WorkerController.ts new file mode 100644 index 0000000..b566d41 --- /dev/null +++ b/src/Modules/Queues/WorkerController.ts @@ -0,0 +1,51 @@ +// src/Modules/Queues/WorkerController.ts + +import { plainToClass } from 'class-transformer'; +import { validateOrReject } from 'class-validator'; +import Container, { Inject, Service } from 'typedi'; +import { logger } from '../../Library/Logger'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; +import { queueToken } from './QueueToken'; +import { WorkerInput, workerInputToken } from './WorkerInput'; + +@Service() +export class WorkerController { + public constructor( + @Inject(serverOptionsToken) + public options: ServerOptions, + ) { + logger.silly(`WorkerController`, { + options, + }); + } + + public static async createWorkerController( + workerInput: WorkerInput, + ): Promise { + const serverOptions = plainToClass( + ServerOptions, + workerInput.serverOptions, + ); + await validateOrReject(serverOptions); + + Container.set({ + id: serverOptionsToken, + global: true, + value: serverOptions, + }); + + Container.set({ + id: queueToken, + global: true, + value: workerInput.queName, + }); + + Container.set({ + id: workerInputToken, + global: true, + value: workerInput, + }); + + return Container.get(WorkerController); + } +} diff --git a/src/Modules/Queues/WorkerInput.ts b/src/Modules/Queues/WorkerInput.ts index 3fa751b..a542cc4 100644 --- a/src/Modules/Queues/WorkerInput.ts +++ b/src/Modules/Queues/WorkerInput.ts @@ -2,12 +2,27 @@ import type { QueueOptions } from 'bullmq'; import { plainToClass, Transform } from 'class-transformer'; import { IsInt, IsString, validateOrReject } from 'class-validator'; +import { Token } from 'typedi'; import { logger } from '../../Library/Logger'; +import { ServerOptions } from '../Server/ServerOptions'; export class WorkerInput { @Transform(({ value }) => { if (typeof value === 'string') { - logger.debug(`WorkerInput.queueOptions is string`, { + logger.silly(`WorkerInput.serverOptions is string`, { + value, + }); + + return JSON.parse(value) as ServerOptions; + } + + return JSON.stringify(value); + }) + public serverOptions: ServerOptions; + + @Transform(({ value }) => { + if (typeof value === 'string') { + logger.silly(`WorkerInput.queueOptions is string`, { value, }); @@ -18,6 +33,9 @@ export class WorkerInput { }) public queueOptions: QueueOptions; + @IsString() + public workerPath: string; + /** * Name of the TypeScript Transpiler Que for this worker */ @@ -46,3 +64,5 @@ export class WorkerInput { return jobInput; } } + +export const workerInputToken = new Token('workerInput'); diff --git a/src/Modules/Redis/RedisController.ts b/src/Modules/Redis/RedisController.ts index 4af35f6..ac62ada 100644 --- a/src/Modules/Redis/RedisController.ts +++ b/src/Modules/Redis/RedisController.ts @@ -8,7 +8,9 @@ import { validateOrReject, } from 'class-validator'; import IORedis from 'ioredis'; +import { Inject, Service } from 'typedi'; import { logger } from '../../Library/Logger'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { RedisType } from './RedisTypes'; export class SetValueInput { @@ -21,6 +23,7 @@ export class SetValueInput { public value: string | string[]; } +@Service() export class RedisController { /** * Redis Instance @@ -53,8 +56,8 @@ export class RedisController { return pipeline.exec(); } - public constructor(options: IORedis.RedisOptions) { - this.IORedis = new IORedis(options); + public constructor(@Inject(serverOptionsToken) options: ServerOptions) { + this.IORedis = new IORedis(options.redis); process.on('exit', () => { // Do some cleanup such as close db diff --git a/src/Modules/SSR/SSRController.tsx b/src/Modules/SSR/SSRController.tsx index 157f062..f803ec9 100644 --- a/src/Modules/SSR/SSRController.tsx +++ b/src/Modules/SSR/SSRController.tsx @@ -1,6 +1,19 @@ // src/Modules/SSR/SSRController.tsx -import { Service } from 'typedi'; +import { renderToString } from 'react-dom/server'; +import { Inject, Service } from 'typedi'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; @Service() -export class SSRController {} +export class SSRController { + @Inject(serverOptionsToken) + public options: ServerOptions; + + public renderApp(): string { + const App = this.options.ssr.appComponent; + + console.log(this.options, App); + + return renderToString(); + } +} diff --git a/src/Modules/SSR/SSROptions.ts b/src/Modules/SSR/SSROptions.ts index e57248f..234694e 100644 --- a/src/Modules/SSR/SSROptions.ts +++ b/src/Modules/SSR/SSROptions.ts @@ -1,5 +1,5 @@ // src/Modules/SSR/SSROptions.ts -import { Exclude } from 'class-transformer'; +import { Exclude, Expose, Transform } from 'class-transformer'; import { IsDefined, IsString } from 'class-validator'; export class SSROptions { @@ -23,5 +23,10 @@ export class SSROptions { public entrypoint: string; @Exclude() + @Transform(({ value }) => { + console.log('Value: ', value); + + return value; + }) public appComponent: () => React.ReactElement; } diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index c21b08f..aeee260 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -1,12 +1,18 @@ // src/Modules/Server/ServerController.ts +import { ApolloServer } from 'apollo-server-fastify'; import { plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; import hyperid from 'hyperid'; +import { resolve } from 'node:path'; import { Container, Inject, Service } from 'typedi'; import { fileURLToPath } from 'url'; +import { getGQLContext } from '../../Library/Context'; import { logger } from '../../Library/Logger'; +import { buildGQLSchema, getResolvers } from '../../Library/Resolvers'; +import { timeout } from '../../Utils/timeout'; import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; +import { SSRController } from '../SSR/SSRController'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; import { WebModuleController } from '../WebModule/WebModuleController'; import { WebModuleJobInput } from '../WebModule/WebModuleJobInput'; @@ -15,18 +21,46 @@ import { ServerOptions, serverOptionsToken } from './ServerOptions'; @Service() export class ServerController { + @Inject() public redisController: RedisController; + @Inject() + public ssrController: SSRController; + // eslint-disable-next-line no-useless-constructor public constructor( private typescriptController: TypeScriptController, private webModuleController: WebModuleController, @Inject(serverOptionsToken) public options: ServerOptions, - ) { - this.redisController = new RedisController({ - host: options.redis.host, + ) {} + + private getPath(relativePath: string): string { + return resolve(this.options.ssr.webRoot, relativePath); + } + + public async createApolloServer(): Promise { + const resolvers = await getResolvers(); + + const schema = await buildGQLSchema( + resolvers, + Container.of(this.options.serverId), + ); + + const gqlServer = new ApolloServer({ + schema, + context: getGQLContext, + introspection: true, + playground: { + settings: { + 'editor.theme': 'light', + 'general.betaUpdates': true, + }, + workspaceName: 'TS-ESWeb', + }, }); + + return gqlServer; } /** @@ -50,6 +84,13 @@ export class ServerController { }); await validateOrReject(serverOptions); + if ( + typeof serverOptions.ssr !== 'undefined' && + typeof options.ssr?.appComponent !== 'undefined' + ) { + serverOptions.ssr.appComponent = options.ssr.appComponent; + } + container.set({ id: serverOptionsToken, global: true, @@ -72,7 +113,7 @@ export class ServerController { throw new Error('Invalid result from Redis'); } - public async getPathModule(filePath: string): Promise { + public async getPathModule(filePath: string): Promise { logger.silly('HelloWorld'); const result = await this.redisController.getValue( @@ -80,9 +121,7 @@ export class ServerController { filePath, ); - logger.silly(`getPath(${filePath})`, { - result, - }); + return result as string; } public async startTypeScript(): Promise { @@ -109,20 +148,6 @@ export class ServerController { await this.typescriptController.waitForTranspileDone(); - logger.silly('Done'); - - await this.getPathModule(filePath); - await this.getPathModule( - '/workspace/node_modules/react-dom/cjs/react-dom.development.js', - ); - - const value = await this.getModuleMap( - '/workspace/node_modules/react-dom/server.js', - ); - logger.debug('React-DOM Server', { - value, - }); - // const typescriptController = this.typescriptController; // async function getAllChildModules(filePath: string): Promise { @@ -145,4 +170,54 @@ export class ServerController { // const moduleFiles = await getAllChildModules(jobOutput.filePath); } + + public async renderHTML(): Promise { + await timeout(10); + + const appHTML = this.ssrController.renderApp(); + + return ` + + TS-ESWeb + + + +
${appHTML}
+ + + `; + } } diff --git a/src/Modules/TypeScript/TypeScriptController.ts b/src/Modules/TypeScript/TypeScriptController.ts index f208d6a..2614178 100644 --- a/src/Modules/TypeScript/TypeScriptController.ts +++ b/src/Modules/TypeScript/TypeScriptController.ts @@ -35,15 +35,15 @@ export class TypeScriptController { const typescriptTranspilerKey = 'typescriptTranspiler'; const moduleMapQueKey = 'typescriptModuleMap'; - this.transpilerQueue = this.queueController.createQueue( - typescriptTranspilerKey, - TranspilerWorkerJobInput, - ); + this.transpilerQueue = this.queueController.createQueue({ + name: typescriptTranspilerKey, + inputClass: TranspilerWorkerJobInput, + }); - this.moduleMapQueue = this.queueController.createQueue( - moduleMapQueKey, - ModuleMapWorkerJobInput, - ); + this.moduleMapQueue = this.queueController.createQueue({ + name: moduleMapQueKey, + inputClass: ModuleMapWorkerJobInput, + }); } /** diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index a8e8a65..461a37f 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -1,5 +1,4 @@ // src/Modules/TypeScript/TypeScriptModuleMapWorker.ts -import { getWorkerData } from '@k-foss/ts-worker'; import { Queue, Worker } from 'bullmq'; import { dirname } from 'path'; import * as ts from 'typescript'; @@ -7,27 +6,36 @@ import { fileURLToPath, pathToFileURL } from 'url'; import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; import { envMode } from '../../Utils/Environment'; -import '../../Utils/Setup'; -import { WorkerInput } from '../Queues/WorkerInput'; +import { workerInputToken } from '../Queues/WorkerInput'; import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { createTypeScriptProgram, isCommonJSImportSplit } from './Utils'; import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; +import Container from 'typedi'; +import { QueueController } from '../Queues/QueueController'; +import { queueToken } from '../Queues/QueueToken'; const logger = coreLogger.child({ labels: { worker: 'TypeScriptModuleMapWorker.ts', workerId: threadId }, }); -logger.info(`Worker starting`); +logger.info(`Worker starting`, { + labels: { + worker: 'TypeScriptModuleMapWorker.ts', + workerId: threadId, + }, +}); -const workerInput = await WorkerInput.createWorkerInput( - getWorkerData(import.meta.url), -); +const queueController = Container.get(QueueController); + +const queueName = Container.get(queueToken); +const workerInput = Container.get(workerInputToken); logger.silly(`workerInput`, { - workerInput, + queueController, + queueName, }); const webModuleMapQue = new Queue('webModuleMapQueue', { @@ -38,13 +46,11 @@ const transpilerQue = new Queue('typescriptTranspiler', { ...workerInput.queueOptions, }); -const moduleMapQue = new Queue(workerInput.queName, { +const moduleMapQue = new Queue(queueName, { ...workerInput.queueOptions, }); -const redisController = new RedisController({ - ...(workerInput.queueOptions.connection as { host: string }), -}); +const redisController = Container.get(RedisController); /** * Discover all imported modules and add to the TypeScript Module Map @@ -216,9 +222,12 @@ async function discoverModuleMap( }), ); + const specifierCore = moduleInput.specifier || moduleInput.filePath; + const specifier = specifierCore.replace('.tsx', ''); + const webModuleJobInputParams: WebModuleMapJobInput = { filePath: moduleInput.filePath, - specifier: moduleInput.specifier, + specifier, importedModules: importedModules.filter(Boolean) as string[], }; @@ -242,6 +251,11 @@ async function discoverModuleMap( JSON.stringify(webModuleJobInputParams), ); + await redisController.IORedis.set( + redisController.getRedisKey(RedisType.MODULE_MAP, specifier), + JSON.stringify(webModuleJobInputParams), + ); + await webModuleMapQue.add('webModuleMapQueue', webModuleJobInput, { jobId: webModuleJobInput.filePath, }); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 96f4a7f..6bbbbb1 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -1,36 +1,45 @@ // src/Modules/TypeScript/TypeScriptTranspilerWorker.ts -import { getWorkerData } from '@k-foss/ts-worker'; -import '../../Utils/Setup'; -import { plainToClass } from 'class-transformer'; import { Worker } from 'bullmq'; -import { logger as coreLogger } from '../../Library/Logger'; +import { cjsToEsmTransformerFactory } from 'cjstoesm'; +import { plainToClass } from 'class-transformer'; +import { validateOrReject } from 'class-validator'; +import { readFile } from 'fs/promises'; +import Container from 'typedi'; import { - transpileModule, + JsxEmit, ModuleKind, - ScriptTarget, ModuleResolutionKind, + ScriptTarget, + transpileModule, } from 'typescript'; -import { WorkerInput } from '../Queues/WorkerInput'; -import { validateOrReject } from 'class-validator'; -import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; -import { readFile } from 'fs/promises'; import { threadId } from 'worker_threads'; +import { logger as coreLogger } from '../../Library/Logger'; +import { processModule } from '../Files/processModule'; +// import { QueueController } from '../Queues/QueueController'; +import { queueToken } from '../Queues/QueueToken'; +import { workerInputToken } from '../Queues/WorkerInput'; import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; +import { ImportTransformer } from '../WebModule/ImportTransformer'; +import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; const logger = coreLogger.child({ labels: { worker: 'TypeScriptTranspilerWorker.ts', workeId: threadId }, }); -logger.info(`Worker starting`); +logger.info(`Worker starting`, { + labels: { + worker: 'TypeScriptTranspilerWorker.ts', + workeId: threadId, + }, +}); + +// const queueController = Container.get(QueueController); -const workerInput = await WorkerInput.createWorkerInput( - getWorkerData(import.meta.url), -); +const queueName = Container.get(queueToken); +const workerInput = Container.get(workerInputToken); -const redisController = new RedisController({ - ...(workerInput.queueOptions.connection as { host: string }), -}); +const redisController = Container.get(RedisController); logger.debug(`Retrieved workerData:`, { objectName: 'workerInput', @@ -51,15 +60,26 @@ async function transformFile(filePath: string): Promise { const file = await readFile(filePath); - const transpiledModule = transpileModule(file.toString(), { + let fileContents = processModule(file.toString()); + + const transpiledModule = transpileModule(fileContents, { + fileName: filePath, + transformers: { + before: [cjsToEsmTransformerFactory()], + after: [new ImportTransformer().after], + afterDeclarations: [ + // ... + ], + }, compilerOptions: { allowJs: true, checkJs: false, module: ModuleKind.ESNext, moduleResolution: ModuleResolutionKind.NodeJs, target: ScriptTarget.ESNext, - isolatedModules: true, inlineSourceMap: true, + jsx: JsxEmit.ReactJSXDev, + jsxFragmentFactory: 'Fragment', }, }); @@ -67,7 +87,7 @@ async function transformFile(filePath: string): Promise { } const transpilerWorker = new Worker( - workerInput.queName, + queueName, async (job) => { logger.info(`Recieved a task for transpilerWorker`, { worker: 'transpilerWorker', diff --git a/src/Modules/WebModule/ImportTransformer.ts b/src/Modules/WebModule/ImportTransformer.ts index f91d3d2..417e339 100644 --- a/src/Modules/WebModule/ImportTransformer.ts +++ b/src/Modules/WebModule/ImportTransformer.ts @@ -19,6 +19,12 @@ export class ImportTransformer extends Transformer { const argument = node.arguments[0]; if (ts.isStringLiteral(argument)) { + logger.silly(`ImportTransformer isCallExpression:`, { + argument: argument.text, + dirName: dirname(sourceFile.fileName), + fileName: sourceFile.fileName, + }); + return context.factory.updateCallExpression( node, node.expression, @@ -65,11 +71,7 @@ export class ImportTransformer extends Transformer { node.moduleSpecifier.text, ); } else { - if (node.moduleSpecifier.text === 'react/jsx-dev-runtime') { - specifier = 'react/cjs/react-jsx-dev-runtime.development'; - } else { - specifier = node.moduleSpecifier.text; - } + specifier = node.moduleSpecifier.text; } let importClause: ts.ImportClause | undefined; diff --git a/src/Modules/WebModule/WebModule.ts b/src/Modules/WebModule/WebModule.ts index 9451923..7d1dfba 100644 --- a/src/Modules/WebModule/WebModule.ts +++ b/src/Modules/WebModule/WebModule.ts @@ -6,7 +6,6 @@ export class WebModule { @Field() public filePath: string; - public constructor(opts: Partial = {}) { - Object.assign(this, opts); - } + @Field() + public code: string; } diff --git a/src/Modules/WebModule/WebModuleController.ts b/src/Modules/WebModule/WebModuleController.ts index 7746aed..f09d1cb 100644 --- a/src/Modules/WebModule/WebModuleController.ts +++ b/src/Modules/WebModule/WebModuleController.ts @@ -9,17 +9,20 @@ import { WebModuleMapJobInput } from './WebModuleMapJobInput'; @Service() export class WebModuleController { + /** + * Redis Queue for Web Module Map Discovery Tasks + */ private webModuleMapQueue: Queue<'webModuleMapQueue', WebModuleMapJobInput>; public constructor( @Inject(() => QueueController) private queueController: QueueController, ) { - this.webModuleMapQueue = queueController.createQueue( - 'webModuleMapQueue', - WebModuleMapJobInput, - true, - ); + this.webModuleMapQueue = queueController.createQueue({ + name: 'webModuleMapQueue', + inputClass: WebModuleMapJobInput, + disableTermination: true, + }); } /** diff --git a/src/Modules/WebModule/WebModuleJobInput.ts b/src/Modules/WebModule/WebModuleJobInput.ts index 453a991..c8a0271 100644 --- a/src/Modules/WebModule/WebModuleJobInput.ts +++ b/src/Modules/WebModule/WebModuleJobInput.ts @@ -9,6 +9,11 @@ export class WebModuleJobInput { @IsString() public sourceText: string; + /** + * Create a new Class from params and validate. + * @param inputParams Input Options for Web Module Job + * @returns Promise resolving to validated class + */ public static async createWebModuleJobInput( inputParams: Partial, ): Promise { diff --git a/src/Modules/WebModule/WebModuleMap.ts b/src/Modules/WebModule/WebModuleMap.ts new file mode 100644 index 0000000..d196eb5 --- /dev/null +++ b/src/Modules/WebModule/WebModuleMap.ts @@ -0,0 +1,30 @@ +// src/Modules/WebModule/WebModuleMap.ts +import { plainToClass } from 'class-transformer'; +import { IsString, validateOrReject } from 'class-validator'; +import { ObjectType } from 'type-graphql'; +import {} from 'typedi'; + +@ObjectType() +export class WebModuleMap { + @IsString() + public filePath: string; + + @IsString({ + each: true, + }) + public imports: string[]; + + /** + * Create a Module Map Class object + * @param inputArgs Input paramters for Web Module Map + * @returns Promise resolving to validated Module Map + */ + public static async createModuleMap( + inputArgs: Partial, + ): Promise { + const webModuleMap = plainToClass(WebModuleMap, inputArgs); + await validateOrReject(webModuleMap); + + return webModuleMap; + } +} diff --git a/src/Modules/WebModule/WebModuleMapController.ts b/src/Modules/WebModule/WebModuleMapController.ts new file mode 100644 index 0000000..371f746 --- /dev/null +++ b/src/Modules/WebModule/WebModuleMapController.ts @@ -0,0 +1,57 @@ +// src/Modules/WebModule/WebModuleMapController.ts +import { Inject, Service } from 'typedi'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; +import { serverOptionsToken, ServerOptions } from '../Server/ServerOptions'; +import { WebModuleMap } from './WebModuleMap'; + +@Service() +export class WebModuleMapController { + public redisController: RedisController; + + public constructor( + @Inject(serverOptionsToken) + public options: ServerOptions, + ) { + this.redisController = new RedisController({ + ...options.redis, + }); + } + + /** + * Set a Module Map Object into Redis + * @param input Web Module Map Input Params + */ + public async setModuleMap(input: WebModuleMap): Promise { + const webModuleMap = await WebModuleMap.createModuleMap(input); + const webModuleMapString = JSON.stringify(webModuleMap); + + return this.redisController.setValue(RedisType.MODULE_MAP, { + key: input.filePath, + value: webModuleMapString, + }); + } + + /** + * Get a Module Map Object from Redis + * @param filePath File Path + */ + public async getModuleMap( + filePath: string, + ): Promise { + const webModuleMapString = await this.redisController.getValue( + RedisType.MODULE_MAP, + filePath, + ); + + if (typeof webModuleMapString === 'string') { + const webModuleMapParams = JSON.parse(webModuleMapString) as WebModuleMap; + + const webModuleMap = await WebModuleMap.createModuleMap( + webModuleMapParams, + ); + + return webModuleMap; + } + } +} diff --git a/src/Modules/WebModule/WebModuleResolver.ts b/src/Modules/WebModule/WebModuleResolver.ts index a09edc1..ee69438 100644 --- a/src/Modules/WebModule/WebModuleResolver.ts +++ b/src/Modules/WebModule/WebModuleResolver.ts @@ -1,85 +1,58 @@ -// src/Modules/WebModule/WebModuleResolver.ts -import { - Arg, - Field, - FieldResolver, - InputType, - Query, - Resolver, - Root, -} from 'type-graphql'; -import { Service } from 'typedi'; +// src/Modules/WebModules/WebModuleResolver.ts +import { Arg, Field, InputType, Query, Resolver } from 'type-graphql'; +import { Inject, Service } from 'typedi'; import { logger } from '../../Library/Logger'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; +import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { WebModule } from './WebModule'; -import { webModuleController } from './WebModuleController'; +import { WebModuleController } from './WebModuleController'; @InputType() class WebModuleFilter { - @Field({ nullable: true }) + @Field() public specifier: string; - - @Field({ nullable: true }) - public filePath: string; } -@Resolver(() => WebModule) +@Resolver() @Service() export class WebModuleResolver { - @Query(() => [WebModule]) - public webModules(): WebModule[] { - return Array.from(webModuleController.modules.values()); - } + @Inject(() => RedisController) + public redisController: RedisController; - @Query(() => WebModule, { nullable: true }) - public webModule( - @Arg('filter') { filePath, specifier }: WebModuleFilter, - ): WebModule | undefined { - if (filePath) { - return webModuleController.getModule(filePath); - } else { - const filePath = webModuleController.specifierTest.get(specifier); + @Inject(serverOptionsToken) + public options: ServerOptions; - logger.debug( - `WebModuleResolver.webModule() result for ${specifier} is filePath: ${ - filePath?.toString() || '' - }`, - ); - - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return webModuleController.getModule(filePath!); - } - } + @Query(() => WebModule) + public async webModule( + @Arg('filter', () => WebModuleFilter) input: WebModuleFilter, + ): Promise { + logger.silly(`WebModuleResolver.webModule(${input.specifier})`); - @FieldResolver(() => [WebModule]) - public dependencies(@Root() webModule: WebModule): WebModule[] { - const deps = Array.from(webModule.dependencies); + const webModuleMapString = await this.redisController.getValue( + RedisType.MODULE_MAP, + input.specifier, + ); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return deps.flatMap( - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - (moduleKey) => webModuleController.getModule(moduleKey)!, - )!; - } + const webModuleMap = JSON.parse(webModuleMapString); - @FieldResolver(() => String) - public specifiers(@Root() webModule: WebModule): string { - return webModuleController.getSpecifiers(webModule.filePath); - } + logger.silly(`WebModuleResolver`, { + webModuleMap, + }); - @Query(() => Boolean) - public helloTest(): boolean { - logger.debug( - `WebModuleResolver.helloTest() Array.from(webModuleController.modules): ${JSON.stringify( - Array.from(webModuleController.modules), - )} - Array.from(webModuleController.specifierMap): ${JSON.stringify( - Array.from(webModuleController.specifierMap), - )} - Array.from(webModuleController.specifierTest): ${JSON.stringify( - Array.from(webModuleController.specifierTest), - )}`, + const webModuleCode = await this.redisController.getValue( + RedisType.MODULE, + webModuleMap?.filePath, ); - return true; + logger.silly(`WebModuleResolver`, { + webModuleMap, + webModuleCode, + }); + + return { + code: webModuleCode, + filePath: '/test', + }; } } diff --git a/src/Web_Test/App.tsx b/src/Web_Test/App.tsx index d5a023e..a144883 100644 --- a/src/Web_Test/App.tsx +++ b/src/Web_Test/App.tsx @@ -1,14 +1,14 @@ // src/App.tsx -import React from 'react'; +import * as React from 'react'; import { Counter } from './Components/Counter/index'; export function App(): React.ReactElement { const sayHello = React.useCallback(() => console.log('HelloWorld'), []); return ( - <> +

HelloWorld

- + ); } diff --git a/src/Web_Test/ServiceWorker.ts b/src/Web_Test/ServiceWorker.ts index ffea884..fa599e9 100644 --- a/src/Web_Test/ServiceWorker.ts +++ b/src/Web_Test/ServiceWorker.ts @@ -31,7 +31,7 @@ workbox.routing.registerRoute( apiHeaders.set('content-type', 'application/json'); const request = new Request('/graphql', { - body: `{"operationName":"webModule","variables":{"moduleSpecifier":"${moduleSpecifier}"},"query":"query webModule($moduleSpecifier: String!) { webModule(filter: {specifier: $moduleSpecifier}) { filePath specifiers code }}"}`, + body: `{"operationName":"webModule","variables":{"moduleSpecifier":"${moduleSpecifier}"},"query":"query webModule($moduleSpecifier: String!) { webModule(filter: {specifier: $moduleSpecifier}) { filePath code }}"}`, method: 'POST', headers: apiHeaders, }); diff --git a/src/index.tsx b/src/index.tsx index 0c57241..917fc69 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,4 +1,5 @@ // src/index.ts +import fastify from 'fastify'; import { resolve } from 'path'; import 'reflect-metadata'; import { logger } from './Library/Logger'; @@ -7,6 +8,8 @@ import { WebAppManifest } from './Modules/WebAppManifest/WebAppManifest'; import { WebAppManfiestController } from './Modules/WebAppManifest/WebAppManifestController'; import { App } from './Web_Test/App'; +const webServer = fastify(); + logger.info(`Starting TS-ESWeb`); logger.debug(`Creating TS-ESWeb SSR Server`); @@ -45,6 +48,10 @@ logger.debug(`Starting the TS-ESWeb SSR Server Transpiler`); await server.startTypeScript(); +const apiServer = await server.createApolloServer(); + +await webServer.register(apiServer.createHandler()); + // await ssrServer.startTranspiler(); // logger.debug(`Creating the Fastify Server`); @@ -55,4 +62,35 @@ await server.startTypeScript(); // logger.debug(`Fastify server is listening ${serverString}`); +webServer.get('/', async function (test, reply) { + reply.type('text/html'); + + return server.renderHTML(); +}); + +webServer.get<{ + Params: { + '*': string; + }; +}>('/Static/*', async function (request, reply) { + const filePath = request.params['*']; + + const contents = await server.getPathModule(filePath); + + if (contents) { + reply.type('application/javascript'); + reply.header('Service-Worker-Allowed', '/'); + + return contents; + } + + reply.code(404); + + reply.send('Error'); +}); + +const bindHost = await webServer.listen(9090, '0.0.0.0'); + +logger.info(`Web server listening on ${bindHost}`); + export {}; From c2898870f7b53e1747b1dd8368ca2763054c4fb7 Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Fri, 26 Feb 2021 06:21:47 +0000 Subject: [PATCH 27/30] feat: Production Minified modules are working now. Fixes the processing regex statement to permit the processing of minifed files where the first exports.XYZ statement isn't on a "newline"/^ but rather allow for ";" as the previous statement --- .vscode/settings.json | 2 +- Extras/Loki/config.yaml | 69 ++++++++ docker-compose.dev.yml | 4 +- src/Library/Transformers.ts | 28 +++ src/Modules/Files/processModule.ts | 64 +++++-- src/Modules/Queues/Queue.ts | 42 +++-- src/Modules/Queues/QueueOptions.ts | 2 +- src/Modules/Queues/QueueToken.ts | 5 + src/Modules/Queues/Worker.ts | 9 +- src/Modules/Queues/WorkerController.ts | 38 ++++- src/Modules/SSR/SSROptions.ts | 7 +- src/Modules/SSR/SSRRoute.ts | 31 ---- src/Modules/Server/SSRServer.tsx | 160 ------------------ src/Modules/Server/ServerController.ts | 20 ++- src/Modules/Server/ServerOptions.ts | 25 ++- src/Modules/Server/createSSRServer.ts | 22 --- src/Modules/Server/index.ts | 2 - src/Modules/TypeScript/Regex.ts | 2 +- src/Modules/TypeScript/TSConfig.ts | 13 +- .../TypeScript/TypeScriptModuleMapWorker.ts | 54 ++++-- .../TypeScript/TypeScriptTranspilerWorker.ts | 34 +++- src/Modules/WebModule/ImportTransformer.ts | 2 +- src/Web_Test/App.tsx | 8 +- src/Web_Test/Imports.ts | 6 - src/index.tsx | 4 +- tsconfig.json | 2 +- 26 files changed, 339 insertions(+), 316 deletions(-) create mode 100644 Extras/Loki/config.yaml delete mode 100644 src/Modules/SSR/SSRRoute.ts delete mode 100644 src/Modules/Server/SSRServer.tsx delete mode 100644 src/Modules/Server/createSSRServer.ts delete mode 100644 src/Modules/Server/index.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 3662b37..25fa621 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,3 @@ { "typescript.tsdk": "node_modules/typescript/lib" -} \ No newline at end of file +} diff --git a/Extras/Loki/config.yaml b/Extras/Loki/config.yaml new file mode 100644 index 0000000..c03f972 --- /dev/null +++ b/Extras/Loki/config.yaml @@ -0,0 +1,69 @@ +auth_enabled: false + + + +server: + http_listen_port: 3100 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/boltdb-shipper-active + cache_location: /loki/boltdb-shipper-cache + cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space + shared_store: filesystem + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/boltdb-shipper-compactor + shared_store: filesystem + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-temp + alertmanager_url: http://localhost:9093 + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index acc8d8a..b50d74a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -30,7 +30,9 @@ services: Loki: image: grafana/loki:latest - command: -config.file=/etc/loki/local-config.yaml + command: -config.file=/etc/loki/config.yaml + volumes: + - ./Extras/Loki/config.yaml:/etc/loki/config.yaml:ro networks: - internalLoki diff --git a/src/Library/Transformers.ts b/src/Library/Transformers.ts index 109d969..20a891e 100644 --- a/src/Library/Transformers.ts +++ b/src/Library/Transformers.ts @@ -8,6 +8,7 @@ import { } from 'typescript'; import { findModuleFiles } from '../Utils/moduleFileFinder'; import { cjsToEsmTransformerFactory } from 'cjstoesm'; +import { logger } from './Logger'; let customTransformers: CustomTransformers; @@ -100,3 +101,30 @@ export async function getTransformers( return customTransformers; } + +export class TypeScriptTransformerController { + public transformers: Transformer[]; + + private async findTransformers(): Promise { + const modules = await findModuleFiles<{ [key: string]: Transformer }>( + /.*Resolver\.(js|ts)/, + ); + + return modules.flatMap((moduleExports) => Object.values(moduleExports)); + } + + /** + * Load transformer modules and return the transformers object + */ + public async loadTransformers(): Promise { + const transformers = await this.findTransformers(); + + logger.silly(`loadTransformers`, { + labels: { + appName: 'TS-ESWeb', + className: 'TypeScriptTransformerController', + }, + transformers, + }); + } +} diff --git a/src/Modules/Files/processModule.ts b/src/Modules/Files/processModule.ts index 7fc1912..9ccc885 100644 --- a/src/Modules/Files/processModule.ts +++ b/src/Modules/Files/processModule.ts @@ -1,5 +1,5 @@ // src/Modules/Files/processModule.ts - +import { isString } from 'class-validator'; import { logger } from '../../Library/Logger'; import { exportsHandler, @@ -10,29 +10,59 @@ import { export function processModule(fileContents: string): string { const exportVars: string[] = []; - logger.silly(`processModule`, { + const processLogger = logger.child({ + function: `processModule`, + }); + + processLogger.silly(`Processing module`, { fileContents, }); - /** - * https://regex101.com/r/uwAq1N/1 - */ + let moduleContents: string = fileContents; - let moduleContents = fileContents - .replace(processNodeReplacement, '$') - .replaceAll(exportsHandler, (test, todo, varName) => { - exportVars.push(varName); - logger.silly(`exportVars:`, { varName }); + function setModuleContents(newValue: string, actionLabel?: string): void { + processLogger.silly(`setModuleContents`, { + newValue, + actionLabel, + }); - return `var ${varName as string}`; - }) - .replace(objectExport, (...args) => { - const { coreCode } = args[args.length - 1]; + moduleContents = newValue; + } - return coreCode.replaceAll('exports.', ''); - }); + setModuleContents( + moduleContents.replace(processNodeReplacement, '$'), + 'processNodeReplacement', + ); + + setModuleContents( + moduleContents.replaceAll( + exportsHandler, + (_test, _todo, varName: string) => { + exportVars.push(varName); + return `var ${varName}`; + }, + ), + 'exportsHandler', + ); + + setModuleContents( + moduleContents.replace(objectExport, (...args) => { + const obj = args[args.length - 1] as { coreCode: string }; + + if (isString(obj.coreCode)) { + return obj.coreCode.replaceAll('exports.', ''); + } + + throw new Error('Invalid object replacement coreCode'); + }), + 'objectExport', + ); + + /** + * https://regex101.com/r/uwAq1N/1 + */ - exportVars.map((exportVar) => { + exportVars.forEach((exportVar) => { moduleContents = moduleContents.replaceAll( `exports.${exportVar}`, `${exportVar}`, diff --git a/src/Modules/Queues/Queue.ts b/src/Modules/Queues/Queue.ts index c5d746d..4118f8e 100644 --- a/src/Modules/Queues/Queue.ts +++ b/src/Modules/Queues/Queue.ts @@ -1,20 +1,20 @@ // src/Modules/Queue/Queue.ts +import { spawnWorker } from '@k-foss/ts-worker'; import { Job, JobsOptions, Queue as BullMQQueue, QueueEvents as BullMQQueueEvents, } from 'bullmq'; +import { ClassConstructor, plainToClass } from 'class-transformer'; +import EventEmitter from 'events'; +import { fileURLToPath } from 'node:url'; +import pEvent from 'p-event'; +import { Logger } from 'winston'; import { Worker } from 'worker_threads'; -import { spawnWorker } from '@k-foss/ts-worker'; import { logger } from '../../Library/Logger'; import { QueueOptions } from './QueueOptions'; -import { ClassConstructor, plainToClass } from 'class-transformer'; -import { Logger } from 'winston'; import { WorkerInput } from './WorkerInput'; -import EventEmitter from 'events'; -import pEvent from 'p-event'; -import { fileURLToPath } from 'node:url'; /** * Task Queue handled by dedicated `worker_threads` created by @@ -156,15 +156,33 @@ export class Queue { */ private startWatching(): void { if (this.options.disableTermination !== true) { - this.handleDrained = (): void => { - this.isRunningJobs().catch(() => { - this.logger.silly('Error isRunningJobs, terminating workers'); + let jobLessCounts = 0; - return this.terminateWorkers(); - }); + const helloWorld = async (): Promise => { + const jobCounts = await this.getCounts(); + + const anyJobs = jobCounts.some((jobCount) => jobCount > 0); + + if (this.hasRun === false) { + if (anyJobs === true) { + this.hasRun = true; + } + + return; + } + + if (anyJobs === false) { + jobLessCounts++; + + if (jobLessCounts > 5) { + return this.terminateWorkers(); + } + } }; - this.queueEvents.on('drained', this.handleDrained); + setInterval(() => { + helloWorld(); + }, 500); } } diff --git a/src/Modules/Queues/QueueOptions.ts b/src/Modules/Queues/QueueOptions.ts index 10c4f32..275c42b 100644 --- a/src/Modules/Queues/QueueOptions.ts +++ b/src/Modules/Queues/QueueOptions.ts @@ -1,6 +1,6 @@ // src/Modules/Queues/QueueOptions.ts -import { IsBoolean, IsDefined, IsOptional, IsString } from 'class-validator'; import type { QueueOptions as BullMQOptions } from 'bullmq'; +import { IsBoolean, IsDefined, IsOptional, IsString } from 'class-validator'; import { ServerOptions } from '../Server/ServerOptions'; export class QueueOptions { diff --git a/src/Modules/Queues/QueueToken.ts b/src/Modules/Queues/QueueToken.ts index 669bdf0..95885c4 100644 --- a/src/Modules/Queues/QueueToken.ts +++ b/src/Modules/Queues/QueueToken.ts @@ -1,4 +1,9 @@ // src/Modules/Queues/QueueToken.ts import { Token } from 'typedi'; +/** + * TypeDI Token for the label/name/key of the Workers Queue + * + * @default queueToken + */ export const queueToken = new Token('queueToken'); diff --git a/src/Modules/Queues/Worker.ts b/src/Modules/Queues/Worker.ts index 0d3d99d..755d7b6 100644 --- a/src/Modules/Queues/Worker.ts +++ b/src/Modules/Queues/Worker.ts @@ -1,10 +1,9 @@ // src/Modules/Queues/Worker.ts import { getWorkerData } from '@k-foss/ts-worker'; -import '../../Utils/Setup'; -import {} from 'worker_threads'; import { logger } from '../../Library/Logger'; -import { WorkerInput } from './WorkerInput'; +import '../../Utils/Setup'; import { WorkerController } from './WorkerController'; +import { WorkerInput } from './WorkerInput'; const workerInput = await WorkerInput.createWorkerInput( getWorkerData(import.meta.url), @@ -18,8 +17,6 @@ const workerController = await WorkerController.createWorkerController( workerInput, ); -logger.silly('WorkerSpawned: ', { - workerController, -}); +workerController.logger.info(`Worker has been spawned`); await import(workerInput.workerPath); diff --git a/src/Modules/Queues/WorkerController.ts b/src/Modules/Queues/WorkerController.ts index b566d41..917e273 100644 --- a/src/Modules/Queues/WorkerController.ts +++ b/src/Modules/Queues/WorkerController.ts @@ -2,21 +2,31 @@ import { plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; -import Container, { Inject, Service } from 'typedi'; -import { logger } from '../../Library/Logger'; +import { basename } from 'node:path'; +import Container, { Inject, Service, Token } from 'typedi'; +import { Logger } from 'winston'; +import { logger as coreLogger } from '../../Library/Logger'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; import { queueToken } from './QueueToken'; import { WorkerInput, workerInputToken } from './WorkerInput'; @Service() export class WorkerController { + public logger: Logger; + public constructor( - @Inject(serverOptionsToken) - public options: ServerOptions, + @Inject(workerInputToken) + public workerInput: WorkerInput, ) { - logger.silly(`WorkerController`, { - options, + this.logger = coreLogger.child({ + labels: { + appName: 'TS-ESWeb', + worker: basename(workerInput.workerPath), + queueName: workerInput.queName, + }, }); + + this.logger.silly(`WorkerController created`); } public static async createWorkerController( @@ -46,6 +56,20 @@ export class WorkerController { value: workerInput, }); - return Container.get(WorkerController); + const workerController = Container.get(WorkerController); + Container.set({ + id: workerControllerToken, + global: true, + value: workerController, + }); + + return Container.get(workerControllerToken); } } + +/** + * TypeDI Token for the current workers WorkerController + */ +export const workerControllerToken = new Token( + 'workerController', +); diff --git a/src/Modules/SSR/SSROptions.ts b/src/Modules/SSR/SSROptions.ts index 234694e..e57248f 100644 --- a/src/Modules/SSR/SSROptions.ts +++ b/src/Modules/SSR/SSROptions.ts @@ -1,5 +1,5 @@ // src/Modules/SSR/SSROptions.ts -import { Exclude, Expose, Transform } from 'class-transformer'; +import { Exclude } from 'class-transformer'; import { IsDefined, IsString } from 'class-validator'; export class SSROptions { @@ -23,10 +23,5 @@ export class SSROptions { public entrypoint: string; @Exclude() - @Transform(({ value }) => { - console.log('Value: ', value); - - return value; - }) public appComponent: () => React.ReactElement; } diff --git a/src/Modules/SSR/SSRRoute.ts b/src/Modules/SSR/SSRRoute.ts deleted file mode 100644 index 187afd8..0000000 --- a/src/Modules/SSR/SSRRoute.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ -/* eslint-disable @typescript-eslint/ban-types */ -// src/Modules/SSR/SSRRoute.ts -import { FastifyInstance, FastifyReply, FastifyRequest } from 'fastify'; -import { Service } from 'typedi'; -import { Route } from '../../Library/Fastify'; -import { ssrServer } from '../Server/createSSRServer'; - -/** - * Route to serve the rendered React SSR Routes - */ -@Service() -export default class SSRRoute implements Route { - public options: Route['options'] = { - method: 'GET', - url: '/Test', - }; - - // eslint-disable-next-line @typescript-eslint/require-await - public async handler( - this: FastifyInstance, - request: FastifyRequest, - reply: FastifyReply, - ): Promise { - reply.type('text/html'); - - const appHTML = ssrServer.renderApp(request.url); - - return appHTML; - } -} diff --git a/src/Modules/Server/SSRServer.tsx b/src/Modules/Server/SSRServer.tsx deleted file mode 100644 index 69e5465..0000000 --- a/src/Modules/Server/SSRServer.tsx +++ /dev/null @@ -1,160 +0,0 @@ -// src/Modules/Server/SSRServer.ts -import fastify, { FastifyInstance } from 'fastify'; -import { resolve } from 'path'; -import React from 'react'; -import { renderToString } from 'react-dom/server'; -import Container, { Service } from 'typedi'; -import { createApolloServer } from '../../Library/Apollo'; -import { getRoutes, Route } from '../../Library/Fastify'; -import { logger } from '../../Library/Logger'; -import { ReactFunction } from '../../Utils/React'; -import { startWebTranspiler } from '../TypeScript'; - -export interface ClientOptions { - appComponent: ReactFunction; - - /** - * Root Directory for Web/Client - * - * @example - * ```ts - * webRoot: path.resolve('./Web') - * ``` - */ - webRoot: string; - - /** - * Entrypoint Path relative to @see webRoot - */ - entrypoint: string; - - /** - * Additional Fastify Routes - */ - serverRoutes: Route[]; -} - -/** - * TS-ESWeb SSR Server Controller - */ -@Service() -export class SSRServer { - /** - * Client Provided Options - */ - public options: ClientOptions = { - appComponent: () =>
HelloWorld
, - entrypoint: 'Imports.ts', - webRoot: 'Web', - serverRoutes: [], - }; - - public webServer: FastifyInstance = fastify(); - - public get entrypoint(): string { - return resolve(this.options.webRoot, this.options.entrypoint); - } - - /** - * Get Absolute Path from a relative path - * @param relativePath Relative Path to Web Root - */ - public getFilePath(relativePath: string): string { - return resolve(this.options.webRoot, relativePath); - } - - /** - * Create a new SSRServer Instance - * @param opt Properties of SSRServer - */ - public constructor(opt: ClientOptions) { - this.options = opt; - } - - public async createFastifyServer(): Promise { - /** - * Get All Route Modules. - */ - const routes = await getRoutes(Container.of()); - - /** - * For each Route Module in routes destructure handler and options and register as a webServer Route. - */ - routes.map((route) => { - return this.webServer.route({ - ...route.options, - handler: async (...params: unknown[]) => route.handler(...params), - }); - }); - - /** - * Apollo GraphQL Server - */ - const gqlServer = await createApolloServer(); - - const apiPlugin = gqlServer.createHandler(); - - /** - * Register the Apollo Server Routes into the Fastify instance - */ - await this.webServer.register(apiPlugin); - - return this.webServer; - } - - public renderApp(path: string): string { - const App = this.options.appComponent; - - logger.debug(`SSRServer.renderApp(${path})`); - - const appHTML = renderToString(); - - const coreHTML = ` - - TS-ESWeb - - - -
${appHTML}
- - - `; - - return coreHTML; - } -} diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index aeee260..6625fbf 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -78,10 +78,16 @@ export class ServerController { const container = Container.of(serverId); - const serverOptions = plainToClass(ServerOptions, { - ...options, - serverId, - }); + const serverOptions = plainToClass( + ServerOptions, + { + ...options, + serverId, + }, + { + strategy: 'exposeAll', + }, + ); await validateOrReject(serverOptions); if ( @@ -139,12 +145,10 @@ export class ServerController { await import.meta.resolve('../../Web_Test/Imports.ts'), ); - const entrypointJob = await this.typescriptController.createModuleMapTask({ + await this.typescriptController.createModuleMapTask({ filePath, }); - logger.silly(`entrypointJob`, { - entrypointJob, - }); + logger.silly(`entrypointJob created`); await this.typescriptController.waitForTranspileDone(); diff --git a/src/Modules/Server/ServerOptions.ts b/src/Modules/Server/ServerOptions.ts index 7a9b525..efc2f93 100644 --- a/src/Modules/Server/ServerOptions.ts +++ b/src/Modules/Server/ServerOptions.ts @@ -1,7 +1,16 @@ // src/Modules/Server/ServerOptions.ts -import { Type } from 'class-transformer'; -import { IsDefined, IsUUID, ValidateNested } from 'class-validator'; +import { Expose, Transform, Type } from 'class-transformer'; +import { + IsDefined, + IsEnum, + IsOptional, + IsString, + IsUUID, + ValidateNested, +} from 'class-validator'; import { Token } from 'typedi'; +import { logger } from '../../Library/Logger'; +import { Environment, envMode } from '../../Utils/Environment'; import { RedisOptions } from '../Redis/RedisOptions'; import { SSROptions } from '../SSR/SSROptions'; @@ -17,6 +26,18 @@ export class ServerOptions { @IsUUID() @IsDefined() public serverId: string; + + @Expose() + @IsEnum(Environment) + @IsOptional() + @Transform(({ value }) => { + if (typeof value === 'undefined') { + return envMode; + } + + return value as Environment; + }) + public envMode: Environment; } export const serverOptionsToken = new Token('serverOptions'); diff --git a/src/Modules/Server/createSSRServer.ts b/src/Modules/Server/createSSRServer.ts deleted file mode 100644 index a1c3c6a..0000000 --- a/src/Modules/Server/createSSRServer.ts +++ /dev/null @@ -1,22 +0,0 @@ -// src/Modules/Server/createSSRServer.ts -import { logger } from '../../Library/Logger'; -import { timeout } from '../../Utils/timeout'; -import { ClientOptions, SSRServer } from './SSRServer'; - -interface SSRServerOptions { - options: ClientOptions; -} - -export let ssrServer: SSRServer; - -export async function createSSRServer({ - options, -}: SSRServerOptions): Promise { - await timeout(50); - - logger.info(`Creating SSR Server with specified configuration`); - - ssrServer = new SSRServer(options); - - return ssrServer; -} diff --git a/src/Modules/Server/index.ts b/src/Modules/Server/index.ts deleted file mode 100644 index 0feac66..0000000 --- a/src/Modules/Server/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -// src/Modules/Server/index.ts -export { createSSRServer } from './createSSRServer'; diff --git a/src/Modules/TypeScript/Regex.ts b/src/Modules/TypeScript/Regex.ts index 1b2b323..e578474 100644 --- a/src/Modules/TypeScript/Regex.ts +++ b/src/Modules/TypeScript/Regex.ts @@ -3,4 +3,4 @@ export const objectExport = /^\{$\n(?((?^\s+)\S+\s+=\s+\{$\n\k export const processNodeReplacement = /if\s+\(process\.env\.NODE_ENV \S+\s"production"\)\s{\n\s+\(function\(\)\s{\n(?(.*\n?)*)}\)\(\);\n}/gim; -export const exportsHandler = /(^exports\.(?\S+)(?=\s+=.*$))(?=.*exports\.\k\s+=)/gmsu; +export const exportsHandler = /((?<=^|;)exports\.(?\S+)(?=\s*=.*;))(?=.*exports\.\k\s*=)/gmsu; diff --git a/src/Modules/TypeScript/TSConfig.ts b/src/Modules/TypeScript/TSConfig.ts index 54f991a..b760920 100644 --- a/src/Modules/TypeScript/TSConfig.ts +++ b/src/Modules/TypeScript/TSConfig.ts @@ -2,20 +2,19 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/unbound-method */ // src/Modules/TypeScript/TSConfig.ts +import { dirname as pathDirname, isAbsolute as isAbsolutePath } from 'path'; import { CompilerOptions, - ScriptTarget, + convertCompilerOptionsFromJson, + findConfigFile, + JsxEmit, ModuleKind, ModuleResolutionKind, - JsxEmit, - findConfigFile, - sys, - convertCompilerOptionsFromJson, readConfigFile, + ScriptTarget, + sys, } from 'typescript'; -import { isAbsolute as isAbsolutePath, dirname as pathDirname } from 'path'; - let tsConfigCache: CompilerOptions; export function getTSConfig(modulePath: string): CompilerOptions { diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index 461a37f..f53aa3a 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -1,21 +1,22 @@ // src/Modules/TypeScript/TypeScriptModuleMapWorker.ts import { Queue, Worker } from 'bullmq'; import { dirname } from 'path'; +import Container from 'typedi'; import * as ts from 'typescript'; import { fileURLToPath, pathToFileURL } from 'url'; import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; import { envMode } from '../../Utils/Environment'; +import { QueueController } from '../Queues/QueueController'; +import { queueToken } from '../Queues/QueueToken'; +import { workerControllerToken } from '../Queues/WorkerController'; import { workerInputToken } from '../Queues/WorkerInput'; +import { RedisController } from '../Redis/RedisController'; +import { RedisType } from '../Redis/RedisTypes'; import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; import { ModuleMapWorkerJobInput } from './ModuleMapWorkerJobInput'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; import { createTypeScriptProgram, isCommonJSImportSplit } from './Utils'; -import { RedisController } from '../Redis/RedisController'; -import { RedisType } from '../Redis/RedisTypes'; -import Container from 'typedi'; -import { QueueController } from '../Queues/QueueController'; -import { queueToken } from '../Queues/QueueToken'; const logger = coreLogger.child({ labels: { worker: 'TypeScriptModuleMapWorker.ts', workerId: threadId }, @@ -30,10 +31,12 @@ logger.info(`Worker starting`, { const queueController = Container.get(QueueController); +const workerController = Container.get(workerControllerToken); + const queueName = Container.get(queueToken); const workerInput = Container.get(workerInputToken); -logger.silly(`workerInput`, { +workerController.logger.silly(`workerInput`, { queueController, queueName, }); @@ -60,11 +63,12 @@ const redisController = Container.get(RedisController); async function discoverModuleMap( moduleInput: ModuleMapWorkerJobInput, ): Promise { - const moduleMapLogger = logger.child({ + const moduleMapLogger = workerController.logger.child({ labels: { filePath: moduleInput.filePath, worker: 'TypeScriptModuleMapWorker.ts', workerId: threadId, + appName: 'TS-ESWeb', }, }); @@ -131,8 +135,22 @@ async function discoverModuleMap( // @ts-expect-error // eslint-disable-next-line array-callback-return (_test, [_filePath, resolvedModule]) => { - if (resolvedModule.packageId?.subModuleName.includes(envMode)) { + workerController.logger.silly(`resolvedArrayReduce`, { + _filePath, + resolvedModule, + includes: workerInput.serverOptions.envMode, + _test, + specifier: moduleInput.specifier, + }); + + if ( + resolvedModule.packageId?.subModuleName.includes( + workerInput.serverOptions.envMode, + ) + ) { return [_filePath, resolvedModule]; + } else { + return _test; } }, ); @@ -157,6 +175,7 @@ async function discoverModuleMap( moduleMapLogger.error('Error during reduce', { // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment err, + specifier: moduleInput.specifier, }); } } @@ -259,13 +278,24 @@ async function discoverModuleMap( await webModuleMapQue.add('webModuleMapQueue', webModuleJobInput, { jobId: webModuleJobInput.filePath, }); + + workerController.logger.silly(`Creating Transpiler Task`, { + jobId: transpilerJobInput.filePath, + transpilerJobInput, + specifier, + }); + await transpilerQue.add('typescriptTranspiler', transpilerJobInput, { jobId: transpilerJobInput.filePath, }); } -const moduleWorkerLogger = logger.child({ +const moduleWorkerLogger = workerController.logger.child({ worker: 'moduleWorker', + labels: { + appName: 'TS-ESWeb', + worker: 'moduleWorker', + }, }); const moduleWorker = new Worker( @@ -295,7 +325,7 @@ const moduleWorker = new Worker( }, ); -logger.silly(`Created moduleWorker`, { - objectName: 'moduleWorker', - moduleWorker, +workerController.logger.silly(`Created moduleWorker`, { + objectName: 'moduleWorker.name', + value: moduleWorker.name, }); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 6bbbbb1..877a67c 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -14,9 +14,12 @@ import { } from 'typescript'; import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; +import { TypeScriptTransformerController } from '../../Library/Transformers'; +import { Environment, envMode } from '../../Utils/Environment'; import { processModule } from '../Files/processModule'; // import { QueueController } from '../Queues/QueueController'; import { queueToken } from '../Queues/QueueToken'; +import { workerControllerToken } from '../Queues/WorkerController'; import { workerInputToken } from '../Queues/WorkerInput'; import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; @@ -36,6 +39,8 @@ logger.info(`Worker starting`, { // const queueController = Container.get(QueueController); +const workerController = Container.get(workerControllerToken); + const queueName = Container.get(queueToken); const workerInput = Container.get(workerInputToken); @@ -46,21 +51,33 @@ logger.debug(`Retrieved workerData:`, { workerInput, }); +const transformerController = new TypeScriptTransformerController(); + /** * Transform provided filepath to ESM with TypeScript Compiler API * @param filePath Path to file to load, transform, and transpile to ESM * @returns Promise resolving to string of tranformed file. */ async function transformFile(filePath: string): Promise { - logger.info(`Transforming ${filePath}`); + workerController.logger.info(`Transforming ${filePath}`); - logger.silly(`Loading File`, { + workerController.logger.silly(`Loading File`, { filePath, }); const file = await readFile(filePath); - let fileContents = processModule(file.toString()); + let fileContents: string; + + fileContents = processModule(file.toString()); + + workerController.logger.silly(`Getting TypeScript Transformers`); + + await transformerController.loadTransformers(); + + workerController.logger.silly(`EnvMode`, { + envMode, + }); const transpiledModule = transpileModule(fileContents, { fileName: filePath, @@ -78,12 +95,15 @@ async function transformFile(filePath: string): Promise { moduleResolution: ModuleResolutionKind.NodeJs, target: ScriptTarget.ESNext, inlineSourceMap: true, - jsx: JsxEmit.ReactJSXDev, + jsx: + workerInput.serverOptions.envMode === Environment.PRODUCTION + ? JsxEmit.ReactJSX + : JsxEmit.ReactJSXDev, jsxFragmentFactory: 'Fragment', }, }); - return transpiledModule.outputText; + return transpiledModule.outputText.replaceAll('exports.', ''); } const transpilerWorker = new Worker( @@ -115,6 +135,8 @@ const transpilerWorker = new Worker( const sourceText = await transformFile(jobInput.filePath); + logger.silly(`Module ${jobInput.filePath} has been transpiled`); + await redisController.setValue(RedisType.MODULE, { key: jobInput.filePath, value: sourceText, @@ -131,5 +153,5 @@ const transpilerWorker = new Worker( logger.silly(`Created transpilerWorker`, { objectName: 'transpilerWorker', - transpilerWorker, + transpilerWorkerName: transpilerWorker.name, }); diff --git a/src/Modules/WebModule/ImportTransformer.ts b/src/Modules/WebModule/ImportTransformer.ts index 417e339..a35b303 100644 --- a/src/Modules/WebModule/ImportTransformer.ts +++ b/src/Modules/WebModule/ImportTransformer.ts @@ -7,7 +7,7 @@ import { Transformer } from '../../Library/Transformers'; const pushDefault = ['react', 'react-dom', 'scheduler', 'scheduler/tracing']; export class ImportTransformer extends Transformer { - public after: Transformer['before'] = (context) => { + public after: Transformer['after'] = (context) => { // const relativePathRegex = /^\.{1,2}[/]/; return (sourceFile: ts.SourceFile): ts.SourceFile => { diff --git a/src/Web_Test/App.tsx b/src/Web_Test/App.tsx index a144883..5154f94 100644 --- a/src/Web_Test/App.tsx +++ b/src/Web_Test/App.tsx @@ -1,14 +1,14 @@ // src/App.tsx -import * as React from 'react'; +import { useCallback } from 'react'; import { Counter } from './Components/Counter/index'; export function App(): React.ReactElement { - const sayHello = React.useCallback(() => console.log('HelloWorld'), []); + const sayHello = useCallback(() => console.log('HelloWorld'), []); return ( - + <>

HelloWorld

- + ); } diff --git a/src/Web_Test/Imports.ts b/src/Web_Test/Imports.ts index ea944a9..984229f 100644 --- a/src/Web_Test/Imports.ts +++ b/src/Web_Test/Imports.ts @@ -13,12 +13,6 @@ export async function importServiceWorker(): Promise< return import('./ServiceWorker'); } -export async function importReactDOMServer(): Promise< - typeof import('react-dom/server') -> { - return import('react-dom/server'); -} - export async function importReactDOM(): Promise { return import('react-dom'); } diff --git a/src/index.tsx b/src/index.tsx index 917fc69..72157e9 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,7 +1,7 @@ // src/index.ts import fastify from 'fastify'; import { resolve } from 'path'; -import 'reflect-metadata'; +import './Utils/Setup'; import { logger } from './Library/Logger'; import { ServerController } from './Modules/Server/ServerController'; import { WebAppManifest } from './Modules/WebAppManifest/WebAppManifest'; @@ -12,7 +12,7 @@ const webServer = fastify(); logger.info(`Starting TS-ESWeb`); -logger.debug(`Creating TS-ESWeb SSR Server`); +logger.debug(`Creating TS-ESWeb Web Manifest`); const manifest: WebAppManifest = { name: 'HelloWorld', diff --git a/tsconfig.json b/tsconfig.json index 178bd4a..ed7d496 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,7 +24,7 @@ "experimentalDecorators": true, "noEmit": false, "emitDecoratorMetadata": true, - "jsx": "react-jsxdev" + "jsx": "react-jsx" }, "include": ["src/**/*", "bin/**/*"], "exclude": ["node_modules"] From 71c68dafd03f589a1bb4896527051ee34776ee6d Mon Sep 17 00:00:00 2001 From: KristianFJones Date: Fri, 26 Feb 2021 20:20:42 +0000 Subject: [PATCH 28/30] feat: Continue work on processing and module maps. Get transformer controller up and running for automatic module import --- package-lock.json | 936 +++++++++++++++++- package.json | 5 +- src/Library/Transformers.ts | 49 +- src/Modules/Files/processModule.ts | 10 + src/Modules/SSR/SSRController.tsx | 9 +- src/Modules/SSR/StaticRouter.tsx | 93 ++ src/Modules/Server/ServerController.ts | 29 +- src/Modules/Server/ServerOptions.ts | 28 +- src/Modules/ServiceWorker/ServiceWorker.ts | 101 -- src/Modules/TypeScript/Regex.ts | 2 + .../TypeScript/TypeScriptModuleMapWorker.ts | 88 +- .../TypeScript/TypeScriptTranspilerWorker.ts | 25 +- src/Modules/TypeScript/Utils.ts | 16 + src/Modules/WebModule/ImportTransformer.ts | 95 +- src/Web_Test/App.tsx | 10 +- src/Web_Test/Client.tsx | 5 +- src/Web_Test/Components/Nav/index.tsx | 13 + src/Web_Test/Components/Routers/index.tsx | 17 + src/Web_Test/Routes/About.tsx | 10 + src/Web_Test/Routes/Home.tsx | 11 +- src/Web_Test/Routes/Lab.tsx | 40 + src/index.tsx | 62 +- 22 files changed, 1416 insertions(+), 238 deletions(-) create mode 100644 src/Modules/SSR/StaticRouter.tsx delete mode 100644 src/Modules/ServiceWorker/ServiceWorker.ts create mode 100644 src/Web_Test/Components/Nav/index.tsx create mode 100644 src/Web_Test/Components/Routers/index.tsx create mode 100644 src/Web_Test/Routes/About.tsx create mode 100644 src/Web_Test/Routes/Lab.tsx diff --git a/package-lock.json b/package-lock.json index 6cff348..387b6ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "hyperid": "^2.1.0", "ioredis": "^4.22.0", "p-event": "^4.2.0", + "react-refresh-typescript": "^1.1.1", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3", @@ -28,6 +29,7 @@ "devDependencies": { "@k-foss/ts-esnode": "^2.0.0", "@k-foss/ts-estests": "^1.0.6", + "@material-ui/core": "^5.0.0-alpha.25", "@types/node": "^14.14.31", "@types/react": "^17.0.0", "@types/react-dom": "^17.0.0", @@ -49,8 +51,9 @@ "prettier": "^2.2.1", "react": "^17.0.1", "react-dom": "^17.0.1", + "react-router-dom": "^6.0.0-beta.0", "typedi": "^0.10.0", - "typescript": "^4.1.3" + "typescript": "^4.2.2" } }, "node_modules/@apollo/protobufjs": { @@ -143,6 +146,15 @@ "node": ">=4" } }, + "node_modules/@babel/runtime": { + "version": "7.13.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.13.7.tgz", + "integrity": "sha512-h+ilqoX998mRVM5FtB5ijRuHUDVt5l3yfoOi2uh18Z/O3hvyaHQ39NpxVkCIG5yFs+mLq/ewFp8Bss6zmWv6ZA==", + "dev": true, + "dependencies": { + "regenerator-runtime": "^0.13.4" + } + }, "node_modules/@dabh/diagnostics": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", @@ -153,6 +165,49 @@ "kuler": "^2.0.0" } }, + "node_modules/@emotion/cache": { + "version": "11.1.3", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.1.3.tgz", + "integrity": "sha512-n4OWinUPJVaP6fXxWZD9OUeQ0lY7DvtmtSuqtRWT0Ofo/sBLCVSgb4/Oa0Q5eFxcwablRKjUXqXtNZVyEwCAuA==", + "dev": true, + "dependencies": { + "@emotion/memoize": "^0.7.4", + "@emotion/sheet": "^1.0.0", + "@emotion/utils": "^1.0.0", + "@emotion/weak-memoize": "^0.2.5", + "stylis": "^4.0.3" + } + }, + "node_modules/@emotion/hash": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", + "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==", + "dev": true + }, + "node_modules/@emotion/memoize": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.5.tgz", + "integrity": "sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ==", + "dev": true + }, + "node_modules/@emotion/sheet": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.0.1.tgz", + "integrity": "sha512-GbIvVMe4U+Zc+929N1V7nW6YYJtidj31lidSmdYcWozwoBIObXBnaJkKNDjZrLm9Nc0BR+ZyHNaRZxqNZbof5g==", + "dev": true + }, + "node_modules/@emotion/utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.0.0.tgz", + "integrity": "sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA==", + "dev": true + }, + "node_modules/@emotion/weak-memoize": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz", + "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA==", + "dev": true + }, "node_modules/@eslint/eslintrc": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.3.0.tgz", @@ -217,6 +272,213 @@ "node": ">= 13.9" } }, + "node_modules/@material-ui/core": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-5.0.0-alpha.25.tgz", + "integrity": "sha512-TtU3m3qw0FKtrDHCPmakJ25/20bGPRkLn8qjDkyCUYfg7kG4RLIwgiI0fKnF5PCDu46qL+s233+nVwAh8Iq+qw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/styled-engine": "5.0.0-alpha.25", + "@material-ui/styles": "5.0.0-alpha.25", + "@material-ui/system": "5.0.0-alpha.25", + "@material-ui/types": "5.1.7", + "@material-ui/unstyled": "5.0.0-alpha.25", + "@material-ui/utils": "5.0.0-alpha.25", + "@popperjs/core": "^2.4.4", + "@types/react-transition-group": "^4.2.0", + "clsx": "^1.0.4", + "csstype": "^3.0.2", + "hoist-non-react-statics": "^3.3.2", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0", + "react-transition-group": "^4.4.0" + }, + "engines": { + "node": ">=10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/material-ui" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0", + "@emotion/styled": "^11.0.0", + "@types/react": "^16.8.6 || ^17.0.0", + "react": "^16.8.0 || ^17.0.0", + "react-dom": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@material-ui/styled-engine": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/styled-engine/-/styled-engine-5.0.0-alpha.25.tgz", + "integrity": "sha512-z9piABOWycFx1yA9Th8MAs/03SfVOobh+4X1owZmOzXVSln3Xm0C3GjsFmfUpYMC8oC0vddFj00IcCCxm0OECg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@emotion/cache": "^11.0.0", + "prop-types": "^15.7.2" + }, + "engines": { + "node": ">=10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/material-ui" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0", + "@emotion/styled": "^11.0.0", + "react": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@material-ui/styles": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/styles/-/styles-5.0.0-alpha.25.tgz", + "integrity": "sha512-VODyi/JT0njmkcyRKiYPacKQmhS11bidg6AMTgAU5KjdmchFPRIl5lwIHFYeRtVEsHX9/9tE0pPlCJIhVYAYsw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@emotion/hash": "^0.8.0", + "@material-ui/types": "5.1.7", + "@material-ui/utils": "5.0.0-alpha.25", + "clsx": "^1.0.4", + "csstype": "^3.0.2", + "hoist-non-react-statics": "^3.3.2", + "jss": "^10.0.3", + "jss-plugin-camel-case": "^10.0.3", + "jss-plugin-default-unit": "^10.0.3", + "jss-plugin-global": "^10.0.3", + "jss-plugin-nested": "^10.0.3", + "jss-plugin-props-sort": "^10.0.3", + "jss-plugin-rule-value-function": "^10.0.3", + "jss-plugin-vendor-prefixer": "^10.0.3", + "prop-types": "^15.7.2" + }, + "engines": { + "node": ">=10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/material-ui" + }, + "peerDependencies": { + "@types/react": "^16.8.6 || ^17.0.0", + "react": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@material-ui/system": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/system/-/system-5.0.0-alpha.25.tgz", + "integrity": "sha512-4TrsBfB9MMsUj8o1jGRBI4Zv1SSASDwg1lfad5SYtlGhL+0LiSDO/XhVvOvYaRi749lguux2rlrVZpIut0jgMA==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "5.0.0-alpha.25", + "csstype": "^3.0.2", + "prop-types": "^15.7.2" + }, + "engines": { + "node": ">=10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/material-ui" + }, + "peerDependencies": { + "@types/react": "^16.8.6 || ^17.0.0", + "react": "^16.8.0 || ^17.0.0", + "react-dom": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@material-ui/types": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/@material-ui/types/-/types-5.1.7.tgz", + "integrity": "sha512-OSpB0gEKZm5h4izTLyipb34PkfazpvusgQMDTmFkSuqcKoChTshfGejEYX6uaZ+4m5xlT5qzihE6eKA+JnjELg==", + "dev": true, + "peerDependencies": { + "@types/react": "*" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@material-ui/unstyled": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/unstyled/-/unstyled-5.0.0-alpha.25.tgz", + "integrity": "sha512-LqAXDhSk2GC3C3SX0U5V3U0WA6gAS5Ayrozo9Br2XtxfEDdPTDmR1R7csT+DruyP3vGBs4k4ILIUz8KfC/kjww==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "5.0.0-alpha.25", + "clsx": "^1.0.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "@types/react": "^16.8.6 || ^17.0.0", + "react": "^16.8.0 || ^17.0.0", + "react-dom": "^16.8.0 || ^17.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@material-ui/utils": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-5.0.0-alpha.25.tgz", + "integrity": "sha512-wgW1ua+ncAU6lrE2bFhd9iU1xmpxj1KL+YRJa0PoFpFgLNVpSTCVHynMlRAERNP+CteazFHj5upkigqJpV406Q==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.4.4", + "@types/prop-types": "^15.7.3", + "@types/react-is": "^16.7.1 || ^17.0.0", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz", @@ -252,6 +514,16 @@ "node": ">= 8" } }, + "node_modules/@popperjs/core": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.8.4.tgz", + "integrity": "sha512-h0lY7g36rhjNV8KVHKS3/BEOgfsxu0AiRI8+ry5IFBGEsQFkpjxtcpVc9ndN8zrKUeMZXAWMc7eQMepfgykpxQ==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -411,6 +683,24 @@ "@types/react": "*" } }, + "node_modules/@types/react-is": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@types/react-is/-/react-is-17.0.0.tgz", + "integrity": "sha512-A0DQ1YWZ0RG2+PV7neAotNCIh8gZ3z7tQnDJyS2xRPDNtAtSPcJ9YyfMP8be36Ha0kQRzbZCrrTMznA4blqO5g==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.1.tgz", + "integrity": "sha512-vIo69qKKcYoJ8wKCJjwSgCTM+z3chw3g18dkrDfVX665tMH7tmbDxEAnPdey4gTlwZz5QuHGzd+hul0OVZDqqQ==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/semver": { "version": "7.3.4", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", @@ -1789,6 +2079,15 @@ "mimic-response": "^1.0.0" } }, + "node_modules/clsx": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.1.1.tgz", + "integrity": "sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -1965,6 +2264,16 @@ "node": ">=8" } }, + "node_modules/css-vendor": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/css-vendor/-/css-vendor-2.0.8.tgz", + "integrity": "sha512-x9Aq0XTInxrkuFeHKbYC7zWY8ai7qJ04Kxd9MnvbC1uO5DagxoHQjm4JvG+vCdXOoFtCjbL2XSZfxmoYa9uQVQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.8.3", + "is-in-browser": "^1.0.2" + } + }, "node_modules/cssfilter": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", @@ -2108,6 +2417,16 @@ "node": ">=6.0.0" } }, + "node_modules/dom-helpers": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.0.tgz", + "integrity": "sha512-Ru5o9+V8CpunKnz5LGgWXkmrH/20cGKwcHwS4m73zIvs54CN9epEmT/HLqFJW3kXpakAFkEdzgy1hzlJe3E4OQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, "node_modules/dot-prop": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", @@ -3368,6 +3687,25 @@ "node": ">=8" } }, + "node_modules/history": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/history/-/history-5.0.0.tgz", + "integrity": "sha512-3NyRMKIiFSJmIPdq7FxkNMJkQ7ZEtVblOQ38VtKaA0zZMW1Eo6Q6W8oDKEflr1kNNTItSnk4JMCO1deeSgbLLg==", + "dev": true, + "peer": true, + "dependencies": { + "@babel/runtime": "^7.7.6" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dev": true, + "dependencies": { + "react-is": "^16.7.0" + } + }, "node_modules/hosted-git-info": { "version": "2.8.8", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", @@ -3397,6 +3735,12 @@ "uuid": "bin/uuid" } }, + "node_modules/hyphenate-style-name": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.4.tgz", + "integrity": "sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==", + "dev": true + }, "node_modules/ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -3446,6 +3790,15 @@ "node": ">=0.8.19" } }, + "node_modules/indefinite-observable": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/indefinite-observable/-/indefinite-observable-2.0.1.tgz", + "integrity": "sha512-G8vgmork+6H9S8lUAg1gtXEj2JxIQTo0g2PbFiYOdjkziSI0F7UYBiVwhZRuixhBCNGczAls34+5HJPyZysvxQ==", + "dev": true, + "dependencies": { + "symbol-observable": "1.2.0" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -3599,6 +3952,12 @@ "node": ">=0.10.0" } }, + "node_modules/is-in-browser": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-in-browser/-/is-in-browser-1.1.3.tgz", + "integrity": "sha1-Vv9NtoOgeMYILrldrX3GLh0E+DU=", + "dev": true + }, "node_modules/is-installed-globally": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", @@ -3786,6 +4145,97 @@ "json5": "lib/cli.js" } }, + "node_modules/jss": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss/-/jss-10.5.1.tgz", + "integrity": "sha512-hbbO3+FOTqVdd7ZUoTiwpHzKXIo5vGpMNbuXH1a0wubRSWLWSBvwvaq4CiHH/U42CmjOnp6lVNNs/l+Z7ZdDmg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "csstype": "^3.0.2", + "indefinite-observable": "^2.0.1", + "is-in-browser": "^1.1.3", + "tiny-warning": "^1.0.2" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/jss" + } + }, + "node_modules/jss-plugin-camel-case": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.5.1.tgz", + "integrity": "sha512-9+oymA7wPtswm+zxVti1qiowC5q7bRdCJNORtns2JUj/QHp2QPXYwSNRD8+D2Cy3/CEMtdJzlNnt5aXmpS6NAg==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "hyphenate-style-name": "^1.0.3", + "jss": "10.5.1" + } + }, + "node_modules/jss-plugin-default-unit": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-default-unit/-/jss-plugin-default-unit-10.5.1.tgz", + "integrity": "sha512-D48hJBc9Tj3PusvlillHW8Fz0y/QqA7MNmTYDQaSB/7mTrCZjt7AVRROExoOHEtd2qIYKOYJW3Jc2agnvsXRlQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "node_modules/jss-plugin-global": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-global/-/jss-plugin-global-10.5.1.tgz", + "integrity": "sha512-jX4XpNgoaB8yPWw/gA1aPXJEoX0LNpvsROPvxlnYe+SE0JOhuvF7mA6dCkgpXBxfTWKJsno7cDSCgzHTocRjCQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "node_modules/jss-plugin-nested": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-nested/-/jss-plugin-nested-10.5.1.tgz", + "integrity": "sha512-xXkWKOCljuwHNjSYcXrCxBnjd8eJp90KVFW1rlhvKKRXnEKVD6vdKXYezk2a89uKAHckSvBvBoDGsfZrldWqqQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/jss-plugin-props-sort": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-props-sort/-/jss-plugin-props-sort-10.5.1.tgz", + "integrity": "sha512-t+2vcevNmMg4U/jAuxlfjKt46D/jHzCPEjsjLRj/J56CvP7Iy03scsUP58Iw8mVnaV36xAUZH2CmAmAdo8994g==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "node_modules/jss-plugin-rule-value-function": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.5.1.tgz", + "integrity": "sha512-3gjrSxsy4ka/lGQsTDY8oYYtkt2esBvQiceGBB4PykXxHoGRz14tbCK31Zc6DHEnIeqsjMUGbq+wEly5UViStQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1", + "tiny-warning": "^1.0.2" + } + }, + "node_modules/jss-plugin-vendor-prefixer": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.5.1.tgz", + "integrity": "sha512-cLkH6RaPZWHa1TqSfd2vszNNgxT1W0omlSjAd6hCFHp3KIocSrW21gaHjlMU26JpTHwkc+tJTCQOmE/O1A4FKQ==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.3.1", + "css-vendor": "^2.0.8", + "jss": "10.5.1" + } + }, "node_modules/jsx-ast-utils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.1.0.tgz", @@ -4908,6 +5358,58 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, + "node_modules/react-refresh-typescript": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/react-refresh-typescript/-/react-refresh-typescript-1.1.1.tgz", + "integrity": "sha512-4H1FREZ9VYAgsNE5Qlt4SaHAcTn3owXd2IVzWlbt9K+b4chNH10G3XOlIOznsf9oB6zAdxy5VpBdE06xUDm3lw==", + "peerDependencies": { + "typescript": "^4" + } + }, + "node_modules/react-router": { + "version": "6.0.0-beta.0", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.0.0-beta.0.tgz", + "integrity": "sha512-VgMdfpVcmFQki/LZuLh8E/MNACekDetz4xqft+a6fBZvvJnVqKbLqebF7hyoawGrV1HcO5tVaUang2Og4W2j1Q==", + "dev": true, + "dependencies": { + "prop-types": "^15.7.2" + }, + "peerDependencies": { + "history": ">=5", + "react": ">=16.8" + } + }, + "node_modules/react-router-dom": { + "version": "6.0.0-beta.0", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.0.0-beta.0.tgz", + "integrity": "sha512-36yNNGMT8RB9FRPL9nKJi6HKDkgOakU+o/2hHpSzR6e37gN70MpOU6QQlmif4oAWWBwjyGc3ZNOMFCsFuHUY5w==", + "dev": true, + "dependencies": { + "prop-types": "^15.7.2", + "react-router": "6.0.0-beta.0" + }, + "peerDependencies": { + "history": ">=5", + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, + "node_modules/react-transition-group": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", + "integrity": "sha512-Djqr7OQ2aPUiYurhPalTrVy9ddmFCCzwhqQmtN+J3+3DzLO209Fdr70QrN8Z3DsglWql6iY1lDWAfpFiBtuKGw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, "node_modules/read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", @@ -4989,6 +5491,12 @@ "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" }, + "node_modules/regenerator-runtime": { + "version": "0.13.7", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==", + "dev": true + }, "node_modules/regexp.prototype.flags": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", @@ -5650,6 +6158,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stylis": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.7.tgz", + "integrity": "sha512-OFFeUXFgwnGOKvEXaSv0D0KQ5ADP0n6g3SVONx6I/85JzNZ3u50FRwB3lVIk1QO2HNdI75tbVzc4Z66Gdp9voA==", + "dev": true + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -5662,6 +6176,15 @@ "node": ">=4" } }, + "node_modules/symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/table": { "version": "6.0.7", "resolved": "https://registry.npmjs.org/table/-/table-6.0.7.tgz", @@ -5797,6 +6320,12 @@ "node": ">=6" } }, + "node_modules/tiny-warning": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", + "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==", + "dev": true + }, "node_modules/to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", @@ -5973,9 +6502,9 @@ "dev": true }, "node_modules/typescript": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.3.tgz", - "integrity": "sha512-B3ZIOf1IKeH2ixgHhj6la6xdwR9QrLC5d1VKeCSY4tvkqhF2eqd9O7txNlS0PO3GrBAFIdr3L1ndNwteUbZLYg==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.2.tgz", + "integrity": "sha512-tbb+NVrLfnsJy3M59lsDgrzWIflR4d4TIUjz+heUnHZwdF7YsrMTKoRERiIvI2lvBG95dfpLxB21WZhys1bgaQ==", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6533,6 +7062,15 @@ } } }, + "@babel/runtime": { + "version": "7.13.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.13.7.tgz", + "integrity": "sha512-h+ilqoX998mRVM5FtB5ijRuHUDVt5l3yfoOi2uh18Z/O3hvyaHQ39NpxVkCIG5yFs+mLq/ewFp8Bss6zmWv6ZA==", + "dev": true, + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, "@dabh/diagnostics": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", @@ -6543,6 +7081,49 @@ "kuler": "^2.0.0" } }, + "@emotion/cache": { + "version": "11.1.3", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.1.3.tgz", + "integrity": "sha512-n4OWinUPJVaP6fXxWZD9OUeQ0lY7DvtmtSuqtRWT0Ofo/sBLCVSgb4/Oa0Q5eFxcwablRKjUXqXtNZVyEwCAuA==", + "dev": true, + "requires": { + "@emotion/memoize": "^0.7.4", + "@emotion/sheet": "^1.0.0", + "@emotion/utils": "^1.0.0", + "@emotion/weak-memoize": "^0.2.5", + "stylis": "^4.0.3" + } + }, + "@emotion/hash": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", + "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==", + "dev": true + }, + "@emotion/memoize": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.5.tgz", + "integrity": "sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ==", + "dev": true + }, + "@emotion/sheet": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.0.1.tgz", + "integrity": "sha512-GbIvVMe4U+Zc+929N1V7nW6YYJtidj31lidSmdYcWozwoBIObXBnaJkKNDjZrLm9Nc0BR+ZyHNaRZxqNZbof5g==", + "dev": true + }, + "@emotion/utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.0.0.tgz", + "integrity": "sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA==", + "dev": true + }, + "@emotion/weak-memoize": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz", + "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA==", + "dev": true + }, "@eslint/eslintrc": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.3.0.tgz", @@ -6585,6 +7166,109 @@ "resolved": "https://registry.npmjs.org/@k-foss/ts-worker/-/ts-worker-1.0.5.tgz", "integrity": "sha512-vCAXlgHDuribOSLXrzMXgjmvlc8fIPbp3qYbRbu56hvJGAhg6WdRV+EWFKrp1WC/rAC0hovFr9gtvnkzXo35Sw==" }, + "@material-ui/core": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-5.0.0-alpha.25.tgz", + "integrity": "sha512-TtU3m3qw0FKtrDHCPmakJ25/20bGPRkLn8qjDkyCUYfg7kG4RLIwgiI0fKnF5PCDu46qL+s233+nVwAh8Iq+qw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@material-ui/styled-engine": "5.0.0-alpha.25", + "@material-ui/styles": "5.0.0-alpha.25", + "@material-ui/system": "5.0.0-alpha.25", + "@material-ui/types": "5.1.7", + "@material-ui/unstyled": "5.0.0-alpha.25", + "@material-ui/utils": "5.0.0-alpha.25", + "@popperjs/core": "^2.4.4", + "@types/react-transition-group": "^4.2.0", + "clsx": "^1.0.4", + "csstype": "^3.0.2", + "hoist-non-react-statics": "^3.3.2", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0", + "react-transition-group": "^4.4.0" + } + }, + "@material-ui/styled-engine": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/styled-engine/-/styled-engine-5.0.0-alpha.25.tgz", + "integrity": "sha512-z9piABOWycFx1yA9Th8MAs/03SfVOobh+4X1owZmOzXVSln3Xm0C3GjsFmfUpYMC8oC0vddFj00IcCCxm0OECg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@emotion/cache": "^11.0.0", + "prop-types": "^15.7.2" + } + }, + "@material-ui/styles": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/styles/-/styles-5.0.0-alpha.25.tgz", + "integrity": "sha512-VODyi/JT0njmkcyRKiYPacKQmhS11bidg6AMTgAU5KjdmchFPRIl5lwIHFYeRtVEsHX9/9tE0pPlCJIhVYAYsw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@emotion/hash": "^0.8.0", + "@material-ui/types": "5.1.7", + "@material-ui/utils": "5.0.0-alpha.25", + "clsx": "^1.0.4", + "csstype": "^3.0.2", + "hoist-non-react-statics": "^3.3.2", + "jss": "^10.0.3", + "jss-plugin-camel-case": "^10.0.3", + "jss-plugin-default-unit": "^10.0.3", + "jss-plugin-global": "^10.0.3", + "jss-plugin-nested": "^10.0.3", + "jss-plugin-props-sort": "^10.0.3", + "jss-plugin-rule-value-function": "^10.0.3", + "jss-plugin-vendor-prefixer": "^10.0.3", + "prop-types": "^15.7.2" + } + }, + "@material-ui/system": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/system/-/system-5.0.0-alpha.25.tgz", + "integrity": "sha512-4TrsBfB9MMsUj8o1jGRBI4Zv1SSASDwg1lfad5SYtlGhL+0LiSDO/XhVvOvYaRi749lguux2rlrVZpIut0jgMA==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "5.0.0-alpha.25", + "csstype": "^3.0.2", + "prop-types": "^15.7.2" + } + }, + "@material-ui/types": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/@material-ui/types/-/types-5.1.7.tgz", + "integrity": "sha512-OSpB0gEKZm5h4izTLyipb34PkfazpvusgQMDTmFkSuqcKoChTshfGejEYX6uaZ+4m5xlT5qzihE6eKA+JnjELg==", + "dev": true, + "requires": {} + }, + "@material-ui/unstyled": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/unstyled/-/unstyled-5.0.0-alpha.25.tgz", + "integrity": "sha512-LqAXDhSk2GC3C3SX0U5V3U0WA6gAS5Ayrozo9Br2XtxfEDdPTDmR1R7csT+DruyP3vGBs4k4ILIUz8KfC/kjww==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@material-ui/utils": "5.0.0-alpha.25", + "clsx": "^1.0.4", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + } + }, + "@material-ui/utils": { + "version": "5.0.0-alpha.25", + "resolved": "https://registry.npmjs.org/@material-ui/utils/-/utils-5.0.0-alpha.25.tgz", + "integrity": "sha512-wgW1ua+ncAU6lrE2bFhd9iU1xmpxj1KL+YRJa0PoFpFgLNVpSTCVHynMlRAERNP+CteazFHj5upkigqJpV406Q==", + "dev": true, + "requires": { + "@babel/runtime": "^7.4.4", + "@types/prop-types": "^15.7.3", + "@types/react-is": "^16.7.1 || ^17.0.0", + "prop-types": "^15.7.2", + "react-is": "^16.8.0 || ^17.0.0" + } + }, "@nodelib/fs.scandir": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz", @@ -6611,6 +7295,12 @@ "fastq": "^1.6.0" } }, + "@popperjs/core": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.8.4.tgz", + "integrity": "sha512-h0lY7g36rhjNV8KVHKS3/BEOgfsxu0AiRI8+ry5IFBGEsQFkpjxtcpVc9ndN8zrKUeMZXAWMc7eQMepfgykpxQ==", + "dev": true + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -6764,6 +7454,24 @@ "@types/react": "*" } }, + "@types/react-is": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@types/react-is/-/react-is-17.0.0.tgz", + "integrity": "sha512-A0DQ1YWZ0RG2+PV7neAotNCIh8gZ3z7tQnDJyS2xRPDNtAtSPcJ9YyfMP8be36Ha0kQRzbZCrrTMznA4blqO5g==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, + "@types/react-transition-group": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.1.tgz", + "integrity": "sha512-vIo69qKKcYoJ8wKCJjwSgCTM+z3chw3g18dkrDfVX665tMH7tmbDxEAnPdey4gTlwZz5QuHGzd+hul0OVZDqqQ==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, "@types/semver": { "version": "7.3.4", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.4.tgz", @@ -7853,6 +8561,12 @@ "mimic-response": "^1.0.0" } }, + "clsx": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-1.1.1.tgz", + "integrity": "sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==", + "dev": true + }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -7994,6 +8708,16 @@ "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", "dev": true }, + "css-vendor": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/css-vendor/-/css-vendor-2.0.8.tgz", + "integrity": "sha512-x9Aq0XTInxrkuFeHKbYC7zWY8ai7qJ04Kxd9MnvbC1uO5DagxoHQjm4JvG+vCdXOoFtCjbL2XSZfxmoYa9uQVQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.8.3", + "is-in-browser": "^1.0.2" + } + }, "cssfilter": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", @@ -8106,6 +8830,16 @@ "esutils": "^2.0.2" } }, + "dom-helpers": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.0.tgz", + "integrity": "sha512-Ru5o9+V8CpunKnz5LGgWXkmrH/20cGKwcHwS4m73zIvs54CN9epEmT/HLqFJW3kXpakAFkEdzgy1hzlJe3E4OQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, "dot-prop": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", @@ -9102,6 +9836,25 @@ "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", "dev": true }, + "history": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/history/-/history-5.0.0.tgz", + "integrity": "sha512-3NyRMKIiFSJmIPdq7FxkNMJkQ7ZEtVblOQ38VtKaA0zZMW1Eo6Q6W8oDKEflr1kNNTItSnk4JMCO1deeSgbLLg==", + "dev": true, + "peer": true, + "requires": { + "@babel/runtime": "^7.7.6" + } + }, + "hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dev": true, + "requires": { + "react-is": "^16.7.0" + } + }, "hosted-git-info": { "version": "2.8.8", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", @@ -9130,6 +9883,12 @@ } } }, + "hyphenate-style-name": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/hyphenate-style-name/-/hyphenate-style-name-1.0.4.tgz", + "integrity": "sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==", + "dev": true + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -9164,6 +9923,15 @@ "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", "dev": true }, + "indefinite-observable": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/indefinite-observable/-/indefinite-observable-2.0.1.tgz", + "integrity": "sha512-G8vgmork+6H9S8lUAg1gtXEj2JxIQTo0g2PbFiYOdjkziSI0F7UYBiVwhZRuixhBCNGczAls34+5HJPyZysvxQ==", + "dev": true, + "requires": { + "symbol-observable": "1.2.0" + } + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -9280,6 +10048,12 @@ "is-extglob": "^2.1.1" } }, + "is-in-browser": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/is-in-browser/-/is-in-browser-1.1.3.tgz", + "integrity": "sha1-Vv9NtoOgeMYILrldrX3GLh0E+DU=", + "dev": true + }, "is-installed-globally": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", @@ -9425,6 +10199,93 @@ "minimist": "^1.2.0" } }, + "jss": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss/-/jss-10.5.1.tgz", + "integrity": "sha512-hbbO3+FOTqVdd7ZUoTiwpHzKXIo5vGpMNbuXH1a0wubRSWLWSBvwvaq4CiHH/U42CmjOnp6lVNNs/l+Z7ZdDmg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "csstype": "^3.0.2", + "indefinite-observable": "^2.0.1", + "is-in-browser": "^1.1.3", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-camel-case": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.5.1.tgz", + "integrity": "sha512-9+oymA7wPtswm+zxVti1qiowC5q7bRdCJNORtns2JUj/QHp2QPXYwSNRD8+D2Cy3/CEMtdJzlNnt5aXmpS6NAg==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "hyphenate-style-name": "^1.0.3", + "jss": "10.5.1" + } + }, + "jss-plugin-default-unit": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-default-unit/-/jss-plugin-default-unit-10.5.1.tgz", + "integrity": "sha512-D48hJBc9Tj3PusvlillHW8Fz0y/QqA7MNmTYDQaSB/7mTrCZjt7AVRROExoOHEtd2qIYKOYJW3Jc2agnvsXRlQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "jss-plugin-global": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-global/-/jss-plugin-global-10.5.1.tgz", + "integrity": "sha512-jX4XpNgoaB8yPWw/gA1aPXJEoX0LNpvsROPvxlnYe+SE0JOhuvF7mA6dCkgpXBxfTWKJsno7cDSCgzHTocRjCQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "jss-plugin-nested": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-nested/-/jss-plugin-nested-10.5.1.tgz", + "integrity": "sha512-xXkWKOCljuwHNjSYcXrCxBnjd8eJp90KVFW1rlhvKKRXnEKVD6vdKXYezk2a89uKAHckSvBvBoDGsfZrldWqqQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-props-sort": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-props-sort/-/jss-plugin-props-sort-10.5.1.tgz", + "integrity": "sha512-t+2vcevNmMg4U/jAuxlfjKt46D/jHzCPEjsjLRj/J56CvP7Iy03scsUP58Iw8mVnaV36xAUZH2CmAmAdo8994g==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1" + } + }, + "jss-plugin-rule-value-function": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.5.1.tgz", + "integrity": "sha512-3gjrSxsy4ka/lGQsTDY8oYYtkt2esBvQiceGBB4PykXxHoGRz14tbCK31Zc6DHEnIeqsjMUGbq+wEly5UViStQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.5.1", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-vendor-prefixer": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.5.1.tgz", + "integrity": "sha512-cLkH6RaPZWHa1TqSfd2vszNNgxT1W0omlSjAd6hCFHp3KIocSrW21gaHjlMU26JpTHwkc+tJTCQOmE/O1A4FKQ==", + "dev": true, + "requires": { + "@babel/runtime": "^7.3.1", + "css-vendor": "^2.0.8", + "jss": "10.5.1" + } + }, "jsx-ast-utils": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.1.0.tgz", @@ -10305,6 +11166,43 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true }, + "react-refresh-typescript": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/react-refresh-typescript/-/react-refresh-typescript-1.1.1.tgz", + "integrity": "sha512-4H1FREZ9VYAgsNE5Qlt4SaHAcTn3owXd2IVzWlbt9K+b4chNH10G3XOlIOznsf9oB6zAdxy5VpBdE06xUDm3lw==", + "requires": {} + }, + "react-router": { + "version": "6.0.0-beta.0", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.0.0-beta.0.tgz", + "integrity": "sha512-VgMdfpVcmFQki/LZuLh8E/MNACekDetz4xqft+a6fBZvvJnVqKbLqebF7hyoawGrV1HcO5tVaUang2Og4W2j1Q==", + "dev": true, + "requires": { + "prop-types": "^15.7.2" + } + }, + "react-router-dom": { + "version": "6.0.0-beta.0", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.0.0-beta.0.tgz", + "integrity": "sha512-36yNNGMT8RB9FRPL9nKJi6HKDkgOakU+o/2hHpSzR6e37gN70MpOU6QQlmif4oAWWBwjyGc3ZNOMFCsFuHUY5w==", + "dev": true, + "requires": { + "prop-types": "^15.7.2", + "react-router": "6.0.0-beta.0" + } + }, + "react-transition-group": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz", + "integrity": "sha512-Djqr7OQ2aPUiYurhPalTrVy9ddmFCCzwhqQmtN+J3+3DzLO209Fdr70QrN8Z3DsglWql6iY1lDWAfpFiBtuKGw==", + "dev": true, + "requires": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + } + }, "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", @@ -10368,6 +11266,12 @@ "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" }, + "regenerator-runtime": { + "version": "0.13.7", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==", + "dev": true + }, "regexp.prototype.flags": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", @@ -10896,6 +11800,12 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "stylis": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.7.tgz", + "integrity": "sha512-OFFeUXFgwnGOKvEXaSv0D0KQ5ADP0n6g3SVONx6I/85JzNZ3u50FRwB3lVIk1QO2HNdI75tbVzc4Z66Gdp9voA==", + "dev": true + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -10905,6 +11815,12 @@ "has-flag": "^3.0.0" } }, + "symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true + }, "table": { "version": "6.0.7", "resolved": "https://registry.npmjs.org/table/-/table-6.0.7.tgz", @@ -11030,6 +11946,12 @@ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz", "integrity": "sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow==" }, + "tiny-warning": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz", + "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==", + "dev": true + }, "to-buffer": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", @@ -11166,9 +12088,9 @@ "dev": true }, "typescript": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.3.tgz", - "integrity": "sha512-B3ZIOf1IKeH2ixgHhj6la6xdwR9QrLC5d1VKeCSY4tvkqhF2eqd9O7txNlS0PO3GrBAFIdr3L1ndNwteUbZLYg==" + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.2.tgz", + "integrity": "sha512-tbb+NVrLfnsJy3M59lsDgrzWIflR4d4TIUjz+heUnHZwdF7YsrMTKoRERiIvI2lvBG95dfpLxB21WZhys1bgaQ==" }, "undefsafe": { "version": "2.0.3", diff --git a/package.json b/package.json index 385f870..0b93f3b 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "devDependencies": { "@k-foss/ts-esnode": "^2.0.0", "@k-foss/ts-estests": "^1.0.6", + "@material-ui/core": "^5.0.0-alpha.25", "@types/node": "^14.14.31", "@types/react": "^17.0.0", "@types/react-dom": "^17.0.0", @@ -38,8 +39,9 @@ "prettier": "^2.2.1", "react": "^17.0.1", "react-dom": "^17.0.1", + "react-router-dom": "^6.0.0-beta.0", "typedi": "^0.10.0", - "typescript": "^4.1.3" + "typescript": "^4.2.2" }, "author": { "email": "me@kristianjones.dev", @@ -59,6 +61,7 @@ "hyperid": "^2.1.0", "ioredis": "^4.22.0", "p-event": "^4.2.0", + "react-refresh-typescript": "^1.1.1", "reflect-metadata": "^0.1.13", "type-graphql": "^1.1.1", "winston": "^3.3.3", diff --git a/src/Library/Transformers.ts b/src/Library/Transformers.ts index 20a891e..13bba30 100644 --- a/src/Library/Transformers.ts +++ b/src/Library/Transformers.ts @@ -9,6 +9,7 @@ import { import { findModuleFiles } from '../Utils/moduleFileFinder'; import { cjsToEsmTransformerFactory } from 'cjstoesm'; import { logger } from './Logger'; +import { ClassType } from 'type-graphql'; let customTransformers: CustomTransformers; @@ -41,6 +42,12 @@ export abstract class Transformer { public afterDeclarations?: TransformerFactory; } +interface TransformersObject { + before: TransformerFactory[]; + + after: TransformerFactory[]; +} + /** * Example Transformer so the findModuleFiles returns non abstract */ @@ -106,25 +113,45 @@ export class TypeScriptTransformerController { public transformers: Transformer[]; private async findTransformers(): Promise { - const modules = await findModuleFiles<{ [key: string]: Transformer }>( - /.*Resolver\.(js|ts)/, + const modules = await findModuleFiles<{ + [key: string]: ClassType; + }>(/.*Transformer\.(js|ts)/); + + return modules.flatMap((moduleExports) => + Object.values(moduleExports).map( + (TransformerClass) => new TransformerClass(), + ), ); - - return modules.flatMap((moduleExports) => Object.values(moduleExports)); } /** * Load transformer modules and return the transformers object */ - public async loadTransformers(): Promise { + public async loadTransformers(): Promise { const transformers = await this.findTransformers(); - logger.silly(`loadTransformers`, { - labels: { - appName: 'TS-ESWeb', - className: 'TypeScriptTransformerController', + return transformers.reduce( + (accumulator, currentValue) => { + const returned = Object.fromEntries( + (Object.keys(accumulator) as ['before', 'after']).map((key) => + typeof currentValue[key] !== 'undefined' + ? [key, [...accumulator[key], currentValue[key]]] + : [key, accumulator[key]], + ), + ) as TransformersObject; + + logger.silly('transformers reduce: ', { + transformers, + accumulator, + currentValue, + }); + + return returned; }, - transformers, - }); + { + before: [cjsToEsmTransformerFactory()], + after: [], + } as TransformersObject, + ); } } diff --git a/src/Modules/Files/processModule.ts b/src/Modules/Files/processModule.ts index 9ccc885..1d00c43 100644 --- a/src/Modules/Files/processModule.ts +++ b/src/Modules/Files/processModule.ts @@ -4,6 +4,7 @@ import { logger } from '../../Library/Logger'; import { exportsHandler, objectExport, + processNodeIfStatement, processNodeReplacement, } from '../TypeScript/Regex'; @@ -34,6 +35,11 @@ export function processModule(fileContents: string): string { 'processNodeReplacement', ); + setModuleContents( + moduleContents.replace(processNodeIfStatement, '$'), + 'processNodeENVIfStatement', + ); + setModuleContents( moduleContents.replaceAll( exportsHandler, @@ -58,6 +64,10 @@ export function processModule(fileContents: string): string { 'objectExport', ); + setModuleContents( + moduleContents.replaceAll('process.env.NODE_ENV', process.env.NODE_ENV), + ); + /** * https://regex101.com/r/uwAq1N/1 */ diff --git a/src/Modules/SSR/SSRController.tsx b/src/Modules/SSR/SSRController.tsx index f803ec9..1e002c0 100644 --- a/src/Modules/SSR/SSRController.tsx +++ b/src/Modules/SSR/SSRController.tsx @@ -1,6 +1,7 @@ // src/Modules/SSR/SSRController.tsx import { renderToString } from 'react-dom/server'; +import { StaticRouter } from './StaticRouter'; import { Inject, Service } from 'typedi'; import { ServerOptions, serverOptionsToken } from '../Server/ServerOptions'; @@ -9,11 +10,15 @@ export class SSRController { @Inject(serverOptionsToken) public options: ServerOptions; - public renderApp(): string { + public renderApp(path: string): string { const App = this.options.ssr.appComponent; console.log(this.options, App); - return renderToString(); + return renderToString( + + + , + ); } } diff --git a/src/Modules/SSR/StaticRouter.tsx b/src/Modules/SSR/StaticRouter.tsx new file mode 100644 index 0000000..71a34ee --- /dev/null +++ b/src/Modules/SSR/StaticRouter.tsx @@ -0,0 +1,93 @@ +import * as React from 'react'; +import { + Action, + Location, + PartialLocation, + To, + createPath, + parsePath, +} from 'history'; +import { Router } from 'react-router-dom'; + +export interface StaticRouterProps { + children?: React.ReactNode; + location?: string | PartialLocation; +} + +/** + * A that may not transition to any other location. This is useful + * on the server where there is no stateful UI. + */ +export function StaticRouter({ + children, + location: loc = '/', +}: StaticRouterProps) { + if (typeof loc === 'string') { + loc = parsePath(loc); + } + + let action = Action.Pop; + let location: Location = { + pathname: loc.pathname || '/', + search: loc.search || '', + hash: loc.hash || '', + state: loc.state || null, + key: loc.key || 'default', + }; + + let staticNavigator = { + createHref(to: To) { + return typeof to === 'string' ? to : createPath(to); + }, + push(to: To) { + throw new Error( + `You cannot use navigator.push() on the server because it is a stateless ` + + `environment. This error was probably triggered when you did a ` + + `\`navigate(${JSON.stringify(to)})\` somewhere in your app.`, + ); + }, + replace(to: To) { + throw new Error( + `You cannot use navigator.replace() on the server because it is a stateless ` + + `environment. This error was probably triggered when you did a ` + + `\`navigate(${JSON.stringify(to)}, { replace: true })\` somewhere ` + + `in your app.`, + ); + }, + go(delta: number) { + throw new Error( + `You cannot use navigator.go() on the server because it is a stateless ` + + `environment. This error was probably triggered when you did a ` + + `\`navigate(${delta})\` somewhere in your app.`, + ); + }, + back() { + throw new Error( + `You cannot use navigator.back() on the server because it is a stateless ` + + `environment.`, + ); + }, + forward() { + throw new Error( + `You cannot use navigator.forward() on the server because it is a stateless ` + + `environment.`, + ); + }, + block() { + throw new Error( + `You cannot use navigator.block() on the server because it is a stateless ` + + `environment.`, + ); + }, + }; + + return ( + + ); +} diff --git a/src/Modules/Server/ServerController.ts b/src/Modules/Server/ServerController.ts index 6625fbf..beb8700 100644 --- a/src/Modules/Server/ServerController.ts +++ b/src/Modules/Server/ServerController.ts @@ -1,6 +1,6 @@ // src/Modules/Server/ServerController.ts import { ApolloServer } from 'apollo-server-fastify'; -import { plainToClass } from 'class-transformer'; +import { classToPlain, plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; import hyperid from 'hyperid'; import { resolve } from 'node:path'; @@ -14,6 +14,7 @@ import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; import { SSRController } from '../SSR/SSRController'; import { TypeScriptController } from '../TypeScript/TypeScriptController'; +import { WebAppManifest } from '../WebAppManifest/WebAppManifest'; import { WebModuleController } from '../WebModule/WebModuleController'; import { WebModuleJobInput } from '../WebModule/WebModuleJobInput'; import { WebModuleMapJobInput } from '../WebModule/WebModuleMapJobInput'; @@ -27,6 +28,11 @@ export class ServerController { @Inject() public ssrController: SSRController; + /** + * Path of the Web App Manifest + */ + private webAppManifestPath = '/Main.webmanifest'; + // eslint-disable-next-line no-useless-constructor public constructor( private typescriptController: TypeScriptController, @@ -63,6 +69,15 @@ export class ServerController { return gqlServer; } + /** + * + * @param options + * @returns + */ + public handleWebAppManifest(): WebAppManifest { + return classToPlain(this.options.manifest) as WebAppManifest; + } + /** * Create a new ServerController with the provided Configuration within the TypeDi Container * @param options Server Configuration @@ -85,9 +100,13 @@ export class ServerController { serverId, }, { - strategy: 'exposeAll', + enableCircularCheck: true, }, ); + logger.silly('serverOptions post transform: ', { + serverOptions, + }); + await validateOrReject(serverOptions); if ( @@ -175,15 +194,15 @@ export class ServerController { // const moduleFiles = await getAllChildModules(jobOutput.filePath); } - public async renderHTML(): Promise { + public async renderHTML(path: string): Promise { await timeout(10); - const appHTML = this.ssrController.renderApp(); + const appHTML = this.ssrController.renderApp(path); return ` TS-ESWeb - +
${appHTML}
diff --git a/src/Modules/Server/ServerOptions.ts b/src/Modules/Server/ServerOptions.ts index efc2f93..7f71a4e 100644 --- a/src/Modules/Server/ServerOptions.ts +++ b/src/Modules/Server/ServerOptions.ts @@ -4,25 +4,45 @@ import { IsDefined, IsEnum, IsOptional, - IsString, IsUUID, ValidateNested, } from 'class-validator'; -import { Token } from 'typedi'; -import { logger } from '../../Library/Logger'; +import Container, { Token } from 'typedi'; import { Environment, envMode } from '../../Utils/Environment'; import { RedisOptions } from '../Redis/RedisOptions'; import { SSROptions } from '../SSR/SSROptions'; +import { + WebAppManifest, + webAppManifestToken, +} from '../WebAppManifest/WebAppManifest'; export class ServerOptions { + @IsDefined() @ValidateNested() @Type(() => RedisOptions) public redis: RedisOptions; - @ValidateNested() + @IsDefined() + @ValidateNested({ + always: true, + }) @Type(() => SSROptions) public ssr: SSROptions; + @IsDefined() + @ValidateNested({ + always: true, + }) + @Type(() => WebAppManifest) + @Transform(({ value, obj }) => { + const container = Container.of((obj as ServerOptions).serverId); + + container.set(webAppManifestToken, value); + + return container.get(webAppManifestToken); + }) + public readonly manifest: WebAppManifest; + @IsUUID() @IsDefined() public serverId: string; diff --git a/src/Modules/ServiceWorker/ServiceWorker.ts b/src/Modules/ServiceWorker/ServiceWorker.ts deleted file mode 100644 index ffea884..0000000 --- a/src/Modules/ServiceWorker/ServiceWorker.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ -/* eslint-disable @typescript-eslint/no-unsafe-member-access */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -// src/Modules/ServiceWorker/ServiceWorker.ts -// src/Web/serviceWorker.ts -const _self = (self as unknown) as ServiceWorkerGlobalScope; -declare const clients: Clients; - -importScripts( - 'https://storage.googleapis.com/workbox-cdn/releases/5.1.2/workbox-sw.js', -); -if (!workbox) throw new Error('Workbox not definied'); - -interface WebModuleQueryResponse { - filePath: string; - specifier: string; - code: string; -} - -type WebModuleQuery = { data: { webModule: WebModuleQueryResponse | null } }; - -workbox.routing.registerRoute( - /\/Static\/import/, - async ({ url, request, event, params }) => { - const moduleSpecifier = url.searchParams.get('specifier'); - - let response: Response; - if (moduleSpecifier) { - console.log(`specifier: ${moduleSpecifier}`); - const apiHeaders = new Headers(); - apiHeaders.set('content-type', 'application/json'); - - const request = new Request('/graphql', { - body: `{"operationName":"webModule","variables":{"moduleSpecifier":"${moduleSpecifier}"},"query":"query webModule($moduleSpecifier: String!) { webModule(filter: {specifier: $moduleSpecifier}) { filePath specifiers code }}"}`, - method: 'POST', - headers: apiHeaders, - }); - - const fetchResponse = await fetch(request); - const queryResult = (await fetchResponse.json()) as WebModuleQuery; - - if (queryResult.data.webModule) { - response = new Response(queryResult.data.webModule.code); - } else { - throw new Error('What?'); - } - } else { - console.log('Getting file: ', url, request, event, params); - response = new Response(`console.log('helloWorld')`); - } - - response.headers.set('Content-Type', 'application/javascript'); - - return response; - }, -); -const channel = new BroadcastChannel('sw-messages'); - -interface ServiceWorkerSkipWaitingMessage { - type: 'SKIP_WAITING'; -} - -interface ServiceWorkerClientClaimMessage { - type: 'CLIENTS_CLAIM'; -} - -type ServiceWorkerMessage = MessageEvent & { - data: ServiceWorkerSkipWaitingMessage | ServiceWorkerClientClaimMessage; -}; - -interface ServiceWorkerDoneMessage { - type: 'DONE'; - command: 'CLIENTS_CLAIM' | 'SKIP_WAITING'; -} - -self.addEventListener('message', async (event: ServiceWorkerMessage) => { - switch (event.data.type) { - case 'SKIP_WAITING': - await _self.skipWaiting(); - - // self.postMessage({ - // type: 'DONE', - // command: 'SKIP_WAITING', - // } as ServiceWorkerDoneMessage); - break; - case 'CLIENTS_CLAIM': - workbox.core.clientsClaim(); - - await clients.claim(); - channel.postMessage({ type: 'READY' }); - // self.postMessage({ - // type: 'DONE', - // command: 'CLIENTS_CLAIM', - // } as ServiceWorkerDoneMessage); - break; - } -}); - -_self.addEventListener('activate', () => { - console.log('SHIJFKDSHLFKJD'); -}); diff --git a/src/Modules/TypeScript/Regex.ts b/src/Modules/TypeScript/Regex.ts index e578474..411ffb3 100644 --- a/src/Modules/TypeScript/Regex.ts +++ b/src/Modules/TypeScript/Regex.ts @@ -3,4 +3,6 @@ export const objectExport = /^\{$\n(?((?^\s+)\S+\s+=\s+\{$\n\k export const processNodeReplacement = /if\s+\(process\.env\.NODE_ENV \S+\s"production"\)\s{\n\s+\(function\(\)\s{\n(?(.*\n?)*)}\)\(\);\n}/gim; +export const processNodeIfStatement = /if\s*\(process\.env\.NODE_ENV\s*\S+\s('|")production("|')\)\s*\{(?.*)\}\s*else\s*{.*\}/gims; + export const exportsHandler = /((?<=^|;)exports\.(?\S+)(?=\s*=.*;))(?=.*exports\.\k\s*=)/gmsu; diff --git a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts index f53aa3a..3a27e7f 100644 --- a/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts +++ b/src/Modules/TypeScript/TypeScriptModuleMapWorker.ts @@ -1,12 +1,14 @@ // src/Modules/TypeScript/TypeScriptModuleMapWorker.ts import { Queue, Worker } from 'bullmq'; +import { isString } from 'class-validator'; +import { readFile } from 'fs/promises'; +import { resolve } from 'node:path'; import { dirname } from 'path'; import Container from 'typedi'; import * as ts from 'typescript'; import { fileURLToPath, pathToFileURL } from 'url'; import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; -import { envMode } from '../../Utils/Environment'; import { QueueController } from '../Queues/QueueController'; import { queueToken } from '../Queues/QueueToken'; import { workerControllerToken } from '../Queues/WorkerController'; @@ -87,6 +89,10 @@ async function discoverModuleMap( checkJs: false, noEmit: true, noEmitHelpers: true, + noLib: true, + isolatedModules: true, + skipLibCheck: true, + skipDefaultLibCheck: true, sourceMap: false, inlineSourceMap: false, }; @@ -102,6 +108,41 @@ async function discoverModuleMap( rootNames: [moduleInput.filePath], compilerOptions, }); + if ( + moduleInput.filePath === + '/workspace/node_modules/@babel/runtime/helpers/extends/_index.mjs' + ) { + const transpilerJobInput = await TranspilerWorkerJobInput.createTranspilerWorkerJobInput( + { + filePath: moduleInput.filePath, + }, + ); + + await transpilerQue.add('typescriptTranspiler', transpilerJobInput, { + jobId: transpilerJobInput.filePath, + }); + + const specifierCore = moduleInput.specifier || moduleInput.filePath; + const specifier = specifierCore.replace(/.(js|ts)x?/, ''); + + const webModuleJobInputParams: WebModuleMapJobInput = { + filePath: moduleInput.filePath, + specifier, + importedModules: [], + }; + + await redisController.IORedis.set( + redisController.getRedisKey(RedisType.MODULE_MAP, moduleInput.filePath), + JSON.stringify(webModuleJobInputParams), + ); + + await redisController.IORedis.set( + redisController.getRedisKey(RedisType.MODULE_MAP, specifier), + JSON.stringify(webModuleJobInputParams), + ); + + return; + } const sourceFile = compilerProgram.getSourceFile(moduleInput.filePath); @@ -111,7 +152,6 @@ async function discoverModuleMap( moduleMapLogger.silly(`sourceFile: `, { objectName: 'sourceFile', - sourceFile: sourceFile.fileName, }); const resolvedArray = Array.from( @@ -181,12 +221,13 @@ async function discoverModuleMap( } const importedModules = await Promise.all( - resolvedArray.map(async ([specifier]) => { + resolvedArray.map(async ([specifier, _test]) => { const parentURI = pathToFileURL(moduleInput.filePath); moduleMapLogger.silly(`Resolving Module`, { specifier, parentURI, + _test, }); const resolvePathURI = await import.meta.resolve( @@ -204,7 +245,35 @@ async function discoverModuleMap( resolvePathURI, }); - const filePath = fileURLToPath(resolvePathURI); + let filePath = fileURLToPath(resolvePathURI); + + const folderPath = dirname(filePath); + + let packageJSON; + + try { + const file = await readFile(resolve(folderPath, 'package.json')); + logger.silly('Package file', { + file: file.toString(), + }); + + packageJSON = JSON.parse(file.toString()); + } catch (err) {} + + if (isString(packageJSON?.module)) { + logger.silly(`Testing HelloWorld`, { + moduleName: packageJSON.module, + filePath: pathToFileURL(folderPath).href, + }); + + filePath = resolve(folderPath, packageJSON.module); + } + + logger.silly('HelloWorld57678', { + filePath, + test1: compilerProgram.getSourceFile(filePath), + packageJSON: packageJSON || 'test', + }); let moduleSpecifier: string; if (ts.isExternalModuleNameRelative(specifier)) { @@ -242,7 +311,7 @@ async function discoverModuleMap( ); const specifierCore = moduleInput.specifier || moduleInput.filePath; - const specifier = specifierCore.replace('.tsx', ''); + let specifier = specifierCore.replace(/\.(js|ts)x?/, ''); const webModuleJobInputParams: WebModuleMapJobInput = { filePath: moduleInput.filePath, @@ -275,6 +344,15 @@ async function discoverModuleMap( JSON.stringify(webModuleJobInputParams), ); + if (specifier.endsWith('/index')) { + specifier = specifier.replace('/index', ''); + } + + await redisController.IORedis.set( + redisController.getRedisKey(RedisType.MODULE_MAP, specifier), + JSON.stringify(webModuleJobInputParams), + ); + await webModuleMapQue.add('webModuleMapQueue', webModuleJobInput, { jobId: webModuleJobInput.filePath, }); diff --git a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts index 877a67c..55c8d89 100644 --- a/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts +++ b/src/Modules/TypeScript/TypeScriptTranspilerWorker.ts @@ -1,6 +1,5 @@ // src/Modules/TypeScript/TypeScriptTranspilerWorker.ts import { Worker } from 'bullmq'; -import { cjsToEsmTransformerFactory } from 'cjstoesm'; import { plainToClass } from 'class-transformer'; import { validateOrReject } from 'class-validator'; import { readFile } from 'fs/promises'; @@ -15,15 +14,13 @@ import { import { threadId } from 'worker_threads'; import { logger as coreLogger } from '../../Library/Logger'; import { TypeScriptTransformerController } from '../../Library/Transformers'; -import { Environment, envMode } from '../../Utils/Environment'; +import { Environment } from '../../Utils/Environment'; import { processModule } from '../Files/processModule'; -// import { QueueController } from '../Queues/QueueController'; import { queueToken } from '../Queues/QueueToken'; import { workerControllerToken } from '../Queues/WorkerController'; import { workerInputToken } from '../Queues/WorkerInput'; import { RedisController } from '../Redis/RedisController'; import { RedisType } from '../Redis/RedisTypes'; -import { ImportTransformer } from '../WebModule/ImportTransformer'; import { TranspilerWorkerJobInput } from './TranspilerWorkerJobInput'; const logger = coreLogger.child({ @@ -67,27 +64,17 @@ async function transformFile(filePath: string): Promise { const file = await readFile(filePath); - let fileContents: string; - - fileContents = processModule(file.toString()); - workerController.logger.silly(`Getting TypeScript Transformers`); - await transformerController.loadTransformers(); + const transformers = await transformerController.loadTransformers(); - workerController.logger.silly(`EnvMode`, { - envMode, + workerController.logger.silly(`Transformers`, { + transformers, }); - const transpiledModule = transpileModule(fileContents, { + const transpiledModule = transpileModule(processModule(file.toString()), { fileName: filePath, - transformers: { - before: [cjsToEsmTransformerFactory()], - after: [new ImportTransformer().after], - afterDeclarations: [ - // ... - ], - }, + transformers, compilerOptions: { allowJs: true, checkJs: false, diff --git a/src/Modules/TypeScript/Utils.ts b/src/Modules/TypeScript/Utils.ts index 122b0f2..7658e5c 100644 --- a/src/Modules/TypeScript/Utils.ts +++ b/src/Modules/TypeScript/Utils.ts @@ -8,6 +8,7 @@ import { } from 'class-validator'; import { parse } from 'path'; import * as ts from 'typescript'; +import { logger } from '../../Library/Logger'; class CreateTypeScriptProgramInput { @IsString() @@ -47,6 +48,12 @@ export async function createTypeScriptProgram( ...inputParams.compilerOptions, }; + logger.silly(`createTypeScriptProgram`, { + inputParams, + input, + options, + }); + const compilierHost = ts.createCompilerHost({ ...options, rootDir: inputParams.rootDir, @@ -76,5 +83,14 @@ export function isCommonJSImportSplit( return true; } + if ( + input.length === 2 && + input.every(([_path, resolvedModule]) => + resolvedModule.packageId?.subModuleName.includes('umd'), + ) + ) { + return true; + } + return false; } diff --git a/src/Modules/WebModule/ImportTransformer.ts b/src/Modules/WebModule/ImportTransformer.ts index a35b303..1061f97 100644 --- a/src/Modules/WebModule/ImportTransformer.ts +++ b/src/Modules/WebModule/ImportTransformer.ts @@ -6,16 +6,46 @@ import { Transformer } from '../../Library/Transformers'; const pushDefault = ['react', 'react-dom', 'scheduler', 'scheduler/tracing']; +/** + * TODO: Setup a type check/`node is XYZ` + * @param node SourceFile Node to test if it's a dynamic import + */ +function isDynamicImport(node: ts.Node): node is ts.ImportCall { + if (ts.isCallExpression(node)) { + return node.expression.kind === ts.SyntaxKind.ImportKeyword; + } + + return false; +} + +function handleImportSpecifier( + sourceSpecifier: string, + sourceDirectory: string, +): string { + let specifier: string; + + const relativeTest = /^\.{0,2}[/]/gm; + + if (relativeTest.test(sourceSpecifier)) { + specifier = resolve(sourceDirectory, sourceSpecifier); + } else { + specifier = sourceSpecifier; + } + + return `/Static/import?specifier=${specifier}`; +} + export class ImportTransformer extends Transformer { public after: Transformer['after'] = (context) => { // const relativePathRegex = /^\.{1,2}[/]/; return (sourceFile: ts.SourceFile): ts.SourceFile => { const visitor: ts.Visitor = (node) => { - if ( - ts.isCallExpression(node) && - node.expression.kind === ts.SyntaxKind.ImportKeyword - ) { + /** + * Handle "Dynamic imports" + */ + + if (isDynamicImport(node)) { const argument = node.arguments[0]; if (ts.isStringLiteral(argument)) { @@ -31,10 +61,10 @@ export class ImportTransformer extends Transformer { node.typeArguments, [ context.factory.createStringLiteral( - `/Static/import?specifier=${resolve( - dirname(sourceFile.fileName), + handleImportSpecifier( argument.text, - )}`, + dirname(sourceFile.fileName), + ), ), ], ); @@ -45,35 +75,10 @@ export class ImportTransformer extends Transformer { ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier) ) { - // let specifier: string; - // if (relativePathRegex.test(node.moduleSpecifier.text)) { - // specifier = resolvePath( - // node.moduleSpecifier.text, - // pathToFileURL(sourceFile.fileName).href, - // ); - // return node; - // } else { - // specifier = - // console.log( - // `Set new specififer to '/Static/import?specifier=${node.moduleSpecifier.text}'`, - // ); - // } - const relativeTest = /^\.{0,2}[/]/gm; - logger.debug( `ImportTransformer Import declaration: ${node.moduleSpecifier.text}`, ); - let specifier: string; - if (relativeTest.test(node.moduleSpecifier.text)) { - specifier = resolve( - dirname(sourceFile.fileName), - node.moduleSpecifier.text, - ); - } else { - specifier = node.moduleSpecifier.text; - } - let importClause: ts.ImportClause | undefined; if ( pushDefault.includes(node.moduleSpecifier.text) && @@ -97,13 +102,20 @@ export class ImportTransformer extends Transformer { importClause = node.importClause; } + logger.debug( + `ImportTransformer Import declaration: ${node.moduleSpecifier.text}`, + ); + return context.factory.updateImportDeclaration( node, node.decorators, node.modifiers, importClause, context.factory.createStringLiteral( - `/Static/import?specifier=${specifier}`, + handleImportSpecifier( + node.moduleSpecifier.text, + dirname(sourceFile.fileName), + ), ), ); } @@ -113,18 +125,6 @@ export class ImportTransformer extends Transformer { node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier) ) { - const relativeTest = /^\.{0,2}[/]/gm; - - let specifier: string; - if (relativeTest.test(node.moduleSpecifier.text)) { - specifier = resolve( - dirname(sourceFile.fileName), - node.moduleSpecifier.text, - ); - } else { - specifier = node.moduleSpecifier.text; - } - /** * Return updated import path to use our Service Worker served path */ @@ -135,7 +135,10 @@ export class ImportTransformer extends Transformer { node.isTypeOnly, node.exportClause, context.factory.createStringLiteral( - `/Static/import?specifier=${specifier}`, + handleImportSpecifier( + node.moduleSpecifier.text, + dirname(sourceFile.fileName), + ), ), ); } diff --git a/src/Web_Test/App.tsx b/src/Web_Test/App.tsx index 5154f94..7abe1e7 100644 --- a/src/Web_Test/App.tsx +++ b/src/Web_Test/App.tsx @@ -1,14 +1,12 @@ // src/App.tsx -import { useCallback } from 'react'; -import { Counter } from './Components/Counter/index'; +import { Nav } from './Components/Nav'; +import { Router } from './Components/Routers'; export function App(): React.ReactElement { - const sayHello = useCallback(() => console.log('HelloWorld'), []); - return ( <> -

HelloWorld

- +