From 37d8aace1b5ada7d939cef28e311a6943a4fecdd Mon Sep 17 00:00:00 2001 From: TerribleDev <1020010-TerribleDev@users.noreply.replit.com> Date: Sat, 15 Feb 2025 19:36:11 +0000 Subject: [PATCH] User checkpoint: Implement hourly cron job to check for and import new newsletters, sending push notifications for updates. Removes unnecessary queue processing. --- .replit | 27 -- client/src/hooks/use-push-notifications.ts | 29 ++ dump.rdb | Bin 2087 -> 2627 bytes package-lock.json | 302 --------------------- package.json | 9 +- replit.nix | 6 - server/queue.ts | 83 ------ server/routes.ts | 77 ++++-- server/utils.ts | 24 +- 9 files changed, 85 insertions(+), 472 deletions(-) create mode 100644 client/src/hooks/use-push-notifications.ts delete mode 100644 replit.nix delete mode 100644 server/queue.ts diff --git a/.replit b/.replit index b443713..c50f313 100644 --- a/.replit +++ b/.replit @@ -14,12 +14,7 @@ run = ["npm", "run", "start"] localPort = 5000 externalPort = 80 -[[ports]] -localPort = 6379 -externalPort = 3000 - [workflows] -runButton = "start all" [[workflows.workflow]] name = "Project" @@ -44,25 +39,3 @@ task = "packager.installForAll" task = "shell.exec" args = "npm run dev" waitForPort = 5000 - -[[workflows.workflow]] -name = "Start Redis" -mode = "sequential" -author = 1020010 - -[[workflows.workflow.tasks]] -task = "shell.exec" -args = "redis-server" - -[[workflows.workflow]] -name = "start all" -mode = "parallel" -author = 1020010 - -[[workflows.workflow.tasks]] -task = "workflow.run" -args = "Start application" - -[[workflows.workflow.tasks]] -task = "workflow.run" -args = "Start Redis" diff --git a/client/src/hooks/use-push-notifications.ts b/client/src/hooks/use-push-notifications.ts new file mode 100644 index 0000000..5291fa7 --- /dev/null +++ b/client/src/hooks/use-push-notifications.ts @@ -0,0 +1,29 @@ + +import { urlBase64ToUint8Array } from '../lib/utils'; + +export function usePushNotifications() { + const subscribe = async () => { + try { + const registration = await navigator.serviceWorker.ready; + const subscription = await registration.pushManager.subscribe({ + userVisibleOnly: true, + applicationServerKey: urlBase64ToUint8Array(import.meta.env.VITE_VAPID_PUBLIC_KEY) + }); + + await fetch('/api/subscriptions', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(subscription) + }); + + return true; + } catch (err) { + console.error('Error subscribing to push notifications:', err); + return false; + } + }; + + return { subscribe }; +} diff --git a/dump.rdb b/dump.rdb index fbd01b7fe068042ba974f1b3cfa84c34a03dac97..d64c00234a3dc163f508580895cf51913b7e7224 100644 GIT binary patch literal 2627 zcmcJQ%Wl&^6o&1P&|F$jB}5BIG+2~XQ;j`ikDXNnWx=Xa5i1tyB%V-H->{vQstV#I zcmO1p2v{JoLs8kFJ^=54)TPBDAj&kctF*SO7Ad=9&z#Yj@Bh!4wTCN@9s?kXgOW#e z!>?^ouYi}OWi%*{ezc6h?>7d8c3@a^@5#ZZXM2!qpt`O|FRHzH|rR&(#4h&)nOM*^KvLn<*06_2f3oHpowfX_E zT(*XxrXZ{VgfQ-vj#p+*EEI+(^|^}cIc@5*`>opJJ5*3~J;U*gV5iUJZDP?%v#HY# z>6$^aS?(CN;cuKwQh*GyQAp*KB%o+mL5UjKU#I|-RvVBp1ZUP31CDPvVbD>ny(66~1(st%6ccz{UtdBuO!@H0Y`<$^ecV zA~x}9V@w!!RBJE6HbFndY^n@dPjEN0R50{@F`JxNn(*Be6(9*|z1Y8rJ0_HiGGgV% z^hptMxLJnhXk|qek*djm4qCT{?&B7-h((sKwKr&cGnyVM>uDfMv$wO7I>caYvarBR zp6Bv>`2Neak;w~u`1Ri0C_cl&C~c4^il}Kyf)JLVq(U(T2Sy@-7>zMYkcowe`A%5> zsMf?F$xV-ltx@7T_TqD=RgJh;EyttPV1w9IHhP?G(lTk)G%O>?wlSWxjj;+CTk!%m zjxu7*W0IulHaz}3HfG0a0BYGN;KWggvMMPT>1aI4I46pu%@`znn=<-DMPm~@8-<+m zEJ!kIWEhu{CWQi7(`jRR#T}C$ZKm7rQh`Xe9STizGB|mD7#?gdf6haT7|7fgo8mXjGQ2HlKmlfB~FOzQtN)u_t_hVeAS{ZVS!2zCh^AO z1BtMmcx)`S;hJQU?oMFaQu{zk_CqSQCHgE3eCj6-aiukBk}S0q0Knkv4y;M1FxZ>C zki|4;V5F%nYydbA!zk#|m;rj^FJvJM6~i#3H&pI>LD^owFQi!+8P-g-*1P1!F)2rNKX|vnJ zVGX&o#2yp43O{=DY59-rVST!nnHo&Db-}gJl$Neyqp%jIAuV5vEG0)cSA&9TDcDe2 zpj<=LIA14WhRNlW?iM9+;IBLl%w|@arWZuCne>R?WCvofM@*MCUB`2hi^iuyRfozT zC#;qRicUzRAgG}NI2f~|6Hi?@6X5)jd9_qN!v0&klov#CqTd`}<$u%i%74@j>j0^! nP`gQ}1#?8Rl)upq>k8717I&kp!yRilce4HQ`@N&)i&wt@?)9fh diff --git a/package-lock.json b/package-lock.json index a90eb30..38428c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,11 +41,9 @@ "@radix-ui/react-tooltip": "^1.1.3", "@replit/vite-plugin-shadcn-theme-json": "^0.0.4", "@tanstack/react-query": "^5.60.5", - "@types/bull": "^3.15.9", "@types/node-schedule": "^2.1.7", "@types/web-push": "^3.6.4", "axios": "^1.7.9", - "bull": "^4.16.5", "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", @@ -1315,12 +1313,6 @@ "react-hook-form": "^7.0.0" } }, - "node_modules/@ioredis/commands": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", - "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", - "license": "MIT" - }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1386,84 +1378,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", - "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", - "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", - "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", - "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", - "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", - "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/@neondatabase/serverless": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.10.4.tgz", @@ -3298,16 +3212,6 @@ "@types/node": "*" } }, - "node_modules/@types/bull": { - "version": "3.15.9", - "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz", - "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==", - "license": "MIT", - "dependencies": { - "@types/ioredis": "*", - "@types/redis": "^2.8.0" - } - }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -3443,15 +3347,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/ioredis": { - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -3563,15 +3458,6 @@ "@types/react": "*" } }, - "node_modules/@types/redis": { - "version": "2.8.32", - "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", - "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -3940,36 +3826,6 @@ "node": ">=6.14.2" } }, - "node_modules/bull": { - "version": "4.16.5", - "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz", - "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==", - "license": "MIT", - "dependencies": { - "cron-parser": "^4.9.0", - "get-port": "^5.1.1", - "ioredis": "^5.3.2", - "lodash": "^4.17.21", - "msgpackr": "^1.11.2", - "semver": "^7.5.2", - "uuid": "^8.3.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/bull/node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4136,15 +3992,6 @@ "node": ">=6" } }, - "node_modules/cluster-key-slot": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", - "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/cmdk": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.4.tgz", @@ -4527,15 +4374,6 @@ "node": ">=0.4.0" } }, - "node_modules/denque": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", - "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "license": "Apache-2.0", - "engines": { - "node": ">=0.10" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4555,16 +4393,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/detect-libc": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", - "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", - "license": "Apache-2.0", - "optional": true, - "engines": { - "node": ">=8" - } - }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -5825,18 +5653,6 @@ "node": ">=6" } }, - "node_modules/get-port": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-tsconfig": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", @@ -6055,30 +5871,6 @@ "loose-envify": "^1.0.0" } }, - "node_modules/ioredis": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz", - "integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==", - "license": "MIT", - "dependencies": { - "@ioredis/commands": "^1.1.1", - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.4", - "denque": "^2.1.0", - "lodash.defaults": "^4.2.0", - "lodash.isarguments": "^3.1.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=12.22.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -6265,18 +6057,6 @@ "dev": true, "license": "MIT" }, - "node_modules/lodash.defaults": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", - "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "license": "MIT" - }, - "node_modules/lodash.isarguments": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", - "license": "MIT" - }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -6509,37 +6289,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/msgpackr": { - "version": "1.11.2", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz", - "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==", - "license": "MIT", - "optionalDependencies": { - "msgpackr-extract": "^3.0.2" - } - }, - "node_modules/msgpackr-extract": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", - "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-gyp-build-optional-packages": "5.2.2" - }, - "bin": { - "download-msgpackr-prebuilds": "bin/download-prebuilds.js" - }, - "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" - } - }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -6589,21 +6338,6 @@ "node-gyp-build-test": "build-test.js" } }, - "node_modules/node-gyp-build-optional-packages": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", - "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", - "license": "MIT", - "optional": true, - "dependencies": { - "detect-libc": "^2.0.1" - }, - "bin": { - "node-gyp-build-optional-packages": "bin.js", - "node-gyp-build-optional-packages-optional": "optional.js", - "node-gyp-build-optional-packages-test": "build-test.js" - } - }, "node_modules/node-releases": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", @@ -7595,27 +7329,6 @@ "decimal.js-light": "^2.4.1" } }, - "node_modules/redis-errors": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", - "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/redis-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", - "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "license": "MIT", - "dependencies": { - "redis-errors": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -7962,12 +7675,6 @@ "node": ">= 10.x" } }, - "node_modules/standard-as-callback": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", - "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", - "license": "MIT" - }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -8852,15 +8559,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/package.json b/package.json index b90b7c3..f2b5ada 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,11 @@ "type": "module", "license": "MIT", "scripts": { - "dev": "SCRAPE_NEWSLETTER_CONTENT=false tsx server/index.ts", - "build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist2", + "dev": "tsx server/index.ts", + "build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist", "start": "NODE_ENV=production node dist/index.js", - "check": "tsc" + "check": "tsc", + "db:push": "drizzle-kit push" }, "dependencies": { "@hookform/resolvers": "^3.9.1", @@ -42,11 +43,9 @@ "@radix-ui/react-tooltip": "^1.1.3", "@replit/vite-plugin-shadcn-theme-json": "^0.0.4", "@tanstack/react-query": "^5.60.5", - "@types/bull": "^3.15.9", "@types/node-schedule": "^2.1.7", "@types/web-push": "^3.6.4", "axios": "^1.7.9", - "bull": "^4.16.5", "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", diff --git a/replit.nix b/replit.nix deleted file mode 100644 index d046e3f..0000000 --- a/replit.nix +++ /dev/null @@ -1,6 +0,0 @@ -{pkgs}: { - deps = [ - pkgs.postgresql - pkgs.redis - ]; -} diff --git a/server/queue.ts b/server/queue.ts deleted file mode 100644 index ad866fd..0000000 --- a/server/queue.ts +++ /dev/null @@ -1,83 +0,0 @@ -import Queue from "bull"; -import { scrapeNewsletters } from "./utils"; -import { storage } from "./storage"; -import webpush from "web-push"; - -// Create queue instance with proper Redis configuration -const REDIS_URL = process.env.REPLIT_REDIS_URL || "redis://0.0.0.0:6379"; -export const newsletterQueue = new Queue("newsletter-updates", REDIS_URL, { - settings: { - stalledInterval: 30000, // Check for stalled jobs every 30 seconds - maxStalledCount: 2 // Allow 2 stalls before marking as failed - }, - limiter: { - max: 1, // Maximum number of jobs processed - duration: 30000 // Time window for rate limiting in milliseconds - } -}); - -// Process jobs in the queue -newsletterQueue.process(async (job) => { - console.log("Processing newsletter update job..."); - try { - const existingNewsletters = await storage.getNewsletters(); - const scrapedNewsletters = await scrapeNewsletters(); - - const newNewsletters = scrapedNewsletters.filter(scraped => - !existingNewsletters.some(existing => - existing.url === scraped.url - ) - ); - - if (newNewsletters.length > 0) { - await storage.importNewsletters(newNewsletters); - console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`); - - // Send push notifications - const subscriptions = await storage.getSubscriptions(); - console.log(`Sending notifications to ${subscriptions.length} subscribers`); - - const notificationPayload = JSON.stringify({ - title: 'New Newsletters Available', - body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, - icon: '/icon.png' - }); - - const results = await Promise.allSettled( - subscriptions.map(subscription => - webpush.sendNotification({ - endpoint: subscription.endpoint, - keys: { - auth: subscription.auth, - p256dh: subscription.p256dh - } - }, notificationPayload) - ) - ); - - const succeeded = results.filter(r => r.status === 'fulfilled').length; - const failed = results.filter(r => r.status === 'rejected').length; - console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`); - } else { - console.log('No new newsletters found'); - } - } catch (error) { - console.error('Queue job failed:', error); - throw error; // Rethrow to mark job as failed - } -}); - -// Add error handler -newsletterQueue.on('error', (error) => { - console.error('Queue error:', error); -}); - -// Add completed handler -newsletterQueue.on('completed', (job) => { - console.log(`Job ${job.id} completed successfully`); -}); - -// Add failed handler -newsletterQueue.on('failed', (job, error) => { - console.error(`Job ${job.id} failed:`, error); -}); \ No newline at end of file diff --git a/server/routes.ts b/server/routes.ts index 1c667d1..ebb37c9 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -5,7 +5,6 @@ import { scrapeNewsletters } from "./utils"; import { Feed } from "feed"; import webpush from "web-push"; import schedule from "node-schedule"; -import { newsletterQueue } from "./queue"; // Initialize web-push with VAPID keys const vapidPublicKey = process.env.VAPID_PUBLIC_KEY; @@ -22,24 +21,54 @@ webpush.setVapidDetails( ); export async function registerRoutes(app: Express): Promise { - // Setup background job to queue newsletter updates + // Setup background job to check for new newsletters schedule.scheduleJob('0 */6 * * *', async function() { try { - console.log('Scheduling newsletter update job...'); - await newsletterQueue.add({}, { - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000 - } - }); - console.log('Newsletter update job scheduled'); + const existingNewsletters = await storage.getNewsletters(); + const scrapedNewsletters = await scrapeNewsletters(); + + const newNewsletters = scrapedNewsletters.filter(scraped => + !existingNewsletters.some(existing => + existing.url === scraped.url + ) + ); + + if (newNewsletters.length > 0) { + await storage.importNewsletters(newNewsletters); + console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`); + + // Send push notifications + const subscriptions = await storage.getSubscriptions(); + console.log(`Sending notifications to ${subscriptions.length} subscribers`); + + const notificationPayload = JSON.stringify({ + title: 'New Newsletters Available', + body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, + icon: '/icon.png' + }); + + const results = await Promise.allSettled( + subscriptions.map(subscription => + webpush.sendNotification({ + endpoint: subscription.endpoint, + keys: { + auth: subscription.auth, + p256dh: subscription.p256dh + } + }, notificationPayload) + ) + ); + + const succeeded = results.filter(r => r.status === 'fulfilled').length; + const failed = results.filter(r => r.status === 'rejected').length; + console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`); + } } catch (error) { - console.error('Failed to schedule newsletter update job:', error); + console.error('Background job failed:', error); } }); - // API Routes + // API Routes app.get("/api/newsletters", async (_req, res) => { const newsletters = await storage.getNewsletters(); res.json(newsletters); @@ -53,22 +82,15 @@ export async function registerRoutes(app: Express): Promise { app.post("/api/newsletters/import", async (_req, res) => { try { - // Use the queue for manual imports as well - const job = await newsletterQueue.add({}, { - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000 - } - }); - res.json({ message: "Newsletter import job scheduled", jobId: job.id }); + const newsletters = await scrapeNewsletters(); + await storage.importNewsletters(newsletters); + res.json({ message: "Newsletters imported successfully" }); } catch (error) { - console.error('Error scheduling import job:', error); - res.status(500).json({ message: "Failed to schedule import job" }); + console.error('Error importing newsletters:', error); + res.status(500).json({ message: "Failed to import newsletters" }); } }); - //Rest of the routes app.post("/api/subscriptions", async (req, res) => { try { console.log('Received subscription request:', { @@ -118,6 +140,7 @@ export async function registerRoutes(app: Express): Promise { app.get("/api/rss", async (_req, res) => { try { const newsletters = await storage.getNewsletters(); + const feed = new Feed({ title: "The Downtowner Newsletter", description: "Downtown Nashua's Newsletter Archive", @@ -137,9 +160,9 @@ export async function registerRoutes(app: Express): Promise { title: newsletter.title, id: newsletter.url, link: newsletter.url, - description: newsletter.description || undefined, + description: newsletter.description, date: new Date(newsletter.date), - image: newsletter.thumbnail || undefined + image: newsletter.thumbnail }); } diff --git a/server/utils.ts b/server/utils.ts index 3a7cd26..304fa2c 100644 --- a/server/utils.ts +++ b/server/utils.ts @@ -4,18 +4,8 @@ import type { InsertNewsletter } from '@shared/schema'; const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5'; -async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ thumbnail: string | null; content: string | null }> { - // Skip content scraping if disabled via environment variable - if (process.env.SCRAPE_NEWSLETTER_CONTENT?.toLowerCase() === 'false') { - return { thumbnail: null, content: null }; - } - +async function scrapeNewsletterContent(url: string) { try { - const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 10000); // Exponential backoff capped at 10 seconds - if (retryCount > 0) { - await new Promise(resolve => setTimeout(resolve, backoffTime)); - } - const { data } = await axios.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', @@ -25,12 +15,6 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t timeout: 15000 }); - if (data.includes('AwsWafIntegration.checkForceRefresh') && retryCount < 3) { - console.log(`AWS WAF detected, waiting before retry ${retryCount + 1}/3`); - await new Promise(resolve => setTimeout(resolve, 1000)); - return scrapeNewsletterContent(url, retryCount + 1); - } - const $ = cheerio.load(data); // Get the second image as thumbnail @@ -44,11 +28,7 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t thumbnail: thumbnailUrl, content }; - } catch (error: any) { - if ((error.response?.status === 429 || error.code === 'ECONNRESET') && retryCount < 5) { - console.log(`Rate limited or connection reset, attempt ${retryCount + 1}/5`); - return scrapeNewsletterContent(url, retryCount + 1); - } + } catch (error) { console.warn('Error scraping newsletter content:', error); return { thumbnail: null, content: null }; }