User checkpoint: Implement hourly cron job to check for and import new newsletters, sending push notifications for updates. Removes unnecessary queue processing.
This commit is contained in:
TerribleDev
27
.replit
27
.replit
@@ -14,12 +14,7 @@ run = ["npm", "run", "start"]
|
||||
localPort = 5000
|
||||
externalPort = 80
|
||||
|
||||
[[ports]]
|
||||
localPort = 6379
|
||||
externalPort = 3000
|
||||
|
||||
[workflows]
|
||||
runButton = "start all"
|
||||
|
||||
[[workflows.workflow]]
|
||||
name = "Project"
|
||||
@@ -44,25 +39,3 @@ task = "packager.installForAll"
|
||||
task = "shell.exec"
|
||||
args = "npm run dev"
|
||||
waitForPort = 5000
|
||||
|
||||
[[workflows.workflow]]
|
||||
name = "Start Redis"
|
||||
mode = "sequential"
|
||||
author = 1020010
|
||||
|
||||
[[workflows.workflow.tasks]]
|
||||
task = "shell.exec"
|
||||
args = "redis-server"
|
||||
|
||||
[[workflows.workflow]]
|
||||
name = "start all"
|
||||
mode = "parallel"
|
||||
author = 1020010
|
||||
|
||||
[[workflows.workflow.tasks]]
|
||||
task = "workflow.run"
|
||||
args = "Start application"
|
||||
|
||||
[[workflows.workflow.tasks]]
|
||||
task = "workflow.run"
|
||||
args = "Start Redis"
|
||||
|
||||
29
client/src/hooks/use-push-notifications.ts
Normal file
29
client/src/hooks/use-push-notifications.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
|
||||
import { urlBase64ToUint8Array } from '../lib/utils';
|
||||
|
||||
export function usePushNotifications() {
|
||||
const subscribe = async () => {
|
||||
try {
|
||||
const registration = await navigator.serviceWorker.ready;
|
||||
const subscription = await registration.pushManager.subscribe({
|
||||
userVisibleOnly: true,
|
||||
applicationServerKey: urlBase64ToUint8Array(import.meta.env.VITE_VAPID_PUBLIC_KEY)
|
||||
});
|
||||
|
||||
await fetch('/api/subscriptions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(subscription)
|
||||
});
|
||||
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('Error subscribing to push notifications:', err);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
return { subscribe };
|
||||
}
|
||||
302
package-lock.json
generated
302
package-lock.json
generated
@@ -41,11 +41,9 @@
|
||||
"@radix-ui/react-tooltip": "^1.1.3",
|
||||
"@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
|
||||
"@tanstack/react-query": "^5.60.5",
|
||||
"@types/bull": "^3.15.9",
|
||||
"@types/node-schedule": "^2.1.7",
|
||||
"@types/web-push": "^3.6.4",
|
||||
"axios": "^1.7.9",
|
||||
"bull": "^4.16.5",
|
||||
"cheerio": "^1.0.0",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
@@ -1315,12 +1313,6 @@
|
||||
"react-hook-form": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@ioredis/commands": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
|
||||
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@isaacs/cliui": {
|
||||
"version": "8.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
|
||||
@@ -1386,84 +1378,6 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||
}
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
|
||||
"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@neondatabase/serverless": {
|
||||
"version": "0.10.4",
|
||||
"resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.10.4.tgz",
|
||||
@@ -3298,16 +3212,6 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/bull": {
|
||||
"version": "3.15.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz",
|
||||
"integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/ioredis": "*",
|
||||
"@types/redis": "^2.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/connect": {
|
||||
"version": "3.4.38",
|
||||
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
|
||||
@@ -3443,15 +3347,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/ioredis": {
|
||||
"version": "4.28.10",
|
||||
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
|
||||
"integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/mime": {
|
||||
"version": "1.3.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
|
||||
@@ -3563,15 +3458,6 @@
|
||||
"@types/react": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/redis": {
|
||||
"version": "2.8.32",
|
||||
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
|
||||
"integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/send": {
|
||||
"version": "0.17.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
|
||||
@@ -3940,36 +3826,6 @@
|
||||
"node": ">=6.14.2"
|
||||
}
|
||||
},
|
||||
"node_modules/bull": {
|
||||
"version": "4.16.5",
|
||||
"resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz",
|
||||
"integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cron-parser": "^4.9.0",
|
||||
"get-port": "^5.1.1",
|
||||
"ioredis": "^5.3.2",
|
||||
"lodash": "^4.17.21",
|
||||
"msgpackr": "^1.11.2",
|
||||
"semver": "^7.5.2",
|
||||
"uuid": "^8.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/bull/node_modules/semver": {
|
||||
"version": "7.7.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
|
||||
"integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
|
||||
"license": "ISC",
|
||||
"bin": {
|
||||
"semver": "bin/semver.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/bytes": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
|
||||
@@ -4136,15 +3992,6 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/cluster-key-slot": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
|
||||
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/cmdk": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.4.tgz",
|
||||
@@ -4527,15 +4374,6 @@
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/denque": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
|
||||
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/depd": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
|
||||
@@ -4555,16 +4393,6 @@
|
||||
"npm": "1.2.8000 || >= 1.4.16"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-libc": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
|
||||
"integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/detect-node-es": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
|
||||
@@ -5825,18 +5653,6 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/get-port": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
|
||||
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/get-tsconfig": {
|
||||
"version": "4.8.1",
|
||||
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz",
|
||||
@@ -6055,30 +5871,6 @@
|
||||
"loose-envify": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ioredis": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz",
|
||||
"integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@ioredis/commands": "^1.1.1",
|
||||
"cluster-key-slot": "^1.1.0",
|
||||
"debug": "^4.3.4",
|
||||
"denque": "^2.1.0",
|
||||
"lodash.defaults": "^4.2.0",
|
||||
"lodash.isarguments": "^3.1.0",
|
||||
"redis-errors": "^1.2.0",
|
||||
"redis-parser": "^3.0.0",
|
||||
"standard-as-callback": "^2.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.22.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/ioredis"
|
||||
}
|
||||
},
|
||||
"node_modules/ipaddr.js": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
|
||||
@@ -6265,18 +6057,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.defaults": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
|
||||
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isarguments": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
|
||||
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lodash.isplainobject": {
|
||||
"version": "4.0.6",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
|
||||
@@ -6509,37 +6289,6 @@
|
||||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/msgpackr": {
|
||||
"version": "1.11.2",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz",
|
||||
"integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"msgpackr-extract": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/msgpackr-extract": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
|
||||
"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"node-gyp-build-optional-packages": "5.2.2"
|
||||
},
|
||||
"bin": {
|
||||
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
@@ -6589,21 +6338,6 @@
|
||||
"node-gyp-build-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build-optional-packages": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
|
||||
"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"node-gyp-build-optional-packages": "bin.js",
|
||||
"node-gyp-build-optional-packages-optional": "optional.js",
|
||||
"node-gyp-build-optional-packages-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.18",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
|
||||
@@ -7595,27 +7329,6 @@
|
||||
"decimal.js-light": "^2.4.1"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-errors": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
|
||||
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/redis-parser": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
|
||||
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"redis-errors": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/regenerator-runtime": {
|
||||
"version": "0.14.1",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
|
||||
@@ -7962,12 +7675,6 @@
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/standard-as-callback": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
|
||||
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/statuses": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
|
||||
@@ -8852,15 +8559,6 @@
|
||||
"node": ">= 0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "8.3.2",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
|
||||
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/vary": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
|
||||
|
||||
@@ -4,10 +4,11 @@
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"dev": "SCRAPE_NEWSLETTER_CONTENT=false tsx server/index.ts",
|
||||
"build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist2",
|
||||
"dev": "tsx server/index.ts",
|
||||
"build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist",
|
||||
"start": "NODE_ENV=production node dist/index.js",
|
||||
"check": "tsc"
|
||||
"check": "tsc",
|
||||
"db:push": "drizzle-kit push"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hookform/resolvers": "^3.9.1",
|
||||
@@ -42,11 +43,9 @@
|
||||
"@radix-ui/react-tooltip": "^1.1.3",
|
||||
"@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
|
||||
"@tanstack/react-query": "^5.60.5",
|
||||
"@types/bull": "^3.15.9",
|
||||
"@types/node-schedule": "^2.1.7",
|
||||
"@types/web-push": "^3.6.4",
|
||||
"axios": "^1.7.9",
|
||||
"bull": "^4.16.5",
|
||||
"cheerio": "^1.0.0",
|
||||
"class-variance-authority": "^0.7.0",
|
||||
"clsx": "^2.1.1",
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
{pkgs}: {
|
||||
deps = [
|
||||
pkgs.postgresql
|
||||
pkgs.redis
|
||||
];
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import Queue from "bull";
|
||||
import { scrapeNewsletters } from "./utils";
|
||||
import { storage } from "./storage";
|
||||
import webpush from "web-push";
|
||||
|
||||
// Create queue instance with proper Redis configuration
|
||||
const REDIS_URL = process.env.REPLIT_REDIS_URL || "redis://0.0.0.0:6379";
|
||||
export const newsletterQueue = new Queue("newsletter-updates", REDIS_URL, {
|
||||
settings: {
|
||||
stalledInterval: 30000, // Check for stalled jobs every 30 seconds
|
||||
maxStalledCount: 2 // Allow 2 stalls before marking as failed
|
||||
},
|
||||
limiter: {
|
||||
max: 1, // Maximum number of jobs processed
|
||||
duration: 30000 // Time window for rate limiting in milliseconds
|
||||
}
|
||||
});
|
||||
|
||||
// Process jobs in the queue
|
||||
newsletterQueue.process(async (job) => {
|
||||
console.log("Processing newsletter update job...");
|
||||
try {
|
||||
const existingNewsletters = await storage.getNewsletters();
|
||||
const scrapedNewsletters = await scrapeNewsletters();
|
||||
|
||||
const newNewsletters = scrapedNewsletters.filter(scraped =>
|
||||
!existingNewsletters.some(existing =>
|
||||
existing.url === scraped.url
|
||||
)
|
||||
);
|
||||
|
||||
if (newNewsletters.length > 0) {
|
||||
await storage.importNewsletters(newNewsletters);
|
||||
console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
|
||||
|
||||
// Send push notifications
|
||||
const subscriptions = await storage.getSubscriptions();
|
||||
console.log(`Sending notifications to ${subscriptions.length} subscribers`);
|
||||
|
||||
const notificationPayload = JSON.stringify({
|
||||
title: 'New Newsletters Available',
|
||||
body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
|
||||
icon: '/icon.png'
|
||||
});
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
subscriptions.map(subscription =>
|
||||
webpush.sendNotification({
|
||||
endpoint: subscription.endpoint,
|
||||
keys: {
|
||||
auth: subscription.auth,
|
||||
p256dh: subscription.p256dh
|
||||
}
|
||||
}, notificationPayload)
|
||||
)
|
||||
);
|
||||
|
||||
const succeeded = results.filter(r => r.status === 'fulfilled').length;
|
||||
const failed = results.filter(r => r.status === 'rejected').length;
|
||||
console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
|
||||
} else {
|
||||
console.log('No new newsletters found');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Queue job failed:', error);
|
||||
throw error; // Rethrow to mark job as failed
|
||||
}
|
||||
});
|
||||
|
||||
// Add error handler
|
||||
newsletterQueue.on('error', (error) => {
|
||||
console.error('Queue error:', error);
|
||||
});
|
||||
|
||||
// Add completed handler
|
||||
newsletterQueue.on('completed', (job) => {
|
||||
console.log(`Job ${job.id} completed successfully`);
|
||||
});
|
||||
|
||||
// Add failed handler
|
||||
newsletterQueue.on('failed', (job, error) => {
|
||||
console.error(`Job ${job.id} failed:`, error);
|
||||
});
|
||||
@@ -5,7 +5,6 @@ import { scrapeNewsletters } from "./utils";
|
||||
import { Feed } from "feed";
|
||||
import webpush from "web-push";
|
||||
import schedule from "node-schedule";
|
||||
import { newsletterQueue } from "./queue";
|
||||
|
||||
// Initialize web-push with VAPID keys
|
||||
const vapidPublicKey = process.env.VAPID_PUBLIC_KEY;
|
||||
@@ -22,24 +21,54 @@ webpush.setVapidDetails(
|
||||
);
|
||||
|
||||
export async function registerRoutes(app: Express): Promise<Server> {
|
||||
// Setup background job to queue newsletter updates
|
||||
// Setup background job to check for new newsletters
|
||||
schedule.scheduleJob('0 */6 * * *', async function() {
|
||||
try {
|
||||
console.log('Scheduling newsletter update job...');
|
||||
await newsletterQueue.add({}, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000
|
||||
}
|
||||
});
|
||||
console.log('Newsletter update job scheduled');
|
||||
const existingNewsletters = await storage.getNewsletters();
|
||||
const scrapedNewsletters = await scrapeNewsletters();
|
||||
|
||||
const newNewsletters = scrapedNewsletters.filter(scraped =>
|
||||
!existingNewsletters.some(existing =>
|
||||
existing.url === scraped.url
|
||||
)
|
||||
);
|
||||
|
||||
if (newNewsletters.length > 0) {
|
||||
await storage.importNewsletters(newNewsletters);
|
||||
console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
|
||||
|
||||
// Send push notifications
|
||||
const subscriptions = await storage.getSubscriptions();
|
||||
console.log(`Sending notifications to ${subscriptions.length} subscribers`);
|
||||
|
||||
const notificationPayload = JSON.stringify({
|
||||
title: 'New Newsletters Available',
|
||||
body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
|
||||
icon: '/icon.png'
|
||||
});
|
||||
|
||||
const results = await Promise.allSettled(
|
||||
subscriptions.map(subscription =>
|
||||
webpush.sendNotification({
|
||||
endpoint: subscription.endpoint,
|
||||
keys: {
|
||||
auth: subscription.auth,
|
||||
p256dh: subscription.p256dh
|
||||
}
|
||||
}, notificationPayload)
|
||||
)
|
||||
);
|
||||
|
||||
const succeeded = results.filter(r => r.status === 'fulfilled').length;
|
||||
const failed = results.filter(r => r.status === 'rejected').length;
|
||||
console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to schedule newsletter update job:', error);
|
||||
console.error('Background job failed:', error);
|
||||
}
|
||||
});
|
||||
|
||||
// API Routes
|
||||
// API Routes
|
||||
app.get("/api/newsletters", async (_req, res) => {
|
||||
const newsletters = await storage.getNewsletters();
|
||||
res.json(newsletters);
|
||||
@@ -53,22 +82,15 @@ export async function registerRoutes(app: Express): Promise<Server> {
|
||||
|
||||
app.post("/api/newsletters/import", async (_req, res) => {
|
||||
try {
|
||||
// Use the queue for manual imports as well
|
||||
const job = await newsletterQueue.add({}, {
|
||||
attempts: 3,
|
||||
backoff: {
|
||||
type: 'exponential',
|
||||
delay: 1000
|
||||
}
|
||||
});
|
||||
res.json({ message: "Newsletter import job scheduled", jobId: job.id });
|
||||
const newsletters = await scrapeNewsletters();
|
||||
await storage.importNewsletters(newsletters);
|
||||
res.json({ message: "Newsletters imported successfully" });
|
||||
} catch (error) {
|
||||
console.error('Error scheduling import job:', error);
|
||||
res.status(500).json({ message: "Failed to schedule import job" });
|
||||
console.error('Error importing newsletters:', error);
|
||||
res.status(500).json({ message: "Failed to import newsletters" });
|
||||
}
|
||||
});
|
||||
|
||||
//Rest of the routes
|
||||
app.post("/api/subscriptions", async (req, res) => {
|
||||
try {
|
||||
console.log('Received subscription request:', {
|
||||
@@ -118,6 +140,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
|
||||
app.get("/api/rss", async (_req, res) => {
|
||||
try {
|
||||
const newsletters = await storage.getNewsletters();
|
||||
|
||||
const feed = new Feed({
|
||||
title: "The Downtowner Newsletter",
|
||||
description: "Downtown Nashua's Newsletter Archive",
|
||||
@@ -137,9 +160,9 @@ export async function registerRoutes(app: Express): Promise<Server> {
|
||||
title: newsletter.title,
|
||||
id: newsletter.url,
|
||||
link: newsletter.url,
|
||||
description: newsletter.description || undefined,
|
||||
description: newsletter.description,
|
||||
date: new Date(newsletter.date),
|
||||
image: newsletter.thumbnail || undefined
|
||||
image: newsletter.thumbnail
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -4,18 +4,8 @@ import type { InsertNewsletter } from '@shared/schema';
|
||||
|
||||
const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5';
|
||||
|
||||
async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ thumbnail: string | null; content: string | null }> {
|
||||
// Skip content scraping if disabled via environment variable
|
||||
if (process.env.SCRAPE_NEWSLETTER_CONTENT?.toLowerCase() === 'false') {
|
||||
return { thumbnail: null, content: null };
|
||||
}
|
||||
|
||||
async function scrapeNewsletterContent(url: string) {
|
||||
try {
|
||||
const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 10000); // Exponential backoff capped at 10 seconds
|
||||
if (retryCount > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
||||
}
|
||||
|
||||
const { data } = await axios.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
@@ -25,12 +15,6 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t
|
||||
timeout: 15000
|
||||
});
|
||||
|
||||
if (data.includes('AwsWafIntegration.checkForceRefresh') && retryCount < 3) {
|
||||
console.log(`AWS WAF detected, waiting before retry ${retryCount + 1}/3`);
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
return scrapeNewsletterContent(url, retryCount + 1);
|
||||
}
|
||||
|
||||
const $ = cheerio.load(data);
|
||||
|
||||
// Get the second image as thumbnail
|
||||
@@ -44,11 +28,7 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t
|
||||
thumbnail: thumbnailUrl,
|
||||
content
|
||||
};
|
||||
} catch (error: any) {
|
||||
if ((error.response?.status === 429 || error.code === 'ECONNRESET') && retryCount < 5) {
|
||||
console.log(`Rate limited or connection reset, attempt ${retryCount + 1}/5`);
|
||||
return scrapeNewsletterContent(url, retryCount + 1);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Error scraping newsletter content:', error);
|
||||
return { thumbnail: null, content: null };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user