From 8d943ed548eefa23addcaae22962e07372f8dc83 Mon Sep 17 00:00:00 2001 From: TerribleDev <1020010-TerribleDev@users.noreply.replit.com> Date: Sat, 15 Feb 2025 18:34:36 +0000 Subject: [PATCH] Agent query: I've set up a queue system for background jobs. Could you try clicking the refresh button to trigger a manual newsletter import and verify if it works? Implement newsletter updates via a Bull queue for asynchronous processing. Screenshot: https://storage.googleapis.com/screenshot-production-us-central1/9dda30b6-4149-4bce-89dc-76333005952c/6b15ec21-707b-471e-b02c-8596307fdb92.jpg --- package-lock.json | 302 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + replit.nix | 5 + server/queue.ts | 73 +++++++++++ server/routes.ts | 77 +++++------- 5 files changed, 409 insertions(+), 50 deletions(-) create mode 100644 replit.nix create mode 100644 server/queue.ts diff --git a/package-lock.json b/package-lock.json index 38428c5..a90eb30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,9 +41,11 @@ "@radix-ui/react-tooltip": "^1.1.3", "@replit/vite-plugin-shadcn-theme-json": "^0.0.4", "@tanstack/react-query": "^5.60.5", + "@types/bull": "^3.15.9", "@types/node-schedule": "^2.1.7", "@types/web-push": "^3.6.4", "axios": "^1.7.9", + "bull": "^4.16.5", "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", @@ -1313,6 +1315,12 @@ "react-hook-form": "^7.0.0" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1378,6 +1386,84 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@neondatabase/serverless": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.10.4.tgz", @@ -3212,6 +3298,16 @@ "@types/node": "*" } }, + "node_modules/@types/bull": { + "version": "3.15.9", + "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz", + "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==", + "license": "MIT", + "dependencies": { + "@types/ioredis": "*", + "@types/redis": "^2.8.0" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -3347,6 +3443,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ioredis": { + "version": "4.28.10", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", + "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -3458,6 +3563,15 @@ "@types/react": "*" } }, + "node_modules/@types/redis": { + "version": "2.8.32", + "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz", + "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/send": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", @@ -3826,6 +3940,36 @@ "node": ">=6.14.2" } }, + "node_modules/bull": { + "version": "4.16.5", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz", + "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.9.0", + "get-port": "^5.1.1", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.11.2", + "semver": "^7.5.2", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/bull/node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3992,6 +4136,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmdk": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.4.tgz", @@ -4374,6 +4527,15 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4393,6 +4555,16 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -5653,6 +5825,18 @@ "node": ">=6" } }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-tsconfig": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", @@ -5871,6 +6055,30 @@ "loose-envify": "^1.0.0" } }, + "node_modules/ioredis": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz", + "integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -6057,6 +6265,18 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -6289,6 +6509,37 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/msgpackr": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz", + "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -6338,6 +6589,21 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-releases": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", @@ -7329,6 +7595,27 @@ "decimal.js-light": "^2.4.1" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", @@ -7675,6 +7962,12 @@ "node": ">= 10.x" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -8559,6 +8852,15 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/package.json b/package.json index f2b5ada..6c9d435 100644 --- a/package.json +++ b/package.json @@ -43,9 +43,11 @@ "@radix-ui/react-tooltip": "^1.1.3", "@replit/vite-plugin-shadcn-theme-json": "^0.0.4", "@tanstack/react-query": "^5.60.5", + "@types/bull": "^3.15.9", "@types/node-schedule": "^2.1.7", "@types/web-push": "^3.6.4", "axios": "^1.7.9", + "bull": "^4.16.5", "cheerio": "^1.0.0", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", diff --git a/replit.nix b/replit.nix new file mode 100644 index 0000000..22199b2 --- /dev/null +++ b/replit.nix @@ -0,0 +1,5 @@ +{pkgs}: { + deps = [ + pkgs.redis + ]; +} diff --git a/server/queue.ts b/server/queue.ts new file mode 100644 index 0000000..3a5cd27 --- /dev/null +++ b/server/queue.ts @@ -0,0 +1,73 @@ +import Queue from "bull"; +import { scrapeNewsletters } from "./utils"; +import { storage } from "./storage"; +import webpush from "web-push"; + +// Create queue instance +export const newsletterQueue = new Queue("newsletter-updates", { + redis: process.env.REDIS_URL || "redis://127.0.0.1:6379" +}); + +// Process jobs in the queue +newsletterQueue.process(async (job) => { + console.log("Processing newsletter update job..."); + try { + const existingNewsletters = await storage.getNewsletters(); + const scrapedNewsletters = await scrapeNewsletters(); + + const newNewsletters = scrapedNewsletters.filter(scraped => + !existingNewsletters.some(existing => + existing.url === scraped.url + ) + ); + + if (newNewsletters.length > 0) { + await storage.importNewsletters(newNewsletters); + console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`); + + // Send push notifications + const subscriptions = await storage.getSubscriptions(); + console.log(`Sending notifications to ${subscriptions.length} subscribers`); + + const notificationPayload = JSON.stringify({ + title: 'New Newsletters Available', + body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, + icon: '/icon.png' + }); + + const results = await Promise.allSettled( + subscriptions.map(subscription => + webpush.sendNotification({ + endpoint: subscription.endpoint, + keys: { + auth: subscription.auth, + p256dh: subscription.p256dh + } + }, notificationPayload) + ) + ); + + const succeeded = results.filter(r => r.status === 'fulfilled').length; + const failed = results.filter(r => r.status === 'rejected').length; + console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`); + } + } catch (error) { + console.error('Queue job failed:', error); + throw error; // Rethrow to mark job as failed + } +}); + +// Add error handler +newsletterQueue.on('error', (error) => { + console.error('Queue error:', error); +}); + +// Add completed handler +newsletterQueue.on('completed', (job) => { + console.log(`Job ${job.id} completed successfully`); +}); + +// Add failed handler +newsletterQueue.on('failed', (job, error) => { + console.error(`Job ${job.id} failed:`, error); +}); diff --git a/server/routes.ts b/server/routes.ts index ebb37c9..1c667d1 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -5,6 +5,7 @@ import { scrapeNewsletters } from "./utils"; import { Feed } from "feed"; import webpush from "web-push"; import schedule from "node-schedule"; +import { newsletterQueue } from "./queue"; // Initialize web-push with VAPID keys const vapidPublicKey = process.env.VAPID_PUBLIC_KEY; @@ -21,54 +22,24 @@ webpush.setVapidDetails( ); export async function registerRoutes(app: Express): Promise { - // Setup background job to check for new newsletters + // Setup background job to queue newsletter updates schedule.scheduleJob('0 */6 * * *', async function() { try { - const existingNewsletters = await storage.getNewsletters(); - const scrapedNewsletters = await scrapeNewsletters(); - - const newNewsletters = scrapedNewsletters.filter(scraped => - !existingNewsletters.some(existing => - existing.url === scraped.url - ) - ); - - if (newNewsletters.length > 0) { - await storage.importNewsletters(newNewsletters); - console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`); - - // Send push notifications - const subscriptions = await storage.getSubscriptions(); - console.log(`Sending notifications to ${subscriptions.length} subscribers`); - - const notificationPayload = JSON.stringify({ - title: 'New Newsletters Available', - body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, - icon: '/icon.png' - }); - - const results = await Promise.allSettled( - subscriptions.map(subscription => - webpush.sendNotification({ - endpoint: subscription.endpoint, - keys: { - auth: subscription.auth, - p256dh: subscription.p256dh - } - }, notificationPayload) - ) - ); - - const succeeded = results.filter(r => r.status === 'fulfilled').length; - const failed = results.filter(r => r.status === 'rejected').length; - console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`); - } + console.log('Scheduling newsletter update job...'); + await newsletterQueue.add({}, { + attempts: 3, + backoff: { + type: 'exponential', + delay: 1000 + } + }); + console.log('Newsletter update job scheduled'); } catch (error) { - console.error('Background job failed:', error); + console.error('Failed to schedule newsletter update job:', error); } }); - // API Routes + // API Routes app.get("/api/newsletters", async (_req, res) => { const newsletters = await storage.getNewsletters(); res.json(newsletters); @@ -82,15 +53,22 @@ export async function registerRoutes(app: Express): Promise { app.post("/api/newsletters/import", async (_req, res) => { try { - const newsletters = await scrapeNewsletters(); - await storage.importNewsletters(newsletters); - res.json({ message: "Newsletters imported successfully" }); + // Use the queue for manual imports as well + const job = await newsletterQueue.add({}, { + attempts: 3, + backoff: { + type: 'exponential', + delay: 1000 + } + }); + res.json({ message: "Newsletter import job scheduled", jobId: job.id }); } catch (error) { - console.error('Error importing newsletters:', error); - res.status(500).json({ message: "Failed to import newsletters" }); + console.error('Error scheduling import job:', error); + res.status(500).json({ message: "Failed to schedule import job" }); } }); + //Rest of the routes app.post("/api/subscriptions", async (req, res) => { try { console.log('Received subscription request:', { @@ -140,7 +118,6 @@ export async function registerRoutes(app: Express): Promise { app.get("/api/rss", async (_req, res) => { try { const newsletters = await storage.getNewsletters(); - const feed = new Feed({ title: "The Downtowner Newsletter", description: "Downtown Nashua's Newsletter Archive", @@ -160,9 +137,9 @@ export async function registerRoutes(app: Express): Promise { title: newsletter.title, id: newsletter.url, link: newsletter.url, - description: newsletter.description, + description: newsletter.description || undefined, date: new Date(newsletter.date), - image: newsletter.thumbnail + image: newsletter.thumbnail || undefined }); }