Agent query: I've set up a queue system for background jobs. Could you try clicking the refresh button to trigger a manual newsletter import and verify if it works?

Implement newsletter updates via a Bull queue for asynchronous processing. Screenshot: https://storage.googleapis.com/screenshot-production-us-central1/9dda30b6-4149-4bce-89dc-76333005952c/6b15ec21-707b-471e-b02c-8596307fdb92.jpg
2025-02-15 18:34:36 +00:00
parent 00785d9df0
commit 8d943ed548
5 changed files with 409 additions and 50 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -41,9 +41,11 @@
        "@radix-ui/react-tooltip": "^1.1.3",
        "@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
        "@tanstack/react-query": "^5.60.5",
+        "@types/bull": "^3.15.9",
        "@types/node-schedule": "^2.1.7",
        "@types/web-push": "^3.6.4",
        "axios": "^1.7.9",
+        "bull": "^4.16.5",
        "cheerio": "^1.0.0",
        "class-variance-authority": "^0.7.0",
        "clsx": "^2.1.1",
@@ -1313,6 +1315,12 @@
        "react-hook-form": "^7.0.0"
      }
    },
+    "node_modules/@ioredis/commands": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
+      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
+      "license": "MIT"
+    },
    "node_modules/@isaacs/cliui": {
      "version": "8.0.2",
      "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -1378,6 +1386,84 @@
        "@jridgewell/sourcemap-codec": "^1.4.14"
      }
    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
+      "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
+      "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
+      "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
+      "cpu": [
+        "arm"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
+      "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
+      "cpu": [
+        "arm64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
+      "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ]
+    },
+    "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
+      "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
+      "cpu": [
+        "x64"
+      ],
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ]
+    },
    "node_modules/@neondatabase/serverless": {
      "version": "0.10.4",
      "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.10.4.tgz",
@@ -3212,6 +3298,16 @@
        "@types/node": "*"
      }
    },
+    "node_modules/@types/bull": {
+      "version": "3.15.9",
+      "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz",
+      "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/ioredis": "*",
+        "@types/redis": "^2.8.0"
+      }
+    },
    "node_modules/@types/connect": {
      "version": "3.4.38",
      "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -3347,6 +3443,15 @@
      "dev": true,
      "license": "MIT"
    },
+    "node_modules/@types/ioredis": {
+      "version": "4.28.10",
+      "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
+      "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/mime": {
      "version": "1.3.5",
      "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -3458,6 +3563,15 @@
        "@types/react": "*"
      }
    },
+    "node_modules/@types/redis": {
+      "version": "2.8.32",
+      "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
+      "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
    "node_modules/@types/send": {
      "version": "0.17.4",
      "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
@@ -3826,6 +3940,36 @@
        "node": ">=6.14.2"
      }
    },
+    "node_modules/bull": {
+      "version": "4.16.5",
+      "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz",
+      "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==",
+      "license": "MIT",
+      "dependencies": {
+        "cron-parser": "^4.9.0",
+        "get-port": "^5.1.1",
+        "ioredis": "^5.3.2",
+        "lodash": "^4.17.21",
+        "msgpackr": "^1.11.2",
+        "semver": "^7.5.2",
+        "uuid": "^8.3.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/bull/node_modules/semver": {
+      "version": "7.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
+      "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
    "node_modules/bytes": {
      "version": "3.1.2",
      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -3992,6 +4136,15 @@
        "node": ">=6"
      }
    },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/cmdk": {
      "version": "1.0.4",
      "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.4.tgz",
@@ -4374,6 +4527,15 @@
        "node": ">=0.4.0"
      }
    },
+    "node_modules/denque": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
    "node_modules/depd": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -4393,6 +4555,16 @@
        "npm": "1.2.8000 || >= 1.4.16"
      }
    },
+    "node_modules/detect-libc": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
+      "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
+      "license": "Apache-2.0",
+      "optional": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/detect-node-es": {
      "version": "1.1.0",
      "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
@@ -5653,6 +5825,18 @@
        "node": ">=6"
      }
    },
+    "node_modules/get-port": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
+      "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
    "node_modules/get-tsconfig": {
      "version": "4.8.1",
      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz",
@@ -5871,6 +6055,30 @@
        "loose-envify": "^1.0.0"
      }
    },
+    "node_modules/ioredis": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz",
+      "integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==",
+      "license": "MIT",
+      "dependencies": {
+        "@ioredis/commands": "^1.1.1",
+        "cluster-key-slot": "^1.1.0",
+        "debug": "^4.3.4",
+        "denque": "^2.1.0",
+        "lodash.defaults": "^4.2.0",
+        "lodash.isarguments": "^3.1.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0",
+        "standard-as-callback": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=12.22.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/ioredis"
+      }
+    },
    "node_modules/ipaddr.js": {
      "version": "1.9.1",
      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6057,6 +6265,18 @@
      "dev": true,
      "license": "MIT"
    },
+    "node_modules/lodash.defaults": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
+      "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
+      "license": "MIT"
+    },
+    "node_modules/lodash.isarguments": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
+      "license": "MIT"
+    },
    "node_modules/lodash.isplainobject": {
      "version": "4.0.6",
      "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
@@ -6289,6 +6509,37 @@
      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
      "license": "MIT"
    },
+    "node_modules/msgpackr": {
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz",
+      "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==",
+      "license": "MIT",
+      "optionalDependencies": {
+        "msgpackr-extract": "^3.0.2"
+      }
+    },
+    "node_modules/msgpackr-extract": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
+      "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "node-gyp-build-optional-packages": "5.2.2"
+      },
+      "bin": {
+        "download-msgpackr-prebuilds": "bin/download-prebuilds.js"
+      },
+      "optionalDependencies": {
+        "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
+        "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
+      }
+    },
    "node_modules/mz": {
      "version": "2.7.0",
      "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -6338,6 +6589,21 @@
        "node-gyp-build-test": "build-test.js"
      }
    },
+    "node_modules/node-gyp-build-optional-packages": {
+      "version": "5.2.2",
+      "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
+      "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
+      "license": "MIT",
+      "optional": true,
+      "dependencies": {
+        "detect-libc": "^2.0.1"
+      },
+      "bin": {
+        "node-gyp-build-optional-packages": "bin.js",
+        "node-gyp-build-optional-packages-optional": "optional.js",
+        "node-gyp-build-optional-packages-test": "build-test.js"
+      }
+    },
    "node_modules/node-releases": {
      "version": "2.0.18",
      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
@@ -7329,6 +7595,27 @@
        "decimal.js-light": "^2.4.1"
      }
    },
+    "node_modules/redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
+      "license": "MIT",
+      "dependencies": {
+        "redis-errors": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
    "node_modules/regenerator-runtime": {
      "version": "0.14.1",
      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
@@ -7675,6 +7962,12 @@
        "node": ">= 10.x"
      }
    },
+    "node_modules/standard-as-callback": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
+      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
+      "license": "MIT"
+    },
    "node_modules/statuses": {
      "version": "2.0.1",
      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -8559,6 +8852,15 @@
        "node": ">= 0.4.0"
      }
    },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
    "node_modules/vary": {
      "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
--- a/package.json
+++ b/package.json
@@ -43,9 +43,11 @@
    "@radix-ui/react-tooltip": "^1.1.3",
    "@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
    "@tanstack/react-query": "^5.60.5",
+    "@types/bull": "^3.15.9",
    "@types/node-schedule": "^2.1.7",
    "@types/web-push": "^3.6.4",
    "axios": "^1.7.9",
+    "bull": "^4.16.5",
    "cheerio": "^1.0.0",
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.1",
--- a/replit.nix
+++ b/replit.nix
@@ -0,0 +1,5 @@
+{pkgs}: {
+  deps = [
+    pkgs.redis
+  ];
+}
--- a/server/queue.ts
+++ b/server/queue.ts
@@ -0,0 +1,73 @@
+import Queue from "bull";
+import { scrapeNewsletters } from "./utils";
+import { storage } from "./storage";
+import webpush from "web-push";
+
+// Create queue instance
+export const newsletterQueue = new Queue("newsletter-updates", {
+  redis: process.env.REDIS_URL || "redis://127.0.0.1:6379"
+});
+
+// Process jobs in the queue
+newsletterQueue.process(async (job) => {
+  console.log("Processing newsletter update job...");
+  try {
+    const existingNewsletters = await storage.getNewsletters();
+    const scrapedNewsletters = await scrapeNewsletters();
+
+    const newNewsletters = scrapedNewsletters.filter(scraped => 
+      !existingNewsletters.some(existing => 
+        existing.url === scraped.url
+      )
+    );
+
+    if (newNewsletters.length > 0) {
+      await storage.importNewsletters(newNewsletters);
+      console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
+
+      // Send push notifications
+      const subscriptions = await storage.getSubscriptions();
+      console.log(`Sending notifications to ${subscriptions.length} subscribers`);
+
+      const notificationPayload = JSON.stringify({
+        title: 'New Newsletters Available',
+        body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
+        icon: '/icon.png'
+      });
+
+      const results = await Promise.allSettled(
+        subscriptions.map(subscription =>
+          webpush.sendNotification({
+            endpoint: subscription.endpoint,
+            keys: {
+              auth: subscription.auth,
+              p256dh: subscription.p256dh
+            }
+          }, notificationPayload)
+        )
+      );
+
+      const succeeded = results.filter(r => r.status === 'fulfilled').length;
+      const failed = results.filter(r => r.status === 'rejected').length;
+      console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
+    }
+  } catch (error) {
+    console.error('Queue job failed:', error);
+    throw error; // Rethrow to mark job as failed
+  }
+});
+
+// Add error handler
+newsletterQueue.on('error', (error) => {
+  console.error('Queue error:', error);
+});
+
+// Add completed handler
+newsletterQueue.on('completed', (job) => {
+  console.log(`Job ${job.id} completed successfully`);
+});
+
+// Add failed handler
+newsletterQueue.on('failed', (job, error) => {
+  console.error(`Job ${job.id} failed:`, error);
+});
--- a/server/routes.ts
+++ b/server/routes.ts
@@ -5,6 +5,7 @@ import { scrapeNewsletters } from "./utils";
 import { Feed } from "feed";
 import webpush from "web-push";
 import schedule from "node-schedule";
+import { newsletterQueue } from "./queue";

 // Initialize web-push with VAPID keys
 const vapidPublicKey = process.env.VAPID_PUBLIC_KEY;
@@ -21,54 +22,24 @@ webpush.setVapidDetails(
 );

 export async function registerRoutes(app: Express): Promise<Server> {
-  // Setup background job to check for new newsletters
+  // Setup background job to queue newsletter updates
  schedule.scheduleJob('0 */6 * * *', async function() {
    try {
-      const existingNewsletters = await storage.getNewsletters();
-      const scrapedNewsletters = await scrapeNewsletters();
-
-      const newNewsletters = scrapedNewsletters.filter(scraped => 
-        !existingNewsletters.some(existing => 
-          existing.url === scraped.url
-        )
-      );
-
-      if (newNewsletters.length > 0) {
-        await storage.importNewsletters(newNewsletters);
-        console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
-
-        // Send push notifications
-        const subscriptions = await storage.getSubscriptions();
-        console.log(`Sending notifications to ${subscriptions.length} subscribers`);
-
-        const notificationPayload = JSON.stringify({
-          title: 'New Newsletters Available',
-          body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
-          icon: '/icon.png'
-        });
-
-        const results = await Promise.allSettled(
-          subscriptions.map(subscription =>
-            webpush.sendNotification({
-              endpoint: subscription.endpoint,
-              keys: {
-                auth: subscription.auth,
-                p256dh: subscription.p256dh
-              }
-            }, notificationPayload)
-          )
-        );
-
-        const succeeded = results.filter(r => r.status === 'fulfilled').length;
-        const failed = results.filter(r => r.status === 'rejected').length;
-        console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
-      }
+      console.log('Scheduling newsletter update job...');
+      await newsletterQueue.add({}, {
+        attempts: 3,
+        backoff: {
+          type: 'exponential',
+          delay: 1000
+        }
+      });
+      console.log('Newsletter update job scheduled');
    } catch (error) {
-      console.error('Background job failed:', error);
+      console.error('Failed to schedule newsletter update job:', error);
    }
  });

-  // API Routes
+  // API Routes 
  app.get("/api/newsletters", async (_req, res) => {
    const newsletters = await storage.getNewsletters();
    res.json(newsletters);
@@ -82,15 +53,22 @@ export async function registerRoutes(app: Express): Promise<Server> {

  app.post("/api/newsletters/import", async (_req, res) => {
    try {
-      const newsletters = await scrapeNewsletters();
-      await storage.importNewsletters(newsletters);
-      res.json({ message: "Newsletters imported successfully" });
+      // Use the queue for manual imports as well
+      const job = await newsletterQueue.add({}, {
+        attempts: 3,
+        backoff: {
+          type: 'exponential',
+          delay: 1000
+        }
+      });
+      res.json({ message: "Newsletter import job scheduled", jobId: job.id });
    } catch (error) {
-      console.error('Error importing newsletters:', error);
-      res.status(500).json({ message: "Failed to import newsletters" });
+      console.error('Error scheduling import job:', error);
+      res.status(500).json({ message: "Failed to schedule import job" });
    }
  });

+  //Rest of the routes
  app.post("/api/subscriptions", async (req, res) => {
    try {
      console.log('Received subscription request:', {
@@ -140,7 +118,6 @@ export async function registerRoutes(app: Express): Promise<Server> {
  app.get("/api/rss", async (_req, res) => {
    try {
      const newsletters = await storage.getNewsletters();
-
      const feed = new Feed({
        title: "The Downtowner Newsletter",
        description: "Downtown Nashua's Newsletter Archive",
@@ -160,9 +137,9 @@ export async function registerRoutes(app: Express): Promise<Server> {
          title: newsletter.title,
          id: newsletter.url,
          link: newsletter.url,
-          description: newsletter.description,
+          description: newsletter.description || undefined,
          date: new Date(newsletter.date),
-          image: newsletter.thumbnail
+          image: newsletter.thumbnail || undefined
        });
      }