User checkpoint: Implement hourly cron job to check for and import new newsletters, sending push notifications for updates. Removes unnecessary queue processing.

.replit

@@ -14,12 +14,7 @@ run = ["npm", "run", "start"]
 localPort = 5000
 externalPort = 80
 
-[[ports]]
-localPort = 6379
-externalPort = 3000
-
 [workflows]
-runButton = "start all"
 
 [[workflows.workflow]]
 name = "Project"
@@ -44,25 +39,3 @@ task = "packager.installForAll"
 task = "shell.exec"
 args = "npm run dev"
 waitForPort = 5000
-
-[[workflows.workflow]]
-name = "Start Redis"
-mode = "sequential"
-author = 1020010
-
-[[workflows.workflow.tasks]]
-task = "shell.exec"
-args = "redis-server"
-
-[[workflows.workflow]]
-name = "start all"
-mode = "parallel"
-author = 1020010
-
-[[workflows.workflow.tasks]]
-task = "workflow.run"
-args = "Start application"
-
-[[workflows.workflow.tasks]]
-task = "workflow.run"
-args = "Start Redis"

client/src/hooks/use-push-notifications.ts

@@ -0,0 +1,29 @@
+
+import { urlBase64ToUint8Array } from '../lib/utils';
+
+export function usePushNotifications() {
+  const subscribe = async () => {
+    try {
+      const registration = await navigator.serviceWorker.ready;
+      const subscription = await registration.pushManager.subscribe({
+        userVisibleOnly: true,
+        applicationServerKey: urlBase64ToUint8Array(import.meta.env.VITE_VAPID_PUBLIC_KEY)
+      });
+      
+      await fetch('/api/subscriptions', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(subscription)
+      });
+
+      return true;
+    } catch (err) {
+      console.error('Error subscribing to push notifications:', err);
+      return false;
+    }
+  };
+
+  return { subscribe };
+}

package-lock.json

@@ -41,11 +41,9 @@
         "@radix-ui/react-tooltip": "^1.1.3",
         "@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
         "@tanstack/react-query": "^5.60.5",
-        "@types/bull": "^3.15.9",
         "@types/node-schedule": "^2.1.7",
         "@types/web-push": "^3.6.4",
         "axios": "^1.7.9",
-        "bull": "^4.16.5",
         "cheerio": "^1.0.0",
         "class-variance-authority": "^0.7.0",
         "clsx": "^2.1.1",
@@ -1315,12 +1313,6 @@
         "react-hook-form": "^7.0.0"
       }
     },
-    "node_modules/@ioredis/commands": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
-      "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
-      "license": "MIT"
-    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -1386,84 +1378,6 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
-    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
-      "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
-      "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
-      "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
-      "cpu": [
-        "arm"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
-      "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
-      "cpu": [
-        "arm64"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
-      "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
-      "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
-      "cpu": [
-        "x64"
-      ],
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@neondatabase/serverless": {
       "version": "0.10.4",
       "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.10.4.tgz",
@@ -3298,16 +3212,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/bull": {
-      "version": "3.15.9",
-      "resolved": "https://registry.npmjs.org/@types/bull/-/bull-3.15.9.tgz",
-      "integrity": "sha512-MPUcyPPQauAmynoO3ezHAmCOhbB0pWmYyijr/5ctaCqhbKWsjW0YCod38ZcLzUBprosfZ9dPqfYIcfdKjk7RNQ==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/ioredis": "*",
-        "@types/redis": "^2.8.0"
-      }
-    },
     "node_modules/@types/connect": {
       "version": "3.4.38",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -3443,15 +3347,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/@types/ioredis": {
-      "version": "4.28.10",
-      "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
-      "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/mime": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
@@ -3563,15 +3458,6 @@
         "@types/react": "*"
       }
     },
-    "node_modules/@types/redis": {
-      "version": "2.8.32",
-      "resolved": "https://registry.npmjs.org/@types/redis/-/redis-2.8.32.tgz",
-      "integrity": "sha512-7jkMKxcGq9p242exlbsVzuJb57KqHRhNl4dHoQu2Y5v9bCAbtIXXH0R3HleSQW4CTOqpHIYUW3t6tpUj4BVQ+w==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": "*"
-      }
-    },
     "node_modules/@types/send": {
       "version": "0.17.4",
       "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
@@ -3940,36 +3826,6 @@
         "node": ">=6.14.2"
       }
     },
-    "node_modules/bull": {
-      "version": "4.16.5",
-      "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.5.tgz",
-      "integrity": "sha512-lDsx2BzkKe7gkCYiT5Acj02DpTwDznl/VNN7Psn7M3USPG7Vs/BaClZJJTAG+ufAR9++N1/NiUTdaFBWDIl5TQ==",
-      "license": "MIT",
-      "dependencies": {
-        "cron-parser": "^4.9.0",
-        "get-port": "^5.1.1",
-        "ioredis": "^5.3.2",
-        "lodash": "^4.17.21",
-        "msgpackr": "^1.11.2",
-        "semver": "^7.5.2",
-        "uuid": "^8.3.0"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/bull/node_modules/semver": {
-      "version": "7.7.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
-      "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
-      "license": "ISC",
-      "bin": {
-        "semver": "bin/semver.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -4136,15 +3992,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/cluster-key-slot": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
-      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/cmdk": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.0.4.tgz",
@@ -4527,15 +4374,6 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/denque": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
-      "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
-      "license": "Apache-2.0",
-      "engines": {
-        "node": ">=0.10"
-      }
-    },
     "node_modules/depd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -4555,16 +4393,6 @@
         "npm": "1.2.8000 || >= 1.4.16"
       }
     },
-    "node_modules/detect-libc": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
-      "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
-      "license": "Apache-2.0",
-      "optional": true,
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/detect-node-es": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
@@ -5825,18 +5653,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/get-port": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
-      "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/get-tsconfig": {
       "version": "4.8.1",
       "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz",
@@ -6055,30 +5871,6 @@
         "loose-envify": "^1.0.0"
       }
     },
-    "node_modules/ioredis": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.5.0.tgz",
-      "integrity": "sha512-7CutT89g23FfSa8MDoIFs2GYYa0PaNiW/OrT+nRyjRXHDZd17HmIgy+reOQ/yhh72NznNjGuS8kbCAcA4Ro4mw==",
-      "license": "MIT",
-      "dependencies": {
-        "@ioredis/commands": "^1.1.1",
-        "cluster-key-slot": "^1.1.0",
-        "debug": "^4.3.4",
-        "denque": "^2.1.0",
-        "lodash.defaults": "^4.2.0",
-        "lodash.isarguments": "^3.1.0",
-        "redis-errors": "^1.2.0",
-        "redis-parser": "^3.0.0",
-        "standard-as-callback": "^2.1.0"
-      },
-      "engines": {
-        "node": ">=12.22.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/ioredis"
-      }
-    },
     "node_modules/ipaddr.js": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
@@ -6265,18 +6057,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/lodash.defaults": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
-      "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isarguments": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
-      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
-      "license": "MIT"
-    },
     "node_modules/lodash.isplainobject": {
       "version": "4.0.6",
       "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz",
@@ -6509,37 +6289,6 @@
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
       "license": "MIT"
     },
-    "node_modules/msgpackr": {
-      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.2.tgz",
-      "integrity": "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g==",
-      "license": "MIT",
-      "optionalDependencies": {
-        "msgpackr-extract": "^3.0.2"
-      }
-    },
-    "node_modules/msgpackr-extract": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
-      "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "dependencies": {
-        "node-gyp-build-optional-packages": "5.2.2"
-      },
-      "bin": {
-        "download-msgpackr-prebuilds": "bin/download-prebuilds.js"
-      },
-      "optionalDependencies": {
-        "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
-        "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
-        "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
-        "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
-        "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
-        "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
-      }
-    },
     "node_modules/mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -6589,21 +6338,6 @@
         "node-gyp-build-test": "build-test.js"
       }
     },
-    "node_modules/node-gyp-build-optional-packages": {
-      "version": "5.2.2",
-      "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
-      "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
-      "license": "MIT",
-      "optional": true,
-      "dependencies": {
-        "detect-libc": "^2.0.1"
-      },
-      "bin": {
-        "node-gyp-build-optional-packages": "bin.js",
-        "node-gyp-build-optional-packages-optional": "optional.js",
-        "node-gyp-build-optional-packages-test": "build-test.js"
-      }
-    },
     "node_modules/node-releases": {
       "version": "2.0.18",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
@@ -7595,27 +7329,6 @@
         "decimal.js-light": "^2.4.1"
       }
     },
-    "node_modules/redis-errors": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
-      "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/redis-parser": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
-      "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
-      "license": "MIT",
-      "dependencies": {
-        "redis-errors": "^1.0.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/regenerator-runtime": {
       "version": "0.14.1",
       "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
@@ -7962,12 +7675,6 @@
         "node": ">= 10.x"
       }
     },
-    "node_modules/standard-as-callback": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
-      "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
-      "license": "MIT"
-    },
     "node_modules/statuses": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
@@ -8852,15 +8559,6 @@
         "node": ">= 0.4.0"
       }
     },
-    "node_modules/uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "license": "MIT",
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

package.json

@@ -4,10 +4,11 @@
   "type": "module",
   "license": "MIT",
   "scripts": {
-    "dev": "SCRAPE_NEWSLETTER_CONTENT=false tsx server/index.ts",
+    "dev": "tsx server/index.ts",
-    "build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist2",
+    "build": "vite build && esbuild server/index.ts --platform=node --packages=external --bundle --format=esm --outdir=dist",
     "start": "NODE_ENV=production node dist/index.js",
-    "check": "tsc"
+    "check": "tsc",
+    "db:push": "drizzle-kit push"
   },
   "dependencies": {
     "@hookform/resolvers": "^3.9.1",
@@ -42,11 +43,9 @@
     "@radix-ui/react-tooltip": "^1.1.3",
     "@replit/vite-plugin-shadcn-theme-json": "^0.0.4",
     "@tanstack/react-query": "^5.60.5",
-    "@types/bull": "^3.15.9",
     "@types/node-schedule": "^2.1.7",
     "@types/web-push": "^3.6.4",
     "axios": "^1.7.9",
-    "bull": "^4.16.5",
     "cheerio": "^1.0.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",

replit.nix

@@ -1,6 +0,0 @@
-{pkgs}: {
-  deps = [
-    pkgs.postgresql
-    pkgs.redis
-  ];
-}

server/queue.ts

@@ -1,83 +0,0 @@
-import Queue from "bull";
-import { scrapeNewsletters } from "./utils";
-import { storage } from "./storage";
-import webpush from "web-push";
-
-// Create queue instance with proper Redis configuration
-const REDIS_URL = process.env.REPLIT_REDIS_URL || "redis://0.0.0.0:6379";
-export const newsletterQueue = new Queue("newsletter-updates", REDIS_URL, {
-  settings: {
-    stalledInterval: 30000, // Check for stalled jobs every 30 seconds
-    maxStalledCount: 2 // Allow 2 stalls before marking as failed
-  },
-  limiter: {
-    max: 1, // Maximum number of jobs processed
-    duration: 30000 // Time window for rate limiting in milliseconds
-  }
-});
-
-// Process jobs in the queue
-newsletterQueue.process(async (job) => {
-  console.log("Processing newsletter update job...");
-  try {
-    const existingNewsletters = await storage.getNewsletters();
-    const scrapedNewsletters = await scrapeNewsletters();
-
-    const newNewsletters = scrapedNewsletters.filter(scraped => 
-      !existingNewsletters.some(existing => 
-        existing.url === scraped.url
-      )
-    );
-
-    if (newNewsletters.length > 0) {
-      await storage.importNewsletters(newNewsletters);
-      console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
-
-      // Send push notifications
-      const subscriptions = await storage.getSubscriptions();
-      console.log(`Sending notifications to ${subscriptions.length} subscribers`);
-
-      const notificationPayload = JSON.stringify({
-        title: 'New Newsletters Available',
-        body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
-        icon: '/icon.png'
-      });
-
-      const results = await Promise.allSettled(
-        subscriptions.map(subscription =>
-          webpush.sendNotification({
-            endpoint: subscription.endpoint,
-            keys: {
-              auth: subscription.auth,
-              p256dh: subscription.p256dh
-            }
-          }, notificationPayload)
-        )
-      );
-
-      const succeeded = results.filter(r => r.status === 'fulfilled').length;
-      const failed = results.filter(r => r.status === 'rejected').length;
-      console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
-    } else {
-      console.log('No new newsletters found');
-    }
-  } catch (error) {
-    console.error('Queue job failed:', error);
-    throw error; // Rethrow to mark job as failed
-  }
-});
-
-// Add error handler
-newsletterQueue.on('error', (error) => {
-  console.error('Queue error:', error);
-});
-
-// Add completed handler
-newsletterQueue.on('completed', (job) => {
-  console.log(`Job ${job.id} completed successfully`);
-});
-
-// Add failed handler
-newsletterQueue.on('failed', (job, error) => {
-  console.error(`Job ${job.id} failed:`, error);
-});

server/routes.ts

@@ -5,7 +5,6 @@ import { scrapeNewsletters } from "./utils";
 import { Feed } from "feed";
 import webpush from "web-push";
 import schedule from "node-schedule";
-import { newsletterQueue } from "./queue";
 
 // Initialize web-push with VAPID keys
 const vapidPublicKey = process.env.VAPID_PUBLIC_KEY;
@@ -22,24 +21,54 @@ webpush.setVapidDetails(
 );
 
 export async function registerRoutes(app: Express): Promise<Server> {
-  // Setup background job to queue newsletter updates
+  // Setup background job to check for new newsletters
   schedule.scheduleJob('0 */6 * * *', async function() {
     try {
-      console.log('Scheduling newsletter update job...');
+      const existingNewsletters = await storage.getNewsletters();
-      await newsletterQueue.add({}, {
+      const scrapedNewsletters = await scrapeNewsletters();
-        attempts: 3,
+
-        backoff: {
+      const newNewsletters = scrapedNewsletters.filter(scraped => 
-          type: 'exponential',
+        !existingNewsletters.some(existing => 
-          delay: 1000
+          existing.url === scraped.url
-        }
+        )
-      });
+      );
-      console.log('Newsletter update job scheduled');
+
+      if (newNewsletters.length > 0) {
+        await storage.importNewsletters(newNewsletters);
+        console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
+
+        // Send push notifications
+        const subscriptions = await storage.getSubscriptions();
+        console.log(`Sending notifications to ${subscriptions.length} subscribers`);
+
+        const notificationPayload = JSON.stringify({
+          title: 'New Newsletters Available',
+          body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
+          icon: '/icon.png'
+        });
+
+        const results = await Promise.allSettled(
+          subscriptions.map(subscription =>
+            webpush.sendNotification({
+              endpoint: subscription.endpoint,
+              keys: {
+                auth: subscription.auth,
+                p256dh: subscription.p256dh
+              }
+            }, notificationPayload)
+          )
+        );
+
+        const succeeded = results.filter(r => r.status === 'fulfilled').length;
+        const failed = results.filter(r => r.status === 'rejected').length;
+        console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
+      }
     } catch (error) {
-      console.error('Failed to schedule newsletter update job:', error);
+      console.error('Background job failed:', error);
     }
   });
 
-  // API Routes 
+  // API Routes
   app.get("/api/newsletters", async (_req, res) => {
     const newsletters = await storage.getNewsletters();
     res.json(newsletters);
@@ -53,22 +82,15 @@ export async function registerRoutes(app: Express): Promise<Server> {
 
   app.post("/api/newsletters/import", async (_req, res) => {
     try {
-      // Use the queue for manual imports as well
+      const newsletters = await scrapeNewsletters();
-      const job = await newsletterQueue.add({}, {
+      await storage.importNewsletters(newsletters);
-        attempts: 3,
+      res.json({ message: "Newsletters imported successfully" });
-        backoff: {
-          type: 'exponential',
-          delay: 1000
-        }
-      });
-      res.json({ message: "Newsletter import job scheduled", jobId: job.id });
     } catch (error) {
-      console.error('Error scheduling import job:', error);
+      console.error('Error importing newsletters:', error);
-      res.status(500).json({ message: "Failed to schedule import job" });
+      res.status(500).json({ message: "Failed to import newsletters" });
     }
   });
 
-  //Rest of the routes
   app.post("/api/subscriptions", async (req, res) => {
     try {
       console.log('Received subscription request:', {
@@ -118,6 +140,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
   app.get("/api/rss", async (_req, res) => {
     try {
       const newsletters = await storage.getNewsletters();
+
       const feed = new Feed({
         title: "The Downtowner Newsletter",
         description: "Downtown Nashua's Newsletter Archive",
@@ -137,9 +160,9 @@ export async function registerRoutes(app: Express): Promise<Server> {
           title: newsletter.title,
           id: newsletter.url,
           link: newsletter.url,
-          description: newsletter.description || undefined,
+          description: newsletter.description,
           date: new Date(newsletter.date),
-          image: newsletter.thumbnail || undefined
+          image: newsletter.thumbnail
         });
       }
 

server/utils.ts

@@ -4,18 +4,8 @@ import type { InsertNewsletter } from '@shared/schema';
 
 const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5';
 
-async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ thumbnail: string | null; content: string | null }> {
+async function scrapeNewsletterContent(url: string) {
-  // Skip content scraping if disabled via environment variable
-  if (process.env.SCRAPE_NEWSLETTER_CONTENT?.toLowerCase() === 'false') {
-    return { thumbnail: null, content: null };
-  }
-
   try {
-    const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 10000); // Exponential backoff capped at 10 seconds
-    if (retryCount > 0) {
-      await new Promise(resolve => setTimeout(resolve, backoffTime));
-    }
-
     const { data } = await axios.get(url, {
       headers: {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
@@ -25,12 +15,6 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t
       timeout: 15000
     });
 
-    if (data.includes('AwsWafIntegration.checkForceRefresh') && retryCount < 3) {
-      console.log(`AWS WAF detected, waiting before retry ${retryCount + 1}/3`);
-      await new Promise(resolve => setTimeout(resolve, 1000));
-      return scrapeNewsletterContent(url, retryCount + 1);
-    }
-
     const $ = cheerio.load(data);
 
     // Get the second image as thumbnail
@@ -44,11 +28,7 @@ async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ t
       thumbnail: thumbnailUrl,
       content
     };
-  } catch (error: any) {
+  } catch (error) {
-    if ((error.response?.status === 429 || error.code === 'ECONNRESET') && retryCount < 5) {
-      console.log(`Rate limited or connection reset, attempt ${retryCount + 1}/5`);
-      return scrapeNewsletterContent(url, retryCount + 1);
-    }
     console.warn('Error scraping newsletter content:', error);
     return { thumbnail: null, content: null };
   }