diff --git a/client/src/pages/home.tsx b/client/src/pages/home.tsx
index 1916830..b469da4 100644
--- a/client/src/pages/home.tsx
+++ b/client/src/pages/home.tsx
@@ -1,23 +1,41 @@
-import { useState } from "react";
+import { useState, useEffect, useRef, useCallback } from "react";
import { format } from "date-fns";
import { Card, CardHeader, CardTitle, CardDescription, CardContent } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import { Button } from "@/components/ui/button";
import { Skeleton } from "@/components/ui/skeleton";
-import { Search, ExternalLink, Calendar, RefreshCw } from "lucide-react";
+import {
+ Search,
+ ExternalLink,
+ Calendar,
+ RefreshCw,
+ Share2,
+ Twitter,
+ Facebook,
+ Rss,
+ Bell,
+ BellOff
+} from "lucide-react";
import { useNewsletters, useNewsletterSearch } from "@/lib/newsletter-data";
import { useToast } from "@/hooks/use-toast";
import { apiRequest } from "@/lib/queryClient";
import { queryClient } from "@/lib/queryClient";
+import { motion, AnimatePresence } from "framer-motion";
+
+const ITEMS_PER_PAGE = 20;
export default function Home() {
const [searchQuery, setSearchQuery] = useState("");
const [isImporting, setIsImporting] = useState(false);
+ const [page, setPage] = useState(1);
+ const [isSubscribed, setIsSubscribed] = useState(false);
+ const loader = useRef(null);
const { data: allNewsletters, isLoading } = useNewsletters();
const { data: searchResults } = useNewsletterSearch(searchQuery);
const { toast } = useToast();
const newsletters = searchQuery ? searchResults : allNewsletters;
+ const paginatedNewsletters = newsletters?.slice(0, page * ITEMS_PER_PAGE);
const handleImport = async () => {
try {
@@ -39,10 +57,88 @@ export default function Home() {
}
};
+ const handleShare = async (newsletter) => {
+ if (navigator.share) {
+ try {
+ await navigator.share({
+ title: newsletter.title,
+ text: newsletter.description || "Check out this newsletter from The Downtowner",
+ url: newsletter.url
+ });
+ } catch (error) {
+ if (error.name !== 'AbortError') {
+ toast({
+ title: "Error",
+ description: "Failed to share newsletter",
+ variant: "destructive",
+ });
+ }
+ }
+ }
+ };
+
+ const handleSubscribe = async () => {
+ try {
+ if (!('serviceWorker' in navigator) || !('Notification' in window)) {
+ throw new Error('Push notifications are not supported');
+ }
+
+ const permission = await Notification.requestPermission();
+ if (permission !== 'granted') {
+ throw new Error('Notification permission denied');
+ }
+
+ const registration = await navigator.serviceWorker.ready;
+ const subscription = await registration.pushManager.subscribe({
+ userVisibleOnly: true,
+ applicationServerKey: process.env.VITE_VAPID_PUBLIC_KEY
+ });
+
+ await apiRequest('POST', '/api/subscriptions', subscription);
+ setIsSubscribed(true);
+ toast({
+ title: "Subscribed!",
+ description: "You'll receive notifications for new newsletters",
+ });
+ } catch (error) {
+ toast({
+ title: "Error",
+ description: error.message || "Failed to subscribe to notifications",
+ variant: "destructive",
+ });
+ }
+ };
+
+ const handleObserver = useCallback((entries) => {
+ const target = entries[0];
+ if (target.isIntersecting && newsletters?.length > page * ITEMS_PER_PAGE) {
+ setPage(prev => prev + 1);
+ }
+ }, [newsletters, page]);
+
+ useEffect(() => {
+ const observer = new IntersectionObserver(handleObserver, {
+ root: null,
+ rootMargin: "20px",
+ threshold: 1.0
+ });
+
+ if (loader.current) {
+ observer.observe(loader.current);
+ }
+
+ return () => observer.disconnect();
+ }, [handleObserver]);
+
return (
-
+
The Downtowner
@@ -68,58 +164,119 @@ export default function Home() {
>
+
+
-
+
+
+
);
diff --git a/package-lock.json b/package-lock.json
index 42d7fbd..ad6de6b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -53,10 +53,12 @@
"embla-carousel-react": "^8.3.0",
"express": "^4.21.2",
"express-session": "^1.18.1",
+ "feed": "^4.2.2",
"framer-motion": "^11.13.1",
"input-otp": "^1.2.4",
"lucide-react": "^0.453.0",
"memorystore": "^1.6.7",
+ "node-schedule": "^2.1.1",
"passport": "^0.7.0",
"passport-local": "^1.0.0",
"react": "^18.3.1",
@@ -69,6 +71,7 @@
"tailwind-merge": "^2.5.4",
"tailwindcss-animate": "^1.0.7",
"vaul": "^1.1.0",
+ "web-push": "^3.6.7",
"wouter": "^3.3.5",
"ws": "^8.18.0",
"zod": "^3.23.8",
@@ -3510,6 +3513,15 @@
"node": ">= 0.6"
}
},
+ "node_modules/agent-base": {
+ "version": "7.1.3",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz",
+ "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 14"
+ }
+ },
"node_modules/ansi-regex": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz",
@@ -3577,6 +3589,18 @@
"integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
"license": "MIT"
},
+ "node_modules/asn1.js": {
+ "version": "5.4.1",
+ "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz",
+ "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==",
+ "license": "MIT",
+ "dependencies": {
+ "bn.js": "^4.0.0",
+ "inherits": "^2.0.1",
+ "minimalistic-assert": "^1.0.0",
+ "safer-buffer": "^2.1.0"
+ }
+ },
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -3650,6 +3674,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/bn.js": {
+ "version": "4.12.1",
+ "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.1.tgz",
+ "integrity": "sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg==",
+ "license": "MIT"
+ },
"node_modules/body-parser": {
"version": "1.20.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
@@ -3749,6 +3779,12 @@
"node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
}
},
+ "node_modules/buffer-equal-constant-time": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
+ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
+ "license": "BSD-3-Clause"
+ },
"node_modules/buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@@ -4052,6 +4088,18 @@
"integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==",
"license": "MIT"
},
+ "node_modules/cron-parser": {
+ "version": "4.9.0",
+ "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
+ "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+ "license": "MIT",
+ "dependencies": {
+ "luxon": "^3.2.1"
+ },
+ "engines": {
+ "node": ">=12.0.0"
+ }
+ },
"node_modules/cross-spawn": {
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -4995,6 +5043,15 @@
"integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
"license": "MIT"
},
+ "node_modules/ecdsa-sig-formatter": {
+ "version": "1.0.11",
+ "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
+ "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "safe-buffer": "^5.0.1"
+ }
+ },
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -5349,6 +5406,18 @@
"reusify": "^1.0.4"
}
},
+ "node_modules/feed": {
+ "version": "4.2.2",
+ "resolved": "https://registry.npmjs.org/feed/-/feed-4.2.2.tgz",
+ "integrity": "sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==",
+ "license": "MIT",
+ "dependencies": {
+ "xml-js": "^1.6.11"
+ },
+ "engines": {
+ "node": ">=0.4.0"
+ }
+ },
"node_modules/fill-range": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -5698,6 +5767,15 @@
"entities": "^4.5.0"
}
},
+ "node_modules/http_ece": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.2.0.tgz",
+ "integrity": "sha512-JrF8SSLVmcvc5NducxgyOrKXe3EsyHMgBFgSaIUGmArKe+rwr0uphRkRXvwiom3I+fpIfoItveHrfudL8/rxuA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=16"
+ }
+ },
"node_modules/http-errors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -5714,6 +5792,19 @@
"node": ">= 0.8"
}
},
+ "node_modules/https-proxy-agent": {
+ "version": "7.0.6",
+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
+ "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
+ "license": "MIT",
+ "dependencies": {
+ "agent-base": "^7.1.2",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 14"
+ }
+ },
"node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@@ -5897,6 +5988,27 @@
"node": ">=6"
}
},
+ "node_modules/jwa": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
+ "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
+ "license": "MIT",
+ "dependencies": {
+ "buffer-equal-constant-time": "1.0.1",
+ "ecdsa-sig-formatter": "1.0.11",
+ "safe-buffer": "^5.0.1"
+ }
+ },
+ "node_modules/jws": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
+ "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
+ "license": "MIT",
+ "dependencies": {
+ "jwa": "^2.0.0",
+ "safe-buffer": "^5.0.1"
+ }
+ },
"node_modules/lilconfig": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz",
@@ -5939,6 +6051,12 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/long-timeout": {
+ "version": "0.1.1",
+ "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
+ "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==",
+ "license": "MIT"
+ },
"node_modules/loose-envify": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
@@ -5970,6 +6088,15 @@
"react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc"
}
},
+ "node_modules/luxon": {
+ "version": "3.5.0",
+ "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz",
+ "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -6081,6 +6208,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/minimalistic-assert": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",
+ "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==",
+ "license": "ISC"
+ },
"node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
@@ -6096,6 +6229,15 @@
"url": "https://github.com/sponsors/isaacs"
}
},
+ "node_modules/minimist": {
+ "version": "1.2.8",
+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
+ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+ "license": "MIT",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/minipass": {
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
@@ -6183,6 +6325,20 @@
"dev": true,
"license": "MIT"
},
+ "node_modules/node-schedule": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.1.tgz",
+ "integrity": "sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==",
+ "license": "MIT",
+ "dependencies": {
+ "cron-parser": "^4.2.0",
+ "long-timeout": "0.1.1",
+ "sorted-array-functions": "^1.3.0"
+ },
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -7292,6 +7448,12 @@
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
+ "node_modules/sax": {
+ "version": "1.4.1",
+ "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz",
+ "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==",
+ "license": "ISC"
+ },
"node_modules/scheduler": {
"version": "0.23.2",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
@@ -7448,6 +7610,12 @@
"url": "https://github.com/sponsors/isaacs"
}
},
+ "node_modules/sorted-array-functions": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
+ "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==",
+ "license": "MIT"
+ },
"node_modules/source-map": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -8905,6 +9073,25 @@
"@esbuild/win32-x64": "0.21.5"
}
},
+ "node_modules/web-push": {
+ "version": "3.6.7",
+ "resolved": "https://registry.npmjs.org/web-push/-/web-push-3.6.7.tgz",
+ "integrity": "sha512-OpiIUe8cuGjrj3mMBFWY+e4MMIkW3SVT+7vEIjvD9kejGUypv8GPDf84JdPWskK8zMRIJ6xYGm+Kxr8YkPyA0A==",
+ "license": "MPL-2.0",
+ "dependencies": {
+ "asn1.js": "^5.3.0",
+ "http_ece": "1.2.0",
+ "https-proxy-agent": "^7.0.0",
+ "jws": "^4.0.0",
+ "minimist": "^1.2.5"
+ },
+ "bin": {
+ "web-push": "src/cli.js"
+ },
+ "engines": {
+ "node": ">= 16"
+ }
+ },
"node_modules/whatwg-encoding": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
@@ -9079,6 +9266,18 @@
}
}
},
+ "node_modules/xml-js": {
+ "version": "1.6.11",
+ "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",
+ "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",
+ "license": "MIT",
+ "dependencies": {
+ "sax": "^1.2.4"
+ },
+ "bin": {
+ "xml-js": "bin/cli.js"
+ }
+ },
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
diff --git a/package.json b/package.json
index 2017e4b..f844720 100644
--- a/package.json
+++ b/package.json
@@ -55,10 +55,12 @@
"embla-carousel-react": "^8.3.0",
"express": "^4.21.2",
"express-session": "^1.18.1",
+ "feed": "^4.2.2",
"framer-motion": "^11.13.1",
"input-otp": "^1.2.4",
"lucide-react": "^0.453.0",
"memorystore": "^1.6.7",
+ "node-schedule": "^2.1.1",
"passport": "^0.7.0",
"passport-local": "^1.0.0",
"react": "^18.3.1",
@@ -71,6 +73,7 @@
"tailwind-merge": "^2.5.4",
"tailwindcss-animate": "^1.0.7",
"vaul": "^1.1.0",
+ "web-push": "^3.6.7",
"wouter": "^3.3.5",
"ws": "^8.18.0",
"zod": "^3.23.8",
diff --git a/server/routes.ts b/server/routes.ts
index bb4ae6d..01a5003 100644
--- a/server/routes.ts
+++ b/server/routes.ts
@@ -2,8 +2,63 @@ import type { Express } from "express";
import { createServer, type Server } from "http";
import { storage } from "./storage";
import { scrapeNewsletters } from "./utils";
+import { Feed } from "feed";
+import webpush from "web-push";
+import schedule from "node-schedule";
+
+// Initialize web-push with VAPID keys
+if (!process.env.VAPID_PUBLIC_KEY || !process.env.VAPID_PRIVATE_KEY) {
+ console.warn('VAPID keys not set. Push notifications will not work.');
+}
+
+webpush.setVapidDetails(
+ 'mailto:team@downtowner.com',
+ process.env.VAPID_PUBLIC_KEY || '',
+ process.env.VAPID_PRIVATE_KEY || ''
+);
export async function registerRoutes(app: Express): Promise {
+ // Setup background job to check for new newsletters
+ schedule.scheduleJob('0 */6 * * *', async function() {
+ try {
+ const existingNewsletters = await storage.getNewsletters();
+ const scrapedNewsletters = await scrapeNewsletters();
+
+ const newNewsletters = scrapedNewsletters.filter(scraped =>
+ !existingNewsletters.some(existing =>
+ existing.url === scraped.url
+ )
+ );
+
+ if (newNewsletters.length > 0) {
+ await storage.importNewsletters(newNewsletters);
+
+ // Send push notifications
+ const subscriptions = await storage.getSubscriptions();
+ const notificationPayload = JSON.stringify({
+ title: 'New Newsletters Available',
+ body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
+ icon: '/icon.png'
+ });
+
+ await Promise.allSettled(
+ subscriptions.map(subscription =>
+ webpush.sendNotification({
+ endpoint: subscription.endpoint,
+ keys: {
+ auth: subscription.auth,
+ p256dh: subscription.p256dh
+ }
+ }, notificationPayload)
+ )
+ );
+ }
+ } catch (error) {
+ console.error('Background job failed:', error);
+ }
+ });
+
+ // API Routes
app.get("/api/newsletters", async (_req, res) => {
const newsletters = await storage.getNewsletters();
res.json(newsletters);
@@ -26,6 +81,58 @@ export async function registerRoutes(app: Express): Promise {
}
});
+ app.post("/api/subscriptions", async (req, res) => {
+ try {
+ const subscription = req.body;
+ await storage.addSubscription({
+ endpoint: subscription.endpoint,
+ auth: subscription.keys.auth,
+ p256dh: subscription.keys.p256dh
+ });
+ res.json({ message: "Subscription added successfully" });
+ } catch (error) {
+ console.error('Error adding subscription:', error);
+ res.status(500).json({ message: "Failed to add subscription" });
+ }
+ });
+
+ app.get("/api/rss", async (_req, res) => {
+ try {
+ const newsletters = await storage.getNewsletters();
+
+ const feed = new Feed({
+ title: "The Downtowner Newsletter",
+ description: "Downtown Nashua's Newsletter Archive",
+ id: "https://downtowner.com/",
+ link: "https://downtowner.com/",
+ language: "en",
+ favicon: "https://downtowner.com/favicon.ico",
+ updated: newsletters[0]?.date ? new Date(newsletters[0].date) : new Date(),
+ generator: "The Downtowner RSS Feed",
+ feedLinks: {
+ rss2: "https://downtowner.com/api/rss"
+ }
+ });
+
+ for (const newsletter of newsletters) {
+ feed.addItem({
+ title: newsletter.title,
+ id: newsletter.url,
+ link: newsletter.url,
+ description: newsletter.description,
+ date: new Date(newsletter.date),
+ image: newsletter.thumbnail
+ });
+ }
+
+ res.type('application/xml');
+ res.send(feed.rss2());
+ } catch (error) {
+ console.error('Error generating RSS feed:', error);
+ res.status(500).json({ message: "Failed to generate RSS feed" });
+ }
+ });
+
const httpServer = createServer(app);
return httpServer;
}
\ No newline at end of file
diff --git a/server/storage.ts b/server/storage.ts
index 165a715..c065f4f 100644
--- a/server/storage.ts
+++ b/server/storage.ts
@@ -1,43 +1,52 @@
-import { type Newsletter, type InsertNewsletter } from "@shared/schema";
+import { type Newsletter, type InsertNewsletter, type Subscription, type InsertSubscription } from "@shared/schema";
+import { db } from "./db";
+import { newsletters, subscriptions } from "@shared/schema";
+import { desc, ilike, or } from "drizzle-orm";
export interface IStorage {
getNewsletters(): Promise;
searchNewsletters(query: string): Promise;
importNewsletters(newsletters: InsertNewsletter[]): Promise;
+ addSubscription(subscription: InsertSubscription): Promise;
+ getSubscriptions(): Promise;
}
-export class MemStorage implements IStorage {
- private newsletters: Newsletter[];
- private currentId: number;
-
- constructor() {
- this.newsletters = [];
- this.currentId = 1;
- }
-
+export class DatabaseStorage implements IStorage {
async getNewsletters(): Promise {
- return this.newsletters.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
+ return await db.select().from(newsletters).orderBy(desc(newsletters.date));
}
async searchNewsletters(query: string): Promise {
const lowercaseQuery = query.toLowerCase();
- return this.newsletters.filter(
- newsletter =>
- newsletter.title.toLowerCase().includes(lowercaseQuery) ||
- (newsletter.description?.toLowerCase() || '').includes(lowercaseQuery)
- ).sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
+ return await db
+ .select()
+ .from(newsletters)
+ .where(
+ or(
+ ilike(newsletters.title, `%${lowercaseQuery}%`),
+ ilike(newsletters.content || '', `%${lowercaseQuery}%`),
+ ilike(newsletters.description || '', `%${lowercaseQuery}%`)
+ )
+ )
+ .orderBy(desc(newsletters.date));
}
- async importNewsletters(newsletters: InsertNewsletter[]): Promise {
- // Ensure description is null if not provided
- const processedNewsletters = newsletters.map(newsletter => ({
- ...newsletter,
- description: newsletter.description ?? null,
- id: this.currentId++
- }));
+ async importNewsletters(newNewsletters: InsertNewsletter[]): Promise {
+ // Insert in batches to avoid overwhelming the database
+ const batchSize = 50;
+ for (let i = 0; i < newNewsletters.length; i += batchSize) {
+ const batch = newNewsletters.slice(i, i + batchSize);
+ await db.insert(newsletters).values(batch);
+ }
+ }
- this.newsletters.push(...processedNewsletters);
+ async addSubscription(subscription: InsertSubscription): Promise {
+ await db.insert(subscriptions).values(subscription);
+ }
+
+ async getSubscriptions(): Promise {
+ return await db.select().from(subscriptions);
}
}
-export const storage = new MemStorage();
\ No newline at end of file
+export const storage = new DatabaseStorage();
\ No newline at end of file
diff --git a/server/utils.ts b/server/utils.ts
index 71992fa..304fa2c 100644
--- a/server/utils.ts
+++ b/server/utils.ts
@@ -4,23 +4,55 @@ import type { InsertNewsletter } from '@shared/schema';
const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5';
+async function scrapeNewsletterContent(url: string) {
+ try {
+ const { data } = await axios.get(url, {
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+ 'Accept-Language': 'en-US,en;q=0.5',
+ },
+ timeout: 15000
+ });
+
+ const $ = cheerio.load(data);
+
+ // Get the second image as thumbnail
+ const images = $('img').toArray();
+ const thumbnailUrl = images.length > 1 ? $(images[1]).attr('src') : null;
+
+ // Extract text content
+ const content = $('body').text().trim();
+
+ return {
+ thumbnail: thumbnailUrl,
+ content
+ };
+ } catch (error) {
+ console.warn('Error scraping newsletter content:', error);
+ return { thumbnail: null, content: null };
+ }
+}
+
export async function scrapeNewsletters(): Promise {
try {
- // Add headers to mimic a browser request
const { data } = await axios.get(ROBLY_ARCHIVE_URL, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
},
- timeout: 10000 // 10 second timeout
+ timeout: 10000
});
const $ = cheerio.load(data);
const newsletters: InsertNewsletter[] = [];
// Find all links that start with /archive?id=
- $('a[href^="/archive?id="]').each((_, element) => {
+ const links = $('a[href^="/archive?id="]');
+ console.log(`Found ${links.length} newsletter links`);
+
+ for (const element of links.toArray()) {
const $element = $(element);
const url = $element.attr('href');
const fullText = $element.parent().text().trim();
@@ -33,18 +65,26 @@ export async function scrapeNewsletters(): Promise {
const [, dateStr, title] = match;
try {
const date = new Date(dateStr).toISOString().split('T')[0];
+ const fullUrl = `https://app.robly.com${url}`;
+
+ // Scrape the newsletter content
+ const { thumbnail, content } = await scrapeNewsletterContent(fullUrl);
newsletters.push({
title: title.trim(),
date,
- url: `https://app.robly.com${url}`,
- description: null
+ url: fullUrl,
+ thumbnail,
+ content,
+ description: content ? content.slice(0, 200) + '...' : null
});
+
+ console.log(`Processed newsletter: ${title}`);
} catch (err) {
console.warn('Error processing date for newsletter:', { dateStr, title }, err);
}
}
- });
+ }
if (newsletters.length === 0) {
console.error('No newsletters found in HTML. First 500 chars of response:', data.slice(0, 500));
diff --git a/shared/schema.ts b/shared/schema.ts
index e570aa4..c023fac 100644
--- a/shared/schema.ts
+++ b/shared/schema.ts
@@ -1,4 +1,4 @@
-import { pgTable, text, serial, date } from "drizzle-orm/pg-core";
+import { pgTable, text, serial, date, timestamp } from "drizzle-orm/pg-core";
import { createInsertSchema } from "drizzle-zod";
import { z } from "zod";
@@ -8,6 +8,9 @@ export const newsletters = pgTable("newsletters", {
date: date("date").notNull(),
url: text("url").notNull(),
description: text("description"),
+ thumbnail: text("thumbnail"),
+ content: text("content"),
+ last_checked: timestamp("last_checked"),
});
export const insertNewsletterSchema = createInsertSchema(newsletters).pick({
@@ -15,7 +18,27 @@ export const insertNewsletterSchema = createInsertSchema(newsletters).pick({
date: true,
url: true,
description: true,
+ thumbnail: true,
+ content: true,
});
export type InsertNewsletter = z.infer;
export type Newsletter = typeof newsletters.$inferSelect;
+
+// Schema for push notification subscriptions
+export const subscriptions = pgTable("subscriptions", {
+ id: serial("id").primaryKey(),
+ endpoint: text("endpoint").notNull(),
+ auth: text("auth").notNull(),
+ p256dh: text("p256dh").notNull(),
+ created_at: timestamp("created_at").defaultNow(),
+});
+
+export const insertSubscriptionSchema = createInsertSchema(subscriptions).pick({
+ endpoint: true,
+ auth: true,
+ p256dh: true,
+});
+
+export type InsertSubscription = z.infer;
+export type Subscription = typeof subscriptions.$inferSelect;
\ No newline at end of file