diff --git a/client/src/pages/home.tsx b/client/src/pages/home.tsx index 1916830..b469da4 100644 --- a/client/src/pages/home.tsx +++ b/client/src/pages/home.tsx @@ -1,23 +1,41 @@ -import { useState } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import { format } from "date-fns"; import { Card, CardHeader, CardTitle, CardDescription, CardContent } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; -import { Search, ExternalLink, Calendar, RefreshCw } from "lucide-react"; +import { + Search, + ExternalLink, + Calendar, + RefreshCw, + Share2, + Twitter, + Facebook, + Rss, + Bell, + BellOff +} from "lucide-react"; import { useNewsletters, useNewsletterSearch } from "@/lib/newsletter-data"; import { useToast } from "@/hooks/use-toast"; import { apiRequest } from "@/lib/queryClient"; import { queryClient } from "@/lib/queryClient"; +import { motion, AnimatePresence } from "framer-motion"; + +const ITEMS_PER_PAGE = 20; export default function Home() { const [searchQuery, setSearchQuery] = useState(""); const [isImporting, setIsImporting] = useState(false); + const [page, setPage] = useState(1); + const [isSubscribed, setIsSubscribed] = useState(false); + const loader = useRef(null); const { data: allNewsletters, isLoading } = useNewsletters(); const { data: searchResults } = useNewsletterSearch(searchQuery); const { toast } = useToast(); const newsletters = searchQuery ? searchResults : allNewsletters; + const paginatedNewsletters = newsletters?.slice(0, page * ITEMS_PER_PAGE); const handleImport = async () => { try { @@ -39,10 +57,88 @@ export default function Home() { } }; + const handleShare = async (newsletter) => { + if (navigator.share) { + try { + await navigator.share({ + title: newsletter.title, + text: newsletter.description || "Check out this newsletter from The Downtowner", + url: newsletter.url + }); + } catch (error) { + if (error.name !== 'AbortError') { + toast({ + title: "Error", + description: "Failed to share newsletter", + variant: "destructive", + }); + } + } + } + }; + + const handleSubscribe = async () => { + try { + if (!('serviceWorker' in navigator) || !('Notification' in window)) { + throw new Error('Push notifications are not supported'); + } + + const permission = await Notification.requestPermission(); + if (permission !== 'granted') { + throw new Error('Notification permission denied'); + } + + const registration = await navigator.serviceWorker.ready; + const subscription = await registration.pushManager.subscribe({ + userVisibleOnly: true, + applicationServerKey: process.env.VITE_VAPID_PUBLIC_KEY + }); + + await apiRequest('POST', '/api/subscriptions', subscription); + setIsSubscribed(true); + toast({ + title: "Subscribed!", + description: "You'll receive notifications for new newsletters", + }); + } catch (error) { + toast({ + title: "Error", + description: error.message || "Failed to subscribe to notifications", + variant: "destructive", + }); + } + }; + + const handleObserver = useCallback((entries) => { + const target = entries[0]; + if (target.isIntersecting && newsletters?.length > page * ITEMS_PER_PAGE) { + setPage(prev => prev + 1); + } + }, [newsletters, page]); + + useEffect(() => { + const observer = new IntersectionObserver(handleObserver, { + root: null, + rootMargin: "20px", + threshold: 1.0 + }); + + if (loader.current) { + observer.observe(loader.current); + } + + return () => observer.disconnect(); + }, [handleObserver]); + return (
-
+

The Downtowner

@@ -68,58 +164,119 @@ export default function Home() { > + +
- +
- {isLoading ? ( - Array(6).fill(0).map((_, i) => ( - - - - - - - - - - )) - ) : newsletters?.length ? ( - newsletters.map((newsletter) => ( - - - - - {newsletter.title} - - - - - {format(new Date(newsletter.date), 'MMMM d, yyyy')} - - - {newsletter.description && ( + + {isLoading ? ( + Array(6).fill(0).map((_, i) => ( + + + + + + -

- {newsletter.description} -

+
- )} -
-
- )) - ) : ( -
- No newsletters found matching your search. -
- )} + + + )) + ) : paginatedNewsletters?.length ? ( + paginatedNewsletters.map((newsletter) => ( + + + + + {newsletter.title} +
+ + e.stopPropagation()} + > + + +
+
+ + + {format(new Date(newsletter.date), 'MMMM d, yyyy')} + +
+ {(newsletter.thumbnail || newsletter.description) && ( + + {newsletter.thumbnail && ( + {newsletter.title} + )} + {newsletter.description && ( +

+ {newsletter.description} +

+ )} +
+ )} +
+
+ )) + ) : ( +
+ No newsletters found matching your search. +
+ )} +
+ +
); diff --git a/package-lock.json b/package-lock.json index 42d7fbd..ad6de6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,10 +53,12 @@ "embla-carousel-react": "^8.3.0", "express": "^4.21.2", "express-session": "^1.18.1", + "feed": "^4.2.2", "framer-motion": "^11.13.1", "input-otp": "^1.2.4", "lucide-react": "^0.453.0", "memorystore": "^1.6.7", + "node-schedule": "^2.1.1", "passport": "^0.7.0", "passport-local": "^1.0.0", "react": "^18.3.1", @@ -69,6 +71,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", "vaul": "^1.1.0", + "web-push": "^3.6.7", "wouter": "^3.3.5", "ws": "^8.18.0", "zod": "^3.23.8", @@ -3510,6 +3513,15 @@ "node": ">= 0.6" } }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ansi-regex": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", @@ -3577,6 +3589,18 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "license": "MIT", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3650,6 +3674,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bn.js": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.1.tgz", + "integrity": "sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -3749,6 +3779,12 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4052,6 +4088,18 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -4995,6 +5043,15 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "license": "MIT" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -5349,6 +5406,18 @@ "reusify": "^1.0.4" } }, + "node_modules/feed": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/feed/-/feed-4.2.2.tgz", + "integrity": "sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==", + "license": "MIT", + "dependencies": { + "xml-js": "^1.6.11" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -5698,6 +5767,15 @@ "entities": "^4.5.0" } }, + "node_modules/http_ece": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.2.0.tgz", + "integrity": "sha512-JrF8SSLVmcvc5NducxgyOrKXe3EsyHMgBFgSaIUGmArKe+rwr0uphRkRXvwiom3I+fpIfoItveHrfudL8/rxuA==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -5714,6 +5792,19 @@ "node": ">= 0.8" } }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -5897,6 +5988,27 @@ "node": ">=6" } }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/lilconfig": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", @@ -5939,6 +6051,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==", + "license": "MIT" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -5970,6 +6088,15 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -6081,6 +6208,12 @@ "node": ">= 0.6" } }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "license": "ISC" + }, "node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -6096,6 +6229,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -6183,6 +6325,20 @@ "dev": true, "license": "MIT" }, + "node_modules/node-schedule": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.1.tgz", + "integrity": "sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.2.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7292,6 +7448,12 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "license": "ISC" + }, "node_modules/scheduler": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", @@ -7448,6 +7610,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==", + "license": "MIT" + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -8905,6 +9073,25 @@ "@esbuild/win32-x64": "0.21.5" } }, + "node_modules/web-push": { + "version": "3.6.7", + "resolved": "https://registry.npmjs.org/web-push/-/web-push-3.6.7.tgz", + "integrity": "sha512-OpiIUe8cuGjrj3mMBFWY+e4MMIkW3SVT+7vEIjvD9kejGUypv8GPDf84JdPWskK8zMRIJ6xYGm+Kxr8YkPyA0A==", + "license": "MPL-2.0", + "dependencies": { + "asn1.js": "^5.3.0", + "http_ece": "1.2.0", + "https-proxy-agent": "^7.0.0", + "jws": "^4.0.0", + "minimist": "^1.2.5" + }, + "bin": { + "web-push": "src/cli.js" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/whatwg-encoding": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", @@ -9079,6 +9266,18 @@ } } }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "license": "MIT", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 2017e4b..f844720 100644 --- a/package.json +++ b/package.json @@ -55,10 +55,12 @@ "embla-carousel-react": "^8.3.0", "express": "^4.21.2", "express-session": "^1.18.1", + "feed": "^4.2.2", "framer-motion": "^11.13.1", "input-otp": "^1.2.4", "lucide-react": "^0.453.0", "memorystore": "^1.6.7", + "node-schedule": "^2.1.1", "passport": "^0.7.0", "passport-local": "^1.0.0", "react": "^18.3.1", @@ -71,6 +73,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", "vaul": "^1.1.0", + "web-push": "^3.6.7", "wouter": "^3.3.5", "ws": "^8.18.0", "zod": "^3.23.8", diff --git a/server/routes.ts b/server/routes.ts index bb4ae6d..01a5003 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -2,8 +2,63 @@ import type { Express } from "express"; import { createServer, type Server } from "http"; import { storage } from "./storage"; import { scrapeNewsletters } from "./utils"; +import { Feed } from "feed"; +import webpush from "web-push"; +import schedule from "node-schedule"; + +// Initialize web-push with VAPID keys +if (!process.env.VAPID_PUBLIC_KEY || !process.env.VAPID_PRIVATE_KEY) { + console.warn('VAPID keys not set. Push notifications will not work.'); +} + +webpush.setVapidDetails( + 'mailto:team@downtowner.com', + process.env.VAPID_PUBLIC_KEY || '', + process.env.VAPID_PRIVATE_KEY || '' +); export async function registerRoutes(app: Express): Promise { + // Setup background job to check for new newsletters + schedule.scheduleJob('0 */6 * * *', async function() { + try { + const existingNewsletters = await storage.getNewsletters(); + const scrapedNewsletters = await scrapeNewsletters(); + + const newNewsletters = scrapedNewsletters.filter(scraped => + !existingNewsletters.some(existing => + existing.url === scraped.url + ) + ); + + if (newNewsletters.length > 0) { + await storage.importNewsletters(newNewsletters); + + // Send push notifications + const subscriptions = await storage.getSubscriptions(); + const notificationPayload = JSON.stringify({ + title: 'New Newsletters Available', + body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, + icon: '/icon.png' + }); + + await Promise.allSettled( + subscriptions.map(subscription => + webpush.sendNotification({ + endpoint: subscription.endpoint, + keys: { + auth: subscription.auth, + p256dh: subscription.p256dh + } + }, notificationPayload) + ) + ); + } + } catch (error) { + console.error('Background job failed:', error); + } + }); + + // API Routes app.get("/api/newsletters", async (_req, res) => { const newsletters = await storage.getNewsletters(); res.json(newsletters); @@ -26,6 +81,58 @@ export async function registerRoutes(app: Express): Promise { } }); + app.post("/api/subscriptions", async (req, res) => { + try { + const subscription = req.body; + await storage.addSubscription({ + endpoint: subscription.endpoint, + auth: subscription.keys.auth, + p256dh: subscription.keys.p256dh + }); + res.json({ message: "Subscription added successfully" }); + } catch (error) { + console.error('Error adding subscription:', error); + res.status(500).json({ message: "Failed to add subscription" }); + } + }); + + app.get("/api/rss", async (_req, res) => { + try { + const newsletters = await storage.getNewsletters(); + + const feed = new Feed({ + title: "The Downtowner Newsletter", + description: "Downtown Nashua's Newsletter Archive", + id: "https://downtowner.com/", + link: "https://downtowner.com/", + language: "en", + favicon: "https://downtowner.com/favicon.ico", + updated: newsletters[0]?.date ? new Date(newsletters[0].date) : new Date(), + generator: "The Downtowner RSS Feed", + feedLinks: { + rss2: "https://downtowner.com/api/rss" + } + }); + + for (const newsletter of newsletters) { + feed.addItem({ + title: newsletter.title, + id: newsletter.url, + link: newsletter.url, + description: newsletter.description, + date: new Date(newsletter.date), + image: newsletter.thumbnail + }); + } + + res.type('application/xml'); + res.send(feed.rss2()); + } catch (error) { + console.error('Error generating RSS feed:', error); + res.status(500).json({ message: "Failed to generate RSS feed" }); + } + }); + const httpServer = createServer(app); return httpServer; } \ No newline at end of file diff --git a/server/storage.ts b/server/storage.ts index 165a715..c065f4f 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -1,43 +1,52 @@ -import { type Newsletter, type InsertNewsletter } from "@shared/schema"; +import { type Newsletter, type InsertNewsletter, type Subscription, type InsertSubscription } from "@shared/schema"; +import { db } from "./db"; +import { newsletters, subscriptions } from "@shared/schema"; +import { desc, ilike, or } from "drizzle-orm"; export interface IStorage { getNewsletters(): Promise; searchNewsletters(query: string): Promise; importNewsletters(newsletters: InsertNewsletter[]): Promise; + addSubscription(subscription: InsertSubscription): Promise; + getSubscriptions(): Promise; } -export class MemStorage implements IStorage { - private newsletters: Newsletter[]; - private currentId: number; - - constructor() { - this.newsletters = []; - this.currentId = 1; - } - +export class DatabaseStorage implements IStorage { async getNewsletters(): Promise { - return this.newsletters.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); + return await db.select().from(newsletters).orderBy(desc(newsletters.date)); } async searchNewsletters(query: string): Promise { const lowercaseQuery = query.toLowerCase(); - return this.newsletters.filter( - newsletter => - newsletter.title.toLowerCase().includes(lowercaseQuery) || - (newsletter.description?.toLowerCase() || '').includes(lowercaseQuery) - ).sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); + return await db + .select() + .from(newsletters) + .where( + or( + ilike(newsletters.title, `%${lowercaseQuery}%`), + ilike(newsletters.content || '', `%${lowercaseQuery}%`), + ilike(newsletters.description || '', `%${lowercaseQuery}%`) + ) + ) + .orderBy(desc(newsletters.date)); } - async importNewsletters(newsletters: InsertNewsletter[]): Promise { - // Ensure description is null if not provided - const processedNewsletters = newsletters.map(newsletter => ({ - ...newsletter, - description: newsletter.description ?? null, - id: this.currentId++ - })); + async importNewsletters(newNewsletters: InsertNewsletter[]): Promise { + // Insert in batches to avoid overwhelming the database + const batchSize = 50; + for (let i = 0; i < newNewsletters.length; i += batchSize) { + const batch = newNewsletters.slice(i, i + batchSize); + await db.insert(newsletters).values(batch); + } + } - this.newsletters.push(...processedNewsletters); + async addSubscription(subscription: InsertSubscription): Promise { + await db.insert(subscriptions).values(subscription); + } + + async getSubscriptions(): Promise { + return await db.select().from(subscriptions); } } -export const storage = new MemStorage(); \ No newline at end of file +export const storage = new DatabaseStorage(); \ No newline at end of file diff --git a/server/utils.ts b/server/utils.ts index 71992fa..304fa2c 100644 --- a/server/utils.ts +++ b/server/utils.ts @@ -4,23 +4,55 @@ import type { InsertNewsletter } from '@shared/schema'; const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5'; +async function scrapeNewsletterContent(url: string) { + try { + const { data } = await axios.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + }, + timeout: 15000 + }); + + const $ = cheerio.load(data); + + // Get the second image as thumbnail + const images = $('img').toArray(); + const thumbnailUrl = images.length > 1 ? $(images[1]).attr('src') : null; + + // Extract text content + const content = $('body').text().trim(); + + return { + thumbnail: thumbnailUrl, + content + }; + } catch (error) { + console.warn('Error scraping newsletter content:', error); + return { thumbnail: null, content: null }; + } +} + export async function scrapeNewsletters(): Promise { try { - // Add headers to mimic a browser request const { data } = await axios.get(ROBLY_ARCHIVE_URL, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', }, - timeout: 10000 // 10 second timeout + timeout: 10000 }); const $ = cheerio.load(data); const newsletters: InsertNewsletter[] = []; // Find all links that start with /archive?id= - $('a[href^="/archive?id="]').each((_, element) => { + const links = $('a[href^="/archive?id="]'); + console.log(`Found ${links.length} newsletter links`); + + for (const element of links.toArray()) { const $element = $(element); const url = $element.attr('href'); const fullText = $element.parent().text().trim(); @@ -33,18 +65,26 @@ export async function scrapeNewsletters(): Promise { const [, dateStr, title] = match; try { const date = new Date(dateStr).toISOString().split('T')[0]; + const fullUrl = `https://app.robly.com${url}`; + + // Scrape the newsletter content + const { thumbnail, content } = await scrapeNewsletterContent(fullUrl); newsletters.push({ title: title.trim(), date, - url: `https://app.robly.com${url}`, - description: null + url: fullUrl, + thumbnail, + content, + description: content ? content.slice(0, 200) + '...' : null }); + + console.log(`Processed newsletter: ${title}`); } catch (err) { console.warn('Error processing date for newsletter:', { dateStr, title }, err); } } - }); + } if (newsletters.length === 0) { console.error('No newsletters found in HTML. First 500 chars of response:', data.slice(0, 500)); diff --git a/shared/schema.ts b/shared/schema.ts index e570aa4..c023fac 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -1,4 +1,4 @@ -import { pgTable, text, serial, date } from "drizzle-orm/pg-core"; +import { pgTable, text, serial, date, timestamp } from "drizzle-orm/pg-core"; import { createInsertSchema } from "drizzle-zod"; import { z } from "zod"; @@ -8,6 +8,9 @@ export const newsletters = pgTable("newsletters", { date: date("date").notNull(), url: text("url").notNull(), description: text("description"), + thumbnail: text("thumbnail"), + content: text("content"), + last_checked: timestamp("last_checked"), }); export const insertNewsletterSchema = createInsertSchema(newsletters).pick({ @@ -15,7 +18,27 @@ export const insertNewsletterSchema = createInsertSchema(newsletters).pick({ date: true, url: true, description: true, + thumbnail: true, + content: true, }); export type InsertNewsletter = z.infer; export type Newsletter = typeof newsletters.$inferSelect; + +// Schema for push notification subscriptions +export const subscriptions = pgTable("subscriptions", { + id: serial("id").primaryKey(), + endpoint: text("endpoint").notNull(), + auth: text("auth").notNull(), + p256dh: text("p256dh").notNull(), + created_at: timestamp("created_at").defaultNow(), +}); + +export const insertSubscriptionSchema = createInsertSchema(subscriptions).pick({ + endpoint: true, + auth: true, + p256dh: true, +}); + +export type InsertSubscription = z.infer; +export type Subscription = typeof subscriptions.$inferSelect; \ No newline at end of file