From dee280d05514013e71d61e7fcde83bb119845c10 Mon Sep 17 00:00:00 2001 From: TerribleDev <1020010-TerribleDev@users.noreply.replit.com> Date: Sat, 15 Feb 2025 17:57:15 +0000 Subject: [PATCH] Agent query: Please verify if: 1. The application loads successfully 2. You can see the newsletter cards with smooth animations 3. The search bar is visible at the top Enhance newsletter archive viewer with mobile UI, social sharing, push notifications, search, RSS feed, and infinite scrolling. Screenshot: https://storage.googleapis.com/screenshot-production-us-central1/9dda30b6-4149-4bce-89dc-76333005952c/fea88e1e-ec20-4c53-8b12-b205f04819b7.jpg --- client/src/pages/home.tsx | 255 ++++++++++++++++++++++++++++++-------- package-lock.json | 199 +++++++++++++++++++++++++++++ package.json | 3 + server/routes.ts | 107 ++++++++++++++++ server/storage.ts | 59 +++++---- server/utils.ts | 52 +++++++- shared/schema.ts | 25 +++- 7 files changed, 619 insertions(+), 81 deletions(-) diff --git a/client/src/pages/home.tsx b/client/src/pages/home.tsx index 1916830..b469da4 100644 --- a/client/src/pages/home.tsx +++ b/client/src/pages/home.tsx @@ -1,23 +1,41 @@ -import { useState } from "react"; +import { useState, useEffect, useRef, useCallback } from "react"; import { format } from "date-fns"; import { Card, CardHeader, CardTitle, CardDescription, CardContent } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; -import { Search, ExternalLink, Calendar, RefreshCw } from "lucide-react"; +import { + Search, + ExternalLink, + Calendar, + RefreshCw, + Share2, + Twitter, + Facebook, + Rss, + Bell, + BellOff +} from "lucide-react"; import { useNewsletters, useNewsletterSearch } from "@/lib/newsletter-data"; import { useToast } from "@/hooks/use-toast"; import { apiRequest } from "@/lib/queryClient"; import { queryClient } from "@/lib/queryClient"; +import { motion, AnimatePresence } from "framer-motion"; + +const ITEMS_PER_PAGE = 20; export default function Home() { const [searchQuery, setSearchQuery] = useState(""); const [isImporting, setIsImporting] = useState(false); + const [page, setPage] = useState(1); + const [isSubscribed, setIsSubscribed] = useState(false); + const loader = useRef(null); const { data: allNewsletters, isLoading } = useNewsletters(); const { data: searchResults } = useNewsletterSearch(searchQuery); const { toast } = useToast(); const newsletters = searchQuery ? searchResults : allNewsletters; + const paginatedNewsletters = newsletters?.slice(0, page * ITEMS_PER_PAGE); const handleImport = async () => { try { @@ -39,10 +57,88 @@ export default function Home() { } }; + const handleShare = async (newsletter) => { + if (navigator.share) { + try { + await navigator.share({ + title: newsletter.title, + text: newsletter.description || "Check out this newsletter from The Downtowner", + url: newsletter.url + }); + } catch (error) { + if (error.name !== 'AbortError') { + toast({ + title: "Error", + description: "Failed to share newsletter", + variant: "destructive", + }); + } + } + } + }; + + const handleSubscribe = async () => { + try { + if (!('serviceWorker' in navigator) || !('Notification' in window)) { + throw new Error('Push notifications are not supported'); + } + + const permission = await Notification.requestPermission(); + if (permission !== 'granted') { + throw new Error('Notification permission denied'); + } + + const registration = await navigator.serviceWorker.ready; + const subscription = await registration.pushManager.subscribe({ + userVisibleOnly: true, + applicationServerKey: process.env.VITE_VAPID_PUBLIC_KEY + }); + + await apiRequest('POST', '/api/subscriptions', subscription); + setIsSubscribed(true); + toast({ + title: "Subscribed!", + description: "You'll receive notifications for new newsletters", + }); + } catch (error) { + toast({ + title: "Error", + description: error.message || "Failed to subscribe to notifications", + variant: "destructive", + }); + } + }; + + const handleObserver = useCallback((entries) => { + const target = entries[0]; + if (target.isIntersecting && newsletters?.length > page * ITEMS_PER_PAGE) { + setPage(prev => prev + 1); + } + }, [newsletters, page]); + + useEffect(() => { + const observer = new IntersectionObserver(handleObserver, { + root: null, + rootMargin: "20px", + threshold: 1.0 + }); + + if (loader.current) { + observer.observe(loader.current); + } + + return () => observer.disconnect(); + }, [handleObserver]); + return (
-
+

The Downtowner

@@ -68,58 +164,119 @@ export default function Home() { > + +
- +
- {isLoading ? ( - Array(6).fill(0).map((_, i) => ( - - - - - - - - - - )) - ) : newsletters?.length ? ( - newsletters.map((newsletter) => ( - - - - - {newsletter.title} - - - - - {format(new Date(newsletter.date), 'MMMM d, yyyy')} - - - {newsletter.description && ( + + {isLoading ? ( + Array(6).fill(0).map((_, i) => ( + + + + + + -

- {newsletter.description} -

+ - )} - - - )) - ) : ( -
- No newsletters found matching your search. -
- )} + + + )) + ) : paginatedNewsletters?.length ? ( + paginatedNewsletters.map((newsletter) => ( + + + + + {newsletter.title} +
+ + e.stopPropagation()} + > + + +
+ + + + {format(new Date(newsletter.date), 'MMMM d, yyyy')} + + + {(newsletter.thumbnail || newsletter.description) && ( + + {newsletter.thumbnail && ( + {newsletter.title} + )} + {newsletter.description && ( +

+ {newsletter.description} +

+ )} + + )} + + + )) + ) : ( +
+ No newsletters found matching your search. +
+ )} +
+ +
); diff --git a/package-lock.json b/package-lock.json index 42d7fbd..ad6de6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,10 +53,12 @@ "embla-carousel-react": "^8.3.0", "express": "^4.21.2", "express-session": "^1.18.1", + "feed": "^4.2.2", "framer-motion": "^11.13.1", "input-otp": "^1.2.4", "lucide-react": "^0.453.0", "memorystore": "^1.6.7", + "node-schedule": "^2.1.1", "passport": "^0.7.0", "passport-local": "^1.0.0", "react": "^18.3.1", @@ -69,6 +71,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", "vaul": "^1.1.0", + "web-push": "^3.6.7", "wouter": "^3.3.5", "ws": "^8.18.0", "zod": "^3.23.8", @@ -3510,6 +3513,15 @@ "node": ">= 0.6" } }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ansi-regex": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", @@ -3577,6 +3589,18 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "license": "MIT", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3650,6 +3674,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bn.js": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.1.tgz", + "integrity": "sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -3749,6 +3779,12 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4052,6 +4088,18 @@ "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -4995,6 +5043,15 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "license": "MIT" }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -5349,6 +5406,18 @@ "reusify": "^1.0.4" } }, + "node_modules/feed": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/feed/-/feed-4.2.2.tgz", + "integrity": "sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==", + "license": "MIT", + "dependencies": { + "xml-js": "^1.6.11" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -5698,6 +5767,15 @@ "entities": "^4.5.0" } }, + "node_modules/http_ece": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http_ece/-/http_ece-1.2.0.tgz", + "integrity": "sha512-JrF8SSLVmcvc5NducxgyOrKXe3EsyHMgBFgSaIUGmArKe+rwr0uphRkRXvwiom3I+fpIfoItveHrfudL8/rxuA==", + "license": "MIT", + "engines": { + "node": ">=16" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -5714,6 +5792,19 @@ "node": ">= 0.8" } }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -5897,6 +5988,27 @@ "node": ">=6" } }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/lilconfig": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", @@ -5939,6 +6051,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w==", + "license": "MIT" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -5970,6 +6088,15 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -6081,6 +6208,12 @@ "node": ">= 0.6" } }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "license": "ISC" + }, "node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -6096,6 +6229,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -6183,6 +6325,20 @@ "dev": true, "license": "MIT" }, + "node_modules/node-schedule": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.1.tgz", + "integrity": "sha512-OXdegQq03OmXEjt2hZP33W2YPs/E5BcFQks46+G2gAxs4gHOIVD1u7EqlYLYSKsaIpyKCK9Gbk0ta1/gjRSMRQ==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.2.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7292,6 +7448,12 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "license": "ISC" + }, "node_modules/scheduler": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", @@ -7448,6 +7610,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==", + "license": "MIT" + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -8905,6 +9073,25 @@ "@esbuild/win32-x64": "0.21.5" } }, + "node_modules/web-push": { + "version": "3.6.7", + "resolved": "https://registry.npmjs.org/web-push/-/web-push-3.6.7.tgz", + "integrity": "sha512-OpiIUe8cuGjrj3mMBFWY+e4MMIkW3SVT+7vEIjvD9kejGUypv8GPDf84JdPWskK8zMRIJ6xYGm+Kxr8YkPyA0A==", + "license": "MPL-2.0", + "dependencies": { + "asn1.js": "^5.3.0", + "http_ece": "1.2.0", + "https-proxy-agent": "^7.0.0", + "jws": "^4.0.0", + "minimist": "^1.2.5" + }, + "bin": { + "web-push": "src/cli.js" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/whatwg-encoding": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", @@ -9079,6 +9266,18 @@ } } }, + "node_modules/xml-js": { + "version": "1.6.11", + "resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz", + "integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==", + "license": "MIT", + "dependencies": { + "sax": "^1.2.4" + }, + "bin": { + "xml-js": "bin/cli.js" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 2017e4b..f844720 100644 --- a/package.json +++ b/package.json @@ -55,10 +55,12 @@ "embla-carousel-react": "^8.3.0", "express": "^4.21.2", "express-session": "^1.18.1", + "feed": "^4.2.2", "framer-motion": "^11.13.1", "input-otp": "^1.2.4", "lucide-react": "^0.453.0", "memorystore": "^1.6.7", + "node-schedule": "^2.1.1", "passport": "^0.7.0", "passport-local": "^1.0.0", "react": "^18.3.1", @@ -71,6 +73,7 @@ "tailwind-merge": "^2.5.4", "tailwindcss-animate": "^1.0.7", "vaul": "^1.1.0", + "web-push": "^3.6.7", "wouter": "^3.3.5", "ws": "^8.18.0", "zod": "^3.23.8", diff --git a/server/routes.ts b/server/routes.ts index bb4ae6d..01a5003 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -2,8 +2,63 @@ import type { Express } from "express"; import { createServer, type Server } from "http"; import { storage } from "./storage"; import { scrapeNewsletters } from "./utils"; +import { Feed } from "feed"; +import webpush from "web-push"; +import schedule from "node-schedule"; + +// Initialize web-push with VAPID keys +if (!process.env.VAPID_PUBLIC_KEY || !process.env.VAPID_PRIVATE_KEY) { + console.warn('VAPID keys not set. Push notifications will not work.'); +} + +webpush.setVapidDetails( + 'mailto:team@downtowner.com', + process.env.VAPID_PUBLIC_KEY || '', + process.env.VAPID_PRIVATE_KEY || '' +); export async function registerRoutes(app: Express): Promise { + // Setup background job to check for new newsletters + schedule.scheduleJob('0 */6 * * *', async function() { + try { + const existingNewsletters = await storage.getNewsletters(); + const scrapedNewsletters = await scrapeNewsletters(); + + const newNewsletters = scrapedNewsletters.filter(scraped => + !existingNewsletters.some(existing => + existing.url === scraped.url + ) + ); + + if (newNewsletters.length > 0) { + await storage.importNewsletters(newNewsletters); + + // Send push notifications + const subscriptions = await storage.getSubscriptions(); + const notificationPayload = JSON.stringify({ + title: 'New Newsletters Available', + body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`, + icon: '/icon.png' + }); + + await Promise.allSettled( + subscriptions.map(subscription => + webpush.sendNotification({ + endpoint: subscription.endpoint, + keys: { + auth: subscription.auth, + p256dh: subscription.p256dh + } + }, notificationPayload) + ) + ); + } + } catch (error) { + console.error('Background job failed:', error); + } + }); + + // API Routes app.get("/api/newsletters", async (_req, res) => { const newsletters = await storage.getNewsletters(); res.json(newsletters); @@ -26,6 +81,58 @@ export async function registerRoutes(app: Express): Promise { } }); + app.post("/api/subscriptions", async (req, res) => { + try { + const subscription = req.body; + await storage.addSubscription({ + endpoint: subscription.endpoint, + auth: subscription.keys.auth, + p256dh: subscription.keys.p256dh + }); + res.json({ message: "Subscription added successfully" }); + } catch (error) { + console.error('Error adding subscription:', error); + res.status(500).json({ message: "Failed to add subscription" }); + } + }); + + app.get("/api/rss", async (_req, res) => { + try { + const newsletters = await storage.getNewsletters(); + + const feed = new Feed({ + title: "The Downtowner Newsletter", + description: "Downtown Nashua's Newsletter Archive", + id: "https://downtowner.com/", + link: "https://downtowner.com/", + language: "en", + favicon: "https://downtowner.com/favicon.ico", + updated: newsletters[0]?.date ? new Date(newsletters[0].date) : new Date(), + generator: "The Downtowner RSS Feed", + feedLinks: { + rss2: "https://downtowner.com/api/rss" + } + }); + + for (const newsletter of newsletters) { + feed.addItem({ + title: newsletter.title, + id: newsletter.url, + link: newsletter.url, + description: newsletter.description, + date: new Date(newsletter.date), + image: newsletter.thumbnail + }); + } + + res.type('application/xml'); + res.send(feed.rss2()); + } catch (error) { + console.error('Error generating RSS feed:', error); + res.status(500).json({ message: "Failed to generate RSS feed" }); + } + }); + const httpServer = createServer(app); return httpServer; } \ No newline at end of file diff --git a/server/storage.ts b/server/storage.ts index 165a715..c065f4f 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -1,43 +1,52 @@ -import { type Newsletter, type InsertNewsletter } from "@shared/schema"; +import { type Newsletter, type InsertNewsletter, type Subscription, type InsertSubscription } from "@shared/schema"; +import { db } from "./db"; +import { newsletters, subscriptions } from "@shared/schema"; +import { desc, ilike, or } from "drizzle-orm"; export interface IStorage { getNewsletters(): Promise; searchNewsletters(query: string): Promise; importNewsletters(newsletters: InsertNewsletter[]): Promise; + addSubscription(subscription: InsertSubscription): Promise; + getSubscriptions(): Promise; } -export class MemStorage implements IStorage { - private newsletters: Newsletter[]; - private currentId: number; - - constructor() { - this.newsletters = []; - this.currentId = 1; - } - +export class DatabaseStorage implements IStorage { async getNewsletters(): Promise { - return this.newsletters.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); + return await db.select().from(newsletters).orderBy(desc(newsletters.date)); } async searchNewsletters(query: string): Promise { const lowercaseQuery = query.toLowerCase(); - return this.newsletters.filter( - newsletter => - newsletter.title.toLowerCase().includes(lowercaseQuery) || - (newsletter.description?.toLowerCase() || '').includes(lowercaseQuery) - ).sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); + return await db + .select() + .from(newsletters) + .where( + or( + ilike(newsletters.title, `%${lowercaseQuery}%`), + ilike(newsletters.content || '', `%${lowercaseQuery}%`), + ilike(newsletters.description || '', `%${lowercaseQuery}%`) + ) + ) + .orderBy(desc(newsletters.date)); } - async importNewsletters(newsletters: InsertNewsletter[]): Promise { - // Ensure description is null if not provided - const processedNewsletters = newsletters.map(newsletter => ({ - ...newsletter, - description: newsletter.description ?? null, - id: this.currentId++ - })); + async importNewsletters(newNewsletters: InsertNewsletter[]): Promise { + // Insert in batches to avoid overwhelming the database + const batchSize = 50; + for (let i = 0; i < newNewsletters.length; i += batchSize) { + const batch = newNewsletters.slice(i, i + batchSize); + await db.insert(newsletters).values(batch); + } + } - this.newsletters.push(...processedNewsletters); + async addSubscription(subscription: InsertSubscription): Promise { + await db.insert(subscriptions).values(subscription); + } + + async getSubscriptions(): Promise { + return await db.select().from(subscriptions); } } -export const storage = new MemStorage(); \ No newline at end of file +export const storage = new DatabaseStorage(); \ No newline at end of file diff --git a/server/utils.ts b/server/utils.ts index 71992fa..304fa2c 100644 --- a/server/utils.ts +++ b/server/utils.ts @@ -4,23 +4,55 @@ import type { InsertNewsletter } from '@shared/schema'; const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5'; +async function scrapeNewsletterContent(url: string) { + try { + const { data } = await axios.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + }, + timeout: 15000 + }); + + const $ = cheerio.load(data); + + // Get the second image as thumbnail + const images = $('img').toArray(); + const thumbnailUrl = images.length > 1 ? $(images[1]).attr('src') : null; + + // Extract text content + const content = $('body').text().trim(); + + return { + thumbnail: thumbnailUrl, + content + }; + } catch (error) { + console.warn('Error scraping newsletter content:', error); + return { thumbnail: null, content: null }; + } +} + export async function scrapeNewsletters(): Promise { try { - // Add headers to mimic a browser request const { data } = await axios.get(ROBLY_ARCHIVE_URL, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', }, - timeout: 10000 // 10 second timeout + timeout: 10000 }); const $ = cheerio.load(data); const newsletters: InsertNewsletter[] = []; // Find all links that start with /archive?id= - $('a[href^="/archive?id="]').each((_, element) => { + const links = $('a[href^="/archive?id="]'); + console.log(`Found ${links.length} newsletter links`); + + for (const element of links.toArray()) { const $element = $(element); const url = $element.attr('href'); const fullText = $element.parent().text().trim(); @@ -33,18 +65,26 @@ export async function scrapeNewsletters(): Promise { const [, dateStr, title] = match; try { const date = new Date(dateStr).toISOString().split('T')[0]; + const fullUrl = `https://app.robly.com${url}`; + + // Scrape the newsletter content + const { thumbnail, content } = await scrapeNewsletterContent(fullUrl); newsletters.push({ title: title.trim(), date, - url: `https://app.robly.com${url}`, - description: null + url: fullUrl, + thumbnail, + content, + description: content ? content.slice(0, 200) + '...' : null }); + + console.log(`Processed newsletter: ${title}`); } catch (err) { console.warn('Error processing date for newsletter:', { dateStr, title }, err); } } - }); + } if (newsletters.length === 0) { console.error('No newsletters found in HTML. First 500 chars of response:', data.slice(0, 500)); diff --git a/shared/schema.ts b/shared/schema.ts index e570aa4..c023fac 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -1,4 +1,4 @@ -import { pgTable, text, serial, date } from "drizzle-orm/pg-core"; +import { pgTable, text, serial, date, timestamp } from "drizzle-orm/pg-core"; import { createInsertSchema } from "drizzle-zod"; import { z } from "zod"; @@ -8,6 +8,9 @@ export const newsletters = pgTable("newsletters", { date: date("date").notNull(), url: text("url").notNull(), description: text("description"), + thumbnail: text("thumbnail"), + content: text("content"), + last_checked: timestamp("last_checked"), }); export const insertNewsletterSchema = createInsertSchema(newsletters).pick({ @@ -15,7 +18,27 @@ export const insertNewsletterSchema = createInsertSchema(newsletters).pick({ date: true, url: true, description: true, + thumbnail: true, + content: true, }); export type InsertNewsletter = z.infer; export type Newsletter = typeof newsletters.$inferSelect; + +// Schema for push notification subscriptions +export const subscriptions = pgTable("subscriptions", { + id: serial("id").primaryKey(), + endpoint: text("endpoint").notNull(), + auth: text("auth").notNull(), + p256dh: text("p256dh").notNull(), + created_at: timestamp("created_at").defaultNow(), +}); + +export const insertSubscriptionSchema = createInsertSchema(subscriptions).pick({ + endpoint: true, + auth: true, + p256dh: true, +}); + +export type InsertSubscription = z.infer; +export type Subscription = typeof subscriptions.$inferSelect; \ No newline at end of file