TerribleDev/DowntownerArchive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restored to '5c89fd2cd9402ccde43ea7be18e5de6d1726a9fc'

Browse Source 
Replit-Restored-To:5c89fd2cd9402ccde43ea7be18e5de6d1726a9fc
This commit is contained in:
TerribleDev
2025-02-18 22:26:43 +00:00
parent 6958c89e72
commit 07497b4648
5 changed files with 88 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
README.md
View File

@@ -1,70 +0,0 @@
# Newsletter Embed Route Documentation
This documentation explains how to embed The Downtowner newsletter content into any website using our embed route.
## Quick Start
Add the following code to your website where you want the newsletter content to appear:
```html
<script>
class NewsletterEmbed extends HTMLElement {
constructor() {
super();
this.attachShadow({ mode: 'open' });
}
async connectedCallback() {
try {
const response = await fetch('https://downtowner.terrible.dev/embed');
const html = await response.text();
this.shadowRoot.innerHTML = html;
} catch (error) {
console.error('Failed to load newsletter content:', error);
this.shadowRoot.innerHTML = '<p>Failed to load newsletter content</p>';
}
}
}
customElements.define('newsletter-embed', NewsletterEmbed);
</script>
<newsletter-embed></newsletter-embed>
```
## Features
- Responsive grid layout
- Dark mode support
- Style isolation using Shadow DOM
- Displays up to 6 recent newsletters
- Each newsletter card includes:
- Title
- Publication date
- Thumbnail image (if available)
- Description
- Read more link
## Customization
The embed route uses CSS variables for theming. You can override these variables in your website's CSS:
```css
newsletter-embed {
--background: #ffffff;
--foreground: #000000;
--card: #ffffff;
--card-foreground: #000000;
--primary: #000000;
--border: #e2e8f0;
}
```
## Demo
You can view a live demo of the embed functionality at `/embed-demo.html` on your Repl.
## CORS
The embed route has CORS enabled, allowing it to be embedded on any domain.

11
client/public/manifest.json
View File

@@ -10,14 +10,7 @@
{
"src": "/icon.png",
"sizes": "192x192",
"type": "image/png",
"purpose": "any maskable"
},
{
"src": "/icon-512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "any maskable"
"type": "image/png"
}
]
}
}

71
client/src/pages/home.tsx
View File

@@ -11,10 +11,11 @@ import {
Calendar,
RefreshCw,
Share2,
Twitter,
Facebook,
Rss,
Bell,
BellOff,
Download,
Rss
BellOff
} from "lucide-react";
import { useNewsletters, useNewsletterSearch } from "@/lib/newsletter-data";
import { useToast } from "@/hooks/use-toast";
@@ -29,8 +30,6 @@ export default function Home() {
const [isImporting, setIsImporting] = useState(false);
const [page, setPage] = useState(1);
const [isSubscribed, setIsSubscribed] = useState(false);
const [deferredPrompt, setDeferredPrompt] = useState<any>(null);
const [isInstallable, setIsInstallable] = useState(false);
const loader = useRef(null);
const { data: allNewsletters, isLoading, isFetching } = useNewsletters();
const { data: searchResults } = useNewsletterSearch(searchQuery);
@@ -40,56 +39,6 @@ export default function Home() {
const paginatedNewsletters = newsletters?.slice(0, page * ITEMS_PER_PAGE);
const isDevelopment = import.meta.env.MODE === 'development';
useEffect(() => {
// Listen for the beforeinstallprompt event
window.addEventListener('beforeinstallprompt', (e) => {
// Prevent Chrome 67 and earlier from automatically showing the prompt
e.preventDefault();
// Stash the event so it can be triggered later
setDeferredPrompt(e);
setIsInstallable(true);
});
// Listen for successful installation
window.addEventListener('appinstalled', () => {
setIsInstallable(false);
setDeferredPrompt(null);
toast({
title: "Success",
description: "The Downtowner has been installed!",
});
});
}, [toast]);
const handleInstall = async () => {
if (!deferredPrompt) return;
try {
// Show the install prompt
deferredPrompt.prompt();
// Wait for the user to respond to the prompt
const { outcome } = await deferredPrompt.userChoice;
if (outcome === 'accepted') {
toast({
title: "Installing...",
description: "The Downtowner is being installed on your device",
});
}
// Clear the deferredPrompt for the next time
setDeferredPrompt(null);
setIsInstallable(false);
} catch (error) {
console.error('Error installing PWA:', error);
toast({
title: "Error",
description: "Failed to install the application",
variant: "destructive",
});
}
};
const handleImport = async () => {
try {
setIsImporting(true);
@@ -248,22 +197,12 @@ export default function Home() {
<Bell className="h-4 w-4" />
)}
</Button>
{isInstallable && (
<Button
variant="outline"
size="icon"
onClick={handleInstall}
title="Install app"
>
<Download className="h-4 w-4" />
</Button>
)}
<Button
variant="outline"
size="icon"
asChild
>
<a href="/api/rss" target="_blank" rel="noopener noreferrer" title="RSS Feed">
<a href="/api/rss" target="_blank" rel="noopener noreferrer">
<Rss className="h-4 w-4" />
</a>
</Button>

181
server/routes.ts
View File

@@ -13,29 +13,25 @@ const vapidPublicKey = process.env.VAPID_PUBLIC_KEY;
const vapidPrivateKey = process.env.VAPID_PRIVATE_KEY;
if (!vapidPublicKey || !vapidPrivateKey) {
throw new Error(
"VAPID keys are required for push notifications. Please set VAPID_PUBLIC_KEY and VAPID_PRIVATE_KEY environment variables.",
);
throw new Error('VAPID keys are required for push notifications. Please set VAPID_PUBLIC_KEY and VAPID_PRIVATE_KEY environment variables.');
}
webpush.setVapidDetails(
"mailto:team@downtowner.com",
'mailto:team@downtowner.com',
vapidPublicKey,
vapidPrivateKey,
vapidPrivateKey
);
export async function registerRoutes(app: Express): Promise<Server> {
// Setup background job to check for new newsletters
schedule.scheduleJob("0 */4 * * *", async function () {
schedule.scheduleJob('0 */6 * * *', async function() {
try {
const existingNewsletters = await storage.getNewsletters();
let newNewslettersCount = 0;
await scrapeNewsletters(async (newsletter) => {
// Check if newsletter already exists
const exists = existingNewsletters.some(
(existing) => existing.url === newsletter.url,
);
const exists = existingNewsletters.some(existing => existing.url === newsletter.url);
if (!exists) {
await storage.importNewsletter(newsletter);
newNewslettersCount++;
@@ -44,52 +40,38 @@ export async function registerRoutes(app: Express): Promise<Server> {
});
if (newNewslettersCount > 0) {
console.log(
`Found ${newNewslettersCount} new newsletters, sending notifications...`,
);
console.log(`Found ${newNewslettersCount} new newsletters, sending notifications...`);
// Send push notifications for new newsletters
const subscriptions = await storage.getActiveSubscriptions();
console.log(
`Sending notifications to ${subscriptions.length} subscribers`,
);
console.log(`Sending notifications to ${subscriptions.length} subscribers`);
const notificationPayload = JSON.stringify({
title: "New Newsletters Available",
body: `${newNewslettersCount} new newsletter${newNewslettersCount > 1 ? "s" : ""} published!`,
icon: "/icon.png",
title: 'New Newsletters Available',
body: `${newNewslettersCount} new newsletter${newNewslettersCount > 1 ? 's' : ''} published!`,
icon: '/icon.png'
});
const results = await Promise.allSettled(
subscriptions.map((subscription) =>
webpush.sendNotification(
{
endpoint: subscription.endpoint,
keys: {
auth: subscription.auth,
p256dh: subscription.p256dh,
},
},
notificationPayload,
),
),
subscriptions.map(subscription =>
webpush.sendNotification({
endpoint: subscription.endpoint,
keys: {
auth: subscription.auth,
p256dh: subscription.p256dh
}
}, notificationPayload)
)
);
const succeeded = results.filter(
(r) => r.status === "fulfilled",
).length;
const failed = results.filter((r) => r.status === "rejected").length;
console.log(
`Push notifications sent: ${succeeded} succeeded, ${failed} failed`,
);
const succeeded = results.filter(r => r.status === 'fulfilled').length;
const failed = results.filter(r => r.status === 'rejected').length;
console.log(`Push notifications sent: ${succeeded} succeeded, ${failed} failed`);
}
// Retry fetching details for newsletters without them
const newslettersWithoutDetails =
await storage.getNewslettersWithoutDetails();
const updatedNewsletters = await retryMissingDetails(
newslettersWithoutDetails,
);
const newslettersWithoutDetails = await storage.getNewslettersWithoutDetails();
const updatedNewsletters = await retryMissingDetails(newslettersWithoutDetails);
for (const newsletter of updatedNewsletters) {
if (newsletter.id) {
@@ -102,8 +84,9 @@ export async function registerRoutes(app: Express): Promise<Server> {
console.log(`Updated details for newsletter: ${newsletter.title}`);
}
}
} catch (error) {
console.error("Background job failed:", error);
console.error('Background job failed:', error);
}
});
@@ -237,43 +220,30 @@ export async function registerRoutes(app: Express): Promise<Server> {
</style>
<div class="newsletter-embed">
<div class="grid">
${newsletters
.slice(0, 6)
.map(
(newsletter) => `
${newsletters.slice(0, 6).map(newsletter => `
<article class="article">
<h2 class="title">${newsletter.title}</h2>
<time class="date">${new Date(newsletter.date).toLocaleDateString()}</time>
${
newsletter.thumbnail
? `
${newsletter.thumbnail ? `
<img src="${newsletter.thumbnail}" alt="${newsletter.title}" class="image">
`
: ""
}
${
newsletter.description
? `
` : ''}
${newsletter.description ? `
<p class="description">${newsletter.description}</p>
`
: ""
}
` : ''}
<a href="${newsletter.url}" target="_blank" rel="noopener noreferrer"
class="link">
Read more
</a>
</article>
`,
)
.join("")}
`).join('')}
</div>
</div>
`;
res.header("Content-Type", "text/html");
res.header('Content-Type', 'text/html');
res.send(content);
} catch (error) {
console.error("Error generating embedded content:", error);
console.error('Error generating embedded content:', error);
res.status(500).json({ message: "Failed to generate embedded content" });
}
});
@@ -285,7 +255,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
});
app.get("/api/newsletters/search", async (req, res) => {
const query = (req.query.q as string) || "";
const query = req.query.q as string || "";
const newsletters = await storage.searchNewsletters(query);
res.json(newsletters);
});
@@ -297,64 +267,55 @@ export async function registerRoutes(app: Express): Promise<Server> {
await storage.importNewsletter(newsletter);
importedCount++;
});
res.json({
message: `Successfully imported ${importedCount} newsletters`,
});
res.json({ message: `Successfully imported ${importedCount} newsletters` });
} catch (error) {
console.error("Error importing newsletters:", error);
console.error('Error importing newsletters:', error);
res.status(500).json({ message: "Failed to import newsletters" });
}
});
app.post("/api/subscriptions", async (req, res) => {
try {
console.log("Received subscription request:", {
console.log('Received subscription request:', {
endpoint: req.body.endpoint,
auth: req.body.keys?.auth ? "[present]" : "[missing]",
p256dh: req.body.keys?.p256dh ? "[present]" : "[missing]",
auth: req.body.keys?.auth ? '[present]' : '[missing]',
p256dh: req.body.keys?.p256dh ? '[present]' : '[missing]'
});
if (
!req.body.endpoint ||
!req.body.keys?.auth ||
!req.body.keys?.p256dh
) {
throw new Error("Invalid subscription data");
if (!req.body.endpoint || !req.body.keys?.auth || !req.body.keys?.p256dh) {
throw new Error('Invalid subscription data');
}
await storage.addSubscription({
endpoint: req.body.endpoint,
auth: req.body.keys.auth,
p256dh: req.body.keys.p256dh,
p256dh: req.body.keys.p256dh
});
// Test the subscription with a welcome notification
try {
await webpush.sendNotification(
{
endpoint: req.body.endpoint,
keys: {
auth: req.body.keys.auth,
p256dh: req.body.keys.p256dh,
},
},
JSON.stringify({
title: "Subscription Successful",
body: "You will now receive notifications for new newsletters!",
icon: "/icon.png",
}),
);
console.log("Welcome notification sent successfully");
await webpush.sendNotification({
endpoint: req.body.endpoint,
keys: {
auth: req.body.keys.auth,
p256dh: req.body.keys.p256dh
}
}, JSON.stringify({
title: 'Subscription Successful',
body: 'You will now receive notifications for new newsletters!',
icon: '/icon.png'
}));
console.log('Welcome notification sent successfully');
} catch (notifError) {
console.error("Failed to send welcome notification:", notifError);
console.error('Failed to send welcome notification:', notifError);
}
res.json({ message: "Subscription added successfully" });
} catch (error) {
console.error("Error adding subscription:", error);
console.error('Error adding subscription:', error);
res.status(500).json({
message: "Failed to add subscription",
error: error instanceof Error ? error.message : "Unknown error",
error: error instanceof Error ? error.message : 'Unknown error'
});
}
});
@@ -363,14 +324,12 @@ export async function registerRoutes(app: Express): Promise<Server> {
try {
const subscriptionId = parseInt(req.params.id);
await storage.saveNotificationSettings(subscriptionId, {
newsletter_notifications: req.body.newsletter_notifications,
newsletter_notifications: req.body.newsletter_notifications
});
res.json({ message: "Notification settings updated successfully" });
} catch (error) {
console.error("Error updating notification settings:", error);
res
.status(500)
.json({ message: "Failed to update notification settings" });
console.error('Error updating notification settings:', error);
res.status(500).json({ message: "Failed to update notification settings" });
}
});
@@ -386,13 +345,11 @@ export async function registerRoutes(app: Express): Promise<Server> {
language: "en",
copyright: "All rights reserved",
favicon: "https://downtowner.com/favicon.ico",
updated: newsletters[0]?.date
? new Date(newsletters[0].date)
: new Date(),
updated: newsletters[0]?.date ? new Date(newsletters[0].date) : new Date(),
generator: "The Downtowner RSS Feed",
feedLinks: {
rss2: "https://downtowner.com/api/rss",
},
rss2: "https://downtowner.com/api/rss"
}
});
for (const newsletter of newsletters) {
@@ -400,20 +357,20 @@ export async function registerRoutes(app: Express): Promise<Server> {
title: newsletter.title,
id: newsletter.url,
link: newsletter.url,
description: newsletter.description || "",
description: newsletter.description || '',
date: new Date(newsletter.date),
image: newsletter.thumbnail || undefined,
image: newsletter.thumbnail || undefined
});
}
res.type("application/xml");
res.type('application/xml');
res.send(feed.rss2());
} catch (error) {
console.error("Error generating RSS feed:", error);
console.error('Error generating RSS feed:', error);
res.status(500).json({ message: "Failed to generate RSS feed" });
}
});
const httpServer = createServer(app);
return httpServer;
}
}

40
server/utils.ts
View File

@@ -8,13 +8,9 @@ const ROBLY_ARCHIVE_URL =
async function scrapeNewsletterContent(
url: string,
retryCount = 0,
): Promise<{
thumbnail: string | null;
content: string | null;
hasDetails: boolean;
}> {
): Promise<{ thumbnail: string | null; content: string | null; hasDetails: boolean }> {
try {
const backoffTime = 60000; // 1 minute
const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 1000);
if (retryCount > 0) {
await new Promise((resolve) => setTimeout(resolve, backoffTime));
}
@@ -54,7 +50,7 @@ async function scrapeNewsletterContent(
} catch (error: any) {
if (
(error.response?.status === 429 || error.code === "ECONNRESET") &&
retryCount < 5
retryCount < 1
) {
console.log(
`Rate limited or connection reset, attempt ${retryCount + 1}/5`,
@@ -67,7 +63,7 @@ async function scrapeNewsletterContent(
}
export async function scrapeNewsletters(
onNewsletterProcessed?: (newsletter: InsertNewsletter) => Promise<void>,
onNewsletterProcessed?: (newsletter: InsertNewsletter) => Promise<void>
): Promise<InsertNewsletter[]> {
try {
const { data } = await axios.get(ROBLY_ARCHIVE_URL, {
@@ -100,8 +96,7 @@ export async function scrapeNewsletters(
const date = new Date(dateStr).toISOString().split("T")[0];
const fullUrl = `https://app.robly.com${url}`;
const { thumbnail, content, hasDetails } =
await scrapeNewsletterContent(fullUrl);
const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(fullUrl);
const newsletter: InsertNewsletter = {
title: title.trim(),
@@ -118,9 +113,7 @@ export async function scrapeNewsletters(
}
newsletters.push(newsletter);
console.log(
`Processed newsletter: ${title} (hasDetails: ${hasDetails})`,
);
console.log(`Processed newsletter: ${title} (hasDetails: ${hasDetails})`);
} catch (err) {
console.warn(
"Error processing date for newsletter:",
@@ -154,21 +147,15 @@ export async function scrapeNewsletters(
}
}
export async function retryMissingDetails(
newsletters: Newsletter[],
): Promise<InsertNewsletter[]> {
const newslettersWithoutDetails = newsletters.filter((n) => !n.hasDetails);
console.log(
`Found ${newslettersWithoutDetails.length} newsletters without details to retry`,
);
export async function retryMissingDetails(newsletters: Newsletter[]): Promise<InsertNewsletter[]> {
const newslettersWithoutDetails = newsletters.filter(n => !n.hasDetails);
console.log(`Found ${newslettersWithoutDetails.length} newsletters without details to retry`);
const updatedNewsletters: InsertNewsletter[] = [];
for (const newsletter of newslettersWithoutDetails) {
try {
const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(
newsletter.url,
);
const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(newsletter.url);
if (hasDetails) {
updatedNewsletters.push({
@@ -181,12 +168,9 @@ export async function retryMissingDetails(
console.log(`Successfully retrieved details for: ${newsletter.title}`);
}
} catch (error) {
console.error(
`Failed to retrieve details for ${newsletter.title}:`,
error,
);
console.error(`Failed to retrieve details for ${newsletter.title}:`, error);
}
}
return updatedNewsletters;
}
}