TerribleDev/DowntownerArchive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Agent query: Could you try the refresh button to import newsletters and check if they are being saved one at a time?

Browse Source 
Refactor newsletter import: Process and save newsletters individually instead of all at once.  Improves efficiency and error handling.

Screenshot: https://storage.googleapis.com/screenshot-production-us-central1/9dda30b6-4149-4bce-89dc-76333005952c/2f9ce22a-3adb-45ce-8465-b6f605458b4d.jpg
This commit is contained in:
TerribleDev
2025-02-18 16:49:51 +00:00
parent 53227f34c0
commit a265360f10
3 changed files with 39 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
server/routes.ts
View File

@@ -25,18 +25,20 @@ export async function registerRoutes(app: Express): Promise<Server> {
schedule.scheduleJob('0 */6 * * *', async function() {
try {
const existingNewsletters = await storage.getNewsletters();
const scrapedNewsletters = await scrapeNewsletters();
let newNewslettersCount = 0;
// Import new newsletters
const newNewsletters = scrapedNewsletters.filter(scraped =>
!existingNewsletters.some(existing =>
existing.url === scraped.url
)
);
await scrapeNewsletters(async (newsletter) => {
// Check if newsletter already exists
const exists = existingNewsletters.some(existing => existing.url === newsletter.url);
if (!exists) {
await storage.importNewsletter(newsletter);
newNewslettersCount++;
console.log(`Imported new newsletter: ${newsletter.title}`);
}
});
if (newNewsletters.length > 0) {
await storage.importNewsletters(newNewsletters);
console.log(`Found ${newNewsletters.length} new newsletters, sending notifications...`);
if (newNewslettersCount > 0) {
console.log(`Found ${newNewslettersCount} new newsletters, sending notifications...`);
// Send push notifications for new newsletters
const subscriptions = await storage.getActiveSubscriptions();
@@ -44,7 +46,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
const notificationPayload = JSON.stringify({
title: 'New Newsletters Available',
body: `${newNewsletters.length} new newsletter${newNewsletters.length > 1 ? 's' : ''} published!`,
body: `${newNewslettersCount} new newsletter${newNewslettersCount > 1 ? 's' : ''} published!`,
icon: '/icon.png'
});
@@ -100,9 +102,12 @@ export async function registerRoutes(app: Express): Promise<Server> {
app.post("/api/newsletters/import", async (_req, res) => {
try {
const newsletters = await scrapeNewsletters();
await storage.importNewsletters(newsletters);
res.json({ message: "Newsletters imported successfully" });
let importedCount = 0;
await scrapeNewsletters(async (newsletter) => {
await storage.importNewsletter(newsletter);
importedCount++;
});
res.json({ message: `Successfully imported ${importedCount} newsletters` });
} catch (error) {
console.error('Error importing newsletters:', error);
res.status(500).json({ message: "Failed to import newsletters" });
@@ -178,6 +183,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
id: "https://downtowner.com/",
link: "https://downtowner.com/",
language: "en",
copyright: "All rights reserved",
favicon: "https://downtowner.com/favicon.ico",
updated: newsletters[0]?.date ? new Date(newsletters[0].date) : new Date(),
generator: "The Downtowner RSS Feed",
@@ -191,9 +197,9 @@ export async function registerRoutes(app: Express): Promise<Server> {
title: newsletter.title,
id: newsletter.url,
link: newsletter.url,
description: newsletter.description,
description: newsletter.description || '',
date: new Date(newsletter.date),
image: newsletter.thumbnail
image: newsletter.thumbnail || undefined
});
}

11
server/storage.ts
View File

@@ -7,6 +7,7 @@ export interface IStorage {
getNewslettersWithoutDetails(): Promise<Newsletter[]>;
searchNewsletters(query: string): Promise<Newsletter[]>;
importNewsletters(newsletters: InsertNewsletter[]): Promise<void>;
importNewsletter(newsletter: InsertNewsletter): Promise<void>;
updateNewsletterDetails(id: number, updates: Partial<InsertNewsletter>): Promise<void>;
addSubscription(subscription: InsertSubscription): Promise<void>;
getSubscriptions(): Promise<Subscription[]>;
@@ -41,11 +42,13 @@ export class DatabaseStorage implements IStorage {
.orderBy(desc(newsletters.date));
}
async importNewsletter(newsletter: InsertNewsletter): Promise<void> {
await db.insert(newsletters).values(newsletter);
}
async importNewsletters(newNewsletters: InsertNewsletter[]): Promise<void> {
const batchSize = 50;
for (let i = 0; i < newNewsletters.length; i += batchSize) {
const batch = newNewsletters.slice(i, i + batchSize);
await db.insert(newsletters).values(batch);
for (const newsletter of newNewsletters) {
await this.importNewsletter(newsletter);
}
}

13
server/utils.ts
View File

@@ -66,7 +66,9 @@ async function scrapeNewsletterContent(
}
}
export async function scrapeNewsletters(): Promise<InsertNewsletter[]> {
export async function scrapeNewsletters(
onNewsletterProcessed?: (newsletter: InsertNewsletter) => Promise<void>
): Promise<InsertNewsletter[]> {
try {
const { data } = await axios.get(ROBLY_ARCHIVE_URL, {
headers: {
@@ -100,7 +102,7 @@ export async function scrapeNewsletters(): Promise<InsertNewsletter[]> {
const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(fullUrl);
newsletters.push({
const newsletter: InsertNewsletter = {
title: title.trim(),
date,
url: fullUrl,
@@ -108,8 +110,13 @@ export async function scrapeNewsletters(): Promise<InsertNewsletter[]> {
content,
description: content ? content.slice(0, 200) + "..." : null,
hasDetails,
});
};
if (onNewsletterProcessed) {
await onNewsletterProcessed(newsletter);
}
newsletters.push(newsletter);
console.log(`Processed newsletter: ${title} (hasDetails: ${hasDetails})`);
} catch (err) {
console.warn(