From 2a331f89f859df30f910bd8e7c0228434f348b37 Mon Sep 17 00:00:00 2001
From: TerribleDev <1020010-TerribleDev@users.noreply.replit.com>
Date: Tue, 18 Feb 2025 22:14:17 +0000
Subject: [PATCH] User checkpoint: Increase newsletter detail retry interval to
 1 minute and maximum retry attempts to 5.

---
 README.md       |  2 +-
 server/utils.ts | 40 ++++++++++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 81277a4..44682c1 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ class NewsletterEmbed extends HTMLElement {
 
     async connectedCallback() {
         try {
-            const response = await fetch('https://your-repl-url/embed');
+            const response = await fetch('https://downtowner.terrible.dev/embed');
             const html = await response.text();
             this.shadowRoot.innerHTML = html;
         } catch (error) {
diff --git a/server/utils.ts b/server/utils.ts
index bff8051..c12f6cb 100644
--- a/server/utils.ts
+++ b/server/utils.ts
@@ -8,9 +8,13 @@ const ROBLY_ARCHIVE_URL =
 async function scrapeNewsletterContent(
   url: string,
   retryCount = 0,
-): Promise<{ thumbnail: string | null; content: string | null; hasDetails: boolean }> {
+): Promise<{
+  thumbnail: string | null;
+  content: string | null;
+  hasDetails: boolean;
+}> {
   try {
-    const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 1000);
+    const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 10000);
     if (retryCount > 0) {
       await new Promise((resolve) => setTimeout(resolve, backoffTime));
     }
@@ -50,7 +54,7 @@ async function scrapeNewsletterContent(
   } catch (error: any) {
     if (
       (error.response?.status === 429 || error.code === "ECONNRESET") &&
-      retryCount < 1
+      retryCount < 5
     ) {
       console.log(
         `Rate limited or connection reset, attempt ${retryCount + 1}/5`,
@@ -63,7 +67,7 @@ async function scrapeNewsletterContent(
 }
 
 export async function scrapeNewsletters(
-  onNewsletterProcessed?: (newsletter: InsertNewsletter) => Promise<void>
+  onNewsletterProcessed?: (newsletter: InsertNewsletter) => Promise<void>,
 ): Promise<InsertNewsletter[]> {
   try {
     const { data } = await axios.get(ROBLY_ARCHIVE_URL, {
@@ -96,7 +100,8 @@ export async function scrapeNewsletters(
           const date = new Date(dateStr).toISOString().split("T")[0];
           const fullUrl = `https://app.robly.com${url}`;
 
-          const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(fullUrl);
+          const { thumbnail, content, hasDetails } =
+            await scrapeNewsletterContent(fullUrl);
 
           const newsletter: InsertNewsletter = {
             title: title.trim(),
@@ -113,7 +118,9 @@ export async function scrapeNewsletters(
           }
 
           newsletters.push(newsletter);
-          console.log(`Processed newsletter: ${title} (hasDetails: ${hasDetails})`);
+          console.log(
+            `Processed newsletter: ${title} (hasDetails: ${hasDetails})`,
+          );
         } catch (err) {
           console.warn(
             "Error processing date for newsletter:",
@@ -147,15 +154,21 @@ export async function scrapeNewsletters(
   }
 }
 
-export async function retryMissingDetails(newsletters: Newsletter[]): Promise<InsertNewsletter[]> {
-  const newslettersWithoutDetails = newsletters.filter(n => !n.hasDetails);
-  console.log(`Found ${newslettersWithoutDetails.length} newsletters without details to retry`);
+export async function retryMissingDetails(
+  newsletters: Newsletter[],
+): Promise<InsertNewsletter[]> {
+  const newslettersWithoutDetails = newsletters.filter((n) => !n.hasDetails);
+  console.log(
+    `Found ${newslettersWithoutDetails.length} newsletters without details to retry`,
+  );
 
   const updatedNewsletters: InsertNewsletter[] = [];
 
   for (const newsletter of newslettersWithoutDetails) {
     try {
-      const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(newsletter.url);
+      const { thumbnail, content, hasDetails } = await scrapeNewsletterContent(
+        newsletter.url,
+      );
 
       if (hasDetails) {
         updatedNewsletters.push({
@@ -168,9 +181,12 @@ export async function retryMissingDetails(newsletters: Newsletter[]): Promise<In
         console.log(`Successfully retrieved details for: ${newsletter.title}`);
       }
     } catch (error) {
-      console.error(`Failed to retrieve details for ${newsletter.title}:`, error);
+      console.error(
+        `Failed to retrieve details for ${newsletter.title}:`,
+        error,
+      );
     }
   }
 
   return updatedNewsletters;
-}
\ No newline at end of file
+}