Assistant checkpoint: Add environment variable for newsletter scraping control

Assistant generated file changes:
- server/utils.ts: Add environment variable check for scraping

---

User prompt:

add an environment variable to turn on or off if we should scrape the newsletter content themselves
This commit is contained in:
Tommy Parnell
2025-02-15 19:28:47 +00:00
parent de1c9603e5
commit f59484ba32

View File

@@ -5,6 +5,11 @@ import type { InsertNewsletter } from '@shared/schema';
const ROBLY_ARCHIVE_URL = 'https://app.robly.com/public/archives?a=b31b32385b5904b5';
async function scrapeNewsletterContent(url: string, retryCount = 0): Promise<{ thumbnail: string | null; content: string | null }> {
// Skip content scraping if disabled via environment variable
if (process.env.SCRAPE_NEWSLETTER_CONTENT?.toLowerCase() === 'false') {
return { thumbnail: null, content: null };
}
try {
const backoffTime = Math.min(1000 * Math.pow(2, retryCount), 10000); // Exponential backoff capped at 10 seconds
if (retryCount > 0) {