feat: add new blog posts, associated images, and automation scripts for content management

2026-04-10 11:55:47 +03:00
parent 83e192ebd6
commit 36ffbea30a
206 changed files with 6626 additions and 727 deletions
@@ -0,0 +1,89 @@
+import fs from 'fs/promises';
+import path from 'path';
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+import TurndownService from 'turndown';
+
+puppeteer.use(StealthPlugin());
+
+const turndownService = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced'
+});
+
+async function main() {
+  const fileContent = await fs.readFile(path.join(process.cwd(), 'docs/mediumLinks.md'), 'utf8');
+  const urls = fileContent.split('\n').map(l => l.trim()).filter(l => l.length > 0);
+  
+  if (urls.length === 0) {
+    console.log('No URLs found in mediumLinks.md');
+    return;
+  }
+  
+  console.log(`Starting to process ${urls.length} articles...`);
+  
+  const browser = await puppeteer.launch({ 
+    headless: 'new', // new headless modes sometimes pass CD clearer
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
+  });
+  
+  const page = await browser.newPage();
+  
+  // Set a realistic user agent
+  await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+  await page.setViewport({ width: 1366, height: 768 });
+  
+  const testUrl = urls[0];
+  console.log(`Testing first URL: ${testUrl}`);
+  
+  try {
+    const response = await page.goto(testUrl, { waitUntil: 'load', timeout: 60000 });
+    
+    // Check if we hit Cloudflare
+    const pageTitle = await page.title();
+    console.log('Page Title:', pageTitle);
+    
+    if (pageTitle.includes('Just a moment')) {
+      console.log('Got caught by Cloudflare still. Let\'s wait 5 seconds...');
+      await new Promise(r => setTimeout(r, 5000));
+    }
+    
+    // Extract metadata
+    const meta = await page.evaluate(() => {
+      const title = document.querySelector('h1')?.innerText || document.querySelector('meta[property="og:title"]')?.content || 'Unknown Title';
+      const excerpt = document.querySelector('meta[name="description"]')?.content || '';
+      const coverImage = document.querySelector('meta[property="og:image"]')?.content || '';
+      const canonical = document.querySelector('link[rel="canonical"]')?.href || '';
+      const date = document.querySelector('meta[property="article:published_time"]')?.content || new Date().toISOString();
+      const readTimeText = document.body.innerText.match(/\d+\s*min\s*read/i);
+      const readTime = readTimeText ? readTimeText[0] : '5 min read';
+      
+      // Get the main article content (excluding headers/footers)
+      let html = '';
+      const sections = document.querySelectorAll('article section');
+      if (sections && sections.length > 0) {
+        sections.forEach(s => { html += s.innerHTML; });
+      } else {
+        html = document.querySelector('article')?.innerHTML || document.body.innerHTML;
+      }
+      
+      return { title, excerpt, coverImage, canonical, date, readTime, html };
+    });
+    
+    console.log('Extracted Metadata:', {
+      title: meta.title,
+      date: meta.date
+    });
+    
+    let markdown = turndownService.turndown(meta.html);
+    markdown = markdown.substring(0, 300) + '...\n[SUCCESSFULLY BYPASSED]';
+    console.log(markdown);
+    
+  } catch (err) {
+    console.error('Error fetching article:', err);
+  } finally {
+    await browser.close();
+  }
+}
+
+main();