Cloudflare Worker Setup Guide

export default {
  async fetch(request, env) {
    const url = new URL(request.url);
    const userAgent = request.headers.get('User-Agent') || '';

    // Bot detection - comprehensive list of search engines, AI crawlers, and social bots
    // Updated for 2025-2026 bot landscape
    const botPatterns = [
      // Search Engines
      'googlebot',
      'bingbot',
      'yandexbot',
      'baiduspider',
      'duckduckbot',
      'applebot',
      // AI Crawlers (Updated 2025-2026)
      'gptbot',
      'google-extended',        // Gemini AI training (2024+)
      'claudebot',
      'claude-web',
      'perplexitybot',
      'anthropic-ai',
      'chatgpt-user',
      'cohere-ai',
      'omgilibot',
      'omgili',
      'meta-externalagent',     // Meta's updated crawler (2024+)
      'facebookbot',            // Legacy Meta crawler
      'facebot',
      'bytespider',             // TikTok/ByteDance crawler
      'amazonbot',              // Amazon Alexa
      'applebot-extended',      // Apple Intelligence AI (2024+)
      'youbot',                 // You.com search
      'diffbot',                // Knowledge graph extraction
      'img2dataset',            // AI training scraper
      // Social Media & Messaging
      'twitterbot',
      'slackbot',
      'discordbot',
      'telegrambot',
      'whatsapp',
      'linkedinbot',
      'pinterestbot',
      // Common Crawlers
      'ccbot',
      'ia_archiver',
      'archive.org_bot'
    ];

    const isBot = botPatterns.some(bot =>
      userAgent.toLowerCase().includes(bot)
    );

    // If not a bot, pass through unchanged
    if (!isBot) {
      return fetch(request);
    }

    // Log bot detection for monitoring (with bot type identification)
    const botType = botPatterns.find(bot => userAgent.toLowerCase().includes(bot));
    console.log(`[geo-butler] Bot detected: ${botType} | User-Agent: ${userAgent.substring(0, 80)} | Path: ${url.pathname}`);

    // Fetch original page
    const response = await fetch(request);

    // Only process HTML responses
    const contentType = response.headers.get('content-type') || '';
    if (!contentType.includes('text/html')) {
      return response;
    }

    // Build schema URL with proper path normalization
    const domain = url.hostname;

    // Remove leading slash, strip trailing slashes, ignore query params
    let path = url.pathname.slice(1) || 'index';
    path = path.replace(/\/+$/, '') || 'index'; // Strip trailing slashes

    // Handle nested paths properly (e.g., /blog/post-1 -> blog/post-1)
    const schemaUrl = `https://schema.baselinelabs.ai/${domain}/${path}.json`;

    // Fetch schema from R2 bucket with timeout
    let schemaData = null;
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), 3000); // 3 second timeout

    try {
      const schemaResponse = await fetch(schemaUrl, {
        signal: controller.signal
      });
      clearTimeout(timeoutId);

      if (schemaResponse.ok) {
        schemaData = await schemaResponse.json();
        console.log(`[geo-butler] Schema fetched successfully: ${schemaUrl}`);
      } else {
        console.log(`[geo-butler] Schema not found (HTTP ${schemaResponse.status}): ${schemaUrl}`);

        // Try fallback to domain-level default schema
        const fallbackUrl = `https://schema.baselinelabs.ai/${domain}/default.json`;
        const fallbackResponse = await fetch(fallbackUrl);
        if (fallbackResponse.ok) {
          schemaData = await fallbackResponse.json();
          console.log(`[geo-butler] Using fallback schema: ${fallbackUrl}`);
        }
      }
    } catch (e) {
      clearTimeout(timeoutId);
      console.log(`[geo-butler] Schema fetch failed: ${e.message}`);
      // Continue without schema - don't block the response
    }

    // If no schema found, return original response
    if (!schemaData) {
      console.log(`[geo-butler] No schema available, returning original HTML`);
      return response;
    }

    // Inject schema into HTML
    let html = await response.text();
    const schemaScript = `
<script type="application/ld+json">
${JSON.stringify(schemaData, null, 2)}
</script>`;

    // Add verification marker for geo-butler
    const verificationMarker = '<!-- geo-butler-schema-active -->';
    const injection = `${verificationMarker}\n${schemaScript}`;

    // Try multiple injection strategies (case-insensitive)
    let injected = false;

    // Strategy 1: Before </head> tag (preferred)
    if (html.match(/<\/head>/i)) {
      html = html.replace(/<\/head>/i, `${injection}\n</head>`);
      injected = true;
    }
    // Strategy 2: After <head> opening tag
    else if (html.match(/<head[^>]*>/i)) {
      html = html.replace(/<head([^>]*)>/i, `<head$1>\n${injection}`);
      injected = true;
    }
    // Strategy 3: After <body> opening tag
    else if (html.match(/<body[^>]*>/i)) {
      html = html.replace(/<body([^>]*)>/i, `<body$1>\n${injection}`);
      injected = true;
    }
    // Strategy 4: Prepend to entire HTML (fallback)
    else {
      html = `${injection}\n${html}`;
      injected = true;
    }

    if (injected) {
      console.log(`[geo-butler] Schema injected successfully`);
    }

    // Create new response with modified HTML
    // CRITICAL: Remove Content-Length header as body size has changed
    const newHeaders = new Headers(response.headers);
    newHeaders.delete('Content-Length');
    newHeaders.delete('content-length');

    return new Response(html, {
      status: response.status,
      statusText: response.statusText,
      headers: newHeaders
    });
  }
};