<?php
/**
 * Content Processor for BoostBot Knowledge Base
 *
 * Processes content using Claude AI for summarization and keyword extraction.
 * Enhanced with Elementor parsing, specs extraction, and price detection.
 *
 * @package BoostBot
 * @since 1.0.0
 */

// phpcs:disable WordPress.DB.PreparedSQL.NotPrepared, WordPress.DB.PreparedSQL.InterpolatedNotPrepared
if ( ! defined( 'ABSPATH' ) ) {
    exit;
}

/**
 * Class BoostBot_Content_Processor
 *
 * Processes and summarizes content with Claude AI.
 */
class BoostBot_Content_Processor {

    /**
     * Claude client instance.
     *
     * @var BoostBot_Claude_Client|null
     */
    private $claude_client = null;

    /**
     * Gemini client instance (optional).
     *
     * @var BoostBot_Gemini_Client|null
     */
    private $gemini_client = null;

    /**
     * Elementor parser instance.
     *
     * @var BoostBot_Elementor_Parser|null
     */
    private $elementor_parser = null;

    /**
     * Specs extractor instance.
     *
     * @var BoostBot_Specs_Extractor|null
     */
    private $specs_extractor = null;

    /**
     * Price extractor instance.
     *
     * @var BoostBot_Price_Extractor|null
     */
    private $price_extractor = null;

    /**
     * Maximum content length for API (in characters).
     *
     * @var int
     */
    private $max_content_length = 8000;

    /**
     * Cache duration for summaries (1 week).
     *
     * @var int
     */
    private $cache_duration = WEEK_IN_SECONDS;

    /**
     * Constructor.
     *
     * @param BoostBot_Claude_Client|null $client Optional Claude client.
     */
    public function __construct( ?BoostBot_Claude_Client $client = null ) {
        $this->claude_client = $client;

        // Initialize extractors.
        if ( class_exists( 'BoostBot_Elementor_Parser' ) ) {
            $this->elementor_parser = new BoostBot_Elementor_Parser();
        }
        if ( class_exists( 'BoostBot_Specs_Extractor' ) ) {
            $this->specs_extractor = new BoostBot_Specs_Extractor();
        }
        if ( class_exists( 'BoostBot_Price_Extractor' ) ) {
            $this->price_extractor = new BoostBot_Price_Extractor();
        }
    }

    /**
     * Get or create Gemini client.
     *
     * @return BoostBot_Gemini_Client|null
     */
    private function get_gemini_client(): ?BoostBot_Gemini_Client {
        if ( null === $this->gemini_client ) {
            if ( ! class_exists( 'BoostBot_Gemini_Client' ) ) {
                return null;
            }
            $this->gemini_client = new BoostBot_Gemini_Client();
        }
        return $this->gemini_client;
    }

    /**
     * Get or create Claude client.
     *
     * @return BoostBot_Claude_Client|null
     */
    private function get_client(): ?BoostBot_Claude_Client {
        if ( null === $this->claude_client ) {
            try {
                $this->claude_client = new BoostBot_Claude_Client();
            } catch ( Exception $e ) {
                BoostBot_Logger::error( 'Failed to create Claude client', array(
                    'error' => $e->getMessage(),
                ) );
                return null;
            }
        }
        return $this->claude_client;
    }

    /**
     * Process content for indexing.
     *
     * @param array $content_data Content data with title, content, etc.
     * @return array Processed data with summary, keywords, intent.
     */
    public function process_content( array $content_data ): array {
        BoostBot_Logger::debug( 'Processing content', array(
            'title' => $content_data['title'] ?? 'Unknown',
        ) );

        // Check cache first
        $cache_key = 'boostbot_summary_' . md5( ( $content_data['content'] ?? '' ) . ( $content_data['title'] ?? '' ) );
        $cached    = get_transient( $cache_key );

        if ( false !== $cached && is_array( $cached ) ) {
            // Only use cache if it contains meaningful data (non-empty summary OR keywords).
            $cached_summary  = isset( $cached['summary'] ) ? trim( (string) $cached['summary'] ) : '';
            $cached_keywords = isset( $cached['keywords'] ) && is_array( $cached['keywords'] ) ? $cached['keywords'] : array();
            if ( '' !== $cached_summary || ! empty( $cached_keywords ) ) {
                BoostBot_Logger::debug( 'Using cached summary' );
                return $cached;
            }
            // Stale/empty cache entry — delete and re-process.
            delete_transient( $cache_key );
            BoostBot_Logger::debug( 'Discarded empty cached summary, re-processing' );
        }

        // Preferred: backend indexing (Option B), then local Gemini, then Claude fallback.
        $provider = (string) BoostBot_Settings::get( 'indexing_llm_provider', 'backend' );

        // 1) Backend attempt (no vendor keys stored in WP)
        if ( 'backend' === $provider && class_exists( 'BoostBot_Indexing_Backend_Client' ) ) {
            $backend = new BoostBot_Indexing_Backend_Client();
            $result  = $backend->summarize( $content_data );
            if ( ! is_wp_error( $result ) && is_array( $result ) ) {
                // If backend returned a summary but no keywords, extract keywords locally.
                $res_summary  = isset( $result['summary'] ) ? trim( (string) $result['summary'] ) : '';
                $res_keywords = isset( $result['keywords'] ) && is_array( $result['keywords'] ) ? $result['keywords'] : array();
                if ( empty( $res_keywords ) && '' !== trim( (string) ( $content_data['content'] ?? '' ) ) ) {
                    $local_keywords = $this->extract_keywords_simple(
                        ( $content_data['title'] ?? '' ) . ' ' . ( $content_data['content'] ?? '' )
                    );
                    if ( ! empty( $local_keywords ) ) {
                        $result['keywords'] = $local_keywords;
                        $res_keywords       = $local_keywords;
                    }
                }
                // Only cache if we got meaningful data.
                if ( '' !== $res_summary || ! empty( $res_keywords ) ) {
                    set_transient( $cache_key, $result, $this->cache_duration );
                }
                return $result;
            }
            if ( is_wp_error( $result ) ) {
                BoostBot_Logger::error( 'Backend summarization failed, falling back', array(
                    'error' => $result->get_error_message(),
                ) );
            }
        }

        // 2) Local Gemini attempt (requires gemini_api_key in WP)
        if ( 'gemini' === $provider ) {
            $gemini = $this->get_gemini_client();
            if ( $gemini && $gemini->is_available() ) {
                try {
                    $result = $this->summarize_with_gemini( $content_data );
                    $res_summary  = isset( $result['summary'] ) ? trim( (string) $result['summary'] ) : '';
                    $res_keywords = isset( $result['keywords'] ) && is_array( $result['keywords'] ) ? $result['keywords'] : array();
                    if ( '' !== $res_summary || ! empty( $res_keywords ) ) {
                        set_transient( $cache_key, $result, $this->cache_duration );
                    }
                    return $result;
                } catch ( Exception $e ) {
                    BoostBot_Logger::error( 'Gemini summarization failed, falling back', array(
                        'error' => $e->getMessage(),
                    ) );
                }
            }
        }

        // 3) Claude fallback
        $client = $this->get_client();
        if ( $client ) {
            try {
                $result = $this->summarize_with_claude( $content_data );
                $res_summary  = isset( $result['summary'] ) ? trim( (string) $result['summary'] ) : '';
                $res_keywords = isset( $result['keywords'] ) && is_array( $result['keywords'] ) ? $result['keywords'] : array();
                if ( '' !== $res_summary || ! empty( $res_keywords ) ) {
                    set_transient( $cache_key, $result, $this->cache_duration );
                }
                return $result;
            } catch ( Exception $e ) {
                BoostBot_Logger::error( 'Claude summarization failed', array(
                    'error' => $e->getMessage(),
                ) );
            }
        }

        // Fallback: simple extraction
        return $this->simple_extraction( $content_data );
    }

    /**
     * Summarize content using Claude.
     *
     * @param array $content_data Content data.
     * @return array Processed result.
     * @throws Exception On API error.
     */
    private function summarize_with_claude( array $content_data ): array {
        $title   = $content_data['title'] ?? '';
        $content = $this->truncate_for_api( $content_data['content'] ?? '' );
        $type    = $content_data['type'] ?? 'page';

        // Build prompt based on content type
        $prompt = $this->build_summarization_prompt( $title, $content, $type );

        $client   = $this->get_client();
        $response = $client->send_message(
            array(
                array(
                    'role'    => 'user',
                    'content' => $prompt,
                ),
            ),
            array(
                'max_tokens'  => 500,
                'temperature' => 0.3,
            )
        );

        // Parse JSON response
        $json_content = $response['content'] ?? '';
        
        // Try to extract JSON from response
        $result = $this->parse_json_response( $json_content );

        if ( null === $result ) {
            // Failed to parse, use fallback
            BoostBot_Logger::warning( 'Failed to parse Claude JSON response' );
            return $this->simple_extraction( $content_data );
        }

        return array(
            'summary'  => $result['summary'] ?? '',
            'keywords' => $result['keywords'] ?? array(),
            'intent'   => $result['intent'] ?? 'general',
        );
    }

    /**
     * Summarize content using Gemini (JSON output).
     *
     * @since 1.7.0
     * @param array $content_data Content data.
     * @return array Processed result.
     * @throws Exception On API error.
     */
    private function summarize_with_gemini( array $content_data ): array {
        $title   = $content_data['title'] ?? '';
        $content = $this->truncate_for_api( $content_data['content'] ?? '' );
        $type    = $content_data['type'] ?? 'page';

        $prompt = $this->build_summarization_prompt( $title, $content, $type );

        $gemini = $this->get_gemini_client();
        if ( ! $gemini ) {
            throw new Exception( 'Gemini client not available' );
        }

        $resp = $gemini->generate_json( $prompt, array(
            'max_output_tokens' => 500,
            'temperature'       => 0.2,
            'top_p'             => 0.9,
        ) );

        // Telemetry: log usage for cost validation (no secrets).
        if ( ! empty( $resp['usage'] ) ) {
            BoostBot_Logger::info( 'Indexing Gemini usage', array(
                'model' => $resp['model'] ?? null,
                'usage' => $resp['usage'],
            ) );
        }

        $result = $this->parse_json_response( $resp['json'] ?? '' );
        if ( null === $result ) {
            BoostBot_Logger::warning( 'Failed to parse Gemini JSON response' );
            return $this->simple_extraction( $content_data );
        }

        return array(
            'summary'  => $result['summary'] ?? '',
            'keywords' => $result['keywords'] ?? array(),
            'intent'   => $result['intent'] ?? 'general',
        );
    }

    /**
     * Build summarization prompt.
     *
     * @param string $title   Content title.
     * @param string $content Content text.
     * @param string $type    Content type.
     * @return string Prompt.
     */
    private function build_summarization_prompt( string $title, string $content, string $type, string $language = '' ): string {
        if ( '' === $language ) {
            $language = BoostBot_Language::get_primary();
        }
        if ( 'en' === $language ) {
            $type_description = array(
                'post'    => 'a blog post',
                'page'    => 'a website page',
                'product' => 'a product',
            );
            $type_text = $type_description[ $type ] ?? 'content';

            return "Summarize the following content in English. This is {$type_text}.\n"
                . "\n"
                . "Title: {$title}\n"
                . "\n"
                . "Content:\n"
                . "{$content}\n"
                . "\n"
                . "Provide a response in JSON format only, with no additional explanations:\n"
                . "{\n"
                . "    \"summary\": \"A 2-3 sentence summary in English describing the content\",\n"
                . "    \"keywords\": [\"keyword1\", \"keyword2\", \"keyword3\", \"keyword4\", \"keyword5\"],\n"
                . "    \"intent\": \"information|product|service|support|contact\"\n"
                . "}\n"
                . "\n"
                . "Rules:\n"
                . "- The summary should be concise and clear\n"
                . "- Choose 5-10 relevant keywords in English\n"
                . "- Choose one intent from the options: information (general info), product (product for sale), service (service), support (support/help), contact (contact us)\n"
                . "- Return valid JSON only!";
        }

        $type_description = array(
            'post'    => 'פוסט בבלוג',
            'page'    => 'עמוד באתר',
            'product' => 'מוצר',
        );
        $type_text = $type_description[ $type ] ?? 'תוכן';

        return "סכם את התוכן הבא בעברית. זהו {$type_text}.\n"
            . "\n"
            . "כותרת: {$title}\n"
            . "\n"
            . "תוכן:\n"
            . "{$content}\n"
            . "\n"
            . "ספק תשובה בפורמט JSON בלבד, ללא הסברים נוספים:\n"
            . "{\n"
            . "    \"summary\": \"סיכום של 2-3 משפטים בעברית שמתאר את התוכן\",\n"
            . "    \"keywords\": [\"מילה1\", \"מילה2\", \"מילה3\", \"מילה4\", \"מילה5\"],\n"
            . "    \"intent\": \"information|product|service|support|contact\"\n"
            . "}\n"
            . "\n"
            . "כללים:\n"
            . "- הסיכום צריך להיות תמציתי וברור\n"
            . "- בחר 5-10 מילות מפתח רלוונטיות בעברית\n"
            . "- בחר intent אחד מהאפשרויות: information (מידע כללי), product (מוצר למכירה), service (שירות), support (תמיכה/עזרה), contact (יצירת קשר)\n"
            . "- החזר JSON תקין בלבד!";
    }

    /**
     * Parse JSON response from Claude.
     *
     * @param string $response Response text.
     * @return array|null Parsed array or null.
     */
    private function parse_json_response( string $response ): ?array {
        // Try to find JSON in response
        $json_start = strpos( $response, '{' );
        $json_end   = strrpos( $response, '}' );

        if ( false === $json_start || false === $json_end ) {
            return null;
        }

        $json_str = substr( $response, $json_start, $json_end - $json_start + 1 );
        $result   = json_decode( $json_str, true );

        if ( JSON_ERROR_NONE !== json_last_error() ) {
            return null;
        }

        return $result;
    }

    /**
     * Simple extraction fallback (no API).
     *
     * @param array $content_data Content data.
     * @return array Basic extraction.
     */
    private function simple_extraction( array $content_data ): array {
        $content = $content_data['content'] ?? '';
        $title   = $content_data['title'] ?? '';

        // Create simple summary
        $summary = wp_trim_words( $content, 50, '...' );

        // Extract keywords (simple approach)
        $keywords = $this->extract_keywords_simple( $title . ' ' . $content );

        // Detect intent
        $intent = $this->detect_intent_simple( $content );

        return array(
            'summary'  => $summary,
            'keywords' => $keywords,
            'intent'   => $intent,
        );
    }

    /**
     * Simple keyword extraction.
     *
     * @param string $text Text to analyze.
     * @return array Keywords.
     */
    private function extract_keywords_simple( string $text ): array {
        // Remove common Hebrew stop words
        $stop_words = array(
            'את', 'של', 'על', 'עם', 'אל', 'מן', 'לא', 'כי', 'הוא', 'היא',
            'הם', 'הן', 'אני', 'אתה', 'את', 'אנחנו', 'הם', 'זה', 'זו', 'זאת',
            'כל', 'גם', 'אם', 'או', 'אבל', 'רק', 'עוד', 'יש', 'אין', 'היה',
            'היתה', 'יהיה', 'להיות', 'the', 'is', 'are', 'was', 'were', 'be',
            'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'a',
            'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of',
            'with', 'by', 'from', 'as', 'into', 'through', 'during', 'before',
            'after', 'above', 'below', 'between', 'under', 'again', 'further',
            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how',
            'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
            'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too',
            'very', 'can', 'will', 'just', 'should', 'now',
        );

        // Clean and split
        $text  = mb_strtolower( $text );
        $text  = preg_replace( '/[^\p{L}\p{N}\s]/u', ' ', $text );
        $words = preg_split( '/\s+/', $text, -1, PREG_SPLIT_NO_EMPTY );

        // Count word frequency
        $word_counts = array();
        foreach ( $words as $word ) {
            if ( mb_strlen( $word ) < 2 || in_array( $word, $stop_words, true ) || preg_match( '/^\d+$/', $word ) ) {
                continue;
            }
            if ( ! isset( $word_counts[ $word ] ) ) {
                $word_counts[ $word ] = 0;
            }
            $word_counts[ $word ]++;
        }

        // Sort by frequency
        arsort( $word_counts );

        // Return top 10
        return array_slice( array_keys( $word_counts ), 0, 10 );
    }

    /**
     * Simple intent detection.
     *
     * @param string $content Content text.
     * @return string Detected intent.
     */
    private function detect_intent_simple( string $content ): string {
        $content_lower = mb_strtolower( $content );

        // Product indicators
        $product_words = array( 'מחיר', 'לקנות', 'מכירה', 'הזמנה', 'משלוח', 'price', 'buy', 'shop', 'cart', '₪', '$' );
        foreach ( $product_words as $word ) {
            if ( false !== mb_strpos( $content_lower, $word ) ) {
                return 'product';
            }
        }

        // Service indicators
        $service_words = array( 'שירות', 'ייעוץ', 'התקנה', 'תיקון', 'service', 'consultation' );
        foreach ( $service_words as $word ) {
            if ( false !== mb_strpos( $content_lower, $word ) ) {
                return 'service';
            }
        }

        // Contact indicators
        $contact_words = array( 'צור קשר', 'התקשר', 'שלח הודעה', 'contact', 'call us', 'email' );
        foreach ( $contact_words as $word ) {
            if ( false !== mb_strpos( $content_lower, $word ) ) {
                return 'contact';
            }
        }

        // Support indicators
        $support_words = array( 'עזרה', 'תמיכה', 'שאלות נפוצות', 'faq', 'help', 'support' );
        foreach ( $support_words as $word ) {
            if ( false !== mb_strpos( $content_lower, $word ) ) {
                return 'support';
            }
        }

        return 'information';
    }

    /**
     * Process multiple items in batch.
     *
     * @param array $items      Array of content items.
     * @param int   $batch_size Items per batch.
     * @return array Results keyed by item ID.
     */
    public function process_batch( array $items, int $batch_size = 5 ): array {
        $results = array();
        $batches = array_chunk( $items, $batch_size );

        foreach ( $batches as $batch_index => $batch ) {
            BoostBot_Logger::debug( 'Processing batch', array(
                'batch' => $batch_index + 1,
                'total' => count( $batches ),
                'items' => count( $batch ),
            ) );

            foreach ( $batch as $item ) {
                $item_id = $item['id'] ?? uniqid();
                $results[ $item_id ] = $this->process_content( $item );
            }

            // Rate limiting between batches
            if ( $batch_index < count( $batches ) - 1 ) {
                sleep( 1 );
            }
        }

        return $results;
    }

    /**
     * Truncate text deterministically to a max length (word boundary when possible).
     *
     * Used by extract_text_at_level() for safe level-1 extraction.
     *
     * @since 1.14.1
     * @param string $text      Text to truncate.
     * @param int    $max_chars Maximum characters.
     * @return string Truncated text.
     */
    private function truncate_text( string $text, int $max_chars ): string {
        if ( $max_chars <= 0 ) {
            return '';
        }
        if ( mb_strlen( $text ) <= $max_chars ) {
            return $text;
        }
        $truncated  = mb_substr( $text, 0, $max_chars );
        $last_space = mb_strrpos( $truncated, ' ' );
        if ( false !== $last_space && $last_space > (int) ( $max_chars * 0.7 ) ) {
            $truncated = mb_substr( $truncated, 0, $last_space );
        }
        return rtrim( $truncated ) . '...';
    }

    /**
     * Truncate text for API.
     *
     * @param string $text      Text to truncate.
     * @param int    $max_chars Maximum characters.
     * @return string Truncated text.
     */
    private function truncate_for_api( string $text, int $max_chars = 0 ): string {
        if ( 0 === $max_chars ) {
            $max_chars = $this->max_content_length;
        }

        if ( mb_strlen( $text ) <= $max_chars ) {
            return $text;
        }

        // Cut at word boundary
        $truncated = mb_substr( $text, 0, $max_chars );
        $last_space = mb_strrpos( $truncated, ' ' );

        if ( false !== $last_space && $last_space > $max_chars * 0.8 ) {
            $truncated = mb_substr( $truncated, 0, $last_space );
        }

        return $truncated . '...';
    }

    /**
     * Process a WordPress post for indexing with full content extraction.
     *
     * Enhanced in Week 24 to detect service pages and categories for
     * the Global Context Awareness feature.
     *
     * @param int $post_id Post ID.
     * @return array|false Processed data or false on failure.
     */
    public function process_post( int $post_id, array $options = array() ) {
        $post = get_post( $post_id );

        if ( ! $post || 'publish' !== $post->post_status ) {
            return false;
        }

        BoostBot_Logger::debug( 'Processing post for indexing', array(
            'post_id' => $post_id,
            'title'   => $post->post_title,
        ) );

        $level = isset( $options['extraction_level'] ) ? (int) $options['extraction_level'] : 5;
        $level = max( 1, min( 5, $level ) );

        // Extract full content (tiered; memory-safe when level < 5).
        if ( $level >= 5 ) {
            $full_content = $this->extract_full_content( $post );
        } else {
            $full_content = $this->extract_text_at_level( $post, $level );
        }

        // =====================================================
        // MEMORY-EFFICIENT PIPELINE
        // Process content in stages, freeing each stage's data
        // before moving to the next. This keeps peak memory low
        // for shared hosting environments (128-256MB).
        // =====================================================

        // Stage 1: Extract specs and price from raw post_content.
        // After this stage, we no longer need the raw post_content.
        $raw_content = $post->post_content;
        $specs = array();
        if ( $this->specs_extractor ) {
            $specs = $this->specs_extractor->extract( $raw_content, $post->post_type, $post_id );
        }
        $price_info = null;
        if ( $this->price_extractor ) {
            $price_info = $this->price_extractor->extract( $post_id, $raw_content );
        }
        unset( $raw_content ); // Free raw content (can be 1MB+ for Elementor pages).

        // Stage 2: Extract clean content and raw content for storage.
        $clean_raw_content = $this->extract_clean_raw_content( $post, $full_content );

        // Stage 3: Generate AI summary (sends content to backend).
        // Cap content sent to AI to prevent excessive memory usage.
        $content_for_ai = $full_content;
        if ( mb_strlen( $content_for_ai ) > 30000 ) {
            $content_for_ai = mb_substr( $content_for_ai, 0, 30000 );
        }
        $processed = $this->process_content( array(
            'title'   => $post->post_title,
            'content' => $content_for_ai,
            'type'    => $post->post_type,
        ) );
        unset( $content_for_ai ); // Free AI input copy.

        // Stage 4: Build final metadata (uses already-extracted data).
        $enhanced_summary = $this->build_enhanced_summary(
            $post,
            $processed['summary'],
            $specs,
            $price_info
        );

        $content_type = $this->determine_content_type( $post, $specs, $price_info );

        $is_service_page = $this->detect_service_page(
            $post->post_title,
            $full_content,
            $processed['keywords'],
            $specs,
            $price_info,
            $post->post_type
        );

        $service_category = $this->detect_service_category(
            $post->post_title,
            $full_content,
            $processed['keywords']
        );

        // Stage 5: Build result, then free intermediate data.
        $result = array(
            'post_id'          => $post_id,
            'post_type'        => $post->post_type,
            'content_type'     => $content_type,
            'title'            => $post->post_title,
            'url'              => urldecode( get_permalink( $post_id ) ),
            'summary'          => $enhanced_summary,
            'full_content'     => $full_content,
            'raw_content'      => $clean_raw_content,
            'technical_specs'  => $specs,
            'price_info'       => $price_info,
            'keywords'         => $processed['keywords'],
            'intent'           => $processed['intent'],
            'is_service_page'  => $is_service_page,
            'service_category' => $service_category,
        );

        // Free all intermediate variables before returning.
        unset( $full_content, $clean_raw_content, $enhanced_summary, $processed, $specs, $price_info );

        return $result;
    }

    /**
     * Week 24.5: Get deterministic canonical text for hashing / estimating deep indexing.
     *
     * This method MUST NOT call any LLMs. It uses the same extraction path as deep indexing
     * (Elementor-aware full text extraction + HTML cleanup), so content hashes are stable.
     *
     * @since 1.8.0
     * @param int $post_id
     * @return string Canonical extracted text (trimmed). Empty string if not indexable.
     */
    public function get_canonical_text_for_indexing( int $post_id ): string {
        $post = get_post( $post_id );
        if ( ! $post || 'publish' !== $post->post_status ) {
            return '';
        }

        $full = $this->extract_full_content( $post );
        $full = trim( (string) $full );
        if ( '' !== $full ) {
            return $full;
        }

        // Fallback: clean raw post_content.
        $raw = isset( $post->post_content ) ? (string) $post->post_content : '';
        $raw = trim( $this->clean_html_content( $raw ) );
        return $raw;
    }

    /**
     * Week 30: Tiered extraction for resilient indexing.
     *
     * This method MUST NOT call any LLMs. It provides multiple extraction levels
     * to adapt to low-memory environments.
     *
     * Levels:
     * 1 - Title + excerpt only
     * 2 - Raw post_content (no filters, basic cleanup)
     * 3 - Elementor-aware extraction (safe) without WP content filters
     * 4 - Core-only light rendering (do_blocks/wpautop) without plugin filters
     * 5 - Full extraction (Elementor-aware + the_content filters)
     *
     * @since 1.13.0
     * @param int $post_id
     * @param int $level 1..5
     * @return array|false {title, content, type, url, level} or false if not indexable.
     */
    public function extract_at_level( int $post_id, int $level ) {
        $post = get_post( $post_id );
        if ( ! $post || 'publish' !== $post->post_status ) {
            return false;
        }

        $level = max( 1, min( 5, (int) $level ) );
        $text  = $this->extract_text_at_level( $post, $level );
        $text  = trim( (string) $text );

        return array(
            'title'   => (string) $post->post_title,
            'content' => $text,
            'type'    => (string) $post->post_type,
            'url'     => urldecode( get_permalink( $post_id ) ),
            'level'   => $level,
        );
    }

    /**
     * Week 30: Deterministic canonical text for a specific extraction level.
     *
     * @since 1.13.0
     * @param int $post_id
     * @param int $level 1..5
     * @return string
     */
    public function get_canonical_text_for_indexing_level( int $post_id, int $level ): string {
        $post = get_post( $post_id );
        if ( ! $post || 'publish' !== $post->post_status ) {
            return '';
        }
        $level = max( 1, min( 5, (int) $level ) );
        $text  = $this->extract_text_at_level( $post, $level );
        return trim( (string) $text );
    }

    /**
     * Extract text for a given level (no LLM calls).
     *
     * @since 1.13.0
     * @param WP_Post $post
     * @param int     $level
     * @return string
     */
    private function extract_text_at_level( WP_Post $post, int $level ): string {
        // Level 1: title + excerpt only (always safe).
        if ( 1 === $level ) {
            $excerpt = isset( $post->post_excerpt ) ? (string) $post->post_excerpt : '';
            if ( '' === trim( $excerpt ) ) {
                $excerpt = isset( $post->post_content ) ? (string) $post->post_content : '';
            }
            $excerpt = $this->clean_html_content( $excerpt );
            $excerpt = $this->truncate_text( $excerpt, 2000 );
            return trim( (string) $post->post_title . "\n\n" . $excerpt );
        }

        // Level 2: raw post_content (no filters; basic cleanup + shortcode stripping).
        if ( 2 === $level ) {
            $raw = isset( $post->post_content ) ? (string) $post->post_content : '';
            if ( function_exists( 'strip_shortcodes' ) ) {
                $raw = strip_shortcodes( $raw );
            }
            $text = trim( $this->clean_html_content( $raw ) );
            unset( $raw );
            return mb_strlen( $text ) > 100000 ? mb_substr( $text, 0, 100000 ) : $text;
        }

        // Level 3: Elementor-aware extraction (safe) without WP filters.
        if ( 3 === $level ) {
            // Try Elementor first when safe (extract_full_content() already guards large _elementor_data).
            if ( $this->elementor_parser ) {
                try {
                    $elementor_content = $this->elementor_parser->extract_content( (int) $post->ID );
                    if ( $elementor_content ) {
                        return trim( $this->clean_html_content( (string) $elementor_content ) );
                    }
                } catch ( Exception $e ) {
                    // Fall through to raw extraction.
                }
            }
            return $this->extract_text_at_level( $post, 2 );
        }

        // Level 4: core-only light rendering (avoid plugin filters).
        if ( 4 === $level ) {
            $content = isset( $post->post_content ) ? (string) $post->post_content : '';
            // Best-effort: render blocks without invoking full the_content filter chain.
            if ( function_exists( 'do_blocks' ) ) {
                $content = do_blocks( $content );
            }
            if ( function_exists( 'wpautop' ) ) {
                $content = wpautop( $content );
            }
            if ( function_exists( 'shortcode_unautop' ) ) {
                $content = shortcode_unautop( $content );
            }
            $text = trim( $this->clean_html_content( $content ) );
            unset( $content );
            return mb_strlen( $text ) > 100000 ? mb_substr( $text, 0, 100000 ) : $text;
        }

        // Level 5: full extraction (Elementor-aware + WP content filters).
        return trim( $this->extract_full_content( $post ) );
    }

    /**
     * Extract clean raw content for detailed queries.
     *
     * This preserves the full text content without AI summarization,
     * useful for technical queries that need specific details.
     *
     * @since 1.6.0 (Week 24)
     * @param WP_Post $post         Post object.
     * @param string  $full_content Already extracted full content.
     * @return string Clean raw content (limited to 10000 chars).
     */
    private function extract_clean_raw_content( WP_Post $post, string $full_content ): string {
        // Use the already-extracted full content if available
        $content = $full_content;

        // If full content is empty, try post_content directly
        if ( empty( $content ) ) {
            $content = $this->clean_html_content( $post->post_content );
        }

        // Limit to reasonable size for database storage
        $max_raw_length = 10000;

        if ( mb_strlen( $content ) > $max_raw_length ) {
            $content = mb_substr( $content, 0, $max_raw_length );
            // Try to cut at a sentence boundary
            $last_period = mb_strrpos( $content, '.' );
            if ( $last_period !== false && $last_period > $max_raw_length * 0.8 ) {
                $content = mb_substr( $content, 0, $last_period + 1 );
            }
        }

        return $content;
    }

    /**
     * Detect if content is a service/product page.
     *
     * Uses heuristics and indicators to determine if this page
     * describes a service or product offering.
     *
     * @since 1.6.0 (Week 24)
     * @param string     $title      Page title.
     * @param string     $content    Page content.
     * @param array      $keywords   Extracted keywords.
     * @param array      $specs      Technical specs.
     * @param array|null $price_info Price information.
     * @param string     $post_type  Post type.
     * @return bool True if this is a service page.
     */
    private function detect_service_page(
        string $title,
        string $content,
        array $keywords,
        array $specs,
        ?array $price_info,
        string $post_type
    ): bool {
        // Products are always service pages
        if ( 'product' === $post_type ) {
            return true;
        }

        // Use Service Catalog helper if available
        if ( class_exists( 'BoostBot_Service_Catalog' ) ) {
            return BoostBot_Service_Catalog::is_service_content( $title, $content, $keywords );
        }

        // Fallback: use heuristics
        $score = 0;

        // Has price info = likely service
        if ( ! empty( $price_info ) ) {
            $score += 3;
        }

        // Has technical specs = likely service
        if ( ! empty( $specs ) ) {
            $score += 2;
        }

        // Check for service-related keywords
        $service_indicators = array(
            'שירות', 'שירותים', 'מחיר', 'מחירון', 'חבילה', 'חבילות',
            'מסלול', 'פתרון', 'הצעה', 'עלות', 'תעריף', 'כולל',
            'service', 'services', 'pricing', 'package', 'solution',
        );

        $combined = mb_strtolower( $title . ' ' . $content );
        foreach ( $service_indicators as $indicator ) {
            if ( mb_strpos( $combined, $indicator ) !== false ) {
                $score++;
            }
        }

        // Threshold
        return $score >= 3;
    }

    /**
     * Detect service category for content.
     *
     * @since 1.6.0 (Week 24)
     * @param string $title    Page title.
     * @param string $content  Page content.
     * @param array  $keywords Extracted keywords.
     * @return string Service category.
     */
    private function detect_service_category( string $title, string $content, array $keywords ): string {
        // Use Service Catalog helper if available
        if ( class_exists( 'BoostBot_Service_Catalog' ) ) {
            // Detect language from content
            $language = $this->detect_content_language( $content );
            return BoostBot_Service_Catalog::detect_category( $title, $content, $language );
        }

        // Fallback to 'general'
        return 'general';
    }

    /**
     * Detect content language.
     *
     * @param string $content Content text.
     * @return string Language code ('he' or 'en').
     */
    private function detect_content_language( string $content ): string {
        // Simple Hebrew detection: check for Hebrew characters
        if ( preg_match( '/[\x{0590}-\x{05FF}]/u', $content ) ) {
            return 'he';
        }
        return 'en';
    }

    /**
     * Extract full content from post.
     *
     * @param WP_Post $post Post object.
     * @return string Full content.
     */
    private function extract_full_content( WP_Post $post ): string {
        // Some Elementor pages can have multi-MB `_elementor_data` which can OOM PHP when parsed/rendered.
        // We detect size via SQL (no need to load the full meta into PHP).
        $skip_wp_content_filters = false;
        $max_elementor_data_chars = (int) apply_filters( 'boostbot_elementor_data_max_chars', 300000 );
        $elementor_data_len       = 0;
        try {
            global $wpdb;
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared
            $elementor_data_len = (int) $wpdb->get_var(
                // phpcs:ignore WordPress.DB.PreparedSQL.NotPrepared
                $wpdb->prepare(
                    // phpcs:ignore WordPress.DB.PreparedSQL.InterpolatedNotPrepared
                    "SELECT LENGTH(meta_value) FROM {$wpdb->postmeta} WHERE post_id = %d AND meta_key = %s LIMIT 1",
                    (int) $post->ID,
                    '_elementor_data'
                )
            );
        } catch ( Exception $e ) {
            $elementor_data_len = 0;
        }

        if ( $elementor_data_len > $max_elementor_data_chars ) {
            BoostBot_Logger::warning( 'Skipping Elementor parse due to large _elementor_data', array(
                'post_id'      => (int) $post->ID,
                'data_len'     => $elementor_data_len,
                'max_data_len' => $max_elementor_data_chars,
            ) );
            // Elementor can hook into `the_content` and still parse/render the large data, so skip filters too.
            $skip_wp_content_filters = true;
        }

        // Try Elementor first (only when safe).
        if ( $this->elementor_parser && ! $skip_wp_content_filters ) {
            $elementor_content = $this->elementor_parser->extract_content( $post->ID );
            if ( $elementor_content ) {
                return $elementor_content;
            }
        }

        // Fall back to regular content.
        $content = $post->post_content;

        // Check memory before applying heavy content filters.
        $mem_limit = function_exists( 'wp_convert_hr_to_bytes' )
            ? wp_convert_hr_to_bytes( ini_get( 'memory_limit' ) )
            : 536870912; // 512MB fallback.
        $mem_used  = memory_get_usage( true );
        if ( $mem_used > (int) ( $mem_limit * 0.6 ) ) { // 60% threshold.
            $skip_wp_content_filters = true;
            BoostBot_Logger::warning( 'Skipping content filters due to high memory usage', array(
                'post_id'   => (int) $post->ID,
                'mem_used'  => round( $mem_used / 1048576 ) . 'MB',
                'mem_limit' => round( $mem_limit / 1048576 ) . 'MB',
            ) );
        }

        // Apply WordPress content filters (unless we intentionally skip them for safety).
        if ( ! $skip_wp_content_filters ) {
            // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- WordPress core filter; not plugin-owned.
            $content = apply_filters( 'the_content', $content );
        }

        // Clean HTML.
        $content = $this->clean_html_content( $content );

        // Cap extracted content to prevent downstream OOM in chunker/AI.
        // 100K chars of visible text is more than enough for meaningful indexing.
        $max_extracted_chars = (int) apply_filters( 'boostbot_max_extracted_content_chars', 100000 );
        if ( mb_strlen( $content ) > $max_extracted_chars ) {
            $content = mb_substr( $content, 0, $max_extracted_chars );
        }

        return $content;
    }

    /**
     * Clean HTML to readable text.
     *
     * @param string $html HTML content.
     * @return string Clean text.
     */
    private function clean_html_content( string $html ): string {
        // Remove scripts, styles, comments.
        $html = preg_replace( '/<script\b[^>]*>.*?<\/script>/is', '', $html );
        $html = preg_replace( '/<style\b[^>]*>.*?<\/style>/is', '', $html );
        $html = preg_replace( '/<!--.*?-->/s', '', $html );

        // Convert block elements to newlines.
        $block_elements = array( 'p', 'div', 'section', 'article', 'header', 'footer', 'h[1-6]', 'li', 'tr' );
        foreach ( $block_elements as $tag ) {
            $html = preg_replace( '/<\/' . $tag . '>/i', "\n", $html );
        }

        // Convert <br> to newlines.
        $html = preg_replace( '/<br\s*\/?>/i', "\n", $html );

        // Strip remaining tags.
        $text = wp_strip_all_tags( $html );

        // Decode entities.
        $text = html_entity_decode( $text, ENT_QUOTES, 'UTF-8' );

        // Normalize whitespace.
        $text = preg_replace( '/[ \t]+/', ' ', $text );
        $text = preg_replace( '/\n\s*\n/', "\n\n", $text );

        return trim( $text );
    }

    /**
     * Build enhanced summary with specs and prices.
     *
     * @param WP_Post    $post       Post object.
     * @param string     $summary    Base summary.
     * @param array      $specs      Technical specs.
     * @param array|null $price_info Price info.
     * @return string Enhanced summary.
     */
    private function build_enhanced_summary( WP_Post $post, string $summary, array $specs, ?array $price_info ): string {
        $lang  = class_exists( 'BoostBot_Language' ) ? BoostBot_Language::get_primary() : 'he';
        $is_he = ( 'he' === $lang );

        $parts = array();

        $parts[] = '# ' . $post->post_title;

        if ( $summary ) {
            $parts[] = $summary;
        }

        if ( ! empty( $specs ) ) {
            $parts[] = $is_he ? "\n## מפרט טכני:" : "\n## Technical Specifications:";
            foreach ( $specs as $key => $value ) {
                $parts[] = "• {$key}: {$value}";
            }
        }

        if ( ! empty( $price_info ) ) {
            $parts[] = $is_he ? "\n## מחיר:" : "\n## Price:";

            if ( isset( $price_info['display_text'] ) ) {
                $parts[] = $price_info['display_text'];
            } elseif ( isset( $price_info['display'] ) ) {
                $parts[] = wp_strip_all_tags( $price_info['display'] );
            }

            if ( ! empty( $price_info['on_sale'] ) ) {
                $parts[] = $is_he ? '🔥 במבצע!' : '🔥 On Sale!';
            }

            if ( isset( $price_info['stock_text'] ) ) {
                $parts[] = $price_info['stock_text'];
            }
        }

        return implode( "\n", $parts );
    }

    /**
     * Determine content type for categorization.
     *
     * @param WP_Post    $post       Post object.
     * @param array      $specs      Technical specs.
     * @param array|null $price_info Price info.
     * @return string Content type.
     */
    private function determine_content_type( WP_Post $post, array $specs, ?array $price_info ): string {
        $post_type = $post->post_type;

        if ( 'product' === $post_type ) {
            return 'product';
        }

        if ( 'post' === $post_type ) {
            return 'article';
        }

        // Check for service pages.
        if ( $price_info && ! empty( $specs ) ) {
            return 'service';
        }

        if ( ! empty( $specs ) ) {
            return 'technical';
        }

        return 'page';
    }

    /**
     * Get full content and metadata for a page.
     *
     * @param int $post_id Post ID.
     * @return array|null Page data or null.
     */
    public function get_page_context( int $post_id ): ?array {
        $post = get_post( $post_id );

        if ( ! $post ) {
            return null;
        }

        // Extract full content.
        $full_content = $this->extract_full_content( $post );

        // Extract specs.
        $specs = array();
        if ( $this->specs_extractor ) {
            $specs = $this->specs_extractor->extract( $post->post_content, $post->post_type, $post_id );
        }

        // Extract price.
        $price_info = null;
        if ( $this->price_extractor ) {
            $price_info = $this->price_extractor->extract( $post_id, $post->post_content );
        }

        return array(
            'title'           => $post->post_title,
            'url'             => urldecode( get_permalink( $post_id ) ),
            'full_content'    => $full_content,
            'technical_specs' => $specs,
            'price_info'      => $price_info,
        );
    }

    /**
     * Format page context for AI prompt.
     *
     * @param array  $context  Page context.
     * @param string $language Language code (default: site primary).
     * @return string Formatted context.
     */
    public function format_context_for_prompt( array $context, string $language = '' ): string {
        if ( '' === $language && class_exists( 'BoostBot_Language' ) ) {
            $language = BoostBot_Language::get_primary();
        }
        $is_he = ( 'he' === $language || '' === $language );

        $parts = array();

        $parts[] = $is_he ? '=== העמוד הנוכחי שבו המשתמש נמצא ===' : '=== Current Page the User Is Viewing ===';
        $parts[] = ( $is_he ? 'כותרת: ' : 'Title: ' ) . ( $context['title'] ?? '' );
        $parts[] = ( $is_he ? 'קישור: ' : 'Link: ' ) . urldecode( (string) ( $context['url'] ?? '' ) );
        $parts[] = '';
        $parts[] = $is_he ? 'תוכן העמוד המלא:' : 'Full page content:';
        $parts[] = $this->truncate_for_api( $context['full_content'] ?? '', 4000 );

        if ( ! empty( $context['technical_specs'] ) ) {
            $parts[] = '';
            $parts[] = $is_he ? 'מפרט טכני:' : 'Technical specifications:';
            foreach ( $context['technical_specs'] as $key => $value ) {
                $parts[] = "• {$key}: {$value}";
            }
        }

        if ( ! empty( $context['price_info'] ) ) {
            $parts[] = '';
            $parts[] = $is_he ? 'מידע מחיר:' : 'Pricing information:';

            if ( isset( $context['price_info']['display_text'] ) ) {
                $parts[] = $context['price_info']['display_text'];
            } elseif ( isset( $context['price_info']['display'] ) ) {
                $parts[] = wp_strip_all_tags( $context['price_info']['display'] );
            }

            if ( ! empty( $context['price_info']['on_sale'] ) ) {
                $parts[] = $is_he ? '(במבצע!)' : '(On sale!)';
            }
        }

        $parts[] = '';
        $parts[] = $is_he ? '=== סוף מידע על העמוד הנוכחי ===' : '=== End of Current Page Information ===';

        return implode( "\n", $parts );
    }
}


