<?php
if ( ! defined( 'ABSPATH' ) ) exit;

/**
 * Get the WPML frontend domain for the current language.
 *
 * @return string|null The domain for the current language or null if WPML is not active
 */
function get_wpml_frontend_domain_for_current_lang() {
    // Check if WPML is active using more common detection methods
    if ( defined('WPML_PLUGIN_PATH') || function_exists('icl_get_languages') ) {
        $current_lang = isset($_GET['lang']) ? sanitize_text_field($_GET['lang']) : null;

        if ( ! $current_lang ) {
            $current_lang = apply_filters( 'wpml_admin_language', null );
        }
        if ( ! $current_lang ) {
            $current_lang = apply_filters( 'wpml_current_language', null );
        }

        // Try different WPML domain functions
        if ( function_exists('wpml_get_domain_for_language') ) {
            return wpml_get_domain_for_language( $current_lang );
        }
        
        // Alternative method using WPML settings
        if ( $current_lang && function_exists('icl_get_languages') ) {
            $languages = icl_get_languages('skip_missing=0');
            if ( isset($languages[$current_lang]) && !empty($languages[$current_lang]['url']) ) {
                return $languages[$current_lang]['url'];
            }
        }
    }

    return null;
}

/**
 * Fetch and cache (per-request) all Morningscore domain entries that match
 * this site's hostname. Returns an array of raw domain objects from the API,
 * each containing at minimum: global_domain_identifier, domain, hl, gl.
 *
 * Uses WPML frontend domain detection when available.
 *
 * Returns WP_Error on API failure.
 *
 * @return array|WP_Error
 */
function rank_fetch_domain_candidates() {
    static $cache = null;
    if ( $cache !== null ) {
        return $cache;
    }

    $wpml_domain = get_wpml_frontend_domain_for_current_lang();
    if ( $wpml_domain ) {
        $site_domain = parse_url( $wpml_domain, PHP_URL_HOST );
    } else {
        $site_domain = parse_url( home_url(), PHP_URL_HOST );
    }
    $site_domain           = preg_replace( '/^www\./', '', $site_domain );
    $api_url               = rank_get_api_base_url() . '/domains';

    $response = wp_remote_get( $api_url, array(
        'headers'    => array(
            'Authorization' => 'Bearer ' . rank_get_api_key(),
            'Accept'        => 'application/json',
        ),
        'timeout'    => 15,
        'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
    ) );

    if ( is_wp_error( $response ) ) {
        $cache = $response;
        return $cache;
    }

    $code = wp_remote_retrieve_response_code( $response );
    $body = json_decode( wp_remote_retrieve_body( $response ), true );

    if ( $code !== 200 || ! is_array( $body ) ) {
        $cache = new WP_Error( 'api_error', 'Failed to retrieve domains. HTTP ' . $code );
        return $cache;
    }

    // Keep only entries matching this site's hostname
    $candidates = array();
    foreach ( $body as $entry ) {
        if ( isset( $entry['domain'] ) ) {
            $normalized = preg_replace( '/^www\./', '', $entry['domain'] );
            if ( $normalized === $site_domain ) {
                $candidates[] = $entry;
            }
        }
    }

    $cache = $candidates;
    return $cache;
}

/**
 * Given a WPML language code (e.g. "en", "da"), return the Morningscore
 * global_domain_identifier whose `hl` field matches it.
 *
 * Falls back to the first candidate for the site's domain if no hl match
 * is found (handles non-WPML sites and unknown language codes gracefully).
 *
 * @param string|null $lang  WPML language code, or null to use current context.
 * @return string|WP_Error
 */
function rank_get_domain_identifier_for_lang( $lang = null ) {
    $candidates = rank_fetch_domain_candidates();
    if ( is_wp_error( $candidates ) ) {
        return $candidates;
    }
    if ( empty( $candidates ) ) {
        return new WP_Error( 'domain_not_found', 'Your website domain was not found in your Morningscore account' );
    }

    // Default to first entry (no-WPML fallback)
    $match = $candidates[0];

    if ( $lang ) {
        foreach ( $candidates as $candidate ) {
            if ( isset( $candidate['hl'] ) && strtolower( $candidate['hl'] ) === strtolower( $lang ) ) {
                $match = $candidate;
                break;
            }
        }
    }

    return $match['global_domain_identifier'];
}

function rank_get_domain_identifier() {
    Rank_Processor::add_debug_log("=== DOMAIN IDENTIFIER REQUEST START ===");

    $wpml_domain = get_wpml_frontend_domain_for_current_lang();
    if ( $wpml_domain ) {
        $site_domain = parse_url( $wpml_domain, PHP_URL_HOST );
        $domain_source = 'WPML frontend domain';
    } else {
        $site_domain = parse_url( home_url(), PHP_URL_HOST );
        $domain_source = 'WordPress home_url()';
    }

    Rank_Processor::add_debug_log("Site domain: $site_domain (source: $domain_source)");

    $candidates = rank_fetch_domain_candidates();
    if ( is_wp_error( $candidates ) ) {
        Rank_Processor::add_debug_log("WP_Error getting domains: " . $candidates->get_error_message());
        Rank_Processor::add_debug_log("Error code: " . $candidates->get_error_code());
        Rank_Processor::add_debug_log("=== DOMAIN IDENTIFIER REQUEST END (ERROR) ===");
        return $candidates;
    }

    if ( empty( $candidates ) ) {
        $normalized_site_domain = preg_replace( '/^www\./', '', $site_domain );
        Rank_Processor::add_debug_log("Domain not found in Morningscore account. Site: $normalized_site_domain");
        Rank_Processor::add_debug_log("=== DOMAIN IDENTIFIER REQUEST END (NOT FOUND) ===");
        return new WP_Error( 'domain_not_found', 'Your website domain was not found in your Morningscore account' );
    }

    $current_lang = null;
    if ( defined( 'WPML_PLUGIN_PATH' ) || function_exists( 'icl_get_languages' ) ) {
        $current_lang = isset( $_GET['lang'] ) ? sanitize_text_field( $_GET['lang'] ) : null;
        if ( ! $current_lang ) {
            $current_lang = apply_filters( 'wpml_admin_language', null );
        }
        if ( ! $current_lang ) {
            $current_lang = apply_filters( 'wpml_current_language', null );
        }
    }

    $normalized_site_domain = preg_replace( '/^www\./', '', $site_domain );
    Rank_Processor::add_debug_log("Looking for domain: $normalized_site_domain (current_lang: " . ( $current_lang ?: 'none' ) . ")");
    Rank_Processor::add_debug_log("Available domains from API: " . json_encode(array_map(function($d) { return ( $d['domain'] ?? 'N/A' ) . ' hl=' . ( $d['hl'] ?? '?' ) . ' gl=' . ( $d['gl'] ?? '?' ); }, $candidates)));

    $matched_candidate = $candidates[0];
    if ( $current_lang ) {
        foreach ( $candidates as $candidate ) {
            if ( isset( $candidate['hl'] ) && strtolower( $candidate['hl'] ) === strtolower( $current_lang ) ) {
                $matched_candidate = $candidate;
                break;
            }
        }
    }

    $domain_identifier = $matched_candidate['global_domain_identifier'];

    Rank_Processor::add_debug_log(
        "Selected domain: " . $matched_candidate['domain'] .
        " hl=" . ( $matched_candidate['hl'] ?? '?' ) .
        " gl=" . ( $matched_candidate['gl'] ?? '?' ) .
        " identifier=" . $domain_identifier .
        " (from " . count( $candidates ) . " candidate(s), current_lang=" . ( $current_lang ?: 'none' ) . ")"
    );

    Rank_Processor::add_debug_log("Domain identifier found: $domain_identifier");
    Rank_Processor::add_debug_log("=== DOMAIN IDENTIFIER REQUEST END (SUCCESS) ===");

    return $domain_identifier;
}

/**
 * Get the AI credits status from the Morningscore API.
 * 
 * @return array|WP_Error Array with credit info or error
 */
function rank_get_ai_credits() {
    $response = wp_remote_get(
        rank_get_api_base_url() . '/credits/get-status?credit_type=credits.ai.monthly_ai_credits',
        array(
            'headers' => array(
                'Authorization' => 'Bearer ' . rank_get_api_key(),
                'Accept' => 'application/json',
            ),
            'timeout' => 15,
            'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
        )
    );

    if ( is_wp_error( $response ) ) {
        return $response;
    }

    $response_code = wp_remote_retrieve_response_code( $response );
    $response_body = wp_remote_retrieve_body( $response );
    $credits = json_decode( $response_body, true );
    
    // Check for inactive credit type error (domain might be inactive)
    if ( is_array( $credits ) && isset( $credits['error'] ) ) {
        $error_message = $credits['error'];
        
        // Check for specific "Inactive credit type" error
        if ( strpos( strtolower( $error_message ), 'inactive' ) !== false ) {
            Rank_Processor::add_debug_log("Credits API returned inactive error: " . $error_message);
            return new WP_Error( 
                'inactive_account', 
                'Your Morningscore account or domain appears to be inactive.',
                array( 
                    'response_body' => $response_body, 
                    'status_code' => $response_code,
                    'original_error' => $error_message
                )
            );
        }
        
        // Generic error from API
        Rank_Processor::add_debug_log("Credits API returned error: " . $error_message);
        return new WP_Error( 
            'api_error', 
            'Error from credits API: ' . $error_message,
            array( 'response_body' => $response_body, 'status_code' => $response_code )
        );
    }
    
    if ( $response_code !== 200 ) {
        return new WP_Error( 
            'api_error', 
            'Failed to retrieve AI credits. Status code: ' . $response_code,
            array( 'response_body' => $response_body, 'status_code' => $response_code )
        );
    }
    
    if ( ! is_array( $credits ) || ! isset( $credits['limit'] ) || ! isset( $credits['usage'] ) ) {
        return new WP_Error( 
            'api_error', 
            'Invalid response format when fetching AI credits',
            array( 'response_body' => $response_body, 'status_code' => $response_code )
        );
    }

    return $credits;
}


/**
 * Get a chunk of issues for pagination.
 *
 * @param string $issue_type The issue type to fetch
 * @param int $limit Number of issues to return
 * @param int $offset Starting offset
 * @param string $domain_identifier Domain identifier
 * @return array Array with 'issues' and 'total' keys
 */
function rank_get_issues_chunk( $issue_type, $limit, $offset, $domain_identifier ) {
    // Calculate which API pages we need to fetch
    $api_per_page = 100;
    $start_page = floor( $offset / $api_per_page ) + 1;
    $end_page = ceil( ( $offset + $limit ) / $api_per_page );
    
    $all_issues = array();
    $skipped_count = 0;
    $total_count = 0;
    
    // Fetch the required API pages
    for ( $page = $start_page; $page <= $end_page; $page++ ) {
        $url = rank_get_api_base_url() . "/{$domain_identifier}/get_issues";
        $query_args = array(
            'filter.issue_type' => $issue_type,
            'filter.resolved' => 'false',
            'filter.ignored' => 'false',
            'pagination.page' => $page,
            'pagination.per_page' => $api_per_page
        );

        $response = wp_remote_get(
            add_query_arg( $query_args, $url ),
            array(
                'headers' => array(
                    'Authorization' => 'Bearer ' . rank_get_api_key(),
                    'Accept' => 'application/json',
                ),
                'timeout' => 60,
                'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
            )
        );

        if ( is_wp_error( $response ) ) {
            return $response;
        }

        $response_code = wp_remote_retrieve_response_code( $response );
        if ( $response_code !== 200 ) {
            return new WP_Error(
                'api_error',
                'Failed to retrieve issues. Status code: ' . $response_code
            );
        }

        $result = json_decode( wp_remote_retrieve_body( $response ), true );
        
        if ( ! is_array( $result ) || ! isset( $result['data'] ) || ! is_array( $result['data'] ) ) {
            return new WP_Error( 'api_error', 'Invalid response format when fetching issues' );
        }

        // Capture total count from first page response
        if ( $page === $start_page && isset( $result['total'] ) ) {
            $total_count = intval( $result['total'] );
        }

        // Light filtering: only check if URL belongs to this WordPress installation (host/path match)
        // Don't do expensive HTTP validation here - that will happen during processing if needed
        $wp_home_url = home_url();
        $wp_base_path = parse_url($wp_home_url, PHP_URL_PATH);
        $wp_host = parse_url($wp_home_url, PHP_URL_HOST);
        
        // Normalize host for comparison (remove www. prefix)
        $normalize_host = function($host) {
            if (empty($host)) return '';
            $host = strtolower($host);
            if (strpos($host, 'www.') === 0) {
                return substr($host, 4);
            }
            return $host;
        };
        $wp_host_normalized = $normalize_host($wp_host);
        
        foreach ($result['data'] as $issue) {
            // Get the page URL from the issue data
            $page_url = null;
            if (isset($issue['page']) && isset($issue['page']['url'])) {
                $page_url = $issue['page']['url'];
            } elseif (isset($issue['page']) && isset($issue['page']['path_full'])) {
                $page_url = $issue['page']['path_full'];
            } elseif (isset($issue['url'])) {
                $page_url = $issue['url'];
            }

            if (!empty($page_url)) {
                // Quick host check only - no expensive HTTP validation
                // Be very lenient with path matching to support WPML and other multilingual setups
                $url_host = parse_url($page_url, PHP_URL_HOST);
                
                // Normalize and check host match (handle www vs non-www)
                $url_host_normalized = $normalize_host($url_host);
                
                // Skip only if host clearly doesn't match
                // Don't do strict path matching here - WPML and other multilingual plugins
                // can have different path structures (e.g., /da/, /en/) that are still valid
                if ($url_host_normalized !== $wp_host_normalized) {
                    $skipped_count++;
                    continue;
                }
                
                // Host matches - add the issue
                // Path validation will happen later during processing if needed
                // This allows WPML language prefixes and other path variations to pass through
                $all_issues[] = $issue;
            } else {
                // No URL found in the issue data, add it anyway
                $all_issues[] = $issue;
            }
        }
    }
    
    // Apply offset and limit to the filtered results
    $chunk_start = $offset % $api_per_page;
    if ( $start_page < $end_page ) {
        // We fetched multiple pages, need to calculate the correct offset
        $chunk_start = $offset - ( ( $start_page - 1 ) * $api_per_page );
    }
    
    $result_issues = array_slice( $all_issues, $chunk_start, $limit );
    
    // Always return both issues and total count since every API response includes the total
    return array(
        'issues' => $result_issues,
        'total' => $total_count
    );
}

/**
 * Get AI fix for a specific issue.
 *
 * @param string $rank_id The rank ID of the issue
 * @param string $domain_identifier Optional domain identifier (if not provided, it will be fetched)
 * @return array|WP_Error The AI fix or error
 */
function rank_get_ai_fix( $rank_id, $domain_identifier = null ) {
    if ( ! $domain_identifier ) {
        $domain_identifier = rank_get_domain_identifier();
        if ( is_wp_error( $domain_identifier ) ) {
            return $domain_identifier;
        }
    }

    $url = rank_get_api_base_url() . "/{$domain_identifier}/get_ai_fix";
    
    $request_body = array(
        'rank_id' => $rank_id,
        'fresh_data' => null
    );
    
    // Initial API call to get task_id
    $response = wp_remote_post(
        $url,
        array(
            'headers' => array(
                'Authorization' => 'Bearer ' . rank_get_api_key(),
                'Content-Type' => 'application/json',
                'Accept' => 'application/json',
            ),
            'body' => wp_json_encode($request_body),
            'timeout' => 15, // Reduced timeout since we're just getting a task ID
            'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
        )
    );

    if ( is_wp_error( $response ) ) {
        return $response;
    }

    $response_code = wp_remote_retrieve_response_code( $response );
    $response_body = wp_remote_retrieve_body( $response );
    
    $decoded_body = json_decode( $response_body, true );

    if ( $response_code !== 200 ) {
        // Specifically check for "OUT_OF_CREDITS"
        if ( isset( $decoded_body['message'] ) && $decoded_body['message'] === 'OUT_OF_CREDITS' ) {
            return new WP_Error(
                'out_of_credits',
                'AI credits exhausted. API Message: OUT_OF_CREDITS',
                array( 'status_code' => $response_code ) // Pass status code
            );
        }
        
        // Handle other API errors
        $error_message = isset( $decoded_body['message'] ) ? $decoded_body['message'] : 'Failed to retrieve AI fix.';
        return new WP_Error(
            'api_error',
            "Failed to retrieve AI fix. Status code: $response_code. Message: $error_message",
            array( 'status_code' => $response_code ) // Pass status code
        );
    }

    // Check if we have a task_id in the response
    if ( ! is_array( $decoded_body ) || ! isset( $decoded_body['task_id'] ) ) {
        return new WP_Error(
            'api_error',
            'Invalid response format when fetching AI fix - missing task_id.',
            array( 'status_code' => $response_code )
        );
    }

    $task_id = $decoded_body['task_id'];

    // Detailed logging so intermittent server-side "failed" results can be
    // traced back to the exact request/task that produced them.
    Rank_Processor::add_debug_log( "=== AI FIX REQUEST ===" );
    Rank_Processor::add_debug_log( "get_ai_fix URL: " . $url );
    Rank_Processor::add_debug_log( "Request rank_id: " . $rank_id );
    Rank_Processor::add_debug_log( "Response code: " . $response_code );
    Rank_Processor::add_debug_log( "Returned task_id: " . ( is_string( $task_id ) ? $task_id : json_encode( $task_id ) ) );
    Rank_Processor::add_debug_log( "Full initial response: " . substr( $response_body, 0, 1000 ) );

    // The task_id is a base64-encoded blob carrying the issue payload. Decode it
    // (best-effort) so the log shows the validator + issue_identifier the task is
    // actually for — invaluable when the server later returns {"status":"failed"}.
    if ( is_string( $task_id ) ) {
        $decoded_task = json_decode( base64_decode( $task_id ), true );
        if ( is_array( $decoded_task ) && isset( $decoded_task['payload'] ) ) {
            $inner = json_decode( $decoded_task['payload'], true );
            if ( is_array( $inner ) ) {
                Rank_Processor::add_debug_log( "Decoded task payload: " . json_encode( $inner ) );
            }
        }
    }

    // Poll for the result
    $result = rank_get_ai_fix_status($task_id);
    
    return $result;
}

/**
 * Forgiving normaliser for the AI fix "content".
 *
 * Models sometimes wrap an otherwise-valid JSON payload in chatter such as a
 * leading "#Output:" label or ```json ``` code fences. Rather than throwing the
 * whole response away, we look for the JSON inside the string and decode it.
 *
 * If the content is already structured (array/object) or is plain text with no
 * embedded JSON, it is returned unchanged.
 *
 * @param mixed $content Raw content from the API response.
 * @return mixed Decoded array/object when JSON is found, otherwise the (trimmed) original.
 */
function rank_clean_ai_fix_content( $content ) {
    // Only strings can carry the "#Output:" / code-fence noise.
    if ( ! is_string( $content ) ) {
        return $content;
    }

    $trimmed = trim( $content );
    if ( $trimmed === '' ) {
        return $content;
    }

    // If it already parses as JSON, use that directly.
    $decoded = json_decode( $trimmed, true );
    if ( json_last_error() === JSON_ERROR_NONE && ( is_array( $decoded ) || is_object( $decoded ) ) ) {
        return $decoded;
    }

    // Otherwise, extract the first JSON array/object embedded in the string.
    // This handles prefixes like "#Output:", "Output:", or ```json fences.
    if ( preg_match( '/(\[.*\]|\{.*\})/s', $trimmed, $m ) ) {
        $candidate = json_decode( $m[0], true );
        if ( json_last_error() === JSON_ERROR_NONE && ( is_array( $candidate ) || is_object( $candidate ) ) ) {
            Rank_Processor::add_debug_log( 'rank_clean_ai_fix_content: recovered JSON from wrapped content.' );
            return $candidate;
        }
    }

    // No usable JSON found — return the trimmed text so downstream string
    // handling still works.
    return $trimmed;
}

/**
 * Poll for AI fix status until completion or timeout.
 *
 * @param string $task_id The task ID to check
 * @param int $max_attempts Maximum number of polling attempts (default: 18 - covers ~35 seconds with 2-second intervals)
 * @param int $poll_interval Interval between polls in seconds (default: 2)
 * @return array|WP_Error The AI fix result or error
 */
function rank_get_ai_fix_status($task_id, $max_attempts = 18, $poll_interval = 2) {
    // Correct endpoint URL format for status check
    $status_url = rank_get_api_base_url() . "/get_ai_fix_status/{$task_id}";
    
    for ($attempt = 1; $attempt <= $max_attempts; $attempt++) {
        // Add retry logic for each poll attempt
        $poll_result = rank_api_request_with_retry($status_url, 3);
        
        if (is_wp_error($poll_result)) {
            return $poll_result;
        }
        
        $response_code = $poll_result['response_code'];
        $result = $poll_result['data'];
        
        if ($response_code !== 200) {
            return new WP_Error(
                'api_error',
                "Failed to retrieve AI fix status. Status code: $response_code.",
                array('status_code' => $response_code)
            );
        }
        
        if (!is_array($result)) {
            return new WP_Error(
                'api_error',
                'Invalid response format when fetching AI fix status.',
                array('status_code' => $response_code)
            );
        }

        // Check if the task is completed or failed
        Rank_Processor::add_debug_log("=== AI FIX STATUS POLL (attempt $attempt) ===");
        Rank_Processor::add_debug_log("Task ID: $task_id");
        Rank_Processor::add_debug_log("Response status: " . ($result['status'] ?? 'NOT_SET'));
        Rank_Processor::add_debug_log("Full response: " . json_encode($result));
        
        if (isset($result['status'])) {
            if ($result['status'] === 'completed') {
                // Format the response to match the expected structure
                if (isset($result['data']) && isset($result['data']['content'])) {
                    Rank_Processor::add_debug_log("AI fix completed successfully. Content: " . substr(json_encode($result['data']['content']), 0, 200));
                    return array(
                        'content' => rank_clean_ai_fix_content( $result['data']['content'] ),
                        'progress' => $result['data']['progress'],
                        'task_id' => $result['data']['task_id'],
                        'identifier' => $result['data']['identifier']
                    );
                } else {
                    // Extract any error message from the response
                    $error_message = 'The AI had an error. Skipping this fix.';
                    
                    // Check if there's a more specific message in the response
                    if (isset($result['data']['message'])) {
                        $error_message = $result['data']['message'];
                    } elseif (isset($result['message'])) {
                        $error_message = $result['message'];
                    } elseif (isset($result['data']['error'])) {
                        $error_message = $result['data']['error'];
                    } elseif (isset($result['error'])) {
                        $error_message = $result['error'];
                    }
                    
                    Rank_Processor::add_debug_log("AI fix completed but no content. Error: " . $error_message);
                    Rank_Processor::add_debug_log("Response data structure: " . json_encode(array_keys($result['data'] ?? [])));
                    
                    return new WP_Error(
                        'api_error',
                        $error_message,
                        array('status_code' => $response_code, 'response' => $result)
                    );
                }
            } elseif ($result['status'] === 'failed') {
                // Handle failed status explicitly
                // When status is 'failed', we stop polling immediately as there's nothing more to do
                $error_message = 'RANK AI Could not find a fix';
                
                // Check if there's a more specific message in the response
                if (isset($result['data']['message'])) {
                    $error_message = $result['data']['message'];
                } elseif (isset($result['message'])) {
                    $error_message = $result['message'];
                } elseif (isset($result['data']['error'])) {
                    $error_message = $result['data']['error'];
                } elseif (isset($result['error'])) {
                    $error_message = $result['error'];
                }
                
                Rank_Processor::add_debug_log("AI fix FAILED. Error: " . $error_message);
                
                // Return error and stop polling - this exits the function and breaks the polling loop
                return new WP_Error(
                    'api_failed',
                    $error_message,
                    array('status_code' => $response_code, 'response' => $result)
                );
            } else {
                Rank_Processor::add_debug_log("AI fix still pending. Status: " . $result['status']);
            }
        } else {
            Rank_Processor::add_debug_log("No status field in response");
        }

        // If not completed, wait before polling again
        if ($attempt < $max_attempts) {
            sleep($poll_interval);
        }
    }

    // If we've reached the maximum number of attempts without completion
    $timeout_seconds = $max_attempts * $poll_interval;
    return new WP_Error(
        'api_timeout',
        "Timed out waiting for AI fix to complete after {$timeout_seconds} seconds. The API did not return a completed or failed status.",
        array('task_id' => $task_id)
    );
}

/**
 * Make an API request with retry logic.
 *
 * @param string $url The URL to request
 * @param int $max_retries Maximum number of retries (default: 3)
 * @param array $args Additional arguments for wp_remote_get
 * @return array|WP_Error The response data or error
 */
function rank_api_request_with_retry($url, $max_retries = 3, $args = array()) {
    $default_args = array(
        'headers' => array(
            'Authorization' => 'Bearer ' . rank_get_api_key(),
            'Accept' => 'application/json',
        ),
        'timeout' => 60,
        'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
    );
    
    $request_args = wp_parse_args($args, $default_args);
    
    $retry_count = 0;
    $last_error = null;
    
    while ($retry_count < $max_retries) {
        $response = wp_remote_get($url, $request_args);
        
        if (!is_wp_error($response)) {
            $response_code = wp_remote_retrieve_response_code($response);
            $response_body = wp_remote_retrieve_body($response);
            $decoded_body = json_decode($response_body, true);
            
            // If we got a valid response, return it
            if ($response_code === 200 && $decoded_body) {
                return array(
                    'response_code' => $response_code,
                    'data' => $decoded_body
                );
            }
        }
        
        // Store the last error
        $last_error = is_wp_error($response) ? $response : new WP_Error(
            'api_error',
            "API request failed with status code: $response_code",
            array('status_code' => $response_code)
        );
        
        // Increment retry count and wait before retrying
        $retry_count++;
        if ($retry_count < $max_retries) {
            sleep(1); // Wait 1 second between retries
        }
    }
    
    // If we've exhausted all retries, return the last error
    return $last_error;
}

// rank_normalize_url() is now defined in config.php (loaded on all requests including frontend).
// Kept here as a reference comment only — do not redefine.

/**
 * Clear the "already fixed" cache for a specific issue
 * This should be called after successfully storing a fix to ensure subsequent checks see the new data
 *
 * @param string $issue_type The type of issue
 * @param string $page_url The URL of the page
 * @param string $issue_identifier Optional. The specific identifier for the issue
 */
function rank_clear_fixed_status_cache($issue_type, $page_url, $issue_identifier = null) {
    // Access the global cache
    global $rank_fixed_status_cache;
    
    if (!isset($rank_fixed_status_cache)) {
        return; // Nothing to clear
    }
    
    // Recreate the same cache key logic as rank_is_issue_already_fixed
    $normalized_page_url = rank_normalize_url($page_url);
    $normalized_identifier = rank_normalize_url($issue_identifier);
    $cache_key = md5($issue_type . '|' . $normalized_page_url . '|' . $normalized_identifier);
    
    // Clear the cache entry
    if (isset($rank_fixed_status_cache[$cache_key])) {
        unset($rank_fixed_status_cache[$cache_key]);
        Rank_Processor::add_debug_log("Cache cleared for: $issue_type | $page_url | " . ($issue_identifier ? basename($issue_identifier) : 'N/A'));
    }
}

/**
 * Check if an issue has already been fixed locally.
 *
 * @param string $issue_type The type of issue (e.g., 'missing_alt_tags')
 * @param string $page_url The URL of the page with the issue
 * @param string $issue_identifier The specific identifier for the issue (e.g., image URL)
 * @return bool True if the issue has already been fixed locally
 */
function rank_is_issue_already_fixed($issue_type, $page_url, $issue_identifier) {
    // Use global cache instead of static so it can be cleared from other functions
    global $rank_attachment_id_cache, $rank_fixed_status_cache, $rank_post_id_cache, $rank_option_cache;
    
    if (!isset($rank_attachment_id_cache)) {
        $rank_attachment_id_cache = [];
    }
    if (!isset($rank_fixed_status_cache)) {
        $rank_fixed_status_cache = [];
    }
    if (!isset($rank_post_id_cache)) {
        $rank_post_id_cache = [];
    }
    if (!isset($rank_option_cache)) {
        $rank_option_cache = [];
    }
    
    // Keep references for backward compatibility with existing code
    $attachment_id_cache = &$rank_attachment_id_cache;
    $fixed_status_cache = &$rank_fixed_status_cache;
    $post_id_cache = &$rank_post_id_cache;
    $option_cache = &$rank_option_cache;
    
    // Debug log the function call
    // Check if issue is fixed
    
    // Normalize URLs first for consistent caching
    $normalized_page_url_for_cache = rank_normalize_url($page_url);
    $normalized_identifier_for_cache = rank_normalize_url($issue_identifier);
    
    // Create a unique cache key using normalized URLs
    $cache_key = md5($issue_type . '|' . $normalized_page_url_for_cache . '|' . $normalized_identifier_for_cache);
    
    // Check if we already have a cached result
    if (isset($fixed_status_cache[$cache_key])) {
        // Using cached result
        return $fixed_status_cache[$cache_key];
    }
    
    // Different logic based on issue type
    $issue_config = rank_get_issue_type_config($issue_type);
    
    if (!$issue_config) {
        // No issue config found
        $fixed_status_cache[$cache_key] = false;
        return false;
    }

    // Special case: large_images is a content_fix (not database_override), but a
    // single physical file may be reported on many pages — once optimized it is
    // fixed everywhere. Match by image_url alone (page-agnostic) to avoid
    // re-spending API credits. This must run before the handler_type switch
    // below because the database_override branch never sees this issue type.
    if ($issue_type === 'large_images' && !empty($issue_identifier)) {
        // If we recently tried this image and the optimizer failed (e.g. the
        // API returned the same-size file or made it bigger), don't burn the
        // API credit again on the next page that references the same image.
        // A short-lived transient caches the failure for the duration of a
        // typical bulk-fix run; it expires so the user can retry later.
        if ( rank_is_image_recently_failed( $issue_identifier ) ) {
            $fixed_status_cache[$cache_key] = true;
            return true;
        }

        global $wpdb;
        $table_name = $wpdb->prefix . RANK_TABLE_NAME;
        $normalized_image_url = rank_normalize_url($issue_identifier);
        $image_url_with_slash = $normalized_image_url . '/';

        Rank_Processor::add_debug_log("=== CHECKING IF ALREADY FIXED (LARGE IMAGES) ===");
        Rank_Processor::add_debug_log("Original image_url: $issue_identifier");
        Rank_Processor::add_debug_log("Normalized image_url: $normalized_image_url");

        $count = $wpdb->get_var($wpdb->prepare(
            "SELECT COUNT(*) FROM `$table_name`
             WHERE override_type = 'image_optimize'
             AND (image_url = %s OR image_url = %s OR image_url = %s)",
            $normalized_image_url, $image_url_with_slash, $issue_identifier
        ));

        Rank_Processor::add_debug_log("Query result count: $count");
        Rank_Processor::add_debug_log("Already fixed: " . ($count > 0 ? 'YES' : 'NO'));
        Rank_Processor::add_debug_log("=== END CHECKING IF ALREADY FIXED (LARGE IMAGES) ===");

        $fixed_status_cache[$cache_key] = ($count > 0);
        return $fixed_status_cache[$cache_key];
    }
    
    // Process issue config
    
    // Get post ID from URL (with caching)
    if (isset($post_id_cache[$page_url])) {
        $post_id = $post_id_cache[$page_url];
    } else {
        $post_id = url_to_postid($page_url);
        if (!$post_id) {
            // Try enhanced URL to post ID conversion
            $post_id = Rank_Processor::enhanced_url_to_postid($page_url);
        }
        $post_id_cache[$page_url] = $post_id;
    }
    
    if ($issue_config['handler_type'] === 'database_override') {
        global $wpdb;
        $table_name = $wpdb->prefix . RANK_TABLE_NAME;
        $db_override_type = '';
        
        // Map issue type to database override type
        switch ($issue_type) {
            case 'missing_meta_description':
            case 'short_meta_description':
            case 'long_meta_description':
            case 'duplicate_meta_description':
                $db_override_type = 'meta_description';
                
                // For meta descriptions, check based on original_page_url (like alt tags)
                // This handles multilingual sites where same post_id can have different URLs
                if (!empty($page_url)) {
                    $normalized_page_url = rank_normalize_url($page_url);
                    $page_url_with_slash = $normalized_page_url . '/';
                    
                    Rank_Processor::add_debug_log("=== CHECKING IF ALREADY FIXED (META) ===");
                    Rank_Processor::add_debug_log("Original page_url: $page_url");
                    Rank_Processor::add_debug_log("Normalized page_url: $normalized_page_url");
                    
                    // Check for: normalized, normalized+slash, AND original raw URL
                    // Filter by issue_type so a prior missing/short fix doesn't mark a long one as fixed
                    $query = $wpdb->prepare(
                        "SELECT COUNT(*) FROM `$table_name`
                         WHERE (original_page_url = %s OR original_page_url = %s OR original_page_url = %s)
                         AND override_type = %s
                         AND (issue_type = %s OR (issue_type IS NULL AND %s = %s))",
                        $normalized_page_url, $page_url_with_slash, $page_url,
                        $db_override_type,
                        $issue_type, $issue_type, $db_override_type
                    );
                    
                    Rank_Processor::add_debug_log("SQL Query: " . $wpdb->remove_placeholder_escape($query));
                    $count = $wpdb->get_var($query);
                    Rank_Processor::add_debug_log("Query result count: $count");
                    Rank_Processor::add_debug_log("Already fixed: " . ($count > 0 ? 'YES' : 'NO'));
                    Rank_Processor::add_debug_log("=== END CHECKING IF ALREADY FIXED (META) ===");
                    
                    $fixed_status_cache[$cache_key] = ($count > 0);
                    return $fixed_status_cache[$cache_key];
                }
                break;
            case 'missing_title_tag':
            case 'short_title_tag':
            case 'long_title_tag':
            case 'duplicate_title':
                $db_override_type = 'title';
                
                // For titles, also check based on original_page_url
                if (!empty($page_url)) {
                    $normalized_page_url = rank_normalize_url($page_url);
                    $page_url_with_slash = $normalized_page_url . '/';
                    
                    Rank_Processor::add_debug_log("=== CHECKING IF ALREADY FIXED (TITLE) ===");
                    Rank_Processor::add_debug_log("Original page_url: $page_url");
                    Rank_Processor::add_debug_log("Normalized page_url: $normalized_page_url");
                    
                    // Check for: normalized, normalized+slash, AND original raw URL
                    // Filter by issue_type so a prior missing/short fix doesn't mark a long one as fixed
                    $query = $wpdb->prepare(
                        "SELECT COUNT(*) FROM `$table_name`
                         WHERE (original_page_url = %s OR original_page_url = %s OR original_page_url = %s)
                         AND override_type = %s
                         AND (issue_type = %s OR (issue_type IS NULL AND %s = %s))",
                        $normalized_page_url, $page_url_with_slash, $page_url,
                        $db_override_type,
                        $issue_type, $issue_type, $db_override_type
                    );
                    
                    Rank_Processor::add_debug_log("SQL Query: " . $wpdb->remove_placeholder_escape($query));
                    $count = $wpdb->get_var($query);
                    Rank_Processor::add_debug_log("Query result count: $count");
                    Rank_Processor::add_debug_log("Already fixed: " . ($count > 0 ? 'YES' : 'NO'));
                    Rank_Processor::add_debug_log("=== END CHECKING IF ALREADY FIXED (TITLE) ===");
                    
                    $fixed_status_cache[$cache_key] = ($count > 0);
                    return $fixed_status_cache[$cache_key];
                }
                break;
            case 'missing_alt_tags':
                $db_override_type = 'image_alt';
                // For alt tags, check based on original_page_url and image URL
                // This is more reliable than post_id which can fail for various reasons
                if (!empty($page_url) && $issue_config['identifier_is_url'] && !empty($issue_identifier)) {
                    // Normalize URLs for comparison - handles encoding and trailing slashes
                    $normalized_page_url = rank_normalize_url($page_url);
                    $normalized_image_url = rank_normalize_url($issue_identifier);

                    // Create URL variations to handle trailing slash differences in database
                    $page_url_with_slash = $normalized_page_url . '/';
                    $image_url_with_slash = $normalized_image_url . '/';

                    // Debug logging
                    Rank_Processor::add_debug_log("=== CHECKING IF ALREADY FIXED ===");
                    Rank_Processor::add_debug_log("Original page_url: $page_url");
                    Rank_Processor::add_debug_log("Normalized page_url: $normalized_page_url");
                    Rank_Processor::add_debug_log("Page URL with slash: $page_url_with_slash");
                    Rank_Processor::add_debug_log("Original image_url: $issue_identifier");
                    Rank_Processor::add_debug_log("Normalized image_url: $normalized_image_url");
                    Rank_Processor::add_debug_log("Image URL with slash: $image_url_with_slash");

                    // Check URL-based storage using original_page_url instead of post_id
                    // Check for: normalized, normalized+slash, AND original raw URL
                    // This ensures we find records regardless of how they were saved
                    $query = $wpdb->prepare(
                        "SELECT COUNT(*) FROM `$table_name`
                         WHERE (original_page_url = %s OR original_page_url = %s OR original_page_url = %s)
                         AND override_type = %s
                         AND (image_url = %s OR image_url = %s OR image_url = %s)",
                        $normalized_page_url, $page_url_with_slash, $page_url,
                        $db_override_type,
                        $normalized_image_url, $image_url_with_slash, $issue_identifier
                    );

                    Rank_Processor::add_debug_log("SQL Query: " . $wpdb->remove_placeholder_escape($query));
                    $count = $wpdb->get_var($query);
                    Rank_Processor::add_debug_log("Query result count: $count");
                    Rank_Processor::add_debug_log("Already fixed: " . ($count > 0 ? 'YES' : 'NO'));
                    Rank_Processor::add_debug_log("=== END CHECKING IF ALREADY FIXED ===");

                    $fixed_status_cache[$cache_key] = ($count > 0);
                    return $fixed_status_cache[$cache_key];
                }

                // If we don't have the required data for alt tags, assume not fixed
                $fixed_status_cache[$cache_key] = false;
                return false;
        }
        
        // All database_override types (meta, title, alt) now use original_page_url checks above
        // If we reach here, it means we don't have the required data
        Rank_Processor::add_debug_log("Missing required data for detection check");
        $fixed_status_cache[$cache_key] = false;
        return false;
    }
    elseif ($issue_config['handler_type'] === 'option_suggestion') {
        // For broken links
        $option_name = '';
        if ($issue_type === 'broken_links_internal') {
            $option_name = 'rank_internal_url_replacements';
        } elseif ($issue_type === 'broken_links_external') {
            $option_name = 'rank_external_url_replacements';
        }
        
        if (!empty($option_name) && !empty($issue_identifier) && !empty($page_url)) {
            // Cache option values to avoid repeated get_option calls
            if (!isset($option_cache[$option_name])) {
                $option_cache[$option_name] = get_option($option_name, []);
            }
            $replacements = $option_cache[$option_name];
            
            if (!is_array($replacements)) {
                $fixed_status_cache[$cache_key] = false;
                return false;
            }
            
            // Normalize URLs to handle encoding and trailing slashes before creating hash
            $normalized_page_url = rank_normalize_url($page_url);
            $normalized_issue_identifier = rank_normalize_url($issue_identifier);
            
            // Check using the same composite key format as storage
            // Format: broken_url::context::md5(page_url)
            // Use normalized URL for consistent hash generation
            $context_hash = md5($normalized_page_url);
            $composite_key = $normalized_issue_identifier . '::context::' . $context_hash;
            
            // Check if this specific broken URL + page context has a replacement
            $result = isset($replacements[$composite_key]);
            $fixed_status_cache[$cache_key] = $result;
            return $result;
        }
    }
    
    $fixed_status_cache[$cache_key] = false;
    return false;
}

/**
 * Get health issue counts from the new /overall endpoint.
 * Simple and fast - no sampling or fix status checking.
 *
 * @return array|WP_Error Array with issue counts by category and crawled_at timestamp or error
 */
function rank_get_health_issues_overview_counts() {
    Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST START ===");
    
    $domain_identifier = rank_get_domain_identifier();
    if (is_wp_error($domain_identifier)) {
        Rank_Processor::add_debug_log("Domain identifier error: " . $domain_identifier->get_error_message());
        Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (DOMAIN ERROR) ===");
        return $domain_identifier;
    }
    
    Rank_Processor::add_debug_log("Domain identifier: $domain_identifier");
    
    // Single API call to get all issue counts
    $url = rank_get_api_base_url() . "/{$domain_identifier}/overall";
    Rank_Processor::add_debug_log("Calling API: $url");
    
    $response = wp_remote_get($url, array(
        'headers' => array(
            'Authorization' => 'Bearer ' . rank_get_api_key(),
            'Accept' => 'application/json',
        ),
        'timeout' => 30,
        'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
    ));
    
    if (is_wp_error($response)) {
        Rank_Processor::add_debug_log("WP_Error getting overview: " . $response->get_error_message());
        Rank_Processor::add_debug_log("Error code: " . $response->get_error_code());
        Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (WP_ERROR) ===");
        return new WP_Error(
            $response->get_error_code(),
            $response->get_error_message(),
            array('original_error' => $response->get_error_data())
        );
    }
    
    $response_code = wp_remote_retrieve_response_code($response);
    $response_body = wp_remote_retrieve_body($response);
    
    Rank_Processor::add_debug_log("API Response code: $response_code");
    Rank_Processor::add_debug_log("API Response body (first 500 chars): " . substr($response_body, 0, 500));
    
    if ($response_code !== 200) {
        Rank_Processor::add_debug_log("Failed to retrieve overview. HTTP $response_code");
        Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (HTTP ERROR) ===");
        return new WP_Error(
            'api_error',
            'Failed to retrieve overview. Status code: ' . $response_code,
            array('response_body' => $response_body, 'status_code' => $response_code)
        );
    }
    
    $result = json_decode($response_body, true);
    if (!is_array($result)) {
        Rank_Processor::add_debug_log("Invalid response format - not an array. Response: " . substr($response_body, 0, 200));
        Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (INVALID FORMAT) ===");
        return new WP_Error(
            'api_error',
            'Invalid overview response format - not an array',
            array('response_body' => $response_body)
        );
    }
    
    // Check if we have the expected structure
    if (!isset($result['categories'])) {
        Rank_Processor::add_debug_log("Missing categories in response. Keys present: " . json_encode(array_keys($result)));
        Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (MISSING CATEGORIES) ===");
        return new WP_Error(
            'api_error',
            'Invalid overview response format - missing categories',
            array('response_body' => $response_body, 'response_keys' => array_keys($result))
        );
    }
    
    // Extract crawled_at timestamp and crawl_status
    $crawled_at = isset($result['crawled_at']) ? $result['crawled_at'] : null;
    $crawl_status = isset($result['crawl_status']) ? $result['crawl_status'] : 'ready';
    
    // Get all known categories for labels
    $all_categories = rank_get_health_check_categories();
    $issue_counts = array();
    
    // Process each category's problems
    foreach ($result['categories'] as $category => $category_data) {
        if (!isset($category_data['problems']) || !is_array($category_data['problems'])) {
            continue;
        }
        
        foreach ($category_data['problems'] as $problem) {
            $validator = $problem['validator'];
            $issues_count = intval($problem['issues_num']);
            
            // Handle empty validator field
            if (empty($validator)) {
                continue; // Skip empty validators
            }
            
            // Map validator to internal issue type (direct mapping)
            $issue_type = $validator;
            $label = isset($all_categories[$issue_type]) ? $all_categories[$issue_type] : $issue_type;
            
            $issue_counts[$issue_type] = array(
                'label' => $label,
                'count' => $issues_count < 0 ? 0 : $issues_count
            );
        }
    }
    
    // Ensure all known issue types are included (even if not in API response)
    foreach ($all_categories as $issue_type => $label) {
        if (!isset($issue_counts[$issue_type])) {
            $issue_counts[$issue_type] = array(
                'label' => $label,
                'count' => 0
            );
        }
    }
    
    Rank_Processor::add_debug_log("Successfully processed overview. Issue types found: " . count($issue_counts));
    Rank_Processor::add_debug_log("Crawl status: $crawl_status, Crawled at: " . ($crawled_at ?: 'N/A'));
    Rank_Processor::add_debug_log("=== HEALTH ISSUES OVERVIEW REQUEST END (SUCCESS) ===");
    
    return array(
        'issues' => $issue_counts,
        'crawled_at' => $crawled_at,
        'crawl_status' => $crawl_status
    );
}



/**
 * Check if the crawled_at timestamp is older than 30 days.
 *
 * @param string $crawled_at ISO 8601 timestamp string (e.g., "2025-07-09T09:54:04.016Z")
 * @return bool True if older than 30 days, false otherwise
 */
function rank_is_crawl_data_too_old($crawled_at) {
    if (empty($crawled_at)) {
        return false; // Consider missing timestamp as fresh
    }
    
    // Parse the ISO 8601 timestamp
    $crawled_timestamp = strtotime($crawled_at);
    if ($crawled_timestamp === false) {
        return false; // Invalid timestamp format, consider fresh
    }
    
    // Calculate 30 days ago from now
    $thirty_days_ago = time() - (30 * 24 * 60 * 60);
    
    // Return true if crawled_at is older than 30 days
    return $crawled_timestamp < $thirty_days_ago;
}


/**
 * Check if a URL is accessible and doesn't redirect.
 *
 * @param string $url The URL to check
 * @return array Result with 'exists' and 'redirects' flags
 */
function rank_validate_url($url) {
    // Use static cache to avoid checking the same URL multiple times
    static $url_cache = array();
    
    // Preserve the original input as the cache key; $url may be rewritten below
    // (host canonicalization), so cache reads/writes must use a stable key.
    $cache_key = $url;
    
    // Return cached result if available
    if (isset($url_cache[$cache_key])) {
        Rank_Processor::add_debug_log("URL validation cache hit for: $url");
        return $url_cache[$cache_key];
    }
    
    Rank_Processor::add_debug_log("=== URL VALIDATION START ===");
    Rank_Processor::add_debug_log("Validating URL: $url");
    
    $result = array(
        'exists' => false,
        'redirects' => false,
        'access_denied' => false,
        'status' => 0,
        'method_used' => ''
    );
    
    // Check if URL belongs to this WordPress installation (subdirectory support)
    $wp_home_url = home_url();
    $wp_base_path = parse_url($wp_home_url, PHP_URL_PATH);
    $wp_host = parse_url($wp_home_url, PHP_URL_HOST);
    
    $url_path = parse_url($url, PHP_URL_PATH);
    $url_host = parse_url($url, PHP_URL_HOST);
    
    Rank_Processor::add_debug_log("WordPress base path: " . ($wp_base_path ?: '/'));
    Rank_Processor::add_debug_log("WordPress host: " . $wp_host);
    Rank_Processor::add_debug_log("URL path: " . ($url_path ?: '/'));
    Rank_Processor::add_debug_log("URL host: " . $url_host);
    
    // Normalize host for comparison (remove www. prefix) to handle www vs non-www
    $normalize_host = function($host) {
        if (empty($host)) return '';
        $host = strtolower($host);
        // Remove www. prefix for comparison
        if (strpos($host, 'www.') === 0) {
            return substr($host, 4);
        }
        return $host;
    };
    $wp_host_normalized = $normalize_host($wp_host);
    $url_host_normalized = $normalize_host($url_host);
    
    // Check if host matches (with www normalization)
    if ($url_host_normalized !== $wp_host_normalized) {
        Rank_Processor::add_debug_log("URL host does not match WordPress host (normalized: $url_host_normalized !== $wp_host_normalized) - skipping");
        $result['exists'] = false;
        $result['method_used'] = 'HOST_MISMATCH';
        $url_cache[$cache_key] = $result;
        Rank_Processor::add_debug_log("=== URL VALIDATION END (HOST MISMATCH) ===");
        return $result;
    }
    
    // Don't do strict path matching here - WPML and other multilingual plugins
    // can have different path structures (e.g., /da/, /en/) that are still valid
    // Path validation will be handled by WordPress url_to_postid() later if needed
    Rank_Processor::add_debug_log("Host matches (normalized) - proceeding with HTTP validation");
    
    // Canonicalize the host before fetching. The API may report a page under a
    // host variant (e.g. www.example.com) that 301-redirects to the site's
    // canonical host (e.g. example.com). Without this rewrite, the HTTP check
    // hits the redirecting variant and the URL is wrongly skipped as "redirects".
    if ($wp_host && $url_host && $url_host !== $wp_host) {
        $canonical_url = preg_replace(
            '#^(https?://)' . preg_quote($url_host, '#') . '#i',
            '$1' . $wp_host,
            $url,
            1
        );
        if ($canonical_url && $canonical_url !== $url) {
            Rank_Processor::add_debug_log("Rewriting host to canonical WordPress host for validation: $url -> $canonical_url");
            $url = $canonical_url;
        }
    }
    
    // Step 1: Try HEAD request first (fast check)
    Rank_Processor::add_debug_log("Attempting HEAD request...");
    $head_response = wp_remote_head($url, array(
        'timeout' => 10, // Increased timeout
        'redirection' => 0, // Don't follow redirects
        'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
        'sslverify' => false // Disable SSL verification to handle sites with certificate issues
    ));
    
    if (is_wp_error($head_response)) {
        $error_message = $head_response->get_error_message();
        Rank_Processor::add_debug_log("HEAD request failed: $error_message");
    } else {
        $head_status = wp_remote_retrieve_response_code($head_response);
        $head_headers = wp_remote_retrieve_headers($head_response);
        
        Rank_Processor::add_debug_log("HEAD response status: $head_status");
        Rank_Processor::add_debug_log("HEAD response headers: " . print_r($head_headers->getAll(), true));
        
        // Check if HEAD request was successful
        if ($head_status >= 200 && $head_status < 300) {
            $result['exists'] = true;
            $result['status'] = $head_status;
            $result['method_used'] = 'HEAD';
            
            Rank_Processor::add_debug_log("HEAD request successful - URL exists");
            Rank_Processor::add_debug_log("=== URL VALIDATION END (HEAD SUCCESS) ===");
            
            // Cache and return successful result
            $url_cache[$cache_key] = $result;
            return $result;
        } else if ($head_status >= 300 && $head_status < 400) {
            $result['redirects'] = true;
            $result['status'] = $head_status;
            Rank_Processor::add_debug_log("HEAD request shows redirect (status: $head_status)");
        } else if ($head_status === 401 || $head_status === 403) {
            // 401/403 means the URL exists but access is denied (WAF, security plugin, bot protection)
            $result['exists'] = true;
            $result['access_denied'] = true;
            $result['status'] = $head_status;
            $result['method_used'] = 'HEAD';
            
            Rank_Processor::add_debug_log("HEAD request returned $head_status - URL exists but access denied (likely WAF/security)");
            Rank_Processor::add_debug_log("=== URL VALIDATION END (HEAD ACCESS DENIED) ===");
            
            // Cache and return - URL exists even though access is denied
            $url_cache[$cache_key] = $result;
            return $result;
        } else {
            Rank_Processor::add_debug_log("HEAD request unsuccessful (status: $head_status)");
        }
    }
    
    // Step 2: Fallback to GET request if HEAD failed or was inconclusive
    Rank_Processor::add_debug_log("Attempting GET request fallback...");
    $get_response = wp_remote_get($url, array(
        'timeout' => 10,
        'redirection' => 0, // Don't follow redirects
        'user-agent' => 'Mozilla/5.0 (compatible; RANK AI/1.0; +https://morningscore.io)',
        'sslverify' => false // Disable SSL verification to handle sites with certificate issues
    ));
    
    if (is_wp_error($get_response)) {
        $error_message = $get_response->get_error_message();
        Rank_Processor::add_debug_log("GET request failed: $error_message");
        $result['exists'] = false;
        $result['method_used'] = 'GET_FAILED';
    } else {
        $get_status = wp_remote_retrieve_response_code($get_response);
        $get_headers = wp_remote_retrieve_headers($get_response);
        
        Rank_Processor::add_debug_log("GET response status: $get_status");
        Rank_Processor::add_debug_log("GET response headers: " . print_r($get_headers->getAll(), true));
        
        $result['status'] = $get_status;
        $result['method_used'] = 'GET';
        
        // Check if it's a redirect (300-399)
        if ($get_status >= 300 && $get_status < 400) {
            $result['redirects'] = true;
            $result['exists'] = false;
            Rank_Processor::add_debug_log("GET request shows redirect (status: $get_status) - marking as non-existent");
        }
        // Check if it exists (200-299)
        else if ($get_status >= 200 && $get_status < 300) {
            $result['exists'] = true;
            Rank_Processor::add_debug_log("GET request successful - URL exists");
        }
        // 401/403 means the URL exists but access is denied (WAF, security plugin, bot protection, etc.)
        // A 403 Forbidden proves the page exists - the server is just refusing access
        else if ($get_status === 401 || $get_status === 403) {
            $result['exists'] = true;
            $result['access_denied'] = true;
            Rank_Processor::add_debug_log("GET request returned $get_status - URL exists but access denied (likely WAF/security plugin blocking automated requests)");
        }
        // Handle other status codes (404, 410, 500, etc.)
        else {
            $result['exists'] = false;
            Rank_Processor::add_debug_log("GET request unsuccessful (status: $get_status) - marking as non-existent");
        }
    }
    
    // Final result logging
    $exists_text = $result['exists'] ? 'EXISTS' : 'DOES NOT EXIST';
    $redirects_text = $result['redirects'] ? 'REDIRECTS' : 'NO REDIRECT';
    $access_text = $result['access_denied'] ? 'ACCESS DENIED' : 'ACCESSIBLE';
    Rank_Processor::add_debug_log("Final result: $exists_text, $redirects_text, $access_text, Status: {$result['status']}, Method: {$result['method_used']}");
    Rank_Processor::add_debug_log("=== URL VALIDATION END ===");
    
    // Cache the result
    $url_cache[$cache_key] = $result;
    
    return $result;
}
