<?php
ini_set('memory_limit', '256M');
ini_set('max_execution_time', 300);

class IntegrityCheckHandler {
    private $db;
    private static $instance = null;
    private $currentState = null;
    private $batchSize = 3;  // Process 3 records per batch to avoid hitting rate limits
    private $maxRecords = 20; // Process 20 records per run (reduced for testing)
    private $forceCleanup = false;

    private function __construct() {
        try {
            $this->db = Database::getInstance();
            $this->ensureAnomalyTable();
        } catch (Exception $e) {
            error_log("Database connection error: " . $e->getMessage());
            throw $e;
        }
    }

    public static function getInstance() {
        if (self::$instance === null) {
            self::$instance = new self();
        }
        return self::$instance;
    }

    /**
     * Ensure the anomaly storage table exists
     */
    private function ensureAnomalyTable() {
    $sql = "CREATE TABLE IF NOT EXISTS panel_anomalies (
        id INT AUTO_INCREMENT PRIMARY KEY,
        panelist_id VARCHAR(10) NOT NULL,
        description TEXT NOT NULL,
        attributes_involved VARCHAR(255) NULL,
        affected_ids TEXT NULL,
        is_processed TINYINT(1) DEFAULT 0,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        INDEX (panelist_id),
        INDEX (is_processed)
    )";
    
    $this->db->query($sql);
}

    public function cleanup() {
        if (!$this->forceCleanup) {
            error_log("Skipping automatic cleanup");
            return true;
        }

        $this->db->query("START TRANSACTION");
        try {
            error_log("Starting forced cleanup process");
            
            $result = $this->db->query("SELECT * FROM panel_analysis_state WHERE is_running = 1");
            if ($result && $result->num_rows > 0) {
                error_log("Found running processes: " . $result->num_rows);
            }
            
            $this->db->query("UPDATE panel_analysis_state SET is_running = 0, status = 'Check terminated' WHERE is_running = 1");
            $this->db->query("DELETE FROM panel_processing_status WHERE status IN ('pending', 'processing')");
            
            $lockFile = sys_get_temp_dir() . '/integrity_check.lock';
            if (file_exists($lockFile)) {
                error_log("Removing lock file");
                unlink($lockFile);
            }

            $this->db->query("COMMIT");
            error_log("Cleanup completed successfully");
            return true;
        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Cleanup error: " . $e->getMessage());
            throw $e;
        }
    }

    public function forceCleanup() {
        $this->forceCleanup = true;
        $result = $this->cleanup();
        $this->forceCleanup = false;
        return $result;
    }

    public function startCheck() {
        try {
            error_log("Starting integrity check");
            
            // Reset any existing checks
            $this->db->query("UPDATE panel_analysis_state SET is_running = 0 WHERE is_running = 1");
            $this->db->query("DELETE FROM panel_processing_status WHERE status IN ('pending', 'processing')");
            $this->db->query("DELETE FROM panel_anomalies"); // Clear previous anomalies

            // Get records that haven't been processed yet, up to $maxRecords
            $result = $this->db->query("
                SELECT pd.* 
                FROM panel_data pd
                LEFT JOIN panel_processing_status pps ON pd.panelist_id = pps.panelist_id
                WHERE pps.panelist_id IS NULL
                ORDER BY pd.id 
                LIMIT {$this->maxRecords}
            ");
            
            if (!$result) {
                error_log("Query error: " . $this->db->getLastError());
                throw new Exception("Failed to fetch records: " . $this->db->getLastError());
            }

            $records = [];
            while ($row = $result->fetch_assoc()) {
                $records[] = $row;
            }

            $totalCount = count($records);
            error_log("Found $totalCount records to process");
            
            if ($totalCount === 0) {
                throw new Exception('No records to process');
            }

            $this->db->query("START TRANSACTION");

            try {
                error_log("Initializing new check process");

                $stmt = $this->db->prepare("
                    INSERT INTO panel_analysis_state 
                    (is_running, is_paused, processed_count, total_count, status, start_time) 
                    VALUES (1, 0, 0, ?, 'Starting phase 1: Anomaly detection', NOW())
                ");
                
                $stmt->bind_param('i', $totalCount);
                
                if (!$stmt->execute()) {
                    throw new Exception("Failed to initialize integrity check");
                }

                $stmt = $this->db->prepare("
                    INSERT INTO panel_processing_status 
                    (panelist_id, status, attempts, last_attempt) 
                    VALUES (?, 'pending', 0, NULL)
                ");

                foreach ($records as $record) {
                    $stmt->bind_param('s', $record['panelist_id']);
                    if (!$stmt->execute()) {
                        error_log("Failed to initialize status for panelist: " . $record['panelist_id']);
                        throw new Exception("Failed to initialize record status");
                    }
                }

                $this->db->query("COMMIT");
                error_log("Check process initialized successfully");
                
                return [
                    'success' => true,
                    'total_count' => $totalCount,
                    'message' => 'Integrity check started'
                ];
                
            } catch (Exception $e) {
                $this->db->query("ROLLBACK");
                error_log("Failed to initialize check: " . $e->getMessage());
                throw $e;
            }
            
        } catch (Exception $e) {
            error_log("Error starting integrity check: " . $e->getMessage());
            return [
                'success' => false, 
                'message' => $e->getMessage()
            ];
        }
    }
    
    public function killExistingProcesses() {
        try {
            error_log("Killing existing processes");
            
            // Force cleanup any existing processes
            $this->db->query("START TRANSACTION");
            
            // Reset all running states
            $this->db->query("UPDATE panel_analysis_state SET is_running = 0, status = 'Process terminated' WHERE is_running = 1");
            
            // Clear all pending processes
            $this->db->query("DELETE FROM panel_processing_status WHERE status IN ('pending', 'processing')");
            
            // Remove any lock files
            $lockFile = sys_get_temp_dir() . '/integrity_check.lock';
            if (file_exists($lockFile)) {
                unlink($lockFile);
            }
            
            // Reset GPT rate limit tracking
            if (method_exists('GptHelper', 'resetRateLimit')) {
                GptHelper::resetRateLimit();
            }
            
            $this->db->query("COMMIT");
            error_log("All existing processes killed");
            return true;
            
        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Error killing processes: " . $e->getMessage());
            throw $e;
        }
    }

    /**
     * Check if phase 1 is complete and all records have been processed
     */
    private function isPhase1Complete() {
        $result = $this->db->query("
            SELECT COUNT(*) as count 
            FROM panel_processing_status 
            WHERE status = 'pending'
        ");
        
        if (!$result) {
            return false;
        }
        
        $row = $result->fetch_assoc();
        return $row['count'] == 0;
    }
    
    /**
     * Start phase 2 - grouping anomalies into directives
     */
    private function startPhase2() {
        $this->db->query("START TRANSACTION");
        
        try {
            // Update the state to indicate we're starting phase 2
            $stmt = $this->db->prepare("
                UPDATE panel_analysis_state 
                SET status = 'Starting phase 2: Grouping anomalies', 
                    last_processed = NOW()
                WHERE is_running = 1
                AND id = (SELECT MAX(id) FROM panel_analysis_state)
            ");
            
            if (!$stmt->execute()) {
                throw new Exception("Failed to update state for phase 2");
            }
            
            $this->db->query("COMMIT");
            $this->currentState = null; // Reset state
            
            // Count anomalies to process
            $result = $this->db->query("SELECT COUNT(*) as count FROM panel_anomalies WHERE is_processed = 0");
            $anomalyCount = $result->fetch_assoc()['count'];
            
            error_log("Starting phase 2 with $anomalyCount anomalies to process");
            return true;
            
        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Failed to start phase 2: " . $e->getMessage());
            return false;
        }
    }
    
    /**
 * Process a batch of anomalies and create/update directives
 */
public function processAnomalieBatch() {
    try {
        error_log("Processing anomaly batch");
        
        // Set a processing timeout
        set_time_limit(60); // 60 seconds should be plenty for a batch
        
        // Check if we have any unprocessed anomalies
        $countQuery = $this->db->query("
            SELECT COUNT(*) as count FROM panel_anomalies WHERE is_processed = 0
        ");
        
        if (!$countQuery) {
            throw new Exception("Failed to check anomaly count: " . $this->db->getLastError());
        }
        
        $count = $countQuery->fetch_assoc()['count'];
        
        if ($count === 0) {
            error_log("No unprocessed anomalies found");
            return [
                'success' => true,
                'status' => 'completed',
                'message' => 'All anomalies processed'
            ];
        }
        
        // Group similar anomalies
        $processed = $this->groupSimilarAnomalies();
        
        if ($processed > 0) {
            error_log("Processed $processed anomalies");
            
            // Check if there are more to process
            $remainingQuery = $this->db->query("
                SELECT COUNT(*) as count FROM panel_anomalies WHERE is_processed = 0
            ");
            
            $remaining = $remainingQuery->fetch_assoc()['count'];
            
            if ($remaining > 0) {
                return [
                    'success' => true,
                    'status' => 'processing',
                    'processed' => $processed,
                    'remaining' => $remaining,
                    'message' => "Processed $processed anomalies, $remaining remaining"
                ];
            } else {
                return [
                    'success' => true,
                    'status' => 'completed',
                    'message' => 'All anomalies processed'
                ];
            }
        } else {
            return [
                'success' => true,
                'status' => 'completed',
                'message' => 'No anomalies to process'
            ];
        }
    } catch (Exception $e) {
        error_log("Error processing anomalies: " . $e->getMessage());
        return [
            'success' => false,
            'message' => $e->getMessage()
        ];
    }
}
    
    /**
 * Group similar anomalies into consolidated directives
 * This should be called during Phase 2 processing
 */
private function groupSimilarAnomalies() {
    try {
        // Mark all anomalies as processed
        $result = $this->db->query("UPDATE panel_anomalies SET is_processed = 1 WHERE is_processed = 0");
        return 1;
    } catch (Exception $e) {
        error_log("Error in groupSimilarAnomalies: " . $e->getMessage());
        return 0;
    }
}

private function createDirectiveFromAnomalySimple($anomaly) {
    // Insert new directive with minimal data
    $stmt = $this->db->prepare("
        INSERT INTO panel_directives (
            description, record_count, status, created_at
        ) VALUES (?, 1, 'pending', NOW())
    ");
    
    $stmt->bind_param('s', $anomaly['description']);
    $stmt->execute();
    
    $newId = $this->db->getLastInsertId();
    error_log("Created new directive #$newId from anomaly");
}

private function addAnomalyToDirectiveSimple($anomaly, $directiveId) {
    try {
        // Simply increment the record count
        $stmt = $this->db->prepare("
            UPDATE panel_directives 
            SET record_count = record_count + 1
            WHERE id = ?
        ");
        
        if (!$stmt) {
            error_log("Failed to prepare statement: " . $this->db->getLastError());
            return false;
        }
        
        $stmt->bind_param('i', $directiveId);
        $result = $stmt->execute();
        
        if (!$result) {
            error_log("Failed to update directive: " . $stmt->error);
            return false;
        }
        
        error_log("Added anomaly to directive $directiveId");
        return true;
    } catch (Exception $e) {
        error_log("Exception adding anomaly to directive: " . $e->getMessage());
        return false;
    }
}

/**
 * Calculate text similarity between two descriptions
 */
private function calculateTextSimilarity($text1, $text2) {
    // Convert text to lowercase and remove punctuation
    $text1 = strtolower(preg_replace('/[^\w\s]/', '', $text1));
    $text2 = strtolower(preg_replace('/[^\w\s]/', '', $text2));
    
    // Extract key concepts that we want to match on
    $concepts = [
        'age' => ['age', 'years', 'child', 'children', 'minor', 'infant', 'baby', 'toddler', 'young', 'old'],
        'income' => ['income', 'rich', 'poor', 'destitute', 'middle', 'class', 'inr', 'salary', 'wage', 'money', 'earn'],
        'employment' => ['employment', 'job', 'work', 'employed', 'self-employed', 'labourer', 'labor', 'profession', 'career', 'occupation', 'salaried', 'wages', 'casual'],
        'education' => ['education', 'literate', 'illiterate', 'school', 'read', 'write', 'literacy', 'educated']
    ];
    
    // Calculate concept matches
    $conceptMatch = 0;
    $conceptTotal = count($concepts);
    
    foreach ($concepts as $category => $terms) {
        $match1 = false;
        $match2 = false;
        
        foreach ($terms as $term) {
            if (strpos($text1, $term) !== false) $match1 = true;
            if (strpos($text2, $term) !== false) $match2 = true;
        }
        
        // If both texts mention the same concept category, increase similarity
        if ($match1 && $match2) {
            $conceptMatch++;
        }
    }
    
    // Calculate basic word similarity (Jaccard)
    $stopWords = ['a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'in', 'to', 'of', 'for', 'with', 'this', 'that', 'would', 'could', 'might', 'may', 'can'];
    $words1 = array_diff(explode(' ', $text1), $stopWords);
    $words2 = array_diff(explode(' ', $text2), $stopWords);
    
    $intersection = array_intersect($words1, $words2);
    $union = array_unique(array_merge($words1, $words2));
    
    $wordSimilarity = empty($union) ? 0 : count($intersection) / count($union);
    
    // Combine concept matching and word similarity
    // Weight concept matching higher since it's more important
    $similarity = ($conceptMatch / $conceptTotal) * 0.7 + $wordSimilarity * 0.3;
    
    return $similarity;
}

/**
 * Add an anomaly to an existing directive
 */
private function addAnomalyToDirectiveSimple($anomaly, $directiveId) {
    try {
        // Simply increment the record count
        $stmt = $this->db->prepare("
            UPDATE panel_directives 
            SET record_count = record_count + 1
            WHERE id = ?
        ");
        
        if (!$stmt) {
            error_log("Failed to prepare statement: " . $this->db->getLastError());
            return false;
        }
        
        $stmt->bind_param('i', $directiveId);
        $result = $stmt->execute();
        
        if (!$result) {
            error_log("Failed to update directive: " . $stmt->error);
            return false;
        }
        
        error_log("Added anomaly to directive $directiveId");
        return true;
    } catch (Exception $e) {
        error_log("Exception adding anomaly to directive: " . $e->getMessage());
        return false;
    }
}

/**
 * Create a new directive from an anomaly
 */
private function createDirectiveFromAnomalySimple($anomaly) {
    try {
        // Insert new directive with minimal data
        $stmt = $this->db->prepare("
            INSERT INTO panel_directives (
                description, record_count, status, created_at
            ) VALUES (?, 1, 'pending', NOW())
        ");
        
        if (!$stmt) {
            error_log("Failed to prepare statement: " . $this->db->getLastError());
            return false;
        }
        
        $stmt->bind_param('s', $anomaly['description']);
        $result = $stmt->execute();
        
        if (!$result) {
            error_log("Failed to create directive: " . $stmt->error);
            return false;
        }
        
        $newId = $this->db->getLastInsertId();
        error_log("Created new directive #$newId from anomaly");
        return true;
    } catch (Exception $e) {
        error_log("Exception creating directive: " . $e->getMessage());
        return false;
    }
}
    
    public function processBatch() {
        try {
            $state = $this->getCurrentState();
            error_log("Processing batch, current state: " . json_encode($state));
            
            if (!$state || $state['is_running'] != '1') {
                error_log("No active check or check not running");
                return ['success' => true, 'status' => 'stopped'];
            }
            
            // Check if check is paused
            if ($state['is_paused'] == '1') {
                error_log("Check is paused");
                return [
                    'success' => true, 
                    'status' => 'paused',
                    'message' => 'Check is paused'
                ];
            }
            
            // Check if we're in phase 1 or phase 2
            if (strpos($state['status'], 'Phase 2') !== false || strpos($state['status'], 'Starting phase 2') !== false) {
                // In Phase 2, process anomalies in batches
                return $this->processAnomalieBatch();
            }

            // Check rate limit before processing
            if (!GptHelper::canMakeRequest()) {
                error_log("Rate limit active, cooling down");
                return $this->handleRateLimit(
                    $state['processed_count'] ?? 0,
                    $state['total_count'] ?? 0
                );
            }

            // Check if phase 1 is complete
            if ($this->isPhase1Complete()) {
                // Start phase 2
                $this->startPhase2();
                
                // Return immediately to let the next call handle phase 2
                return [
                    'success' => true,
                    'status' => 'switching_phase',
                    'message' => 'Starting phase 2: Grouping anomalies'
                ];
            }

            // Phase 1: Get next batch of pending records
            $stmt = $this->db->prepare("
                SELECT pd.*
                FROM panel_processing_status pps
                JOIN panel_data pd ON pps.panelist_id = pd.panelist_id
                WHERE pps.status = 'pending'
                ORDER BY pps.panelist_id
                LIMIT ?
            ");
            $stmt->bind_param('i', $this->batchSize);
            $stmt->execute();
            $result = $stmt->get_result();
            
            if (!$result) {
                throw new Exception("Failed to fetch pending records");
            }
            
            $records = [];
            while ($row = $result->fetch_assoc()) {
                $records[] = $row;
            }
            
            if (empty($records)) {
                // No more pending records - check if we're done
                $remainingResult = $this->db->query("
                    SELECT COUNT(*) as count 
                    FROM panel_processing_status 
                    WHERE status = 'pending'
                ");
                
                $remaining = $remainingResult->fetch_assoc()['count'];
                
                if ($remaining == 0) {
                    // Start phase 2
                    $this->startPhase2();
                    
                    return [
                        'success' => true,
                        'status' => 'switching_phase',
                        'message' => 'Starting phase 2: Grouping anomalies'
                    ];
                }
                
                return [
                    'success' => true,
                    'status' => 'waiting',
                    'message' => 'Waiting for more records'
                ];
            }
            
            // Process this batch of records
            $processed = 0;
            $successful = $state['processed_count'] ?? 0;
            
            // Get attributes only once for the batch
            $attributes = $this->getAttributeMetadata();
            
            foreach ($records as $record) {
                try {
                    // Mark record as processing
                    $this->markRecordStatus($record['panelist_id'], 'processing');
                    
                    // Check if we can make a request
                    if (!GptHelper::canMakeRequest()) {
                        // Return immediately to avoid wasting time
                        return $this->handleRateLimit(
                            $successful,
                            $state['total_count'] ?? 0,
                            $record['panelist_id']
                        );
                    }
                    
                    // Check record for logical inconsistencies
                    $inconsistencies = $this->checkRecordIntegrity($record, $attributes);
                    
                    // Save any inconsistencies found
                    foreach ($inconsistencies as $description) {
                        error_log("Found inconsistency for {$record['panelist_id']}: $description");
                        $this->saveAnomaly($record['panelist_id'], $description);
                    }
                    
                    // Mark as completed regardless of whether inconsistencies were found
                    $this->markRecordStatus($record['panelist_id'], 'completed');
                    $successful++;
                    $processed++;
                    
                } catch (Exception $e) {
                    error_log("Error processing record: " . $e->getMessage());
                    $this->markRecordStatus($record['panelist_id'], 'failed');
                    $processed++;
                    
                    // Check if this is a rate limit error
                    if (stripos($e->getMessage(), 'rate limit') !== false) {
                        return $this->handleRateLimit(
                            $successful,
                            $state['total_count'] ?? 0,
                            $record['panelist_id']
                        );
                    }
                }
            }
            
            // Update state with progress
            $totalPanel = $state['total_count'] ?? 0;
            $progress = round(($successful / $totalPanel) * 100, 2);
            
            $this->updateState([
                'processed_count' => $successful,
                'status' => "Phase 1: Processed $successful of $totalPanel records ($progress%)",
                'last_processed' => date('Y-m-d H:i:s')
            ]);
            
            return [
                'success' => true,
                'status' => 'processing',
                'progress' => $progress,
                'processed' => $successful,
                'total' => $totalPanel
            ];
            
        } catch (Exception $e) {
            error_log("Batch processing error: " . $e->getMessage());
            return ['success' => false, 'message' => $e->getMessage()];
        }
    }
    
    private function handleRateLimit($successful, $totalPanel, $panelistId = null) {
        if ($panelistId) {
            $this->markRecordStatus($panelistId, 'pending');
        }
        
        $progress = round(($successful / $totalPanel) * 100, 2);
        $this->updateState([
            'is_running' => '1',  // Keep process running
            'processed_count' => $successful,
            'status' => "Rate limit cooling down... Progress: $progress% (Successful: $successful)",
            'last_processed' => date('Y-m-d H:i:s')
        ]);
        
        return [
            'success' => true,
            'status' => 'cooling',
            'message' => 'Rate limit cooling down',
            'progress' => $progress
        ];
    }

    /**
     * Check a record for logical inconsistencies
     * Returns array of inconsistency descriptions
     */
    public function checkRecordIntegrity($record, $attributes) {
        $maxRetries = GPT_MAX_RETRIES;
        $attempt = 0;
        
        do {
            try {
                if ($attempt > 0) {
                    error_log("Retrying integrity check for record {$record['panelist_id']}, attempt $attempt");
                    sleep(GPT_RETRY_DELAY);
                }

                $attributeValues = json_decode($record['attribute_values'], true);
                if (!$attributeValues) {
                    throw new Exception("Invalid attribute values format");
                }
                
                $profileData = [];
                foreach ($attributeValues as $attrId => $value) {
                    if (isset($attributes[$attrId])) {
                        $profileData[$attributes[$attrId]['name']] = $value;
                    }
                }

                if (empty($profileData)) {
                    throw new Exception("No valid attributes found");
                }

                if (!GptHelper::canMakeRequest()) {
                    sleep(GPT_COOLDOWN_PERIOD);
                    continue;
                }

                $messages = [
                    [
                        'role' => 'system',
                        'content' => 'You are an expert demographic analyst focused on identifying logical inconsistencies in demographic profiles. You are analyzing synthetic panel data to find combinations of demographic attributes that would not be plausible in the real world.'
                    ],
                    [
                        'role' => 'user',
                        'content' => $this->prepareGptPrompt($profileData)
                    ]
                ];

                $response = GptHelper::makeRequest($messages);
                if (empty($response)) {
                    throw new Exception("Empty response from GPT");
                }
                
                $inconsistencies = $this->parseGptResponse($response);
                return $inconsistencies;

            } catch (Exception $e) {
                error_log("GPT request failed: " . $e->getMessage());
                
                if (stripos($e->getMessage(), 'rate limit') !== false) {
                    $attempt++;
                    if ($attempt >= $maxRetries) {
                        error_log("Final failure checking record {$record['panelist_id']}: " . $e->getMessage());
                        return [];
                    }
                    sleep(GPT_COOLDOWN_PERIOD);
                    continue;
                }
                
                throw $e;
            }
        } while ($attempt < $maxRetries);
        
        return [];
    }

    private function prepareGptPrompt($profileData) {
        $prompt = "Analyze this demographic profile for logical inconsistencies or impossible combinations of attributes:\n\n";
        
        foreach ($profileData as $attribute => $value) {
            $prompt .= "$attribute: $value\n";
        }
        
        $prompt .= "\nIdentify any combinations of attributes that would be logically inconsistent or impossible in the real world. ";
        $prompt .= "For example, children under 5 years old cannot have jobs, a person cannot be both married and single simultaneously, etc.\n\n";
        $prompt .= "Format your response as follows:\n";
        $prompt .= "1. If you find ANY logical inconsistencies, start each one with 'INCONSISTENCY:' followed by a clear explanation.\n";
        $prompt .= "2. If the profile is logically consistent, respond with 'CONSISTENT: No logical inconsistencies found.'\n\n";
        $prompt .= "Be thorough and check for all possible inconsistencies between age, education, employment, marital status, etc. Focus only on logical impossibilities, not statistical unlikelihood.";

        return $prompt;
    }

    private function parseGptResponse($response) {
        error_log("Parsing GPT response: " . $response);
        
        $inconsistencies = [];
        $lines = explode("\n", $response);
        
        foreach ($lines as $line) {
            $line = trim($line);
            if (empty($line)) continue;
            
            // Skip if explicitly marked as consistent
            if (stripos($line, 'CONSISTENT:') === 0) {
                error_log("Profile marked as consistent");
                continue;
            }
            
            // Extract inconsistency descriptions
            if (preg_match('/^INCONSISTENCY:\s*(.+)$/i', $line, $matches)) {
                $description = $this->standardizeDescription(trim($matches[1]));
                error_log("Found inconsistency: $description");
                $inconsistencies[] = $description;
            }
        }
        
        error_log("Total inconsistencies found: " . count($inconsistencies));
        return $inconsistencies;
    }

    /**
     * Save an anomaly for later grouping
     */
    private function saveAnomaly($panelistId, $description) {
    try {
        error_log("Saving anomaly for panelist $panelistId: $description");
        
        // Extract attribute names from the description
        $attributePattern = '/\b(Gender|Age|Employment Type|Urbanization|Literacy|Gross Annual Income|Religion|Education|Marital Status)\b/i';
        preg_match_all($attributePattern, $description, $matches);
        $attributesInvolved = implode(', ', array_unique($matches[0]));
        
        $affectedIds = json_encode([$panelistId]);
        
        $stmt = $this->db->prepare("
            INSERT INTO panel_anomalies (panelist_id, description, attributes_involved, affected_ids)
            VALUES (?, ?, ?, ?)
        ");
        
        if (!$stmt) {
            error_log("Failed to prepare statement: " . $this->db->getLastError());
            return false;
        }
        
        $stmt->bind_param('ssss', $panelistId, $description, $attributesInvolved, $affectedIds);
        $result = $stmt->execute();
        
        if (!$result) {
            error_log("Failed to save anomaly: " . $stmt->error);
            return false;
        }
        
        error_log("Successfully saved anomaly for $panelistId");
        return true;
    } catch (Exception $e) {
        error_log("Exception saving anomaly: " . $e->getMessage());
        return false;
    }
}

    private function standardizeDescription($description) {
        // Remove patterns like "Profile indicates" or "This profile shows"
        $description = preg_replace('/^(the\s+)?(profile|record|data)\s+(indicates|shows|has|contains|suggests|presents|features)\s+/i', '', $description);
        
        // Use present tense and declarative form
        $description = preg_replace('/^there\s+is\s+an?\s+inconsistency\s+(with|between|in)\s+/i', '', $description);
        
        // Normalize ages
        $description = preg_replace('/\ba(?:ge)?\s+of\s+(\d+)\b/i', '$1 years old', $description);
        
        // Normalize general inconsistency phrases
        $description = preg_replace('/^(it\s+is|this\s+is)\s+(impossible|not\s+possible|inconsistent)\s+for\s+/i', '', $description);
        
        return trim($description);
    }

    private function findSimilarDirective($description) {
        $sql = "SELECT id, description FROM panel_directives WHERE status != 'resolved'";
        $result = $this->db->query($sql);
        
        if (!$result) {
            error_log("Failed to query directives: " . $this->db->getLastError());
            return null;
        }
        
        while ($row = $result->fetch_assoc()) {
            similar_text($description, $row['description'], $percent);
            if ($percent > 70) {  // 70% similarity threshold
                return $row;
            }
        }
        
        return null;
    }

    private function markRecordStatus($panelistId, $status) {
        $sql = "
            INSERT INTO panel_processing_status (panelist_id, status, attempts, last_attempt) 
            VALUES (?, ?, 1, NOW())
            ON DUPLICATE KEY UPDATE 
                status = VALUES(status),
                attempts = attempts + 1,
                last_attempt = NOW()
        ";
        $stmt = $this->db->prepare($sql);
        $stmt->bind_param('ss', $panelistId, $status);
        $stmt->execute();
    }

    public function getAttributeMetadata() {
        $result = $this->db->query("SELECT id, name, choices FROM attributes ORDER BY id");
        $attributes = [];
        while ($row = $result->fetch_assoc()) {
            $attributes[$row['id']] = [
                'name' => $row['name'],
                'choices' => json_decode($row['choices'], true)
            ];
        }
        return $attributes;
    }

    public function getCurrentState() {
        if ($this->currentState === null) {
            $result = $this->db->query("
                SELECT * FROM panel_analysis_state 
                WHERE id = (SELECT MAX(id) FROM panel_analysis_state)
            ");
            if (!$result) {
                error_log("Failed to fetch current state: " . $this->db->getLastError());
                return null;
            }
            $this->currentState = $result->fetch_assoc();
        }
        return $this->currentState;
    }
    
    private function updateState($updates) {
        $setClauses = [];
        foreach ($updates as $field => $value) {
            $setClauses[] = "$field = " . 
                (is_numeric($value) ? $value : "'" . $this->db->escape($value) . "'");
        }
        
        $sql = "UPDATE panel_analysis_state 
                SET " . implode(', ', $setClauses) . " 
                WHERE id = (SELECT MAX(id) FROM panel_analysis_state)";
                
        $success = $this->db->query($sql);
        
        if ($success) {
            $this->currentState = null;
        }
        
        return [
            'success' => $success,
            'message' => $success ? 'State updated' : 'Failed to update state'
        ];
    }

    public function pauseCheck() {
        try {
            $sql = "UPDATE panel_analysis_state 
                    SET is_paused = 1, 
                        status = 'Check paused'
                    WHERE is_running = 1 
                    AND id = (SELECT MAX(id) FROM panel_analysis_state)";
            
            $this->db->query($sql);
            $this->currentState = null; // Reset current state
            return [
                'success' => true,
                'message' => 'Check paused successfully'
            ];
        } catch (Exception $e) {
            error_log("Error pausing check: " . $e->getMessage());
            return [
                'success' => false,
                'message' => 'Failed to pause check: ' . $e->getMessage()
            ];
        }
    }
    
    public function resumeCheck() {
        try {
            $sql = "UPDATE panel_analysis_state 
                    SET is_paused = 0, 
                        status = 'Check resumed'
                    WHERE is_running = 1 
                    AND is_paused = 1
                    AND id = (SELECT MAX(id) FROM panel_analysis_state)";
            
            $this->db->query($sql);
            $this->currentState = null; // Reset current state
            return [
                'success' => true,
                'message' => 'Check resumed successfully'
            ];
        } catch (Exception $e) {
            error_log("Error resuming check: " . $e->getMessage());
            return [
                'success' => false,
                'message' => 'Failed to resume check: ' . $e->getMessage()
            ];
        }
    }

    public function addDirective($description) {
        try {
            $description = trim($description);
            if (empty($description)) {
                throw new Exception("Directive description cannot be empty");
            }

            // Start transaction
            $this->db->query("START TRANSACTION");

            // Insert new directive
            $stmt = $this->db->prepare("
                INSERT INTO panel_directives (description, record_count, status) 
                VALUES (?, 0, 'pending')
            ");
            
            $stmt->bind_param('s', $description);
            
            if (!$stmt->execute()) {
                throw new Exception("Failed to create directive");
            }

            $directiveId = $this->db->getLastInsertId();
            
            $this->db->query("COMMIT");
            
            return [
                'success' => true,
                'directive_id' => $directiveId,
                'message' => 'Directive added successfully'
            ];
            
        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Error creating directive: " . $e->getMessage());
            return [
                'success' => false,
                'message' => $e->getMessage()
            ];
        }
    }

    public function getDirectives() {
        try {
            error_log("Getting directives - start");
            
            $sql = "SELECT id, description, record_count, status, last_resolved,
                   DATE_FORMAT(created_at, '%b %d, %Y %H:%i') as created_date 
                FROM panel_directives 
                ORDER BY created_at DESC";
            
            $result = $this->db->query($sql);
            
            if ($result === false) {
                error_log("Database query failed: " . $this->db->getLastError());
                throw new Exception("Failed to fetch directives");
            }
            
            $directives = [];
            while ($row = $result->fetch_assoc()) {
                $directives[] = [
                    'id' => $row['id'],
                    'description' => $row['description'],
                    'record_count' => (int)($row['record_count'] ?? 0),
                    'status' => $row['status'],
                    'last_resolved' => $row['last_resolved'],
                    'created_date' => $row['created_date']
                ];
            }
            
            $response = [
                'success' => true,
                'directives' => $directives
            ];
            
            return $response;
            
        } catch (Exception $e) {
            error_log("Error in getDirectives: " . $e->getMessage());
            error_log("Stack trace: " . $e->getTraceAsString());
            return [
                'success' => false,
                'message' => $e->getMessage()
            ];
        }
    }

    public function handleDirectiveAction($directiveId, $action) {
        try {
            error_log("Handling directive action: $action for directive ID: $directiveId");
            
            $this->db->query("START TRANSACTION");
            
            $status = '';
            switch($action) {
                case 'approve':
                    $status = 'approved';
                    break;
                case 'reject':
                    $status = 'rejected';
                    break;
                case 'resolve':
                    $status = 'resolved';
                    break;
                default:
                    throw new Exception("Invalid action");
            }

            error_log("Setting status to: $status");

            $sql = "UPDATE panel_directives SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?";
            $stmt = $this->db->prepare($sql);
            
            if (!$stmt) {
                throw new Exception("Failed to prepare statement: " . $this->db->getLastError());
            }
            
            $stmt->bind_param('si', $status, $directiveId);
            
            if (!$stmt->execute()) {
                throw new Exception("Failed to update directive status: " . $stmt->error);
            }

            error_log("Rows affected: " . $stmt->affected_rows);

            $this->db->query("COMMIT");

            return [
                'success' => true,
                'message' => "Directive {$status} successfully"
            ];

        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Error handling directive action: " . $e->getMessage());
            return [
                'success' => false,
                'message' => $e->getMessage()
            ];
        }
    }

    public function checkAffectedMembers($directiveId) {
        try {
            error_log("Starting checkAffectedMembers for directive: " . $directiveId);
            
            // Get all panel members with this directive
            $sql = "SELECT panelist_id FROM directive_records WHERE directive_id = ?";
            $stmt = $this->db->prepare($sql);
            $stmt->bind_param('i', $directiveId);
            $stmt->execute();
            $result = $stmt->get_result();
            
            $affectedMembers = [];
            while ($row = $result->fetch_assoc()) {
                $affectedMembers[] = $row['panelist_id'];
            }
            
            // Get the directive description
            $stmt = $this->db->prepare("SELECT description FROM panel_directives WHERE id = ?");
            $stmt->bind_param('i', $directiveId);
            $stmt->execute();
            $result = $stmt->get_result();
            $directive = $result->fetch_assoc();

            if (!$directive) {
                throw new Exception("Directive not found");
            }

            $count = count($affectedMembers);
            error_log("Found {$count} affected members for directive {$directiveId}");

            return [
                'success' => true,
                'affected_count' => $count,
                'description' => $directive['description'],
                'affected_members' => $affectedMembers
            ];

        } catch (Exception $e) {
            error_log("Error in checkAffectedMembers: " . $e->getMessage());
            return [
                'success' => false,
                'message' => $e->getMessage()
            ];
        }
    }

    public function applyResolveAction($directiveId) {
        try {
            $this->db->query("START TRANSACTION");

            // Get affected panelist IDs
            $sql = "SELECT DISTINCT panelist_id FROM directive_records WHERE directive_id = ?";
            $stmt = $this->db->prepare($sql);
            $stmt->bind_param('i', $directiveId);
            $stmt->execute();
            $result = $stmt->get_result();
            
            $affected = [];
            while ($row = $result->fetch_assoc()) {
                $affected[] = $this->db->escape($row['panelist_id']);
            }

            if (!empty($affected)) {
                // Delete affected panel members
                $panelist_ids = "'" . implode("','", $affected) . "'";
                $delete_sql = "DELETE FROM panel_data WHERE panelist_id IN ($panelist_ids)";
                if (!$this->db->query($delete_sql)) {
                    throw new Exception("Failed to delete affected panel members");
                }
            }

            // Update directive status and last_resolved
            $sql = "UPDATE panel_directives 
                    SET status = 'resolved', 
                        last_resolved = CURRENT_TIMESTAMP 
                    WHERE id = ?";
            $stmt = $this->db->prepare($sql);
            $stmt->bind_param('i', $directiveId);
            
            if (!$stmt->execute()) {
                throw new Exception("Failed to update directive status");
            }

            $this->db->query("COMMIT");
            
            return [
                'success' => true,
                'message' => 'Directive resolved successfully',
                'affected_count' => count($affected)
            ];

        } catch (Exception $e) {
            $this->db->query("ROLLBACK");
            error_log("Error in applyResolveAction: " . $e->getMessage());
            return [
                'success' => false,
                'message' => $e->getMessage()
            ];
        }
    }
    
    public function resetStuckProcess() {
        try {
            $sql = "UPDATE panel_analysis_state 
                    SET is_running = 0, 
                        status = 'Check terminated due to inactivity'
                    WHERE is_running = 1 
                    AND last_processed < DATE_SUB(NOW(), INTERVAL 5 MINUTE)";
            
            $this->db->query($sql);
            return true;
        } catch (Exception $e) {
            error_log("Error resetting stuck process: " . $e->getMessage());
            return false;
        }
    }
}