#!/usr/bin/env python3
"""
Data Verification Script
Verifies downloaded datasets are readable and documents their structure
"""

import pandas as pd
import os
from pathlib import Path

def verify_dataset(filepath, dataset_name):
    """
    Verify a dataset file and return its structure information.

    Parameters:
    -----------
    filepath : str
        Path to the dataset file
    dataset_name : str
        Human-readable name for the dataset

    Returns:
    --------
    dict : Dictionary containing dataset metadata
    """
    print(f"\n{'='*60}")
    print(f"Verifying: {dataset_name}")
    print(f"{'='*60}")

    if not os.path.exists(filepath):
        print(f"❌ File not found: {filepath}")
        return None

    file_size = os.path.getsize(filepath)
    print(f"File size: {file_size:,} bytes ({file_size/1024:.2f} KB)")

    try:
        # Read the dataset with a preview
        df = pd.read_csv(filepath, nrows=5)
        print(f"✅ Successfully loaded (preview of 5 rows)")
        print(f"\nShape (preview): {df.shape}")
        print(f"Columns ({len(df.columns)}): {list(df.columns)}")
        print(f"\nColumn Data Types:")
        print(df.dtypes)
        print(f"\nFirst 3 rows:")
        print(df.head(3).to_string())

        # Count total rows (efficiently)
        total_rows = sum(1 for _ in open(filepath)) - 1  # subtract header
        print(f"\nTotal rows in dataset: {total_rows:,}")

        # Check for missing values in preview
        missing = df.isnull().sum()
        if missing.any():
            print(f"\nMissing values in preview:")
            print(missing[missing > 0])

        return {
            'name': dataset_name,
            'filepath': filepath,
            'file_size': file_size,
            'columns': list(df.columns),
            'total_rows': total_rows,
            'dtypes': df.dtypes.to_dict(),
            'sample_data': df.head(3).to_dict(),
            'status': 'success'
        }

    except Exception as e:
        print(f"❌ Error reading file: {str(e)}")
        return {
            'name': dataset_name,
            'filepath': filepath,
            'status': 'error',
            'error': str(e)
        }

def main():
    """Main verification workflow"""
    print("="*60)
    print("Dataset Verification Tool")
    print("="*60)

    data_dir = Path("/app/sandbox/session_20251229_081931_9f2d364070e9/workflow/data")

    # Define datasets to verify
    datasets = [
        {
            'file': data_dir / 'crop_production_district.csv',
            'name': 'District-wise Crop Production (India)'
        },
        {
            'file': data_dir / 'crop_state_2017.csv',
            'name': 'State-wise Agricultural Data (2017)'
        },
        {
            'file': data_dir / 'soil_fertility_data.csv',
            'name': 'Soil Fertility Data'
        }
    ]

    results = []
    for ds in datasets:
        result = verify_dataset(str(ds['file']), ds['name'])
        if result:
            results.append(result)

    # Summary
    print(f"\n{'='*60}")
    print("VERIFICATION SUMMARY")
    print(f"{'='*60}")
    successful = sum(1 for r in results if r['status'] == 'success')
    print(f"✅ Successfully verified: {successful}/{len(results)} datasets")

    if successful < len(results):
        print(f"❌ Failed: {len(results) - successful} datasets")

    return results

if __name__ == "__main__":
    results = main()