#!/usr/bin/env python3 """ Data Acquisition Script for Infrastructure & Gender Analysis ============================================================ Attempts to download Census 2011 and NFHS-5 district-level data from public repositories. Target Data: 1. Census 2011: District-level Water, Sanitation, Electricity indicators 2. NFHS-5: District-level Female Labor Force Participation and Empowerment metrics CRITICAL: Does NOT generate synthetic data. Creates MISSING_DATA.md if downloads fail. """ import sys import os from pathlib import Path import requests import pandas as pd from io import StringIO import time # Configuration SESSION_DIR = Path("/app/sandbox/session_20251229_112053_41c35bf38d4f") DATA_RAW_DIR = SESSION_DIR / "data" / "raw" TIMEOUT = 30 # seconds for HTTP requests # Ensure data directory exists DATA_RAW_DIR.mkdir(parents=True, exist_ok=True) print("=" * 70) print("DATA ACQUISITION SCRIPT") print("=" * 70) print(f"Target directory: {DATA_RAW_DIR}") print() # Track download status download_status = { "census_2011": {"success": False, "file": None, "message": ""}, "nfhs_5": {"success": False, "file": None, "message": ""} } def download_file(url, save_path, dataset_name): """Download file from URL with progress tracking.""" print(f"[{dataset_name}] Attempting to download from: {url}") try: response = requests.get(url, timeout=TIMEOUT, stream=True) response.raise_for_status() # Get file size if available total_size = int(response.headers.get('content-length', 0)) with open(save_path, 'wb') as f: if total_size == 0: f.write(response.content) print(f"[{dataset_name}] ✓ Downloaded (size unknown)") else: downloaded = 0 chunk_size = 8192 for chunk in response.iter_content(chunk_size=chunk_size): if chunk: f.write(chunk) downloaded += len(chunk) percent = (downloaded / total_size) * 100 if downloaded % (chunk_size * 100) == 0: # Print every ~800KB print(f"[{dataset_name}] Progress: {percent:.1f}% ({downloaded}/{total_size} bytes)") file_size = os.path.getsize(save_path) print(f"[{dataset_name}] ✓ Successfully downloaded: {file_size:,} bytes") return True, f"Downloaded successfully ({file_size:,} bytes)" except requests.exceptions.Timeout: return False, f"Download timed out (>{TIMEOUT}s)" except requests.exceptions.ConnectionError: return False, "Connection error - unable to reach server" except requests.exceptions.HTTPError as e: return False, f"HTTP error: {e.response.status_code} - {e.response.reason}" except Exception as e: return False, f"Download failed: {str(e)}" def verify_census_data(file_path): """Verify Census 2011 data contains required columns.""" print(f"\n[Census 2011] Verifying data structure...") try: # Try reading with different encodings and delimiters df = None for encoding in ['utf-8', 'latin-1', 'iso-8859-1']: try: df = pd.read_csv(file_path, encoding=encoding, nrows=10) break except UnicodeDecodeError: continue if df is None: return False, "Unable to read file with common encodings" print(f"[Census 2011] Shape (first 10 rows): {df.shape}") print(f"[Census 2011] Columns: {list(df.columns)[:10]}") # Show first 10 columns # Check for required columns (case-insensitive) cols_lower = [str(col).lower() for col in df.columns] required_keywords = ['district', 'water', 'sanitation', 'electricity', 'toilet', 'latrine', 'drinking'] found_keywords = [] for keyword in required_keywords: if any(keyword in col for col in cols_lower): found_keywords.append(keyword) print(f"[Census 2011] Found keywords: {found_keywords}") # At minimum, need district identifier and at least one infrastructure indicator has_district = any('district' in col or 'name' in col for col in cols_lower) has_infrastructure = any(kw in cols_lower for kw in ['water', 'sanitation', 'electricity', 'toilet', 'latrine']) if has_district and has_infrastructure: return True, f"Valid data - contains district info and infrastructure indicators" else: missing = [] if not has_district: missing.append("district identifier") if not has_infrastructure: missing.append("infrastructure indicators (water/sanitation/electricity)") return False, f"Missing required columns: {', '.join(missing)}" except Exception as e: return False, f"Verification failed: {str(e)}" def verify_nfhs_data(file_path): """Verify NFHS-5 data contains required columns.""" print(f"\n[NFHS-5] Verifying data structure...") try: # Try reading with different encodings df = None for encoding in ['utf-8', 'latin-1', 'iso-8859-1']: try: df = pd.read_csv(file_path, encoding=encoding, nrows=10) break except UnicodeDecodeError: continue if df is None: return False, "Unable to read file with common encodings" print(f"[NFHS-5] Shape (first 10 rows): {df.shape}") print(f"[NFHS-5] Columns: {list(df.columns)[:10]}") # Show first 10 columns # Check for required columns (case-insensitive) cols_lower = [str(col).lower() for col in df.columns] required_keywords = ['district', 'female', 'women', 'labor', 'labour', 'employment', 'empowerment', 'work'] found_keywords = [] for keyword in required_keywords: if any(keyword in col for col in cols_lower): found_keywords.append(keyword) print(f"[NFHS-5] Found keywords: {found_keywords}") # At minimum, need district identifier and gender/empowerment indicators has_district = any('district' in col or 'name' in col for col in cols_lower) has_gender = any(kw in cols_lower for kw in ['female', 'women', 'labor', 'labour', 'employment', 'empowerment', 'work']) if has_district and has_gender: return True, f"Valid data - contains district info and gender/empowerment indicators" else: missing = [] if not has_district: missing.append("district identifier") if not has_gender: missing.append("gender/empowerment indicators") return False, f"Missing required columns: {', '.join(missing)}" except Exception as e: return False, f"Verification failed: {str(e)}" # ============================================================================= # CENSUS 2011 DATA ACQUISITION # ============================================================================= print("\n" + "=" * 70) print("ATTEMPTING TO DOWNLOAD CENSUS 2011 DATA") print("=" * 70) census_sources = [ { "name": "pratapvardhan/India-Census-2011 (District PCA)", "url": "https://raw.githubusercontent.com/pratapvardhan/India-Census-2011/master/district-pca.csv", "filename": "census_2011_district_pca.csv" }, { "name": "pratapvardhan/India-Census-2011 (Houselisting)", "url": "https://raw.githubusercontent.com/pratapvardhan/India-Census-2011/master/district-houselisting.csv", "filename": "census_2011_district_houselisting.csv" }, { "name": "Alternative GitHub source", "url": "https://raw.githubusercontent.com/pratapvardhan/districtdatabase/master/census2011/district-pca-2011.csv", "filename": "census_2011_district_alt.csv" } ] for i, source in enumerate(census_sources, 1): print(f"\n[Census 2011] Attempt {i}/{len(census_sources)}") print(f"[Census 2011] Source: {source['name']}") save_path = DATA_RAW_DIR / source['filename'] success, message = download_file(source['url'], save_path, "Census 2011") if success: # Verify the downloaded data valid, verify_msg = verify_census_data(save_path) print(f"[Census 2011] Verification: {verify_msg}") if valid: download_status["census_2011"]["success"] = True download_status["census_2011"]["file"] = str(save_path) download_status["census_2011"]["message"] = "Successfully downloaded and verified" print(f"[Census 2011] ✓✓ Data acquired successfully!") break else: print(f"[Census 2011] ✗ Data verification failed, trying next source...") download_status["census_2011"]["message"] = verify_msg else: print(f"[Census 2011] ✗ {message}") download_status["census_2011"]["message"] = message time.sleep(1) # Brief delay between attempts if not download_status["census_2011"]["success"]: print(f"\n[Census 2011] ✗✗ All download attempts failed") # ============================================================================= # NFHS-5 DATA ACQUISITION # ============================================================================= print("\n" + "=" * 70) print("ATTEMPTING TO DOWNLOAD NFHS-5 DATA") print("=" * 70) nfhs_sources = [ { "name": "NFHS India GitHub Repository (District Factsheet)", "url": "https://raw.githubusercontent.com/nfhs-india/nfhs5-data/main/NFHS5_District_Factsheet.csv", "filename": "nfhs_5_district_factsheet.csv" }, { "name": "Alternative NFHS-5 source", "url": "https://raw.githubusercontent.com/pratapvardhan/nfhs-india/main/data/nfhs5-district.csv", "filename": "nfhs_5_district_alt.csv" }, { "name": "Data.gov.in mirror (if available)", "url": "https://raw.githubusercontent.com/datameet/india-data/master/nfhs/nfhs5_district.csv", "filename": "nfhs_5_district_datameet.csv" } ] for i, source in enumerate(nfhs_sources, 1): print(f"\n[NFHS-5] Attempt {i}/{len(nfhs_sources)}") print(f"[NFHS-5] Source: {source['name']}") save_path = DATA_RAW_DIR / source['filename'] success, message = download_file(source['url'], save_path, "NFHS-5") if success: # Verify the downloaded data valid, verify_msg = verify_nfhs_data(save_path) print(f"[NFHS-5] Verification: {verify_msg}") if valid: download_status["nfhs_5"]["success"] = True download_status["nfhs_5"]["file"] = str(save_path) download_status["nfhs_5"]["message"] = "Successfully downloaded and verified" print(f"[NFHS-5] ✓✓ Data acquired successfully!") break else: print(f"[NFHS-5] ✗ Data verification failed, trying next source...") download_status["nfhs_5"]["message"] = verify_msg else: print(f"[NFHS-5] ✗ {message}") download_status["nfhs_5"]["message"] = message time.sleep(1) # Brief delay between attempts if not download_status["nfhs_5"]["success"]: print(f"\n[NFHS-5] ✗✗ All download attempts failed") # ============================================================================= # FINAL SUMMARY AND MISSING DATA REPORT # ============================================================================= print("\n" + "=" * 70) print("DATA ACQUISITION SUMMARY") print("=" * 70) census_success = download_status["census_2011"]["success"] nfhs_success = download_status["nfhs_5"]["success"] print(f"Census 2011: {'✓ SUCCESS' if census_success else '✗ FAILED'}") if census_success: print(f" File: {download_status['census_2011']['file']}") else: print(f" Reason: {download_status['census_2011']['message']}") print(f"\nNFHS-5: {'✓ SUCCESS' if nfhs_success else '✗ FAILED'}") if nfhs_success: print(f" File: {download_status['nfhs_5']['file']}") else: print(f" Reason: {download_status['nfhs_5']['message']}") # If any downloads failed, create MISSING_DATA.md if not census_success or not nfhs_success: print("\n" + "=" * 70) print("CREATING MISSING_DATA.md") print("=" * 70) missing_data_content = """# Missing Data Report This file lists datasets that could not be automatically downloaded from public repositories. ## Data Acquisition Status """ if not census_success: missing_data_content += """### Census 2011 - District Level Infrastructure Data **Status**: ❌ Download Failed **Required Data**: District-level indicators for: - Water supply (piped water, tap water access) - Sanitation facilities (toilets, latrines) - Electricity access **Attempted Sources**: - https://github.com/pratapvardhan/India-Census-2011 - Alternative GitHub repositories **Failure Reason**: {census_reason} **What to do**: Please manually download the Census 2011 District Primary Census Abstract (PCA) or Houselisting data containing infrastructure indicators from: 1. Official Census website: https://censusindia.gov.in/census.website/ 2. Data.gov.in: https://data.gov.in/ 3. Any other verified government source Then upload the CSV file to: `/app/sandbox/session_20251229_112053_41c35bf38d4f/user_data/census_2011_district.csv` **Required Columns** (or similar): - District Name/Code - Water supply indicators - Sanitation/Toilet indicators - Electricity access indicators --- """.format(census_reason=download_status["census_2011"]["message"]) if not nfhs_success: missing_data_content += """### NFHS-5 - District Level Gender & Empowerment Data **Status**: ❌ Download Failed **Required Data**: District-level indicators for: - Female Labor Force Participation Rate (FLFPR) - Women's empowerment metrics - Gender-related socioeconomic indicators **Attempted Sources**: - GitHub repositories for NFHS data - Open data portals **Failure Reason**: {nfhs_reason} **What to do**: Please manually download the NFHS-5 District Factsheets from: 1. Official NFHS website: http://rchiips.org/nfhs/factsheet_NFHS-5.shtml 2. Data.gov.in 3. Any other verified government source Then upload the CSV file to: `/app/sandbox/session_20251229_112053_41c35bf38d4f/user_data/nfhs_5_district.csv` **Required Columns** (or similar): - District Name/Code - Female labor force participation rate - Women's empowerment indicators - Employment/work-related metrics for women --- """.format(nfhs_reason=download_status["nfhs_5"]["message"]) missing_data_content += """## Next Steps 1. Download the missing datasets from the official government sources listed above 2. Place them in the `user_data/` directory with the suggested filenames 3. Re-run the analysis pipeline ## Important Notes - **DO NOT use synthetic or randomly generated data** - only use official government datasets - Ensure the data is at the **district level** (not state or national level) - The data should be in **CSV format** or easily convertible to CSV - Column names may vary - the analysis script will adapt to different naming conventions --- *Generated by: 01_data_acquisition.py* *Date: {date}* """.format(date=time.strftime("%Y-%m-%d %H:%M:%S")) missing_data_path = SESSION_DIR / "MISSING_DATA.md" with open(missing_data_path, 'w') as f: f.write(missing_data_content) print(f"✓ Created: {missing_data_path}") print("\nPlease refer to MISSING_DATA.md for instructions on manually acquiring the data.") # Exit with error code to indicate incomplete data acquisition sys.exit(1) else: print("\n✓✓ All datasets successfully acquired!") print("Ready to proceed with data processing and analysis.") sys.exit(0)