#!/usr/bin/env python3
"""
Merge keywords and subject areas from EMBO members file into candidates file.

This script enriches data/embo_membership_candidates_with_author_ids.xlsx
with keywords and subject areas from data/emboplanet_allmem_cleaned_2026-01-23.xlsx
using the mapping: embo_person_id <-> mem_joi_PSUB__Person::_kpln__ID
"""

import pandas as pd
from pathlib import Path

def merge_keywords_to_candidates():
    """Merge keywords and subject areas into candidates file."""
    
    print("="*70)
    print("MERGING KEYWORDS & SUBJECT AREAS TO CANDIDATES FILE")
    print("="*70)
    print()
    
    # Load files
    print("📂 Loading files...")
    candidates_file = "data/embo_membership_candidates_with_author_ids.xlsx"
    members_file = "data/emboplanet_allmem_cleaned_2026-01-23.xlsx"
    
    candidates_df = pd.read_excel(candidates_file)
    members_df = pd.read_excel(members_file)
    
    print(f"✓ Loaded {len(candidates_df)} rows from candidates file")
    print(f"✓ Loaded {len(members_df)} rows from members file")
    print()
    
    # Prepare members data for merge
    print("🔗 Preparing merge...")
    members_subset = members_df[[
        'mem_joi_PSUB__Person::_kpln__ID',
        'mem_joi_PSUB__Person::SC_ML_OfficialKeywords',
        'mem_joi_PSUB__Person::SC_SubjectAreas',
        'mem_joi_psub_add_INST__Institution#CurrentOnly::IN_FullName_Print__lct'
    ]].copy()
    
    # Rename for clarity
    members_subset.columns = [
        'embo_person_id',
        'keywords',
        'subject_areas',
        'affiliation'
    ]
    
    # Convert keywords and subject_areas from slash-separated to comma-separated
    print("🔄 Converting keywords format (slash → comma)...")
    members_subset['keywords'] = members_subset['keywords'].apply(
        lambda x: x.replace(' / ', ', ') if pd.notna(x) and isinstance(x, str) else x
    )
    members_subset['subject_areas'] = members_subset['subject_areas'].apply(
        lambda x: x.replace(' / ', ', ') if pd.notna(x) and isinstance(x, str) else x
    )
    
    # Remove duplicates (keep first)
    members_subset = members_subset.drop_duplicates(subset=['embo_person_id'], keep='first')
    
    print(f"✓ Prepared {len(members_subset)} unique members for merge")
    print()
    
    # Check for existing columns
    cols_to_add = ['keywords', 'subject_areas', 'affiliation']
    existing_cols = [col for col in cols_to_add if col in candidates_df.columns]
    
    if existing_cols:
        print(f"⚠️  Columns already exist: {existing_cols}")
        print("   These will be overwritten with new data")
        candidates_df = candidates_df.drop(columns=existing_cols)
    
    # Merge
    print("🔄 Merging data...")
    enriched_df = candidates_df.merge(
        members_subset,
        on='embo_person_id',
        how='left'
    )
    
    # Statistics
    matched = enriched_df['keywords'].notna().sum()
    total = len(enriched_df)
    
    print(f"✓ Merge complete!")
    print()
    print("📊 Statistics:")
    print(f"  Total candidates: {total}")
    print(f"  Matched with keywords: {matched} ({matched/total*100:.1f}%)")
    print(f"  Keywords missing: {total - matched} ({(total-matched)/total*100:.1f}%)")
    print()
    
    # Show sample
    print("📋 Sample of enriched data:")
    sample = enriched_df[enriched_df['keywords'].notna()].head(3)
    for idx, row in sample.iterrows():
        print(f"\n  {row['first_name_plus_initials']} {row['last_name']}")
        print(f"    Affiliation: {row.get('affiliation', 'N/A')}")
        keywords = str(row.get('keywords', ''))[:60]
        print(f"    Keywords: {keywords}...")
        break
    print()
    
    # Save
    output_file = candidates_file  # Overwrite original
    print(f"💾 Saving enriched file to: {output_file}")
    enriched_df.to_excel(output_file, index=False)
    print("✓ Saved successfully!")
    print()
    
    print("="*70)
    print("✅ MERGE COMPLETE!")
    print("="*70)
    print()
    print(f"The file {candidates_file} now includes:")
    print("  - keywords (from mem_joi_PSUB__Person::SC_ML_OfficialKeywords)")
    print("  - subject_areas (from mem_joi_PSUB__Person::SC_SubjectAreas)")
    print("  - affiliation (from mem_joi_psub_add_INST__Institution...)")
    print()
    print("You can now run benchmarks with full context!")


if __name__ == "__main__":
    merge_keywords_to_candidates()
