from pathlib import Path

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer


# 1. LOAD
#    user_items.csv    : items x users  (rows=items, cols=users)
#    item_features.csv : items x binary-features + description column
DATA_PATH = Path(__file__).resolve().parent / "user_items.csv"
FEAT_PATH = Path(__file__).resolve().parent / "item_features.csv"

raw      = pd.read_csv(DATA_PATH, index_col=0)
feat_raw = pd.read_csv(FEAT_PATH, index_col=0)

print("Original Rating Matrix (items x users):")
print(raw.to_string())
print("\nItem Feature + Description Table:")
print(feat_raw.to_string())


# 2. FIND MISSING VALUE (?) LOCATION  —  identical to CF
mask = raw == "?"
first_missing = mask.stack()[mask.stack()].index[0]
target_item = str(first_missing[0])   # row  -> item
target_user = str(first_missing[1])   # col  -> user
print(f"\nTarget: item='{target_item}', user='{target_user}'")


# 3. CLEAN & CONVERT TO NUMERIC
raw.replace("?", np.nan, inplace=True)
raw = raw.apply(pd.to_numeric)
raw.index   = raw.index.astype(str)
raw.columns = raw.columns.astype(str)
feat_raw.index = feat_raw.index.astype(str)

# Split descriptions from binary features
descriptions = feat_raw["description"].astype(str)         # Series: item -> text
feat = feat_raw.drop(columns=["description"]).apply(pd.to_numeric)
feat.columns = feat.columns.astype(str)

# Align feature matrix rows to rating matrix items
feat         = feat.reindex(raw.index).fillna(0)
descriptions = descriptions.reindex(raw.index).fillna("")

df_iu = raw.copy()    # items x users
df_ui = raw.T.copy()  # users x items

print("\nCleaned Rating Matrix:")
print(df_iu.to_string())
print("\nAligned Binary Feature Matrix:")
print(feat.to_string())
print("\nDescriptions:")
print(descriptions.to_string())


# 4. MEANS
item_means = df_iu.mean(axis=1)   # mean across users  (per item)
user_means = df_ui.mean(axis=1)   # mean across items  (per user)
print("\nItem means:")
print(item_means.round(4).to_string())
print("\nUser means:")
print(user_means.round(4).to_string())


# ─── HELPER: weighted deviation prediction ────────────────────────────────────
def predict_weighted_dev(sim_series, rating_series, u_mean, label,
                         threshold=0.2, top_k=3):
    """
    r̂(u,i) = mean(u) + Σ[sim(i,j)*(r(u,j)-mean(u))] / Σ|sim(i,j)|
    Filters: positive sims → above threshold → top-k.
    Returns (prediction, filtered_sim, filtered_rat).
    """
    nb = sim_series.index.intersection(rating_series.index)
    sim = sim_series[nb]
    rat = rating_series[nb]
    print(f"  Neighbour items : {list(nb)}")
    print(f"  Similarities    : {sim.round(4).values}")
    print(f"  Ratings         : {rat.values}")

    if nb.empty:
        pred = u_mean
        print(f"  No neighbours. Fallback to user mean: {pred:.4f}")
        return pred, sim, rat

    sim = sim[sim > 0]
    sim = sim[sim > threshold]
    sim = sim.sort_values(ascending=False).head(top_k)
    rat = rat[sim.index]
    print(f"  Filtered (pos, >{threshold}, top-{top_k}): {list(sim.index)}")
    print(f"  Similarities: {sim.round(4).values}")
    print(f"  Ratings     : {rat.values}")

    if sim.empty:
        pred = u_mean
        print(f"  All filtered out. Fallback to user mean: {pred:.4f}")
        return pred, sim, rat

    dev = rat - u_mean
    num = (sim * dev).sum()
    den = sim.abs().sum()
    pred = u_mean + (num / den) if den != 0 else u_mean

    print(f"\n  --- {label} ---")
    print(f"  user mean ('{target_user}')      : {u_mean:.4f}")
    print(f"  numerator  Σ sim*(r-mean)        : {num:.4f}")
    print(f"  denominator Σ|sim|               : {den:.4f}")
    print(f"  predicted r('{target_user}', item '{target_item}') = {pred:.4f}")
    return pred, sim, rat
# ──────────────────────────────────────────────────────────────────────────────


# ==========================================================================
# METHOD 1 — COSINE SIMILARITY on RAW binary feature vectors
# ==========================================================================
print("\n" + "=" * 55)
print("METHOD 1: Item-Feature Cosine Similarity (Raw Binary)")
print("=" * 55)

cos_raw = cosine_similarity(feat)
item_sim_cos = pd.DataFrame(cos_raw, index=feat.index, columns=feat.index).astype(str).apply(
    lambda c: pd.to_numeric(c, errors="coerce"))
item_sim_cos = pd.DataFrame(cosine_similarity(feat), index=feat.index, columns=feat.index)
print("\nItem-Item Cosine Similarity (raw features):")
print(item_sim_cos.round(4).to_string())

# Alternative: Pearson correlation on feature rows
item_sim_pear = feat.T.corr()
print("\nItem-Item Pearson Correlation (raw features) [alternative]:")
print(item_sim_pear.round(4).to_string())

# Alternative: Jaccard (for binary features)
def jaccard_matrix(mat):
    b = (mat > 0).astype(float).values
    inter = b @ b.T
    sums  = b.sum(axis=1, keepdims=True)
    union = sums + sums.T - inter
    with np.errstate(invalid="ignore"):
        jac = np.where(union == 0, 0.0, inter / union)
    return pd.DataFrame(jac, index=mat.index, columns=mat.index)

item_sim_jac = jaccard_matrix(feat)
print("\nItem-Item Jaccard Similarity (binary features) [alternative]:")
print(item_sim_jac.round(4).to_string())

# ----- PREDICTION -----------------------------------------------------------
u_mean_m1 = user_means[target_user]
rated_by_user_m1 = df_iu[target_user].dropna().drop(target_item, errors="ignore")
sims_m1 = item_sim_cos[target_item].drop(target_item).dropna()

pred_m1, _, _ = predict_weighted_dev(
    sims_m1, rated_by_user_m1, u_mean_m1,
    "Method 1 Result (Cosine Raw, Weighted Deviation)")

# Alternative: simple weighted average (no mean-centering)
# nb = sims_m1.index.intersection(rated_by_user_m1.index)
# pred_wa = (sims_m1[nb] * rated_by_user_m1[nb]).sum() / sims_m1[nb].abs().sum()


# ==========================================================================
# METHOD 2 — TF-IDF WEIGHTED COSINE SIMILARITY on binary features
# Upweights rare/discriminating features (IDF penalises common ones)
# ==========================================================================
print("\n" + "=" * 55)
print("METHOD 2: Item-Feature Cosine Similarity (TF-IDF Weighted)")
print("=" * 55)

tfidf = TfidfTransformer(smooth_idf=True, use_idf=True)
feat_tfidf_arr = tfidf.fit_transform(feat.values).toarray()
feat_tfidf = pd.DataFrame(feat_tfidf_arr, index=feat.index, columns=feat.columns)
print("\nTF-IDF Feature Matrix:")
print(feat_tfidf.round(4).to_string())

item_sim_tfidf = pd.DataFrame(
    cosine_similarity(feat_tfidf_arr), index=feat.index, columns=feat.index)
print("\nItem-Item Cosine Similarity (TF-IDF):")
print(item_sim_tfidf.round(4).to_string())

rated_by_user_m2 = df_iu[target_user].dropna().drop(target_item, errors="ignore")
sims_m2 = item_sim_tfidf[target_item].drop(target_item).dropna()

pred_m2, _, _ = predict_weighted_dev(
    sims_m2, rated_by_user_m2, user_means[target_user],
    "Method 2 Result (TF-IDF Cosine, Weighted Deviation)")


# ==========================================================================
# METHOD 3 — USER PROFILE APPROACH
# Build explicit taste fingerprint, score target item against it
# ==========================================================================
print("\n" + "=" * 55)
print("METHOD 3: User Profile → Item Score (Cosine)")
print("=" * 55)

rated_m3  = df_iu[target_user].dropna().drop(target_item, errors="ignore")
feat_rated = feat.loc[rated_m3.index]
u_mean_m3 = user_means[target_user]

if rated_m3.empty:
    pred_m3 = item_means[target_item]
    print(f"  No ratings at all. Fallback to item mean: {pred_m3:.4f}")
else:
    deviations = rated_m3 - u_mean_m3
    pos_dev    = deviations[deviations > 0]

    if pos_dev.empty:
        # all ratings at/below mean → simple weighted avg
        profile = feat_rated.T.dot(rated_m3) / rated_m3.sum()
        print("  [No items above user mean — using simple weighted avg profile]")
    else:
        # Mean-centered profile: weight by positive deviation only
        profile = feat_rated.loc[pos_dev.index].T.dot(pos_dev) / pos_dev.abs().sum()

    profile   = pd.Series(profile, index=feat.columns)
    profile_A = feat_rated.T.dot(rated_m3) / rated_m3.sum()  # simple avg (alternative)

    print("\nUser Profile Vector (mean-centered, positive-deviation weights):")
    print(profile.round(4).to_string())
    print("\nUser Profile Vector (simple weighted avg) [alternative]:")
    print(profile_A.round(4).to_string())

    target_feat  = feat.loc[[target_item]].values
    cos_m3       = cosine_similarity(profile.values.reshape(1, -1), target_feat)[0, 0]
    dot_m3       = profile.values @ target_feat.flatten()
    print(f"\n  Cosine(profile, '{target_item}')   = {cos_m3:.4f}")
    print(f"  Dot-product score [alternative]   = {dot_m3:.4f}")

    # Score → rating: linear mapping [-1,1] → [r_min, r_max]
    r_min, r_max = 1, 5
    pred_m3_map  = r_min + (cos_m3 + 1) / 2 * (r_max - r_min)

    # Weighted deviation using profile-vs-rated-items similarity
    rated_cos  = cosine_similarity(profile.values.reshape(1, -1), feat_rated.values)[0]
    sim_prof   = pd.Series(rated_cos, index=feat_rated.index)
    dev_prof   = rated_m3 - u_mean_m3
    num_m3 = (sim_prof * dev_prof).sum()
    den_m3 = sim_prof.abs().sum()
    pred_m3_dev = u_mean_m3 + (num_m3 / den_m3) if den_m3 != 0 else u_mean_m3

    print(f"\n--- Method 3 Result (User Profile + Cosine) ---")
    print(f"  user mean ('{target_user}')              : {u_mean_m3:.4f}")
    print(f"  cosine(profile, target item)             : {cos_m3:.4f}")
    print(f"  predicted [score→rating linear mapping]  : {pred_m3_map:.4f}")
    print(f"  predicted [weighted deviation]            : {pred_m3_dev:.4f}")
    pred_m3 = pred_m3_dev


# ==========================================================================
# METHOD 4 — BAG OF WORDS (BoW) FROM ITEM DESCRIPTIONS
# Extracts features directly from text; no manual labelling needed
# ==========================================================================
print("\n" + "=" * 55)
print("METHOD 4: BoW / TF-IDF from Item Descriptions")
print("=" * 55)

# 4a. CountVectorizer → raw term-frequency matrix  (items x vocab)
#     stop_words='english' removes 'with', 'and', 'the', etc.
cv = CountVectorizer(stop_words="english", min_df=1)
bow_arr  = cv.fit_transform(descriptions).toarray()
vocab    = cv.get_feature_names_out()
bow_df   = pd.DataFrame(bow_arr, index=descriptions.index, columns=vocab)
print("\nBoW Term-Frequency Matrix (items x vocab):")
print(bow_df.to_string())

# Alternative: Binary BoW (presence/absence, ignore frequency)
bow_bin_df = (bow_df > 0).astype(int)
print("\nBinary BoW Matrix [alternative — ignores term frequency]:")
print(bow_bin_df.to_string())

# 4b. TF-IDF on BoW vectors (standard text-retrieval approach)
tfidf_bow = TfidfTransformer(smooth_idf=True, use_idf=True)
bow_tfidf_arr = tfidf_bow.fit_transform(bow_arr).toarray()
bow_tfidf_df  = pd.DataFrame(bow_tfidf_arr, index=descriptions.index, columns=vocab)
print("\nTF-IDF BoW Matrix:")
print(bow_tfidf_df.round(4).to_string())

# 4c. Item-item similarity: cosine on TF-IDF BoW
item_sim_bow = pd.DataFrame(
    cosine_similarity(bow_tfidf_arr), index=descriptions.index, columns=descriptions.index)
print("\nItem-Item Cosine Similarity (TF-IDF BoW):")
print(item_sim_bow.round(4).to_string())

# Alternative: cosine on raw BoW counts
item_sim_bow_raw = pd.DataFrame(
    cosine_similarity(bow_arr), index=descriptions.index, columns=descriptions.index)
print("\nItem-Item Cosine Similarity (raw BoW counts) [alternative]:")
print(item_sim_bow_raw.round(4).to_string())

# Alternative: Jaccard on binary BoW
item_sim_bow_jac = jaccard_matrix(bow_bin_df)
print("\nItem-Item Jaccard Similarity (binary BoW) [alternative]:")
print(item_sim_bow_jac.round(4).to_string())

# 4d. PREDICTION via weighted deviation (same formula, BoW similarities)
rated_by_user_m4 = df_iu[target_user].dropna().drop(target_item, errors="ignore")
sims_m4 = item_sim_bow[target_item].drop(target_item).dropna()

pred_m4, _, _ = predict_weighted_dev(
    sims_m4, rated_by_user_m4, user_means[target_user],
    "Method 4 Result (TF-IDF BoW, Weighted Deviation)")

# 4e. USER PROFILE from BoW vectors (mirrors Method 3 but in text space)
print(f"\n--- Method 4b: BoW User Profile ---")
feat_bow_rated = bow_tfidf_df.loc[rated_by_user_m4.index]
u_mean_m4 = user_means[target_user]

if feat_bow_rated.empty:
    pred_m4b = u_mean_m4
    print(f"  No rated items. Fallback to user mean: {pred_m4b:.4f}")
else:
    dev_m4b   = rated_by_user_m4 - u_mean_m4
    pos_m4b   = dev_m4b[dev_m4b > 0]
    if pos_m4b.empty:
        wts = rated_by_user_m4
        profile_bow = feat_bow_rated.T.dot(wts) / wts.sum()
    else:
        profile_bow = feat_bow_rated.loc[pos_m4b.index].T.dot(pos_m4b) / pos_m4b.abs().sum()

    target_bow  = bow_tfidf_df.loc[[target_item]].values
    cos_m4b     = cosine_similarity(profile_bow.values.reshape(1, -1), target_bow)[0, 0]

    sim_bow_prof = pd.Series(
        cosine_similarity(profile_bow.values.reshape(1, -1), feat_bow_rated.values)[0],
        index=feat_bow_rated.index)
    dev_bow = rated_by_user_m4 - u_mean_m4
    num_m4b = (sim_bow_prof * dev_bow).sum()
    den_m4b = sim_bow_prof.abs().sum()
    pred_m4b = u_mean_m4 + (num_m4b / den_m4b) if den_m4b != 0 else u_mean_m4

    print(f"  user mean ('{target_user}')               : {u_mean_m4:.4f}")
    print(f"  cosine(BoW profile, target item)          : {cos_m4b:.4f}")
    print(f"  predicted r('{target_user}', item '{target_item}')  : {pred_m4b:.4f}")


# FINAL SUMMARY

print("\n" + "=" * 65)
print("FINAL SUMMARY")
print("=" * 65)
print(f"  Target                              : user='{target_user}', item='{target_item}'")
print(f"  M1 — Cosine (raw binary features)  : {pred_m1:.4f}")
print(f"  M2 — Cosine (TF-IDF features)      : {pred_m2:.4f}")
print(f"  M3 — User Profile Cosine            : {pred_m3:.4f}")
print(f"  M4 — TF-IDF BoW from descriptions  : {pred_m4:.4f}")
print(f"  M4b— BoW User Profile               : {pred_m4b:.4f}")
print("=" * 65)


# VISUALISATION

fig, axes = plt.subplots(2, 3, figsize=(20, 10))
fig.suptitle("Content-Based Filtering — Visualisation", fontsize=13, fontweight="bold")

def annotate_heatmap(ax, df, fmt=".2f"):
    for i in range(df.shape[0]):
        for j in range(df.shape[1]):
            val = df.iloc[i, j]
            if not (isinstance(val, float) and np.isnan(val)):
                ax.text(j, i, f"{val:{fmt}}", ha="center", va="center", fontsize=7)

def set_tick_labels(ax, df):
    ax.set_xticks(range(len(df.columns))); ax.set_xticklabels(df.columns, rotation=45, ha="right", fontsize=8)
    ax.set_yticks(range(len(df.index)));   ax.set_yticklabels(df.index, fontsize=8)

# 1. Rating matrix
ax = axes[0, 0]
ti = list(df_iu.index).index(target_item)
tu = list(df_iu.columns).index(target_user)
im = ax.imshow(df_iu.values.astype(float), cmap="YlGn", aspect="auto", vmin=1, vmax=5)
for i in range(df_iu.shape[0]):
    for j in range(df_iu.shape[1]):
        val = df_iu.iloc[i, j]
        if i == ti and j == tu:
            ax.add_patch(plt.Rectangle((j-.5, i-.5), 1, 1, fill=True, color="#4fc3f7", zorder=2))
            ax.text(j, i, "?", ha="center", va="center", fontsize=9, fontweight="bold", color="navy", zorder=3)
        elif pd.notna(val):
            ax.text(j, i, int(val), ha="center", va="center", fontsize=8)
set_tick_labels(ax, df_iu)
ax.set_title("Rating matrix (blue = target ?)", fontsize=10)
plt.colorbar(im, ax=ax, shrink=0.8, label="rating")

# 2. Binary feature matrix
ax = axes[0, 1]
im2 = ax.imshow(feat.values.astype(float), cmap="Blues", aspect="auto", vmin=0, vmax=1)
annotate_heatmap(ax, feat, fmt=".0f")
ti_f = list(feat.index).index(target_item)
ax.add_patch(plt.Rectangle((-.5, ti_f-.5), feat.shape[1], 1, fill=False, edgecolor="red", lw=2))
set_tick_labels(ax, feat)
ax.set_title(f"Binary features (red = target '{target_item}')", fontsize=10)
plt.colorbar(im2, ax=ax, shrink=0.8)

# 3. Item-item cosine (raw binary)
ax = axes[0, 2]
im3 = ax.imshow(item_sim_cos.values.astype(float), cmap="coolwarm", vmin=-1, vmax=1, aspect="auto")
annotate_heatmap(ax, item_sim_cos)
set_tick_labels(ax, item_sim_cos)
ax.set_title("Item-item cosine (raw binary)", fontsize=10)
plt.colorbar(im3, ax=ax, shrink=0.8, label="cosine")

# 4. TF-IDF item-item similarity
ax = axes[1, 0]
im4 = ax.imshow(item_sim_tfidf.values.astype(float), cmap="coolwarm", vmin=-1, vmax=1, aspect="auto")
annotate_heatmap(ax, item_sim_tfidf)
set_tick_labels(ax, item_sim_tfidf)
ax.set_title("Item-item cosine (TF-IDF features)", fontsize=10)
plt.colorbar(im4, ax=ax, shrink=0.8, label="cosine")

# 5. BoW TF-IDF item-item similarity
ax = axes[1, 1]
im5 = ax.imshow(item_sim_bow.values.astype(float), cmap="coolwarm", vmin=-1, vmax=1, aspect="auto")
annotate_heatmap(ax, item_sim_bow)
set_tick_labels(ax, item_sim_bow)
ax.set_title("Item-item cosine (TF-IDF BoW from desc)", fontsize=10)
plt.colorbar(im5, ax=ax, shrink=0.8, label="cosine")

# 6. BoW term-frequency heatmap (items x top-10 vocab)
ax = axes[1, 2]
top_terms = bow_df.sum().sort_values(ascending=False).head(10).index
bow_top   = bow_df[top_terms]
im6 = ax.imshow(bow_top.values.astype(float), cmap="Purples", aspect="auto")
annotate_heatmap(ax, bow_top, fmt=".0f")
set_tick_labels(ax, bow_top)
ax.set_title("BoW term frequencies (top-10 terms)", fontsize=10)
plt.colorbar(im6, ax=ax, shrink=0.8, label="count")

plt.tight_layout()
plt.show()


# 7. EVALUATION METRICS (All methods, leave-one-out on observed ratings)

print("\n" + "=" * 65)
print("STEP 7: EVALUATION METRICS REPORT (ALL METHODS)")
print("=" * 65)


def weighted_dev_from_similarity(sim_series, rating_series, u_mean, threshold=0.2, top_k=3):
    """Return weighted-deviation prediction with the same filters used above."""
    nb = sim_series.index.intersection(rating_series.index)
    if len(nb) == 0:
        return float(u_mean)

    sim = sim_series[nb]
    rat = rating_series[nb]

    sim = sim[sim > 0]
    sim = sim[sim > threshold]
    sim = sim.sort_values(ascending=False).head(top_k)
    if sim.empty:
        return float(u_mean)

    rat = rat[sim.index]
    dev = rat - u_mean
    num = (sim * dev).sum()
    den = sim.abs().sum()
    return float(u_mean + (num / den)) if den != 0 else float(u_mean)


def predict_m1(train_iu, user_id, item_id):
    user_mean_local = train_iu.T.mean(axis=1).get(user_id, np.nan)
    sims = item_sim_cos[item_id].drop(item_id, errors="ignore").dropna()
    rated = train_iu[user_id].dropna().drop(item_id, errors="ignore")
    return weighted_dev_from_similarity(sims, rated, user_mean_local, threshold=0.2, top_k=3)


def predict_m2(train_iu, user_id, item_id):
    user_mean_local = train_iu.T.mean(axis=1).get(user_id, np.nan)
    sims = item_sim_tfidf[item_id].drop(item_id, errors="ignore").dropna()
    rated = train_iu[user_id].dropna().drop(item_id, errors="ignore")
    return weighted_dev_from_similarity(sims, rated, user_mean_local, threshold=0.2, top_k=3)


def predict_m3(train_iu, user_id, item_id):
    user_mean_local = train_iu.T.mean(axis=1).get(user_id, np.nan)
    rated = train_iu[user_id].dropna().drop(item_id, errors="ignore")

    if rated.empty:
        return float(train_iu.mean(axis=1).get(item_id, user_mean_local))

    feat_rated_local = feat.loc[rated.index]
    deviations = rated - user_mean_local
    pos_dev = deviations[deviations > 0]

    if pos_dev.empty:
        profile = feat_rated_local.T.dot(rated) / rated.sum() if rated.sum() != 0 else feat_rated_local.mean(axis=0)
    else:
        denom = pos_dev.abs().sum()
        profile = (
            feat_rated_local.loc[pos_dev.index].T.dot(pos_dev) / denom
            if denom != 0
            else feat_rated_local.mean(axis=0)
        )

    sim_prof = pd.Series(
        cosine_similarity(profile.values.reshape(1, -1), feat_rated_local.values)[0],
        index=feat_rated_local.index,
    )
    dev_prof = rated - user_mean_local
    num = (sim_prof * dev_prof).sum()
    den = sim_prof.abs().sum()
    return float(user_mean_local + (num / den)) if den != 0 else float(user_mean_local)


def predict_m4(train_iu, user_id, item_id):
    user_mean_local = train_iu.T.mean(axis=1).get(user_id, np.nan)
    sims = item_sim_bow[item_id].drop(item_id, errors="ignore").dropna()
    rated = train_iu[user_id].dropna().drop(item_id, errors="ignore")
    return weighted_dev_from_similarity(sims, rated, user_mean_local, threshold=0.2, top_k=3)


def predict_m4b(train_iu, user_id, item_id):
    user_mean_local = train_iu.T.mean(axis=1).get(user_id, np.nan)
    rated = train_iu[user_id].dropna().drop(item_id, errors="ignore")

    if rated.empty:
        return float(user_mean_local)

    feat_rated_local = bow_tfidf_df.loc[rated.index]
    dev = rated - user_mean_local
    pos_dev = dev[dev > 0]

    if pos_dev.empty:
        wts = rated
        profile = feat_rated_local.T.dot(wts) / wts.sum() if wts.sum() != 0 else feat_rated_local.mean(axis=0)
    else:
        denom = pos_dev.abs().sum()
        profile = (
            feat_rated_local.loc[pos_dev.index].T.dot(pos_dev) / denom
            if denom != 0
            else feat_rated_local.mean(axis=0)
        )

    sim_prof = pd.Series(
        cosine_similarity(profile.values.reshape(1, -1), feat_rated_local.values)[0],
        index=feat_rated_local.index,
    )
    num = (sim_prof * dev).sum()
    den = sim_prof.abs().sum()
    return float(user_mean_local + (num / den)) if den != 0 else float(user_mean_local)


def compute_rmse(frame, pred_col):
    if frame.empty:
        return np.nan
    return float(np.sqrt(np.mean((frame["true"] - frame[pred_col]) ** 2)))


def compute_rank_corr(frame, pred_col):
    if frame.empty:
        return np.nan
    if frame["true"].nunique() < 2 or frame[pred_col].nunique() < 2:
        return np.nan
    return float(frame["true"].corr(frame[pred_col], method="spearman"))


def topk_precision_recall(frame, pred_col, k, relevance_threshold):
    p_scores = []
    r_scores = []

    for user_id, grp in frame.groupby("user"):
        grp = grp.sort_values(pred_col, ascending=False)
        if grp.empty:
            continue

        top_n = min(k, len(grp))
        if top_n == 0:
            continue

        top_items = set(grp.head(top_n)["item"])
        relevant_items = set(grp.loc[grp["true"] >= relevance_threshold, "item"])
        hits = len(top_items.intersection(relevant_items))

        p_scores.append(hits / top_n)
        if len(relevant_items) > 0:
            r_scores.append(hits / len(relevant_items))

    precision_k = float(np.mean(p_scores)) if p_scores else np.nan
    recall_k = float(np.mean(r_scores)) if r_scores else np.nan
    return precision_k, recall_k


records = []

for item_id in df_iu.index:
    for user_id in df_iu.columns:
        true_rating = df_iu.loc[item_id, user_id]
        if pd.isna(true_rating):
            continue

        train_iu = df_iu.copy()
        train_iu.loc[item_id, user_id] = np.nan

        records.append(
            {
                "user": str(user_id),
                "item": str(item_id),
                "true": float(true_rating),
                "pred_m1": predict_m1(train_iu, str(user_id), str(item_id)),
                "pred_m2": predict_m2(train_iu, str(user_id), str(item_id)),
                "pred_m3": predict_m3(train_iu, str(user_id), str(item_id)),
                "pred_m4": predict_m4(train_iu, str(user_id), str(item_id)),
                "pred_m4b": predict_m4b(train_iu, str(user_id), str(item_id)),
            }
        )

eval_df = pd.DataFrame(records)

eval_k = 3
relevance_threshold = 4.0

methods = {
    "M1-CosineRaw": "pred_m1",
    "M2-TFIDF": "pred_m2",
    "M3-Profile": "pred_m3",
    "M4-BoW": "pred_m4",
    "M4b-BoWProfile": "pred_m4b",
}

metric_rows = [
    "RMSE",
    f"Precision at top-{eval_k}",
    "Rank correlation (Spearman)",
    f"Precision@{eval_k}",
    f"Recall@{eval_k}",
]

report = pd.DataFrame({"Metric": metric_rows})

for method_name, col_name in methods.items():
    valid = eval_df.dropna(subset=[col_name]).copy()
    rmse_val = compute_rmse(valid, col_name)
    rank_val = compute_rank_corr(valid, col_name)
    p_k, r_k = topk_precision_recall(valid, col_name, eval_k, relevance_threshold)

    report[method_name] = [rmse_val, p_k, rank_val, p_k, r_k]

print(f"Evaluated known ratings: {len(eval_df)}")
print(f"Top-K for ranking metrics: K={eval_k}, relevance threshold >= {relevance_threshold}")
print("\nComparison report:")
print(report.round(4).to_string(index=False))
print("=" * 65)

