# dm1

import re
from collections import defaultdict

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.cluster import DBSCAN, AgglomerativeClustering
from sklearn.preprocessing import StandardScaler


# 🔹 1. PREPROCESSING
def load_and_clean(filepath, remove_cols=None):
    data = pd.read_csv(filepath)

    if remove_cols:
        data.drop(columns=remove_cols, inplace=True)

    # Fill missing values instead of dropping
    for col in data.columns:
        if data[col].dtype == "object":
            data[col].fillna(data[col].mode()[0], inplace=True)
        else:
            data[col].fillna(data[col].mean(), inplace=True)

    # Manual label encoding using category codes
    cat_cols = data.select_dtypes(include="object").columns
    for col in cat_cols:
        data[col] = pd.Categorical(data[col]).codes

    return data


def normalize(matrix):
    scaler = StandardScaler()
    return scaler.fit_transform(matrix)


# 🔹 2. VISUALIZATION
def scatter_clusters(features, cluster_labels, chart_title):
    unique_labels = np.unique(cluster_labels)
    colors = plt.cm.tab10(np.linspace(0, 1, len(unique_labels)))
    for lbl, col in zip(unique_labels, colors):
        mask = np.array(cluster_labels) == lbl
        plt.scatter(
            features[mask, 0],
            features[mask, 1],
            color=col,
            label=f"Cluster {lbl}",
            s=20,
        )
    plt.title(chart_title)
    plt.xlabel("Feature 1")
    plt.ylabel("Feature 2")
    plt.legend(fontsize=6)
    plt.show()


def show_dendrogram(data_matrix):
    Z = linkage(data_matrix, method="ward")
    dendrogram(Z)
    plt.title("Dendrogram")
    plt.show()


def bar_chart(x_vals, y_vals, x_label, y_label, title):
    plt.bar(range(len(x_vals)), y_vals, tick_label=x_vals if len(x_vals) < 30 else None)
    plt.xlabel(x_label)
    plt.ylabel(y_label)
    plt.title(title)
    plt.show()


def pie_chart(dataframe, col_name):
    freq = dataframe[col_name].value_counts()
    plt.pie(freq.values, labels=freq.index.tolist(), autopct="%1.1f%%")
    plt.title(f"{col_name} Distribution")
    plt.show()


# 🔹 3. AGGLOMERATIVE CLUSTERING
def run_agglomerative(file):
    data = load_and_clean(file, remove_cols=["CustomerID"])

    feature_cols = ["Age", "Annual Income (k$)", "Spending Score (1-100)"]
    X = data[feature_cols].values
    X_norm = normalize(X)

    pie_chart(data, "Gender")
    bar_chart(
        data["Age"].values,
        data["Annual Income (k$)"].values,
        "Age",
        "Income",
        "Age vs Income",
    )

    # Ward linkage agglomerative clustering with 5 clusters
    clusterer = AgglomerativeClustering(n_clusters=5, linkage="ward")
    cluster_labels = clusterer.fit_predict(X_norm)

    scatter_clusters(X_norm, cluster_labels, "Agglomerative Clustering")
    show_dendrogram(X_norm)


# 🔹 4. DBSCAN (CUSTOM)
def compute_distance_matrix(X):
    # Vectorized pairwise Euclidean distances using broadcasting
    diff = X[:, np.newaxis, :] - X[np.newaxis, :, :]
    return np.sqrt((diff**2).sum(axis=2))


def custom_dbscan(X, eps=0.5, min_pts=5):
    dist_matrix = compute_distance_matrix(X)
    n = len(X)
    # -1 = noise, 0 = unvisited
    labels = [0] * n
    current_cluster = 0

    for point_idx in range(n):
        if labels[point_idx] != 0:
            continue

        # Get neighbors using precomputed distances
        neighbor_indices = list(np.where(dist_matrix[point_idx] <= eps)[0])

        if len(neighbor_indices) < min_pts:
            labels[point_idx] = -1  # noise
            continue

        current_cluster += 1
        labels[point_idx] = current_cluster

        seed_set = set(neighbor_indices) - {point_idx}

        while seed_set:
            q = seed_set.pop()

            if labels[q] == -1:
                labels[q] = current_cluster

            if labels[q] != 0:
                continue

            labels[q] = current_cluster
            q_neighbors = list(np.where(dist_matrix[q] <= eps)[0])

            if len(q_neighbors) >= min_pts:
                seed_set.update(q_neighbors)

    return labels


def run_dbscan(file):
    data = load_and_clean(file, remove_cols=["Channel", "Region"])

    # Display first few records
    print("First 5 records:\n", data.head())

    features = data[["Grocery", "Milk"]].values
    features_scaled = normalize(features)

    plt.scatter(features_scaled[:, 0], features_scaled[:, 1])
    plt.title("Normalized Data")
    plt.show()

    # Custom DBSCAN
    my_labels = custom_dbscan(features_scaled, eps=0.5, min_pts=15)
    scatter_clusters(features_scaled, my_labels, "Custom DBSCAN")

    # Sklearn DBSCAN (for comparison)
    sk_labels = DBSCAN(eps=0.5, min_samples=15).fit_predict(features_scaled)
    scatter_clusters(features_scaled, sk_labels, "Sklearn DBSCAN")


# 🔹 5. MS-GSP (FULL)
def parse_sequences(filepath):
    all_seqs = []
    with open(filepath) as f:
        for line in f:
            itemsets = re.findall(r"\{(.*?)\}", line.strip())
            seq = [frozenset(map(int, s.split(","))) for s in itemsets]
            if seq:
                all_seqs.append(seq)
    return all_seqs


def parse_params(filepath):
    mis_vals = {}
    sdc_val = 0.0
    with open(filepath) as f:
        for line in f:
            line = line.strip()
            if "MIS" in line:
                item_id = int(re.search(r"\((\d+)\)", line).group(1))
                mis_vals[item_id] = float(line.split("=")[-1].strip())
            elif "SDC" in line:
                sdc_val = float(line.split("=")[-1].strip())
    return mis_vals, sdc_val


def item_support(sequences):
    total = len(sequences)
    freq = defaultdict(set)
    for sid, seq in enumerate(sequences):
        seen = set()
        for itemset in seq:
            seen |= itemset
        for item in seen:
            freq[item].add(sid)
    return {item: len(sids) / total for item, sids in freq.items()}


def pattern_contains(pattern, sequence):
    # Check if pattern is a subsequence of sequence
    pos = 0
    for itemset in sequence:
        if pos < len(pattern) and pattern[pos].issubset(itemset):
            pos += 1
    return pos == len(pattern)


def support_of(pattern, sequences):
    return sum(1 for seq in sequences if pattern_contains(pattern, seq))


def all_items_in(pattern):
    return set().union(*pattern)


def sdc_ok(pattern, sup_map, sdc):
    items = list(all_items_in(pattern))
    for a in range(len(items)):
        for b in range(a + 1, len(items)):
            if abs(sup_map[items[a]] - sup_map[items[b]]) > sdc:
                return False
    return True


def MSGSP(file_data, file_para):
    sequences = parse_sequences(file_data)
    MIS, SDC = parse_params(file_para)
    sup_map = item_support(sequences)
    total = len(sequences)

    # Plot item support
    plt.bar(list(sup_map.keys()), list(sup_map.values()))
    plt.title("Support Distribution")
    plt.show()

    # --- F1: items sorted by MIS that meet their own threshold ---
    sorted_items = sorted(MIS.keys(), key=lambda x: MIS[x])
    freq_items = [i for i in sorted_items if sup_map.get(i, 0) >= MIS[i]]
    print("L:", freq_items)

    F1 = [[frozenset([i])] for i in freq_items]
    print("\nF1:", F1)

    F = [F1]

    # --- F2: candidate 2-sequences and 2-itemsets ---
    C2 = []
    for i in range(len(freq_items)):
        for j in range(i + 1, len(freq_items)):
            a, b = freq_items[i], freq_items[j]
            if abs(sup_map[a] - sup_map[b]) <= SDC:
                C2.append([frozenset([a, b])])  # itemset
                C2.append([frozenset([a]), frozenset([b])])  # sequence

    F2 = []
    for cand in C2:
        items = all_items_in(cand)
        min_mis = min(MIS[i] for i in items)
        s = support_of(cand, sequences) / total
        if s >= min_mis and sdc_ok(cand, sup_map, SDC) and cand not in F2:
            F2.append(cand)

    print("\nF2:", F2)
    F.append(F2)

    # --- Higher-order levels ---
    level = 3
    while True:
        prev = F[level - 2]
        candidates = []

        for p in prev:
            for q in prev:
                if p[1:] == q[:-1]:
                    extended = p + [q[-1]]
                    if extended not in candidates:
                        candidates.append(extended)

        if not candidates:
            break

        Fk = []
        for cand in candidates:
            items = all_items_in(cand)
            min_mis = min(MIS[i] for i in items)
            s = support_of(cand, sequences) / total
            if s >= min_mis and sdc_ok(cand, sup_map, SDC) and cand not in Fk:
                Fk.append(cand)

        if not Fk:
            break

        print(f"\nF{level}:", Fk)
        F.append(Fk)
        level += 1

    # --- Final output ---
    print("\nFinal Patterns:\n")
    for level_patterns in F:
        for pat in level_patterns:
            cnt = support_of(pat, sequences)
            pat_str = (
                "<" + "".join("{" + ",".join(map(str, s)) + "}" for s in pat) + ">"
            )
            print(f"Pattern: {pat_str}  count: {cnt}")


# 🔥 RUN EVERYTHING

# Part 1
run_agglomerative("filepath")

# Part 2
run_dbscan("/filepath")

# Part 3
MSGSP("/file1path", "/file2path")


import matplotlib.pyplot as plt
import numpy as np
# dm2
import pandas as pd
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn.cluster import AgglomerativeClustering
from sklearn.preprocessing import LabelEncoder

# Load dataset
df = pd.read_csv("FinalDM/Dataset/Mall_Customers.csv")

# 1. Remove CustomerID
df = df.drop("CustomerID", axis=1)

# 2. Check missing values
print("Missing values:\n", df.isnull().sum())

# 3. Encode Gender (Male=1, Female=0)
df["Gender"] = df["Gender"].map({"Male": 1, "Female": 0})

# 4. Pie chart (Male vs Female)
gender_counts = df["Gender"].value_counts()
labels = ["Female", "Male"]
plt.pie(gender_counts, labels=labels, autopct="%1.1f%%")
plt.title("Gender Distribution")
plt.show()

# 5. Bar graph (Age & Income)
plt.bar(df["Age"], df["Annual Income (k$)"])
plt.xlabel("Age")
plt.ylabel("Income")
plt.title("Age vs Income")
plt.show()

# 6. Agglomerative Clustering
X = df[["Age", "Annual Income (k$)", "Spending Score (1-100)"]]

model = AgglomerativeClustering(n_clusters=5, linkage="ward")
labels = model.fit_predict(X)

# Plot clusters
plt.scatter(X.iloc[:, 0], X.iloc[:, 1], c=labels)
plt.xlabel("Age")
plt.ylabel("Income")
plt.title("Agglomerative Clustering")
plt.show()

# 7. Dendrogram
linked = linkage(X, method="ward")

plt.figure(figsize=(10, 5))
dendrogram(linked)
plt.title("Dendrogram")
plt.show()


import matplotlib.pyplot as plt
import numpy as np
# =========================================================
# problem 1 - using manual implementation
# =========================================================
import pandas as pd
from sklearn.preprocessing import StandardScaler

# Load dataset
df = pd.read_csv("FinalDM/Dataset/Wholesale customers data.csv")

# Drop columns
df = df.drop(["Channel", "Region"], axis=1)

# 1. Display first few records
print("First 5 records:\n", df.head())

# Select Grocery & Milk
X = df[["Grocery", "Milk"]]

# Normalize
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# 2. Visualize normalized dataset
plt.scatter(X_scaled[:, 0], X_scaled[:, 1])
plt.xlabel("Grocery (Normalized)")
plt.ylabel("Milk (Normalized)")
plt.title("Normalized Dataset")
plt.show()


# -------- CUSTOM DBSCAN --------
def euclidean(p1, p2):
    return np.sqrt(np.sum((p1 - p2) ** 2))


def get_neighbors(X, point_idx, eps):
    return [i for i in range(len(X)) if euclidean(X[point_idx], X[i]) <= eps]


def dbscan(X, eps, min_pts):
    labels = [-1] * len(X)
    cluster_id = 0

    for i in range(len(X)):
        if labels[i] != -1:
            continue

        neighbors = get_neighbors(X, i, eps)

        if len(neighbors) < min_pts:
            labels[i] = 0  # noise
        else:
            cluster_id += 1
            labels[i] = cluster_id

            j = 0
            while j < len(neighbors):
                n = neighbors[j]

                if labels[n] == 0:
                    labels[n] = cluster_id

                if labels[n] == -1:
                    labels[n] = cluster_id
                    new_neighbors = get_neighbors(X, n, eps)

                    if len(new_neighbors) >= min_pts:
                        neighbors += new_neighbors

                j += 1

    return labels


# Apply DBSCAN
labels = dbscan(X_scaled, eps=0.5, min_pts=15)

# 3. Plot cluster results
plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=labels)
plt.xlabel("Grocery")
plt.ylabel("Milk")
plt.title("Custom DBSCAN Clustering")
plt.show()

from sklearn.cluster import DBSCAN
# problem 2 -- Using Built-in
from sklearn.datasets import make_moons

# Generate moons dataset
X, _ = make_moons(n_samples=2000, noise=0.05)

# Apply DBSCAN
db = DBSCAN(eps=0.2, min_samples=5)
labels = db.fit_predict(X)

# Plot
plt.scatter(X[:, 0], X[:, 1], c=labels)
plt.title("DBSCAN on Moons Dataset")
plt.show()

# Add noise
noise = np.random.uniform(low=-1.5, high=2.5, size=(200, 2))
X_noisy = np.vstack([X, noise])

labels_noisy = db.fit_predict(X_noisy)

plt.scatter(X_noisy[:, 0], X_noisy[:, 1], c=labels_noisy)
plt.title("DBSCAN with Noise")
plt.show()


# =========================================================
# FULL MS-GSP
# =========================================================
import re
from collections import defaultdict

import matplotlib.pyplot as plt


# -------------------------------
# 1. READ DATA
# -------------------------------
def read_data(file):
    sequences = []
    with open(file, "r") as f:
        for line in f:
            sets = re.findall(r"\{(.*?)\}", line)
            seq = [set(map(int, s.split(","))) for s in sets]
            sequences.append(seq)
    return sequences


# -------------------------------
# 2. READ PARAMETERS
# -------------------------------
def read_params(file):
    MIS = {}
    SDC = 0

    with open(file, "r") as f:
        for line in f:
            if "MIS" in line:
                item = int(re.findall(r"\((\d+)\)", line)[0])
                MIS[item] = float(line.split("=")[1])
            elif "SDC" in line:
                SDC = float(line.split("=")[1])

    return MIS, SDC


# -------------------------------
# 3. SUPPORT
# -------------------------------
def get_support(sequences):
    count = defaultdict(int)
    total = len(sequences)

    for seq in sequences:
        items = set()
        for s in seq:
            items |= s
        for item in items:
            count[item] += 1

    support = {k: v / total for k, v in count.items()}
    return support, count


# -------------------------------
# 4. INIT PASS (L)
# -------------------------------
def init_pass(MIS, support):
    sorted_items = sorted(MIS.keys(), key=lambda x: MIS[x])
    L = []

    for item in sorted_items:
        if support.get(item, 0) >= MIS[item]:
            L.append(item)

    return L


# -------------------------------
# 5. F1
# -------------------------------
def generate_F1(L):
    return [[{item}] for item in L]


# -------------------------------
# 6. SUBSEQUENCE CHECK
# -------------------------------
def is_subsequence(pattern, sequence):
    i = 0
    for s in sequence:
        if i < len(pattern) and pattern[i].issubset(s):
            i += 1
    return i == len(pattern)


# -------------------------------
# 7. COUNT SUPPORT
# -------------------------------
def count_support(pattern, sequences):
    return sum(is_subsequence(pattern, seq) for seq in sequences)


# -------------------------------
# 8. GET ITEMS
# -------------------------------
def get_items(pattern):
    items = set()
    for s in pattern:
        items |= s
    return items


# -------------------------------
# 9. SDC CHECK
# -------------------------------
def check_SDC(pattern, support, SDC):
    items = list(get_items(pattern))
    for i in range(len(items)):
        for j in range(i + 1, len(items)):
            if abs(support[items[i]] - support[items[j]]) > SDC:
                return False
    return True


# -------------------------------
# 10. C2 GENERATION
# -------------------------------
def generate_C2(L, support, MIS, SDC):
    C2 = []

    for i in range(len(L)):
        for j in range(i + 1, len(L)):
            i1, i2 = L[i], L[j]

            if support[i2] >= MIS[i1] and abs(support[i2] - support[i1]) <= SDC:
                C2.append([{i1, i2}])  # same itemset
                C2.append([{i1}, {i2}])  # sequence

    return C2


# -------------------------------
# 11. JOIN STEP
# -------------------------------
def join_step(Fk_1):
    Ck = []

    for p in Fk_1:
        for q in Fk_1:
            if p[1:] == q[:-1]:
                candidate = p + [q[-1]]
                if candidate not in Ck:
                    Ck.append(candidate)

    return Ck


# -------------------------------
# 12. PRUNE STEP
# -------------------------------
def prune(Ck, Fk_1):
    pruned = []

    for c in Ck:
        valid = True

        for i in range(len(c)):
            sub = c[:i] + c[i + 1 :]
            if sub not in Fk_1:
                valid = False
                break

        if valid:
            pruned.append(c)

    return pruned


# -------------------------------
# 13. MS-GSP MAIN
# -------------------------------
def MSGSP(sequences, MIS, SDC):
    support, raw_count = get_support(sequences)

    # Plot support
    plt.bar(list(support.keys()), list(support.values()))
    plt.title("Support Distribution")
    plt.show()

    L = init_pass(MIS, support)
    print("L:", L)

    F = []

    # F1
    F1 = generate_F1(L)
    print("\nF1:", F1)
    F.append(F1)

    # F2
    C2 = generate_C2(L, support, MIS, SDC)

    F2 = []
    for c in C2:
        count = count_support(c, sequences)
        sup = count / len(sequences)

        min_mis = min(MIS[item] for item in get_items(c))

        if sup >= min_mis and check_SDC(c, support, SDC):
            if c not in F2:
                F2.append(c)

    print("\nF2:", F2)
    F.append(F2)

    # Fk
    k = 3
    while True:
        Ck = join_step(F[k - 2])
        Ck = prune(Ck, F[k - 2])

        if not Ck:
            break

        Fk = []
        for c in Ck:
            count = count_support(c, sequences)
            sup = count / len(sequences)

            min_mis = min(MIS[item] for item in get_items(c))

            if sup >= min_mis and check_SDC(c, support, SDC):
                if c not in Fk:
                    Fk.append(c)

        if not Fk:
            break

        print(f"\nF{k}:", Fk)
        F.append(Fk)

        k += 1

    return F


# -------------------------------
# 14. PRINT OUTPUT
# -------------------------------
def print_patterns(F, sequences):
    print("\nFinal Patterns:\n")

    for level in F:
        for pattern in level:
            count = count_support(pattern, sequences)

            pattern_str = "<"
            for s in pattern:
                pattern_str += "{" + ",".join(map(str, s)) + "}"
            pattern_str += ">"

            print(f"Pattern: {pattern_str}  count: {count}")


# -------------------------------
# RUN
# -------------------------------
sequences = read_data("FinalDM/Dataset/data (1).txt")
MIS, SDC = read_params("FinalDM/Dataset/para.txt")

F = MSGSP(sequences, MIS, SDC)
print_patterns(F, sequences)


# =========================================================
# GSP
# =========================================================
import re
from collections import defaultdict
from itertools import combinations

import matplotlib.pyplot as plt


# -------------------------------
# 1. READ DATA
# -------------------------------
def read_data(file):
    sequences = []
    with open(file, "r") as f:
        for line in f:
            sets = re.findall(r"\{(.*?)\}", line)
            seq = [set(map(int, s.split(","))) for s in sets]
            sequences.append(seq)
    return sequences


# -------------------------------
# 2. READ PARAMETERS (MIS + SDC)
# -------------------------------
def read_params(file):
    MIS = {}
    SDC = 0

    with open(file, "r") as f:
        for line in f:
            if "MIS" in line:
                item = int(re.findall(r"\((\d+)\)", line)[0])
                val = float(line.split("=")[1])
                MIS[item] = val
            elif "SDC" in line:
                SDC = float(line.split("=")[1])

    return MIS, SDC


# -------------------------------
# 3. SUPPORT CALCULATION
# -------------------------------
def get_support(sequences):
    count = defaultdict(int)
    total = len(sequences)

    for seq in sequences:
        unique_items = set()
        for s in seq:
            unique_items |= s
        for item in unique_items:
            count[item] += 1

    support = {k: v / total for k, v in count.items()}
    return support, count


# -------------------------------
# 4. PLOT SUPPORT GRAPH
# -------------------------------
def plot_support(support):
    items = list(support.keys())
    values = list(support.values())

    plt.bar(items, values)
    plt.xlabel("Items")
    plt.ylabel("Support")
    plt.title("Support Distribution")
    plt.show()


# -------------------------------
# 5. FREQUENT 1-ITEMSETS
# -------------------------------
def frequent_1_itemsets(support, MIS):
    return {item for item in support if support[item] >= MIS[item]}


# -------------------------------
# 6. CHECK SUBSEQUENCE
# -------------------------------
def is_subsequence(pattern, sequence):
    i = 0
    for s in sequence:
        if pattern[i].issubset(s):
            i += 1
            if i == len(pattern):
                return True
    return False


# -------------------------------
# 7. COUNT PATTERN SUPPORT
# -------------------------------
def count_pattern(pattern, sequences):
    count = 0
    for seq in sequences:
        if is_subsequence(pattern, seq):
            count += 1
    return count


# -------------------------------
# 8. GENERATE CANDIDATES (GSP STYLE)
# -------------------------------
def generate_candidates(prev_patterns):
    candidates = []

    for p1 in prev_patterns:
        for p2 in prev_patterns:
            if p1[1:] == p2[:-1]:
                new_pattern = p1 + [p2[-1]]
                candidates.append(new_pattern)

    return candidates


# -------------------------------
# 9. FILTER USING MIS + SDC
# -------------------------------
def filter_patterns(candidates, sequences, MIS, SDC):
    total = len(sequences)
    valid_patterns = []

    for pattern in candidates:
        count = count_pattern(pattern, sequences)
        support = count / total

        items = set()
        for p in pattern:
            items |= p

        # MIS condition
        if all(support >= MIS[item] for item in items):

            # SDC condition (approx)
            supports = [support]
            if max(supports) - min(supports) <= SDC:
                valid_patterns.append((pattern, count))

    return valid_patterns


# -------------------------------
# 10. PRINT PATTERNS
# -------------------------------
def print_patterns(patterns):
    for pattern, count in patterns:
        pattern_str = "<"
        for p in pattern:
            pattern_str += "{" + ",".join(map(str, p)) + "}"
        pattern_str += ">"

        print(f"Pattern: {pattern_str}  count: {count}")


# -------------------------------
# MAIN EXECUTION
# -------------------------------
sequences = read_data("FinalDM/Dataset/data (1).txt")
MIS, SDC = read_params("FinalDM/Dataset/para.txt")

# Support
support, raw_count = get_support(sequences)

print("SUPPORT VALUES:")
for k, v in support.items():
    print(f"Item {k}: {v:.2f}")

# Plot graph
plot_support(support)

# Frequent 1-itemsets
F1 = frequent_1_itemsets(support, MIS)
print("\nFrequent 1-itemsets:", F1)

# Convert F1 to sequence format
patterns = [[{item}] for item in F1]

k = 2
all_patterns = []

while patterns:
    print(f"\nGenerating patterns of length {k}...")

    candidates = generate_candidates(patterns)
    valid_patterns = filter_patterns(candidates, sequences, MIS, SDC)

    if not valid_patterns:
        break

    print_patterns(valid_patterns)
    all_patterns.extend(valid_patterns)

    # Prepare next iteration
    patterns = [pattern for pattern, _ in valid_patterns]
    k += 1

# Final Output
print("\nFINAL FREQUENT SEQUENTIAL PATTERNS:")
print_patterns(all_patterns)


import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
# IR1
import pandas as pd
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# -------------------- STOPWORDS --------------------
stop_words = {
    "is",
    "am",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    "a",
    "an",
    "the",
    "and",
    "or",
    "not",
    "in",
    "on",
    "at",
    "to",
    "for",
    "with",
    "by",
    "of",
    "that",
    "this",
    "it",
    "as",
    "from",
    "but",
    "about",
    "into",
    "over",
    "after",
    "before",
    "between",
    "out",
    "up",
    "down",
    "so",
    "than",
    "too",
    "very",
    "can",
    "will",
}


def simple_stem(word):
    suffixes = ["ing", "ed", "ly", "es", "s", "ment"]
    for suf in suffixes:
        if word.endswith(suf) and len(word) > len(suf) + 2:
            return word[: -len(suf)]
    return word


def preprocess(text):
    tokens = text.lower().split()
    tokens = [t.strip(".,!?:;()[]{}\"'") for t in tokens]
    tokens = [t for t in tokens if t and t not in stop_words]
    tokens = [simple_stem(t) for t in tokens]
    return " ".join(tokens)


# -------------------- LOAD DATASET --------------------
df = pd.read_csv("/content/drive/MyDrive/Colab Notebooks/IRdata.csv")
df["clean_text"] = df["text"].astype(str).apply(preprocess)
docs = df["clean_text"].tolist()

print("\n--- Preprocessed Documents ---")
print(df[["id", "clean_text"]])

# ===============================================================
# 1️⃣ CONTENT-BASED RECOMMENDATION
# ===============================================================

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(docs)
similarity_matrix = cosine_similarity(X)


def content_recommend(doc_id, top_n=3):
    scores = list(enumerate(similarity_matrix[doc_id]))
    scores = sorted(scores, key=lambda x: x[1], reverse=True)
    scores = scores[1 : top_n + 1]
    print(f"\n--- Content-Based Recommendations for Document {doc_id+1} ---")
    for idx, score in scores:
        print(f"Doc {idx+1} (Score={score:.3f}): {df.iloc[idx]['text']}")


content_recommend(0)

# ---------------- VISUALIZATION: Similarity Heatmap ----------------
plt.figure(figsize=(8, 6))
sns.heatmap(similarity_matrix, annot=True, cmap="Blues")
plt.title("Document Similarity Matrix (Content-Based)")
plt.xlabel("Document ID")
plt.ylabel("Document ID")
plt.show()

# ===============================================================
# 2️⃣ COLLABORATIVE FILTERING (USER–ITEM)
# ===============================================================
ratings = pd.DataFrame(
    {
        "user": ["u1", "u1", "u2", "u2", "u3", "u3", "u4", "u4"],
        "item": [1, 2, 2, 3, 3, 4, 4, 5],
        "rating": [5, 4, 4, 5, 3, 4, 5, 4],
    }
)

user_item_matrix = ratings.pivot_table(index="user", columns="item", values="rating")
user_item_matrix = user_item_matrix.fillna(0)

user_sim = cosine_similarity(user_item_matrix)
user_sim_df = pd.DataFrame(
    user_sim, index=user_item_matrix.index, columns=user_item_matrix.index
)


def recommend_item(user, top_n=3):
    similar_users = user_sim_df[user].sort_values(ascending=False).index[1 : top_n + 1]
    print(f"\n--- Collaborative Filtering Recommendation for {user} ---")
    print("Similar Users:", list(similar_users))


recommend_item("u1")

# rating recommendation - given user and item:
# ===============================================================
# 🔹 PREDICT MISSING RATING (USER-BASED CF)
# ===============================================================


def predict_rating(user, item):
    numerator = 0
    denominator = 0

    for other_user in user_item_matrix.index:
        # consider only users who rated this item
        if user_item_matrix.loc[other_user, item] > 0:
            sim = user_sim_df.loc[user, other_user]
            rating = user_item_matrix.loc[other_user, item]

            numerator += sim * rating
            denominator += abs(sim)

    if denominator == 0:
        return 0  # no similar users found
    return numerator / denominator


print("\n--- Predicted Rating ---")
print("u1 rating for item 3:", predict_rating("u1", 3))


# Optional - rating for all missing values:
def predict_rating(user, item):
    numerator = 0
    denominator = 0

    for other_user in user_item_matrix.index:
        if user_item_matrix.loc[other_user, item] > 0:
            sim = user_sim_df.loc[user, other_user]
            rating = user_item_matrix.loc[other_user, item]

            numerator += sim * rating
            denominator += abs(sim)

    if denominator == 0:
        # fallback: average rating of the item
        item_ratings = user_item_matrix[item]
        non_zero_ratings = item_ratings[item_ratings > 0]

        if len(non_zero_ratings) == 0:
            return 0
        return non_zero_ratings.mean()

    return numerator / denominator


def fill_missing_ratings():
    filled = user_item_matrix.copy()
    for user in user_item_matrix.index:
        for item in user_item_matrix.columns:
            if user_item_matrix.loc[user, item] == 0:
                filled.loc[user, item] = predict_rating(user, item)
    return filled


filled_matrix = fill_missing_ratings()
print("\n--- Filled User-Item Matrix ---")
print(filled_matrix)

# ---------------- VISUALIZATION: User Similarity Heatmap ----------------
plt.figure(figsize=(6, 4))
sns.heatmap(user_sim_df, annot=True, cmap="Greens")
plt.title("User Similarity Matrix (Collaborative Filtering)")
plt.show()

# ===============================================================
# 3️⃣ PAGE RANK ALGORITHM
# ===============================================================

G = nx.Graph()
for i in range(len(docs)):
    for j in range(i + 1, len(docs)):
        if similarity_matrix[i][j] > 0.2:
            G.add_edge(i, j, weight=similarity_matrix[i][j])

pr = nx.pagerank(G)

print("\n--- PageRank Scores for Documents ---")
for i, score in pr.items():
    print(f"Doc {i+1}: Score = {score:.4f}")

# ---------------- VISUALIZATION: PageRank Graph ----------------
plt.figure(figsize=(8, 6))
pos = nx.spring_layout(G, seed=7)
sizes = [5000 * pr[i] for i in G.nodes()]
nx.draw(
    G, pos, with_labels=True, node_size=sizes, node_color="skyblue", edge_color="gray"
)
plt.title("Document Graph Based on Similarity (PageRank Size = Score)")
plt.show()

# ===============================================================
# 4️⃣ DIMENSION REDUCTION (PCA)
# ===============================================================

pca = PCA(n_components=2)
X_reduced = pca.fit_transform(X.toarray())

print("\n--- PCA Dimensionality Reduction (2D) ---")
for i, vec in enumerate(X_reduced):
    print(f"Doc {i+1}: {vec}")

# ---------------- VISUALIZATION: PCA Scatter Plot ----------------
plt.figure(figsize=(8, 6))
plt.scatter(X_reduced[:, 0], X_reduced[:, 1], s=120, color="purple")

for i in range(len(X_reduced)):
    plt.text(X_reduced[i, 0] + 0.02, X_reduced[i, 1] + 0.02, f"Doc {i+1}")

plt.title("PCA - 2D Document Vector Visualization")
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.grid(True)
plt.show()

import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
# IR2
import pandas as pd
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# -------------------- STOPWORDS --------------------
stop_words = {
    "is",
    "am",
    "are",
    "was",
    "were",
    "be",
    "been",
    "being",
    "a",
    "an",
    "the",
    "and",
    "or",
    "not",
    "in",
    "on",
    "at",
    "to",
    "for",
    "with",
    "by",
    "of",
    "that",
    "this",
    "it",
    "as",
    "from",
    "but",
    "about",
    "into",
    "over",
    "after",
    "before",
    "between",
    "out",
    "up",
    "down",
    "so",
    "than",
    "too",
    "very",
    "can",
    "will",
}


# Different style: uses a loop with early return instead of checking all suffixes
def simple_stem(word):
    for suf in ["ing", "ed", "ly", "es", "s", "ment"]:
        if word.endswith(suf) and len(word) > len(suf) + 2:
            return word[: -len(suf)]
    return word


# Different style: uses list comprehension chain instead of step-by-step
def preprocess(text):
    raw_tokens = text.lower().split()
    cleaned = [t.strip(".,!?:;()[]{}\"'") for t in raw_tokens]
    filtered = [t for t in cleaned if t and t not in stop_words]
    stemmed = [simple_stem(t) for t in filtered]
    return " ".join(stemmed)


# -------------------- LOAD DATASET --------------------
df = pd.read_csv("/content/drive/MyDrive/Colab Notebooks/IRdata.csv")
df["clean_text"] = df["text"].astype(str).apply(preprocess)
docs = df["clean_text"].tolist()

print("\n--- Preprocessed Documents ---")
print(df[["id", "clean_text"]])

# ===============================================================
# 1️⃣ CONTENT-BASED RECOMMENDATION
# ===============================================================

# Same CountVectorizer — different style: fit and transform separated
cv = CountVectorizer()
cv.fit(docs)
X = cv.transform(docs)

# Manual cosine similarity — same result as sklearn's cosine_similarity
# dot(A, B) / (||A|| * ||B||) — done via matrix multiplication on normalized vectors
X_dense = X.toarray().astype(float)
norms = np.linalg.norm(X_dense, axis=1, keepdims=True)
norms[norms == 0] = 1e-10
X_normed = X_dense / norms
similarity_matrix = X_normed @ X_normed.T  # identical to cosine_similarity(X)


def content_recommend(doc_id, top_n=3):
    # Different style: uses dictionary then sorts, instead of list of tuples
    score_dict = {
        i: similarity_matrix[doc_id][i] for i in range(len(docs)) if i != doc_id
    }
    top_docs = sorted(score_dict, key=score_dict.get, reverse=True)[:top_n]

    print(f"\n--- Content-Based Recommendations for Document {doc_id+1} ---")
    for idx in top_docs:
        print(f"Doc {idx+1} (Score={score_dict[idx]:.3f}): {df.iloc[idx]['text']}")


content_recommend(0)

# ---------------- VISUALIZATION: Similarity Heatmap ----------------
plt.figure(figsize=(8, 6))
sns.heatmap(similarity_matrix, annot=True, cmap="Blues")
plt.title("Document Similarity Matrix (Content-Based)")
plt.xlabel("Document ID")
plt.ylabel("Document ID")
plt.show()

# ===============================================================
# 2️⃣ COLLABORATIVE FILTERING (USER–ITEM)
# ===============================================================

# Different style: build as list of tuples first, then DataFrame
user_item_data = [
    ("u1", 1, 5),
    ("u1", 2, 4),
    ("u2", 2, 4),
    ("u2", 3, 5),
    ("u3", 3, 3),
    ("u3", 4, 4),
    ("u4", 4, 5),
    ("u4", 5, 4),
]
ratings = pd.DataFrame(user_item_data, columns=["user", "item", "rating"])

# Build user-item matrix — same pivot logic, different variable names
user_item_matrix = ratings.pivot_table(
    index="user", columns="item", values="rating"
).fillna(0)

# User similarity — same cosine_similarity call, wrapped differently
ui_array = user_item_matrix.values.astype(float)
raw_sim = cosine_similarity(ui_array)
user_sim_df = pd.DataFrame(
    raw_sim, index=user_item_matrix.index, columns=user_item_matrix.index
)


def recommend_item(user, top_n=3):
    # Different style: drops the user itself first, then takes top_n
    others = user_sim_df[user].drop(labels=[user])
    similar_users = others.sort_values(ascending=False).head(top_n).index.tolist()
    print(f"\n--- Collaborative Filtering Recommendation for {user} ---")
    print("Similar Users:", similar_users)


recommend_item("u1")

# ===============================================================
# 🔹 PREDICT MISSING RATING (USER-BASED CF)
# ===============================================================


def predict_rating(user, item):
    num = 0.0
    den = 0.0
    for other in user_item_matrix.index:
        r = user_item_matrix.loc[other, item]
        if r > 0:
            s = user_sim_df.loc[user, other]
            num += s * r
            den += abs(s)
    return num / den if den != 0 else 0


print("\n--- Predicted Rating ---")
print("u1 rating for item 3:", predict_rating("u1", 3))


# Redefine with fallback
def predict_rating(user, item):
    num = 0.0
    den = 0.0
    for other in user_item_matrix.index:
        r = user_item_matrix.loc[other, item]
        if r > 0:
            s = user_sim_df.loc[user, other]
            num += s * r
            den += abs(s)

    if den == 0:
        col = user_item_matrix[item]
        non_zero = col[col > 0]
        return non_zero.mean() if len(non_zero) > 0 else 0

    return num / den


def fill_missing_ratings():
    filled = user_item_matrix.copy()
    for u in user_item_matrix.index:
        for it in user_item_matrix.columns:
            if user_item_matrix.loc[u, it] == 0:
                filled.loc[u, it] = predict_rating(u, it)
    return filled


filled_matrix = fill_missing_ratings()
print("\n--- Filled User-Item Matrix ---")
print(filled_matrix)

# ---------------- VISUALIZATION: User Similarity Heatmap ----------------
plt.figure(figsize=(6, 4))
sns.heatmap(user_sim_df, annot=True, cmap="Greens")
plt.title("User Similarity Matrix (Collaborative Filtering)")
plt.show()

# ===============================================================
# 3️⃣ PAGE RANK ALGORITHM
# ===============================================================

# Different style: builds edge list first, then adds all at once
edge_list = [
    (i, j, similarity_matrix[i][j])
    for i in range(len(docs))
    for j in range(i + 1, len(docs))
    if similarity_matrix[i][j] > 0.2
]

G = nx.Graph()
G.add_weighted_edges_from(edge_list)

pr = nx.pagerank(G)

print("\n--- PageRank Scores for Documents ---")
for node, score in pr.items():
    print(f"Doc {node+1}: Score = {score:.4f}")

# ---------------- VISUALIZATION: PageRank Graph ----------------
plt.figure(figsize=(8, 6))
pos = nx.spring_layout(G, seed=7)
sizes = [5000 * pr[n] for n in G.nodes()]
nx.draw(
    G, pos, with_labels=True, node_size=sizes, node_color="skyblue", edge_color="gray"
)
plt.title("Document Graph Based on Similarity (PageRank Size = Score)")
plt.show()

# ===============================================================
# 4️⃣ DIMENSION REDUCTION (PCA)
# ===============================================================

# Different style: PCA fit and transform on separate lines
reducer = PCA(n_components=2)
reducer.fit(X.toarray())
X_reduced = reducer.transform(X.toarray())

print("\n--- PCA Dimensionality Reduction (2D) ---")
for i, vec in enumerate(X_reduced):
    print(f"Doc {i+1}: {vec}")

# ---------------- VISUALIZATION: PCA Scatter Plot ----------------
plt.figure(figsize=(8, 6))
plt.scatter(X_reduced[:, 0], X_reduced[:, 1], s=120, color="purple")

for i in range(len(X_reduced)):
    plt.text(X_reduced[i, 0] + 0.02, X_reduced[i, 1] + 0.02, f"Doc {i+1}")

plt.title("PCA - 2D Document Vector Visualization")
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.grid(True)
plt.show()
