# -*- coding: utf-8 -*-
"""ir dm final code sem.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1pShgFSMDWpmjt0iCqFTyic4hYGP5vgFc
"""

# -------------------------------------------------------------------------------------------------------#
# IR CODE#
# -------------------------------------------------------------------------------------------------------#

import os

import matplotlib.pyplot as plt
import networkx as nx
import pandas as pd
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.preprocessing import StandardScaler

sns.set()

figures = []

print("Current Directory:", os.getcwd())
print("Files in folder:", os.listdir())

# ============================================================

# 1. CONTENT-BASED RECOMMENDER SYSTEM

# ============================================================

try:
    print("\nCONTENT-BASED RECOMMENDER SYSTEM")

    movies = pd.read_csv("movies.csv")
    movies.columns = movies.columns.str.lower().str.strip()

    title_col = [c for c in movies.columns if "title" in c][0]
    genre_col = [c for c in movies.columns if "genre" in c][0]

    tfidf = TfidfVectorizer()
    tfidf_matrix = tfidf.fit_transform(movies[genre_col])
    cosine_sim = cosine_similarity(tfidf_matrix)

    movie_idx = 0
    scores = sorted(
        list(enumerate(cosine_sim[movie_idx])), key=lambda x: x[1], reverse=True
    )[1:4]

    print("Recommendations for:", movies[title_col].iloc[movie_idx])
    for i in scores:
        print(movies[title_col].iloc[i[0]])

    fig1 = plt.figure()
    sns.heatmap(cosine_sim, cmap="Blues")
    plt.title("Content Similarity")
    figures.append(fig1)


except Exception as e:
    print("Error in Content-Based:", e)

# ============================================================

# 2. COLLABORATIVE FILTERING

# ============================================================

try:
    print("\nCOLLABORATIVE FILTERING")

    ratings = pd.read_csv("data.csv")
    ratings.columns = ratings.columns.str.lower().str.strip()

    user_col = [c for c in ratings.columns if "user" in c][0]
    item_col = [c for c in ratings.columns if "movie" in c or "item" in c][0]
    rating_col = [c for c in ratings.columns if "rating" in c][0]

    user_item = ratings.pivot(
        index=user_col, columns=item_col, values=rating_col
    ).fillna(0)
    user_sim = cosine_similarity(user_item)

    print("User Similarity Matrix:\n", user_sim)

    fig2 = plt.figure()
    sns.heatmap(user_item, cmap="coolwarm")
    plt.title("User-Item Matrix")
    figures.append(fig2)


except Exception as e:
    print("Error in Collaborative:", e)

# ============================================================

# 3. PCA

# ============================================================

try:
    print("\nPCA")

    data = pd.read_csv("data.csv")
    data.columns = data.columns.str.lower().str.strip()

    scaler = StandardScaler()
    scaled = scaler.fit_transform(data)

    pca = PCA(n_components=2)
    result = pca.fit_transform(scaled)

    print("Reduced Data:\n", result)

    fig3 = plt.figure()
    plt.scatter(result[:, 0], result[:, 1])
    plt.title("PCA Projection")
    figures.append(fig3)


except Exception as e:
    print("Error in PCA:", e)

# ============================================================

# 4. PAGERANK

# ============================================================

try:
    print("\nPAGERANK")

    graph = pd.read_csv("graph.csv")
    graph.columns = graph.columns.str.lower().str.strip()

    source_col = graph.columns[0]
    target_col = graph.columns[1]

    G = nx.from_pandas_edgelist(
        graph, source_col, target_col, create_using=nx.DiGraph()
    )
    pr = nx.pagerank(G)

    print("PageRank Scores:")
    for k, v in pr.items():
        print(k, ":", round(v, 4))

    fig4 = plt.figure()
    nx.draw(G, with_labels=True, node_color="lightblue")
    plt.title("Web Graph")
    figures.append(fig4)

except Exception as e:
    print("Error in PageRank:", e)

# ============================================================

# 5. INVERTED INDEX

# ============================================================

try:
    print("\nINVERTED INDEX")

    docs = pd.read_csv("documents.csv")
    docs.columns = docs.columns.str.lower().str.strip()

    id_col = [c for c in docs.columns if "id" in c][0]
    text_col = [c for c in docs.columns if "content" in c or "text" in c][0]

    index = {}

    for _, row in docs.iterrows():
        words = str(row[text_col]).lower().split()
        for w in words:
            if w not in index:
                index[w] = []
            index[w].append(row[id_col])

    query = "data"
    print("Search:", query)

    if query in index:
        for doc_id in index[query]:
            print(docs[docs[id_col] == doc_id][text_col].values[0])
    else:
        print("No results")


except Exception as e:
    print("Error in IR:", e)

# ============================================================

# SHOW ALL PLOTS

# ============================================================
print("\nShowing all plots...")

# Activate all figures created
for num in plt.get_fignums():
    plt.figure(num)

# Display all together
plt.show()

# -------------------------------------------------------------------------------------------------------#
# DM CODE#
# -------------------------------------------------------------------------------------------------------#
# ============================================================
# PROBLEM SHEET 13 – AGGLOMERATIVE CLUSTERING (WARD)
# ============================================================

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.cluster.hierarchy import dendrogram, linkage

print("\n===== AGGLOMERATIVE CLUSTERING =====")

# Load dataset
df = pd.read_csv("Mall_Customers.csv")

# Remove CustomerID
df = df.drop("CustomerID", axis=1)

# Missing values
print("Missing values:\n", df.isnull().sum())

# Convert Gender
df["Gender"] = df["Gender"].map({"Male": 1, "Female": 0})

# Pie chart
gender_counts = df["Gender"].value_counts()
plt.figure()
plt.pie(gender_counts, labels=["Male", "Female"], autopct="%1.1f%%")
plt.title("Male vs Female Ratio")
plt.show()

# Bar graphs
plt.figure()
plt.bar(range(len(df)), df["Age"])
plt.title("Age Distribution")
plt.show()

plt.figure()
plt.bar(range(len(df)), df["Annual Income (k$)"])
plt.title("Income Distribution")
plt.show()

# Ward linkage clustering
data = df[["Age", "Annual Income (k$)", "Spending Score (1-100)"]].values
Z = linkage(data, method="ward")

plt.figure(figsize=(10, 5))
dendrogram(Z)
plt.title("Dendrogram (Ward Linkage)")
plt.xlabel("Data Points")
plt.ylabel("Distance")
plt.show()


# ============================================================
# PROBLEM SHEET 14 – DBSCAN
# ============================================================

from sklearn.cluster import DBSCAN
from sklearn.datasets import make_moons
from sklearn.preprocessing import StandardScaler

print("\n===== DBSCAN CLUSTERING =====")

# Load dataset
df = pd.read_csv("Wholesale customers data.csv")
df = df.drop(["Channel", "Region"], axis=1)

print("First few records:\n", df.head())

# Normalize
data = df[["Grocery", "Milk"]].values
scaler = StandardScaler()
data_scaled = scaler.fit_transform(data)

plt.scatter(data_scaled[:, 0], data_scaled[:, 1])
plt.title("Normalized Data")
plt.xlabel("Grocery")
plt.ylabel("Milk")
plt.show()

# Manual DBSCAN
EPS = 0.5
MINPTS = 15

UNVISITED = 0
NOISE = -1

labels = [UNVISITED] * len(data_scaled)
cluster_id = 0


def euclidean(p1, p2):
    return np.sqrt(np.sum((p1 - p2) ** 2))


def region_query(idx):
    neighbors = []
    for i in range(len(data_scaled)):
        if euclidean(data_scaled[idx], data_scaled[i]) <= EPS:
            neighbors.append(i)
    return neighbors


def expand_cluster(idx, neighbors, cluster_id):
    labels[idx] = cluster_id

    i = 0
    while i < len(neighbors):
        point = neighbors[i]

        if labels[point] == UNVISITED:
            labels[point] = cluster_id
            new_neighbors = region_query(point)

            if len(new_neighbors) >= MINPTS:
                neighbors += new_neighbors

        elif labels[point] == NOISE:
            labels[point] = cluster_id

        i += 1


# Main loop
for i in range(len(data_scaled)):
    if labels[i] != UNVISITED:
        continue

    neighbors = region_query(i)

    if len(neighbors) < MINPTS:
        labels[i] = NOISE
    else:
        cluster_id += 1
        expand_cluster(i, neighbors, cluster_id)

# Plot clusters
plt.scatter(data_scaled[:, 0], data_scaled[:, 1], c=labels, cmap="rainbow")
plt.title("Manual DBSCAN Clustering")
plt.show()

# Built-in DBSCAN (moon dataset)
X, _ = make_moons(n_samples=2000, noise=0.05)

model = DBSCAN(eps=0.2, min_samples=5)
labels_moon = model.fit_predict(X)

plt.scatter(X[:, 0], X[:, 1], c=labels_moon, cmap="rainbow")
plt.title("DBSCAN on Moon Dataset")
plt.show()

# Add noise
noise = np.random.uniform(low=-1.5, high=2.5, size=(200, 2))
X_noisy = np.vstack([X, noise])

labels_noisy = model.fit_predict(X_noisy)

plt.scatter(X_noisy[:, 0], X_noisy[:, 1], c=labels_noisy, cmap="rainbow")
plt.title("DBSCAN with Noise")
plt.show()


# ============================================================
# PROBLEM SHEET 15 – MS-GSP (FULLY CORRECT)
# ============================================================

import re

print("\n===== MS-GSP SEQUENTIAL PATTERN MINING =====")

# Read parameters
MIS = {}
SDC = 0


def convert(x):
    try:
        return int(x)
    except:
        return x


with open("para.txt") as f:
    for line in f:
        if "MIS" in line:
            item = re.findall(r"\((.*?)\)", line)[0]
            item = convert(item)
            MIS[item] = float(line.split("=")[1])
        elif "SDC" in line:
            SDC = float(line.split("=")[1])


# Read data
def parse_sequence(line):
    sets = re.findall(r"\{(.*?)\}", line)
    return [set(convert(x) for x in s.split(",")) for s in sets]


sequences = []
with open("data (1).txt") as f:
    for line in f:
        sequences.append(parse_sequence(line.strip()))

N = len(sequences)


# Support
def item_support(item):
    return sum(1 for seq in sequences if any(item in s for s in seq)) / N


support = {i: item_support(i) for i in MIS}


# Subsequence check
def is_subsequence(pattern, sequence):
    i = 0
    for itemset in sequence:
        if pattern[i].issubset(itemset):
            i += 1
            if i == len(pattern):
                return True
    return False


def sequence_support(pattern):
    return sum(1 for seq in sequences if is_subsequence(pattern, seq))


# Init pass
L = sorted(MIS.keys(), key=lambda x: MIS[x])
F = []

F1 = []
for item in L:
    if support[item] >= MIS[item]:
        F1.append([{item}])

F.append(F1)
print("F1:", F1)


# Join
def join_step(Fk):
    candidates = []
    for s1 in Fk:
        for s2 in Fk:
            if s1[1:] == s2[:-1]:
                candidates.append(s1 + [s2[-1]])

            if len(s1) == len(s2) and s1[:-1] == s2[:-1]:
                new_seq = s1[:-1] + [s1[-1] | s2[-1]]
                if new_seq not in candidates:
                    candidates.append(new_seq)
    return candidates


# Prune
def prune_step(candidates, Fk):
    pruned = []
    Fk_set = [tuple(tuple(sorted(s)) for s in seq) for seq in Fk]

    for c in candidates:
        valid = True
        for i in range(len(c)):
            sub = c[:i] + c[i + 1 :]
            sub_tuple = tuple(tuple(sorted(s)) for s in sub)

            if sub_tuple not in Fk_set:
                valid = False
                break

        if valid:
            pruned.append(c)

    return pruned


# Main loop
k = 1
while True:
    if len(F[k - 1]) == 0:
        break

    Ck = prune_step(join_step(F[k - 1]), F[k - 1])
    Fk = []

    for c in Ck:
        count = sequence_support(c)

        items = set()
        for s in c:
            items.update(s)

        min_mis = min(MIS[i] for i in items)

        if (count / N) >= min_mis:
            valid = True
            item_list = list(items)

            for i in range(len(item_list)):
                for j in range(i + 1, len(item_list)):
                    if abs(support[item_list[i]] - support[item_list[j]]) > SDC:
                        valid = False
                        break

            if valid:
                Fk.append(c)

    if not Fk:
        break

    print(f"F{k+1}:", Fk)
    F.append(Fk)
    k += 1


# Output
def format_pattern(p):
    return "<" + "".join(["{" + ",".join(map(str, sorted(s))) + "}" for s in p]) + ">"


print("\nFINAL PATTERNS:")
for level in F:
    for p in level:
        print(f"Pattern :{format_pattern(p)} count: {sequence_support(p)}")
