2.1  Clustering

import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score, davies_bouldin_score

np.random.seed(42)

# ── Simulate Customer Segmentation Data ───────────────────────
def make_segment(n, inc_mu, bal_mu, freq_mu, label):
    return pd.DataFrame({
        'income':    np.random.normal(inc_mu, 10, n),
        'balance':   np.random.normal(bal_mu, bal_mu*0.15, n),
        'freq':      np.random.normal(freq_mu, 3, n),
        'true_seg':  [label]*n
    })

df = pd.concat([
    make_segment(1000, 28,  2.1, 8,  'Basic'),
    make_segment(800,  65,  12,  15, 'Mid-tier'),
    make_segment(500,  95,  42,  12, 'Affluent'),
    make_segment(300,  120, 85,  9,  'Premium'),
    make_segment(400,  45,  5,   22, 'Young-active')
]).reset_index(drop=True)

X = df[['income','balance','freq']].values
scaler = StandardScaler()
X_sc = scaler.fit_transform(X)

# ── Elbow Method ──────────────────────────────────────────────
print("=== K-Means — Elbow Method ===")
print(f"{'K':>4} | {'Inertia':>12} | {'Silhouette':>12} | {'DB Score':>10}")
print("-"*46)
for k in range(2, 9):
    km = KMeans(n_clusters=k, init='k-means++', n_init=10, random_state=42)
    labels = km.fit_predict(X_sc)
    sil  = silhouette_score(X_sc, labels)
    db   = davies_bouldin_score(X_sc, labels)
    print(f"{k:4d} | {km.inertia_:12.2f} | {sil:12.4f} | {db:10.4f}")

# ── Fit with K=5 ─────────────────────────────────────────────
km5 = KMeans(n_clusters=5, init='k-means++', n_init=20, random_state=42)
df['cluster'] = km5.fit_predict(X_sc)

# ── Cluster profiles ──────────────────────────────────────────
print("\nCluster Profiles:")
print(df.groupby('cluster')[['income','balance','freq']].mean().round(2))

# ── Agreement with true segments ─────────────────────────────
from sklearn.metrics import adjusted_rand_score
ari = adjusted_rand_score(df['true_seg'], df['cluster'])
print(f"\nAdjusted Rand Index vs True Segments: {ari:.4f}")

import numpy as np
import pandas as pd
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics import adjusted_rand_score

np.random.seed(42)

# ── Simulate Financial Transaction Data (2D: amount, frequency) ──
def ring_cluster(n, cx, cy, r, noise):
    angles = np.random.uniform(0, 2*np.pi, n)
    return np.column_stack([cx + r*np.cos(angles) + np.random.normal(0,noise,n),
                             cy + r*np.sin(angles) + np.random.normal(0,noise,n)])

# Normal transactions: two dense blobs
normal_a = np.random.multivariate_normal([80, 2],  [[400,2],[2,0.5]], 2000)
normal_b = np.random.multivariate_normal([150, 4], [[900,5],[5,1.0]], 1500)

# Fraud: sparse, unusual region
fraud    = np.random.multivariate_normal([3200, 18], [[1e5,50],[50,8]], 50)

# Noise transactions
noise_pts = np.column_stack([np.random.uniform(500, 2000, 30),
                              np.random.uniform(8, 14, 30)])

X      = np.vstack([normal_a, normal_b, fraud, noise_pts])
y_true = np.array([0]*2000 + [0]*1500 + [1]*50 + [-1]*30)  # 0=normal,1=fraud,-1=noise

X_sc = StandardScaler().fit_transform(X)

# ── k-dist plot to choose eps ─────────────────────────────────
nbrs = NearestNeighbors(n_neighbors=5).fit(X_sc)
dists, _ = nbrs.kneighbors(X_sc)
k_dists  = np.sort(dists[:, 4])[::-1]
print("Top-10 k-distances (for eps selection):")
print(np.round(k_dists[:10], 3))

# ── DBSCAN ────────────────────────────────────────────────────
db = DBSCAN(eps=0.4, min_samples=10).fit(X_sc)
labels = db.labels_
n_clusters  = len(set(labels)) - (1 if -1 in labels else 0)
n_noise     = np.sum(labels == -1)
noise_pct   = n_noise / len(labels) * 100

print(f"\n=== DBSCAN — Transaction Clustering ===")
print(f"Clusters found : {n_clusters}")
print(f"Noise points   : {n_noise} ({noise_pct:.2f}%)")
print(f"Cluster sizes  : {dict(zip(*np.unique(labels, return_counts=True)))}")

# ── Fraud recall ──────────────────────────────────────────────
fraud_idx    = np.where(y_true == 1)[0]
fraud_labels = labels[fraud_idx]
print(f"\nFraud transactions label distribution:")
print(dict(zip(*np.unique(fraud_labels, return_counts=True))))

# ── Try different eps values ──────────────────────────────────
print("\nSensitivity to eps (min_samples=10):")
for eps in [0.2, 0.3, 0.4, 0.5, 0.7]:
    lb = DBSCAN(eps=eps, min_samples=10).fit_predict(X_sc)
    nc = len(set(lb)) - (1 if -1 in lb else 0)
    nn = np.sum(lb==-1)
    print(f"  eps={eps:.1f}: {nc} clusters, {nn} noise pts")

import numpy as np
import pandas as pd
from scipy.cluster.hierarchy import linkage, dendrogram, fcluster, cophenet
from scipy.spatial.distance import pdist
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import silhouette_score

np.random.seed(42)

# ── Simulate Lupus Patient Biomarker Data ─────────────────────
def subtype(n, means, label):
    cov = np.diag([1.2]*len(means))
    return pd.DataFrame(
        np.random.multivariate_normal(means, cov, n),
        columns=[f'bio{i}' for i in range(len(means))]
    ).assign(true=label)

n_bio = 8
df = pd.concat([
    subtype(68, [2.1, 1.8, 0.5, 0.3, 1.2, 0.8, 0.4, 0.6], 'Skin+Joints'),
    subtype(42, [0.8, 0.6, 2.9, 2.4, 0.7, 1.1, 0.3, 0.4], 'Renal'),
    subtype(35, [0.4, 0.5, 0.6, 0.3, 2.8, 2.4, 1.8, 1.2], 'Neuro'),
    subtype(55, [1.9, 1.7, 1.8, 1.6, 1.7, 1.5, 1.6, 1.8], 'Multi-organ')
]).reset_index(drop=True)

X = StandardScaler().fit_transform(df.drop('true', axis=1))

# ── Linkage methods comparison ────────────────────────────────
print("Linkage   | Cophenetic r | Silhouette (k=4)")
print("----------|--------------|-----------------")
for method in ['ward','complete','average','single']:
    Z = linkage(X, method=method)
    c, _ = cophenet(Z, pdist(X))
    labels = fcluster(Z, t=4, criterion='maxclust')
    sil = silhouette_score(X, labels)
    print(f"{method:10s}| {c:.4f}       | {sil:.4f}")

# ── Ward linkage dendrogram cut at k=4 ───────────────────────
Z = linkage(X, method='ward')
labels4 = fcluster(Z, t=4, criterion='maxclust')
df['cluster'] = labels4

# ── Cluster purity ────────────────────────────────────────────
from sklearn.metrics import adjusted_rand_score
ari = adjusted_rand_score(df['true'], labels4)
print(f"\nAdjusted Rand Index (ward, k=4): {ari:.4f}")

# ── Cluster sizes ─────────────────────────────────────────────
print("Cluster sizes:", dict(zip(*np.unique(labels4, return_counts=True))))
print("\nCross-tabulation (cluster vs true subtype):")
print(pd.crosstab(df['cluster'], df['true']))