Analisis Log & Deteksi Anomali
Berbasis AI

# ================================================================
# S11409 - Sesi 11: Log Anomaly Detection Pipeline
# Dosen: Riadi Marta Dinata, S.Ti., M.Kom. | ISTN Jakarta
# pip install scikit-learn pandas numpy matplotlib
# ================================================================

import re, json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from collections import Counter, defaultdict
from datetime import datetime, timedelta
import warnings; warnings.filterwarnings('ignore')

# ── STEP 1: GENERATE SYNTHETIC LOG DATA ──────────────────────
np.random.seed(42)

def generate_logs(n_normal=500, n_anomaly=30):
    """Generate simulated payment service logs"""
    logs = []
    base_time = datetime(2024, 3, 15, 8, 0, 0)

    # Normal logs - gaussian distributed latency
    for i in range(n_normal):
        t = base_time + timedelta(seconds=i * 0.5)
        latency = max(10, int(np.random.normal(120, 30)))  # ~120ms normal
        tx_id = f"TX-{9000 + i}"
        user_id = f"U-{np.random.randint(1000, 9999)}"
        amount = np.random.choice([50000, 75000, 100000, 150000, 200000])
        level = "INFO" if latency < 200 else "WARN"
        msg = f"Transaction {tx_id} processed for {user_id}, amount={amount}, latency={latency}ms"
        logs.append({"timestamp": t, "level": level, "latency": latency,
                     "msg": msg, "is_anomaly": 0})

    # Anomaly block - spike in errors and latency
    anomaly_start = base_time + timedelta(seconds=n_normal * 0.5 + 10)
    for i in range(n_anomaly):
        t = anomaly_start + timedelta(seconds=i * 0.3)
        latency = int(np.random.normal(800, 150))  # 800ms anomaly spike
        tx_id = f"TX-{9500 + i}"
        user_id = f"U-{np.random.randint(1000, 9999)}"
        level = np.random.choice(["ERROR", "WARN", "ERROR"], p=[0.6, 0.2, 0.2])
        if level == "ERROR":
            msg = f"Transaction {tx_id} FAILED: connection timeout after {latency}ms"
        else:
            msg = f"Transaction {tx_id} slow warning: latency={latency}ms exceeds threshold"
        logs.append({"timestamp": t, "level": level, "latency": latency,
                     "msg": msg, "is_anomaly": 1})

    df = pd.DataFrame(logs).sort_values("timestamp").reset_index(drop=True)
    return df

df = generate_logs()
print("=" * 60)
print("    LOG ANOMALY DETECTION PIPELINE")
print("    S11409 ISTN Jakarta | Sesi 11")
print("=" * 60)
print(f"\nTotal logs   : {len(df)}")
print(f"Normal logs  : {(df['is_anomaly']==0).sum()}")
print(f"Anomaly logs : {(df['is_anomaly']==1).sum()}")

# ── STEP 2: FEATURE ENGINEERING FROM LOGS ────────────────────
def engineer_features(df, window_size=20):
    """Extract time-window features for each log entry"""
    features = []
    for i in range(len(df)):
        start = max(0, i - window_size)
        window = df.iloc[start:i+1]

        feat = {
            "latency": df.iloc[i]["latency"],
            "is_error": 1 if df.iloc[i]["level"] == "ERROR" else 0,
            "is_warn":  1 if df.iloc[i]["level"] == "WARN"  else 0,
            # Window-based features
            "window_error_rate":  (window["level"] == "ERROR").mean(),
            "window_warn_rate":   (window["level"] == "WARN").mean(),
            "window_mean_latency": window["latency"].mean(),
            "window_max_latency":  window["latency"].max(),
            "window_std_latency":  window["latency"].std() if len(window) > 1 else 0,
            "window_p95_latency":  window["latency"].quantile(0.95),
        }
        features.append(feat)

    return pd.DataFrame(features)

print("\nEngineering features from log windows...")
X = engineer_features(df, window_size=20)
y_true = df["is_anomaly"].values

# ── STEP 3: ISOLATION FOREST ANOMALY DETECTION ───────────────
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

# contamination = expected fraction of anomalies
iso_forest = IsolationForest(
    n_estimators=200,
    contamination=0.07,   # ~7% anomali yang kita harapkan
    max_samples="auto",
    random_state=42,
    n_jobs=-1
)

print("Training Isolation Forest...")
y_pred_raw = iso_forest.fit_predict(X_scaled)
y_pred = (y_pred_raw == -1).astype(int)  # -1 = anomali, 1 = normal
anomaly_scores = -iso_forest.score_samples(X_scaled)  # makin tinggi = lebih anomali

# ── STEP 4: EVALUATE ─────────────────────────────────────────
from sklearn.metrics import classification_report, confusion_matrix

print("\n--- Anomaly Detection Results ---")
print(classification_report(y_true, y_pred,
      target_names=["Normal", "Anomaly"]))

tp = ((y_pred==1) & (y_true==1)).sum()
fp = ((y_pred==1) & (y_true==0)).sum()
fn = ((y_pred==0) & (y_true==1)).sum()
tn = ((y_pred==0) & (y_true==0)).sum()
print(f"  True Positives  (caught anomalies): {tp}")
print(f"  False Positives (false alarms)    : {fp}")
print(f"  False Negatives (missed anomalies): {fn}")

# ── STEP 5: VISUALIZE ─────────────────────────────────────────
fig, axes = plt.subplots(2, 2, figsize=(14, 8))
fig.patch.set_facecolor('#0A0608')
fig.suptitle('Log Anomaly Detection Dashboard — S11409 ISTN',
             color='white', fontsize=13, fontweight='bold', y=1.01)

# Plot 1: Latency timeline with anomalies
ax1 = axes[0, 0]; ax1.set_facecolor('#160C0E')
ax1.plot(df.index, df['latency'], color='#9B7A80', alpha=0.5, linewidth=0.8, label='Latency')
anomaly_idx = np.where(y_pred == 1)[0]
ax1.scatter(anomaly_idx, df['latency'].iloc[anomaly_idx],
            color='#E8374F', s=25, zorder=5, label='Detected Anomaly')
ax1.axhline(y=200, color='#D4A017', linestyle='--', alpha=0.6, label='Threshold 200ms')
ax1.set_title('Response Latency Timeline', color='white')
ax1.tick_params(colors='#9B7A80')
ax1.legend(fontsize=8, facecolor='#160C0E', labelcolor='white')

# Plot 2: Anomaly score distribution
ax2 = axes[0, 1]; ax2.set_facecolor('#160C0E')
ax2.hist(anomaly_scores[y_true==0], bins=40, color='#22C55E', alpha=0.6, label='Normal')
ax2.hist(anomaly_scores[y_true==1], bins=20, color='#E8374F', alpha=0.7, label='Anomaly')
ax2.set_title('Anomaly Score Distribution', color='white')
ax2.tick_params(colors='#9B7A80')
ax2.legend(fontsize=9, facecolor='#160C0E', labelcolor='white')

# Plot 3: Log level distribution
ax3 = axes[1, 0]; ax3.set_facecolor('#160C0E')
level_counts = df['level'].value_counts()
colors_bar = ['#22C55E', '#F97316', '#E8374F'][:len(level_counts)]
ax3.bar(level_counts.index, level_counts.values, color=colors_bar, edgecolor='none')
ax3.set_title('Log Level Distribution', color='white')
ax3.tick_params(colors='#9B7A80')

# Plot 4: Window error rate
ax4 = axes[1, 1]; ax4.set_facecolor('#160C0E')
ax4.fill_between(range(len(X)), X['window_error_rate'],
                 color='#A78BFA', alpha=0.4, label='Error Rate (window)')
ax4.scatter(anomaly_idx, X['window_error_rate'].iloc[anomaly_idx],
            color='#E8374F', s=20, zorder=5, label='Anomaly Points')
ax4.set_title('Window Error Rate', color='white')
ax4.tick_params(colors='#9B7A80')
ax4.legend(fontsize=8, facecolor='#160C0E', labelcolor='white')

plt.tight_layout()
plt.savefig('log_anomaly_detection.png', dpi=150,
            facecolor='#0A0608', bbox_inches='tight')
print("\nVisualisasi tersimpan: log_anomaly_detection.png")

# ── STEP 6: ALERTING SIMULATION ──────────────────────────────
print("\n--- Alert Simulation (last 10 anomalies) ---")
anomaly_logs = df[y_pred == 1].tail(10)
for _, row in anomaly_logs.iterrows():
    score = anomaly_scores[row.name]
    severity = "CRITICAL" if score > 0.6 else "HIGH"
    print(f"  [{severity}] {row['timestamp'].strftime('%H:%M:%S')} "
          f"| Latency={row['latency']}ms | {row['level']} "
          f"| Score={score:.3f}")

print("\nDone! Jalankan dengan: python log_anomaly.py")