Reinforcement Learning
untuk Kontrol Adaptif

Exploration vs Exploitation: Dilema Utama RL

import numpy as np
import matplotlib.pyplot as plt

# ==========================================
# Q-LEARNING: KONTROL SUHU SEDERHANA
# State: suhu diskret [16..30] °C
# Action: {0=AC kuat, 1=AC sedang, 2=AC mati}
# Target: jaga suhu di 22°C
# ==========================================

np.random.seed(42)

# === ENVIRONMENT DEFINISI ===
T_min, T_max  = 16, 30
T_target      = 22
n_states      = T_max - T_min + 1   # 15 state (16°C..30°C)
n_actions     = 3                    # 0=kuat, 1=sedang, 2=mati
action_effect = [-1.5, -0.5, 0.8]  # ΔT per step tiap aksi

def state_to_temp(s): return s + T_min
def temp_to_state(T): return int(np.clip(T - T_min, 0, n_states-1))

def step(s, a):
    T_now = state_to_temp(s)
    # Dinamika suhu + gangguan acak (panas dari luar)
    T_next = T_now + action_effect[a] + 0.3*np.random.randn() + 0.1
    s_next = temp_to_state(T_next)
    
    # Reward: hukum deviasi dari target + hemat energi AC kuat
    err    = abs(T_next - T_target)
    reward = -err - 0.5*(a == 0)     # AC kuat dikurangi reward 0.5
    if err < 0.8: reward += 2.0        # bonus berada dekat target
    return s_next, reward

# === Q-TABLE: 15 states × 3 actions ===
Q = np.zeros((n_states, n_actions))

# === HYPERPARAMETERS ===
alpha   = 0.15   # learning rate
gamma   = 0.95   # discount factor
eps_max = 1.0
eps_min = 0.05
eps_dec = 0.003

n_episodes  = 800
steps_per_ep = 100
ep_rewards   = []

# === TRAINING LOOP ===
eps = eps_max
for ep in range(n_episodes):
    s = temp_to_state(28)    # mulai dari suhu 28°C (panas)
    total_r = 0
    for _ in range(steps_per_ep):
        # ε-greedy action selection
        if np.random.rand() < eps:
            a = np.random.randint(n_actions)   # eksplorasi
        else:
            a = np.argmax(Q[s])                # eksploitasi
        
        s_next, r = step(s, a)
        
        # Q-Learning update (TD Error)
        td_err = r + gamma*np.max(Q[s_next]) - Q[s, a]
        Q[s, a] += alpha * td_err
        
        s = s_next
        total_r += r
    
    ep_rewards.append(total_r / steps_per_ep)
    eps = max(eps_min, eps * (1 - eps_dec))

# === TEST POLICY yang sudah dipelajari ===
s = temp_to_state(28); T_trace = [state_to_temp(s)]
a_trace = []; r_trace = []
for _ in range(120):
    a = np.argmax(Q[s])                    # policy greedy (tanpa eksplorasi)
    s, r = step(s, a)
    T_trace.append(state_to_temp(s))
    a_trace.append(a); r_trace.append(r)

# === PLOT ===
fig, axes = plt.subplots(3, 1, figsize=(12, 9))

# Smooth reward curve
smooth = np.convolve(ep_rewards, np.ones(30)/30, mode='valid')
axes[0].plot(ep_rewards, alpha=0.3, color='orange')
axes[0].plot(smooth, color='orange', lw=2, label='Reward rata-rata (smoothed)')
axes[0].set_ylabel('Total Reward / Episode')
axes[0].set_title('Q-Learning: Kurva Belajar Agent Kontrol Suhu')
axes[0].legend(); axes[0].grid(alpha=0.3)

axes[1].plot(T_trace, 'cyan', lw=2, label='Suhu Aktual (°C)')
axes[1].axhline(T_target, color='orange', ls='--', lw=2, label=f'Target {T_target}°C')
axes[1].fill_between(range(len(T_trace)), T_target-1, T_target+1,
                      alpha=0.1, color='orange', label='Zona ±1°C')
axes[1].set_ylabel('Suhu (°C)')
axes[1].set_title('Test Policy: Suhu dikendalikan mendekati 22°C')
axes[1].legend(fontsize=9); axes[1].grid(alpha=0.3)

action_labels = {0:'AC Kuat', 1:'AC Sedang', 2:'AC Mati'}
colors_a = {0:'#3b82f6', 1:'#f97316', 2:'#22c55e'}
for ai, label in action_labels.items():
    idx = [i for i,a in enumerate(a_trace) if a==ai]
    axes[2].scatter(idx, [ai]*len(idx), c=colors_a[ai], s=30, label=label)
axes[2].set_yticks([0,1,2]); axes[2].set_yticklabels(['AC Kuat','AC Sedang','AC Mati'])
axes[2].set_xlabel('Langkah')
axes[2].set_title('Urutan Aksi Policy RL yang Dipelajari')
axes[2].legend(ncol=3, fontsize=9); axes[2].grid(alpha=0.2)

plt.tight_layout(); plt.show()

print(f"\n📊 Q-Table yang Dipelajari (baris=suhu, kolom=aksi):")
for s_i in range(n_states):
    best = np.argmax(Q[s_i])
    T    = state_to_temp(s_i)
    print(f"  T={T}°C → best action: {action_labels[best]:10s} | Q={Q[s_i]}")