[Modeling] Random Forest

1. 기본 구현

# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

df = pd.read_csv('heart_disease_missing_processed.csv')

X = df.drop('Heart Disease Status', axis=1)
y = df['Heart Disease Status']

x_train, x_test, y_train, y_test, = train_test_split(X, y, test_size=0.2, random_state=42)

rf_model = RandomForestClassifier(n_estimators=100, class_weight='balanced', random_state=42)

rf_model.fit(x_train, y_train)

predictions = rf_model.predict(x_test)

print("=== 🌲 Random Forest 결과 ===")
train_predictions = rf_model.predict(x_train)
print(f"Train Accuracy: {accuracy_score(y_train, train_predictions):.4f}")
print(f"Test Accuracy: {accuracy_score(y_test, predictions):.4f}\n")

print("Confusion Matrix:")
print(confusion_matrix(y_test, predictions))

print("\nClassification_report:")
print(classification_report(y_test, predictions))

 

=== 🌲 Random Forest 결과 ===
Train Accuracy: 1.0000
Test Accuracy: 0.8065

Confusion Matrix:
[[1613    0]
 [ 387    0]]

Classification_report:
C:\Users\limye\anaconda3\envs\damvenv\Lib\site-packages\sklearn\metrics\_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
C:\Users\limye\anaconda3\envs\damvenv\Lib\site-packages\sklearn\metrics\_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
C:\Users\limye\anaconda3\envs\damvenv\Lib\site-packages\sklearn\metrics\_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
              precision    recall  f1-score   support

           0       0.81      1.00      0.89      1613
           1       0.00      0.00      0.00       387

    accuracy                           0.81      2000
   macro avg       0.40      0.50      0.45      2000
weighted avg       0.65      0.81      0.72      2000

 

=> depth에 제한을 걸지 않아서 training set에 너무 딱 맞아 버림.

 

2. Pruning, Smote

# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from imblearn.over_sampling import SMOTE

df = pd.read_csv('heart_disease_missing_processed.csv')

X = df.drop('Heart Disease Status', axis=1)
y = df['Heart Disease Status']

x_train, x_test, y_train, y_test, = train_test_split(X, y, test_size=0.2, random_state=42)

smote = SMOTE(random_state=42)
x_train_smote, y_train_smote = smote.fit_resample(x_train, y_train)

rf_model = RandomForestClassifier(
    n_estimators=100, 
    max_depth=10,
    min_samples_leaf=5,
    class_weight='balanced', 
    random_state=42)

rf_model.fit(x_train, y_train)

predictions = rf_model.predict(x_test)

print("=== Random Forest 결과 ===")
train_predictions = rf_model.predict(x_train)
print(f"Train Accuracy: {accuracy_score(y_train, train_predictions):.4f}")
print(f"Test Accuracy: {accuracy_score(y_test, predictions):.4f}\n")

print("Confusion Matrix:")
print(confusion_matrix(y_test, predictions))

print("\nClassification_report:")
print(classification_report(y_test, predictions))

# 중요 변수 확인
importances = pd.DataFrame(
    {'Feature': X.columns, 'Importance': rf_model.feature_importances_}
).sort_values(by='Importance', ascending=False)
print("\n=== Top 5 중요 특징 ===")
print(importances.head(5))

 

=== Random Forest 결과 ===
Train Accuracy: 0.9431
Test Accuracy: 0.7460

Confusion Matrix:
[[1456  157]
 [ 351   36]]

Classification_report:
              precision    recall  f1-score   support

           0       0.81      0.90      0.85      1613
           1       0.19      0.09      0.12       387

    accuracy                           0.75      2000
   macro avg       0.50      0.50      0.49      2000
weighted avg       0.69      0.75      0.71      2000


=== Top 5 중요 특징 ===
              Feature  Importance
10          CRP Level    0.133253
6                 BMI    0.132744
8         Sleep Hours    0.130935
2   Cholesterol Level    0.113541
0                 Age    0.110103

1. f1-score가 logistic regression의 최고 f1-score를 넘지 못함.

2. 거의 다 0으로 맞혀서 정확도가 높은 경우임

3. feature의 중요도가 거의 비슷해서 분류하기 어려운 경우

 

3. SMOTE 제외, RandomForest + Threshold

# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score

df = pd.read_csv('heart_disease_missing_processed.csv')

X = df.drop('Heart Disease Status', axis=1)
y = df['Heart Disease Status']

x_train, x_test, y_train, y_test, = train_test_split(X, y, test_size=0.2, random_state=42)

rf_model = RandomForestClassifier(
    n_estimators=100, 
    max_depth=5, # max depth 제한 10 -> 5
    class_weight='balanced', 
    random_state=42
    )

rf_model.fit(x_train, y_train)

probabilities = rf_model.predict_proba(x_test)[:, 1]
best_f1 = 0

for threshold in np.arange(0.1, 0.9, 0.01):
    custom_predictions = (probabilities >= threshold).astype(int)
    current_f1 = f1_score(y_test, custom_predictions)
    
    if current_f1 > best_f1:
        best_f1 = current_f1
        best_threshold = threshold

final_predictions = (probabilities >= best_threshold).astype(int)
print("=== 튜닝된 Random Forest 최종 결과 ===")
print("Confusion Matrix:")
print(confusion_matrix(y_test, final_predictions))
print("\nClassification_report:")
print(classification_report(y_test, final_predictions))

=== 튜닝된 Random Forest 최종 결과 ===
Confusion Matrix:
[[  32 1581]
 [   4  383]]

Classification_report:
              precision    recall  f1-score   support

           0       0.89      0.02      0.04      1613
           1       0.20      0.99      0.33       387

    accuracy                           0.21      2000
   macro avg       0.54      0.50      0.18      2000
weighted avg       0.75      0.21      0.09      2000

1. f1-score가 이전보다 높아짐. logistic regression의 0.32와 비슷함.

2. 전부 다 1로 맞혀서 0 label의 false positive가 양산됨.