Ensemble Methods

1from sklearn.ensemble import RandomForestClassifier
2from sklearn.model_selection import train_test_split, GridSearchCV
3from sklearn.metrics import classification_report, confusion_matrix
4import numpy as np
5
6# Load data
7from sklearn.datasets import load_breast_cancer
8data = load_breast_cancer()
9X, y = data.data, data.target
10
11X_train, X_test, y_train, y_test = train_test_split(
12    X, y, test_size=0.2, random_state=42
13)
14
15# Basic Random Forest
16rf = RandomForestClassifier(n_estìmators=100, random_state=42)
17rf.fit(X_train, y_train)
18
19# Evaluate
20y_pred = rf.predict(X_test)
21print(classification_report(y_test, y_pred))
22
23# Feature Importance
24importance = rf.feature_importances_
25indices = np.argsort(importance)[::-1][:10]
26
27print("\nTop 10 Features:")
28for i, idx in enumerate(indices):
29    print(f"{i+1}. {data.feature_names[idx]}: {importance[idx]:.4f}")

1param_grid = {
2    'n_estìmators': [100, 200, 300],
3    'max_depth': [10, 20, None],
4    'min_samples_split': [2, 5, 10],
5    'max_features': ['sqrt', 'log2']
6}
7
8grid_search = GridSearchCV(
9    RandomForestClassifier(random_state=42),
10    param_grid, cv=5, scoring='f1', n_jobs=-1
11)
12grid_search.fit(X_train, y_train)
13
14print(f"Best params: {grid_search.best_params_}")
15print(f"Best CV F1: {grid_search.best_score_:.4f}")

1import xgboost as xgb
2from sklearn.model_selection import train_test_split
3from sklearn.metrics import classification_report
4
5# Prepare data
6X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
7
8# XGBoost Classifier
9xgb_clf = xgb.XGBClassifier(
10    n_estìmators=200,
11    learning_rate=0.1,
12    max_depth=5,
13    subsample=0.8,
14    colsample_bytree=0.8,
15    reg_lambda=1,
16    random_state=42,
17    eval_metric='logloss'
18)
19
20# Early Stopping
21xgb_clf.fit(
22    X_train, y_train,
23    eval_set=[(X_test, y_test)],
24    verbose=False
25)
26
27# Evaluate
28y_pred = xgb_clf.predict(X_test)
29print(classification_report(y_test, y_pred))
30
31# Feature Importance
32xgb.plot_importance(xgb_clf, max_num_features=10)

1from sklearn.ensemble import StackingClassifier, RandomForestClassifier
2from sklearn.linear_model import LogisticRegression
3from sklearn.svm import SVC
4import xgboost as xgb
5
6# Define base models
7base_models = [
8    ('rf', RandomForestClassifier(n_estìmators=100, random_state=42)),
9    ('xgb', xgb.XGBClassifier(n_estìmators=100, random_state=42)),
10    ('svc', SVC(probability=True, random_state=42))
11]
12
13# Stacking with Logistic Regression as meta model
14stacking_clf = StackingClassifier(
15    estìmators=base_models,
16    final_estìmator=LogisticRegression(),
17    cv=5
18)
19
20stacking_clf.fit(X_train, y_train)
21y_pred = stacking_clf.predict(X_test)
22
23print(classification_report(y_test, y_pred))

1import lightgbm as lgb
2
3lgb_clf = lgb.LGBMClassifier(
4    n_estìmators=200,
5    learning_rate=0.1,
6    max_depth=5,
7    num_leaves=31,
8    random_state=42
9)
10
11lgb_clf.fit(X_train, y_train)
12y_pred = lgb_clf.predict(X_test)
13print(classification_report(y_test, y_pred))

1from sklearn.ensemble import RandomForestRegressor
2from sklearn.datasets import fetch_california_housing
3from sklearn.model_selection import train_test_split
4from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
5import numpy as np
6
7# 1. Load Data
8housing = fetch_california_housing()
9X, y = housing.data, housing.target
10
11# 2. Split
12X_train, X_test, y_train, y_test = train_test_split(
13    X, y, test_size=0.2, random_state=42
14)
15
16# 3. Train Random Forest Regressor
17rf_reg = RandomForestRegressor(
18    n_estimators=200,
19    max_depth=15,
20    min_samples_split=5,
21    max_features='sqrt',
22    random_state=42,
23    n_jobs=-1
24)
25rf_reg.fit(X_train, y_train)
26
27# 4. Predict
28y_pred = rf_reg.predict(X_test)
29
30# 5. Evaluation
31mae = mean_absolute_error(y_test, y_pred)
32mse = mean_squared_error(y_test, y_pred)
33rmse = np.sqrt(mse)
34r2 = r2_score(y_test, y_pred)
35
36print("=== Regression Metrics ===")
37print(f"MAE:  {mae:.4f}")
38print(f"MSE:  {mse:.4f}")
39print(f"RMSE: {rmse:.4f}")
40print(f"R²:   {r2:.4f}")
41
42# 6. Feature Importance
43importances = pd.DataFrame({
44    'feature': housing.feature_names,
45    'importance': rf_reg.feature_importances_
46}).sort_values('importance', ascending=False)
47
48print("\n=== Feature Importances ===")
49print(importances)

1# ✅ Good Practice
2rf = RandomForestClassifier(
3    n_estimators=200,        # Nhiều trees = stable
4    max_depth=15,            # Limit depth tránh overfit
5    min_samples_split=10,    # Regularization
6    max_features='sqrt',     # Feature randomness
7    n_jobs=-1,               # Parallel processing
8    random_state=42,         # Reproducibility
9    oob_score=True           # Out-of-bag validation
10)

1# ✅ Good Practice
2gb = GradientBoostingClassifier(
3    n_estimators=100,        # Moderate số trees
4    learning_rate=0.1,       # Vừa phải
5    max_depth=5,             # Shallow trees
6    subsample=0.8,           # Stochastic GB
7    min_samples_split=10,    # Regularization
8    validation_fraction=0.2, # Early stopping
9    n_iter_no_change=10,     # Stop if no improve
10    random_state=42
11)

1from sklearn.ensemble import GradientBoostingClassifier
2from sklearn.model_selection import train_test_split
3
4# Split thêm validation set
5X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.3)
6X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5)
7
8# Gradient Boosting với early stopping
9gb = GradientBoostingClassifier(
10    n_estimators=1000,        # Nhiều trees
11    learning_rate=0.05,       # Low LR
12    max_depth=5,
13    validation_fraction=0.2,  # 20% for early stopping
14    n_iter_no_change=20,      # Stop if no improve in 20 iterations
15    tol=0.0001,               # Tolerance
16    verbose=1,
17    random_state=42
18)
19
20gb.fit(X_train, y_train)
21
22print(f"Best iteration: {gb.n_estimators_}")
23print(f"Training stopped at: {gb.n_estimators_}")