Building ML-Powered Threat Intelligence with Honeypot Datasets: From Raw Data to Production Models#

1
def create_temporal_features(df):
2
    """
3
    Create time-based features for attack pattern detection
4

5
    Args:
6
        df: DataFrame with timestamp column
7

8
    Returns:
9
        DataFrame with added temporal features
10
    """
11
    if 'timestamp' not in df.columns:
12
        raise ValueError("Timestamp column required for temporal features")
13

14
    # Basic time components
15
    df['hour'] = df['timestamp'].dt.hour
16
    df['day_of_week'] = df['timestamp'].dt.dayofweek
17
    df['day_of_month'] = df['timestamp'].dt.day
18
    df['month'] = df['timestamp'].dt.month
19

20
    # Working hours indicator (9 AM - 5 PM on weekdays)
21
    df['is_working_hours'] = (
22
        (df['hour'] >= 9) &
23
        (df['hour'] < 17) &
24
        (df['day_of_week'] < 5)
25
    ).astype(int)
26

27
    # Weekend indicator
28
    df['is_weekend'] = df['day_of_week'].isin([5, 6]).astype(int)
29

30
    # Night time indicator (10 PM - 6 AM)
31
    df['is_night'] = (
32
        (df['hour'] >= 22) |
33
        (df['hour'] < 6)
34
    ).astype(int)
35

36
    # Time since first connection from same source
37
    df['first_seen'] = df.groupby('src_ip')['timestamp'].transform('min')
38
    df['hours_since_first_seen'] = (
39
        (df['timestamp'] - df['first_seen']).dt.total_seconds() / 3600
40
    )
41

42
    # Connection frequency features
43
    df['daily_connection_count'] = df.groupby([
44
        'src_ip',
45
        df['timestamp'].dt.date
46
    ])['src_ip'].transform('count')
47

48
    # Hourly connection rate
49
    df['hourly_connection_count'] = df.groupby([
50
        'src_ip',
51
        df['timestamp'].dt.floor('H')
52
    ])['src_ip'].transform('count')
53

54
    # Time between connections (session gaps)
55
    df_sorted = df.sort_values(['src_ip', 'timestamp'])
56
    df_sorted['time_since_last_connection'] = df_sorted.groupby('src_ip')['timestamp'].diff().dt.total_seconds()
57
    df['time_since_last_connection'] = df_sorted['time_since_last_connection']
58

59
    # Fill NaN (first connection has no previous connection)
60
    df['time_since_last_connection'].fillna(0, inplace=True)
61

62
    print(f"Created temporal features: hour, day_of_week, is_weekend, connection frequency metrics")
63

64
    return df

1
def create_behavioral_features(df):
2
    """
3
    Create behavioral aggregation features based on source IP patterns
4

5
    Args:
6
        df: DataFrame with network events
7

8
    Returns:
9
        DataFrame with behavioral features
10
    """
11
    print("Creating behavioral aggregation features...")
12

13
    # Per-source IP aggregations
14
    source_stats = df.groupby('src_ip').agg({
15
        'dst_port': ['nunique', 'count'],
16
        'bytes_sent': ['mean', 'std', 'max', 'sum'],
17
        'bytes_received': ['mean', 'std', 'max', 'sum'],
18
        'session_duration': ['mean', 'median', 'max'],
19
        'protocol': lambda x: x.mode().iloc[0] if len(x) > 0 else 'Unknown'
20
    }).reset_index()
21

22
    # Flatten column names
23
    source_stats.columns = [
24
        'src_ip',
25
        'unique_ports', 'total_connections',
26
        'avg_bytes_sent', 'std_bytes_sent', 'max_bytes_sent', 'total_bytes_sent',
27
        'avg_bytes_received', 'std_bytes_received', 'max_bytes_received', 'total_bytes_received',
28
        'avg_duration', 'median_duration', 'max_duration',
29
        'primary_protocol'
30
    ]
31

32
    # Merge back to original dataset
33
    df = df.merge(source_stats, on='src_ip', how='left')
34

35
    # Port scanning indicators
36
    df['port_diversity'] = df['unique_ports'] / df['total_connections']
37
    df['is_port_scanner'] = (df['unique_ports'] > 10).astype(int)
38

39
    # Connection pattern features
40
    df['avg_connection_size'] = (df['avg_bytes_sent'] + df['avg_bytes_received'])
41

42
    # Bandwidth usage patterns
43
    df['total_bandwidth'] = df['total_bytes_sent'] + df['total_bytes_received']
44
    df['bandwidth_asymmetry'] = (
45
        abs(df['total_bytes_sent'] - df['total_bytes_received']) /
46
        (df['total_bandwidth'] + 1)  # +1 to avoid division by zero
47
    )
48

49
    # Session behavior patterns
50
    df['connection_rate'] = df['total_connections'] / (df['hours_since_first_seen'] + 1)
51

52
    # Consistency metrics (lower std dev means more consistent behavior)
53
    df['bytes_consistency'] = df['std_bytes_sent'] / (df['avg_bytes_sent'] + 1)
54

55
    print(f"Created {len(source_stats.columns) - 1} behavioral features")
56

57
    return df

1
def create_geographic_features(df):
2
    """
3
    Create geography-based threat intelligence features
4

5
    Args:
6
        df: DataFrame with geolocation data
7

8
    Returns:
9
        DataFrame with geographic features
10
    """
11
    print("Creating geographic and network features...")
12

13
    # Country-level threat scoring
14
    if 'country' in df.columns:
15
        country_threat_scores = df.groupby('country').agg({
16
            'src_ip': 'nunique',
17
            'attack_type': lambda x: (x != 'benign').sum() if 'attack_type' in df.columns else 0
18
        }).reset_index()
19

20
        country_threat_scores.columns = ['country', 'unique_ips_from_country', 'attack_count_from_country']
21

22
        country_threat_scores['country_threat_ratio'] = (
23
            country_threat_scores['attack_count_from_country'] /
24
            (country_threat_scores['unique_ips_from_country'] + 1)
25
        )
26

27
        df = df.merge(
28
            country_threat_scores[['country', 'country_threat_ratio', 'unique_ips_from_country']],
29
            on='country',
30
            how='left'
31
        )
32

33
    # ASN-based features
34
    if 'asn' in df.columns:
35
        asn_stats = df.groupby('asn').agg({
36
            'src_ip': 'nunique',
37
            'bytes_sent': 'mean',
38
            'total_connections': 'sum'
39
        }).reset_index()
40

41
        asn_stats.columns = ['asn', 'unique_ips_per_asn', 'avg_bytes_per_asn', 'total_conn_per_asn']
42

43
        df = df.merge(asn_stats, on='asn', how='left', suffixes=('', '_asn'))
44

45
    # Protocol distribution features
46
    if 'protocol' in df.columns:
47
        protocol_dummies = pd.get_dummies(df['protocol'], prefix='protocol')
48
        df = pd.concat([df, protocol_dummies], axis=1)
49

50
    # Distance from known malicious ranges (simplified example)
51
    # In production, integrate with threat intelligence feeds
52
    known_malicious_countries = ['CN', 'RU', 'KP']  # Example
53
    if 'country' in df.columns:
54
        df['from_high_risk_country'] = df['country'].isin(known_malicious_countries).astype(int)
55

56
    print("Geographic features created successfully")
57

58
    return df

1
def create_target_features(df):
2
    """
3
    Create features based on attack targets (ports, services)
4

5
    Args:
6
        df: DataFrame with destination port information
7

8
    Returns:
9
        DataFrame with target-based features
10
    """
11
    print("Creating target-based features...")
12

13
    # Common service ports mapping
14
    common_ports = {
15
        22: 'SSH',
16
        23: 'Telnet',
17
        80: 'HTTP',
18
        443: 'HTTPS',
19
        3306: 'MySQL',
20
        3389: 'RDP',
21
        5432: 'PostgreSQL',
22
        8080: 'HTTP-Alt'
23
    }
24

25
    if 'dst_port' in df.columns:
26
        df['target_service'] = df['dst_port'].map(common_ports).fillna('Other')
27

28
        # Service targeting patterns
29
        df['targets_ssh'] = (df['dst_port'] == 22).astype(int)
30
        df['targets_rdp'] = (df['dst_port'] == 3389).astype(int)
31
        df['targets_web'] = df['dst_port'].isin([80, 443, 8080]).astype(int)
32
        df['targets_database'] = df['dst_port'].isin([3306, 5432, 1433]).astype(int)
33

34
        # Well-known vs ephemeral ports
35
        df['targets_wellknown_port'] = (df['dst_port'] < 1024).astype(int)
36
        df['targets_registered_port'] = ((df['dst_port'] >= 1024) & (df['dst_port'] < 49152)).astype(int)
37
        df['targets_ephemeral_port'] = (df['dst_port'] >= 49152).astype(int)
38

39
    print("Target features created successfully")
40

41
    return df

1
def engineer_all_features(df):
2
    """
3
    Complete feature engineering pipeline
4

5
    Args:
6
        df: Raw preprocessed DataFrame
7

8
    Returns:
9
        DataFrame with all engineered features
10
    """
11
    print("\n" + "=" * 60)
12
    print("STARTING FEATURE ENGINEERING PIPELINE")
13
    print("=" * 60 + "\n")
14

15
    # Create copies to avoid modifying original
16
    df_features = df.copy()
17

18
    # Apply feature engineering steps
19
    df_features = create_temporal_features(df_features)
20
    df_features = create_behavioral_features(df_features)
21
    df_features = create_geographic_features(df_features)
22
    df_features = create_target_features(df_features)
23

24
    print("\n" + "=" * 60)
25
    print("FEATURE ENGINEERING COMPLETE")
26
    print("=" * 60)
27
    print(f"Total features: {len(df_features.columns)}")
28
    print(f"Total records: {len(df_features)}")
29
    print("=" * 60 + "\n")
30

31
    return df_features

1
from sklearn.ensemble import RandomForestClassifier
2
from sklearn.model_selection import train_test_split, cross_val_score
3
from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score
4
import joblib
5

6
def train_binary_classifier(df, save_model=True):
7
    """
8
    Train binary classifier to detect attacks vs benign traffic
9

10
    Args:
11
        df: DataFrame with engineered features
12
        save_model: Whether to save trained model to disk
13

14
    Returns:
15
        Trained model and performance metrics
16
    """
17
    print("\n" + "=" * 60)
18
    print("TRAINING BINARY CLASSIFIER (Attack vs Benign)")
19
    print("=" * 60 + "\n")
20

21
    # Define feature columns
22
    feature_cols = [
23
        # Temporal features
24
        'hour', 'day_of_week', 'is_weekend', 'is_night',
25
        'hours_since_first_seen', 'daily_connection_count',
26

27
        # Behavioral features
28
        'unique_ports', 'total_connections', 'port_diversity',
29
        'avg_bytes_sent', 'avg_bytes_received', 'avg_duration',
30
        'connection_rate', 'bandwidth_asymmetry',
31

32
        # Geographic features
33
        'country_threat_ratio', 'from_high_risk_country',
34

35
        # Target features
36
        'targets_ssh', 'targets_web', 'targets_database'
37
    ]
38

39
    # Filter to available columns
40
    available_features = [col for col in feature_cols if col in df.columns]
41

42
    print(f"Using {len(available_features)} features for training")
43

44
    # Prepare features and labels
45
    X = df[available_features].fillna(0)
46

47
    # Create binary label (assuming 'attack_type' column exists)
48
    if 'attack_type' in df.columns:
49
        y = (df['attack_type'] != 'benign').astype(int)
50
    elif 'label' in df.columns:
51
        y = (df['label'] != 0).astype(int)
52
    else:
53
        raise ValueError("No label column found (need 'attack_type' or 'label')")
54

55
    # Check class distribution
56
    print(f"Class distribution:")
57
    print(f"  Benign: {(y == 0).sum()} ({(y == 0).sum() / len(y) * 100:.2f}%)")
58
    print(f"  Attack: {(y == 1).sum()} ({(y == 1).sum() / len(y) * 100:.2f}%)")
59

60
    # Split data (stratified to maintain class distribution)
61
    X_train, X_test, y_train, y_test = train_test_split(
62
        X, y,
63
        test_size=0.2,
64
        random_state=42,
65
        stratify=y
66
    )
67

68
    print(f"\nTraining set: {len(X_train)} samples")
69
    print(f"Test set: {len(X_test)} samples")
70

71
    # Train Random Forest model
72
    print("\nTraining Random Forest classifier...")
73
    rf_model = RandomForestClassifier(
74
        n_estimators=100,
75
        max_depth=10,
76
        min_samples_split=5,
77
        min_samples_leaf=2,
78
        class_weight='balanced',  # Handle imbalanced data
79
        random_state=42,
80
        n_jobs=-1  # Use all CPU cores
81
    )
82

83
    rf_model.fit(X_train, y_train)
84

85
    # Evaluate on test set
86
    print("\nEvaluating model...")
87
    y_pred = rf_model.predict(X_test)
88
    y_pred_proba = rf_model.predict_proba(X_test)[:, 1]
89

90
    # Print classification report
91
    print("\nClassification Report:")
92
    print(classification_report(
93
        y_test, y_pred,
94
        target_names=['Benign', 'Attack']
95
    ))
96

97
    # Calculate ROC-AUC
98
    roc_auc = roc_auc_score(y_test, y_pred_proba)
99
    print(f"ROC-AUC Score: {roc_auc:.4f}")
100

101
    # Feature importance
102
    feature_importance = pd.DataFrame({
103
        'feature': available_features,
104
        'importance': rf_model.feature_importances_
105
    }).sort_values('importance', ascending=False)
106

107
    print("\nTop 10 Most Important Features:")
108
    print(feature_importance.head(10).to_string(index=False))
109

110
    # Save model
111
    if save_model:
112
        model_path = 'models/binary_classifier_rf.pkl'
113
        joblib.dump(rf_model, model_path)
114
        print(f"\nModel saved to: {model_path}")
115

116
        # Save feature list
117
        feature_path = 'models/binary_classifier_features.pkl'
118
        joblib.dump(available_features, feature_path)
119
        print(f"Feature list saved to: {feature_path}")
120

121
    return rf_model, {
122
        'roc_auc': roc_auc,
123
        'features': available_features,
124
        'feature_importance': feature_importance
125
    }

1
from sklearn.preprocessing import LabelEncoder
2
from xgboost import XGBClassifier
3
import warnings
4
warnings.filterwarnings('ignore')
5

6
def train_multiclass_classifier(df, save_model=True):
7
    """
8
    Train multi-class classifier to categorize attack types
9

10
    Args:
11
        df: DataFrame with engineered features
12
        save_model: Whether to save trained model
13

14
    Returns:
15
        Trained model, label encoder, and metrics
16
    """
17
    print("\n" + "=" * 60)
18
    print("TRAINING MULTI-CLASS CLASSIFIER (Attack Type Categorization)")
19
    print("=" * 60 + "\n")
20

21
    # Encode attack types
22
    le = LabelEncoder()
23

24
    if 'attack_type' in df.columns:
25
        df['attack_label'] = le.fit_transform(df['attack_type'])
26
    elif 'label' in df.columns:
27
        df['attack_label'] = le.fit_transform(df['label'])
28
    else:
29
        raise ValueError("No attack type column found")
30

31
    print(f"Attack types found: {len(le.classes_)}")
32
    for idx, attack_type in enumerate(le.classes_):
33
        count = (df['attack_label'] == idx).sum()
34
        print(f"  {attack_type}: {count} samples")
35

36
    # Feature selection
37
    feature_cols = [
38
        # Temporal
39
        'hour', 'day_of_week', 'is_weekend', 'is_night',
40
        'daily_connection_count', 'hourly_connection_count',
41

42
        # Behavioral
43
        'unique_ports', 'total_connections', 'port_diversity',
44
        'avg_bytes_sent', 'std_bytes_sent',
45
        'avg_bytes_received', 'std_bytes_received',
46
        'avg_duration', 'median_duration',
47
        'connection_rate', 'bytes_consistency',
48

49
        # Geographic
50
        'country_threat_ratio', 'from_high_risk_country',
51

52
        # Target
53
        'targets_ssh', 'targets_rdp', 'targets_web', 'targets_database'
54
    ]
55

56
    available_features = [col for col in feature_cols if col in df.columns]
57

58
    X = df[available_features].fillna(0)
59
    y = df['attack_label']
60

61
    # Split data
62
    X_train, X_test, y_train, y_test = train_test_split(
63
        X, y,
64
        test_size=0.2,
65
        random_state=42,
66
        stratify=y
67
    )
68

69
    # Train XGBoost for multi-class
70
    print(f"\nTraining XGBoost classifier with {len(available_features)} features...")
71

72
    xgb_model = XGBClassifier(
73
        n_estimators=200,
74
        max_depth=6,
75
        learning_rate=0.1,
76
        subsample=0.8,
77
        colsample_bytree=0.8,
78
        random_state=42,
79
        n_jobs=-1,
80
        eval_metric='mlogloss'
81
    )
82

83
    xgb_model.fit(
84
        X_train, y_train,
85
        eval_set=[(X_test, y_test)],
86
        verbose=False
87
    )
88

89
    # Evaluate
90
    y_pred = xgb_model.predict(X_test)
91

92
    print("\nMulti-Class Classification Report:")
93
    print(classification_report(
94
        y_test, y_pred,
95
        target_names=le.classes_
96
    ))
97

98
    # Confusion matrix
99
    cm = confusion_matrix(y_test, y_pred)
100
    print("\nConfusion Matrix:")
101
    print(cm)
102

103
    # Save model
104
    if save_model:
105
        model_path = 'models/multiclass_classifier_xgb.pkl'
106
        joblib.dump(xgb_model, model_path)
107
        print(f"\nModel saved to: {model_path}")
108

109
        encoder_path = 'models/label_encoder.pkl'
110
        joblib.dump(le, encoder_path)
111
        print(f"Label encoder saved to: {encoder_path}")
112

113
    return xgb_model, le, {'confusion_matrix': cm}

1
from sklearn.ensemble import IsolationForest
2
from sklearn.preprocessing import StandardScaler
3

4
def train_anomaly_detector(df, contamination=0.05, save_model=True):
5
    """
6
    Train anomaly detection model for zero-day threat detection
7

8
    Args:
9
        df: DataFrame with engineered features
10
        contamination: Expected proportion of anomalies (0.05 = 5%)
11
        save_model: Whether to save trained model
12

13
    Returns:
14
        Trained model, scaler, and anomaly scores
15
    """
16
    print("\n" + "=" * 60)
17
    print("TRAINING ANOMALY DETECTOR (Zero-Day Threat Detection)")
18
    print("=" * 60 + "\n")
19

20
    # Use only benign traffic for training (if available)
21
    if 'attack_type' in df.columns:
22
        benign_data = df[df['attack_type'] == 'benign']
23
        print(f"Training on {len(benign_data)} benign samples")
24
    else:
25
        benign_data = df
26
        print(f"Training on {len(benign_data)} total samples (no labels available)")
27

28
    # Feature selection for anomaly detection
29
    feature_cols = [
30
        'unique_ports', 'total_connections', 'port_diversity',
31
        'avg_bytes_sent', 'avg_bytes_received',
32
        'avg_duration', 'connection_rate',
33
        'hours_since_first_seen', 'daily_connection_count'
34
    ]
35

36
    available_features = [col for col in feature_cols if col in df.columns]
37

38
    X_benign = benign_data[available_features].fillna(0)
39

40
    # Scale features (important for distance-based anomaly detection)
41
    scaler = StandardScaler()
42
    X_scaled = scaler.fit_transform(X_benign)
43

44
    print(f"Training Isolation Forest with contamination={contamination}...")
45

46
    # Train Isolation Forest
47
    iso_forest = IsolationForest(
48
        n_estimators=100,
49
        contamination=contamination,
50
        max_samples='auto',
51
        random_state=42,
52
        n_jobs=-1
53
    )
54

55
    iso_forest.fit(X_scaled)
56

57
    # Test on full dataset
58
    print("\nTesting on full dataset...")
59
    X_all = df[available_features].fillna(0)
60
    X_all_scaled = scaler.transform(X_all)
61

62
    # Get anomaly scores (lower = more anomalous)
63
    anomaly_scores = iso_forest.decision_function(X_all_scaled)
64
    predictions = iso_forest.predict(X_all_scaled)
65

66
    df['anomaly_score'] = anomaly_scores
67
    df['is_anomaly'] = (predictions == -1).astype(int)
68

69
    # Statistics
70
    anomaly_count = df['is_anomaly'].sum()
71
    anomaly_percentage = (anomaly_count / len(df)) * 100
72

73
    print(f"\nAnomalies detected: {anomaly_count} ({anomaly_percentage:.2f}%)")
74

75
    # If we have labels, evaluate performance
76
    if 'attack_type' in df.columns:
77
        actual_attacks = (df['attack_type'] != 'benign').astype(int)
78
        detected_anomalies = df['is_anomaly']
79

80
        from sklearn.metrics import precision_score, recall_score, f1_score
81

82
        precision = precision_score(actual_attacks, detected_anomalies)
83
        recall = recall_score(actual_attacks, detected_anomalies)
84
        f1 = f1_score(actual_attacks, detected_anomalies)
85

86
        print(f"\nPerformance vs Known Attacks:")
87
        print(f"  Precision: {precision:.4f}")
88
        print(f"  Recall: {recall:.4f}")
89
        print(f"  F1-Score: {f1:.4f}")
90

91
    # Save model
92
    if save_model:
93
        model_path = 'models/anomaly_detector_isoforest.pkl'
94
        joblib.dump(iso_forest, model_path)
95
        print(f"\nModel saved to: {model_path}")
96

97
        scaler_path = 'models/anomaly_scaler.pkl'
98
        joblib.dump(scaler, scaler_path)
99
        print(f"Scaler saved to: {scaler_path}")
100

101
    return iso_forest, scaler, df[['anomaly_score', 'is_anomaly']]

1
import joblib
2
from datetime import datetime
3
import numpy as np
4

5
class ThreatDetectionPipeline:
6
    """
7
    Production-ready threat detection pipeline
8
    """
9

10
    def __init__(self, model_path, scaler_path=None, feature_list_path=None):
11
        """
12
        Initialize pipeline with trained models
13

14
        Args:
15
            model_path: Path to trained model file
16
            scaler_path: Path to feature scaler (optional)
17
            feature_list_path: Path to feature list file (optional)
18
        """
19
        self.model = joblib.load(model_path)
20
        self.scaler = joblib.load(scaler_path) if scaler_path else None
21
        self.feature_list = joblib.load(feature_list_path) if feature_list_path else None
22

23
        print(f"Loaded model from: {model_path}")
24
        if self.scaler:
25
            print(f"Loaded scaler from: {scaler_path}")
26
        if self.feature_list:
27
            print(f"Using {len(self.feature_list)} features")
28

29
    def preprocess_event(self, event):
30
        """
31
        Convert raw security event to feature vector
32

33
        Args:
34
            event: Dictionary with security event data
35

36
        Returns:
37
            Feature vector ready for model prediction
38
        """
39
        # Extract temporal features
40
        if 'timestamp' in event:
41
            dt = datetime.fromisoformat(event['timestamp'].replace('Z', '+00:00'))
42
            hour = dt.hour
43
            day_of_week = dt.weekday()
44
            is_weekend = 1 if day_of_week >= 5 else 0
45
            is_night = 1 if (hour >= 22 or hour < 6) else 0
46
        else:
47
            hour = datetime.now().hour
48
            day_of_week = datetime.now().weekday()
49
            is_weekend = 1 if day_of_week >= 5 else 0
50
            is_night = 1 if (hour >= 22 or hour < 6) else 0
51

52
        # Build feature dictionary
53
        features = {
54
            # Temporal
55
            'hour': hour,
56
            'day_of_week': day_of_week,
57
            'is_weekend': is_weekend,
58
            'is_night': is_night,
59

60
            # Behavioral
61
            'unique_ports': event.get('unique_ports', 1),
62
            'total_connections': event.get('connection_count', 1),
63
            'port_diversity': event.get('port_diversity', 0),
64
            'avg_bytes_sent': event.get('bytes_sent', 0),
65
            'avg_bytes_received': event.get('bytes_received', 0),
66
            'avg_duration': event.get('session_duration', 0),
67
            'connection_rate': event.get('connection_rate', 0),
68

69
            # Geographic
70
            'country_threat_ratio': event.get('country_threat_ratio', 0),
71
            'from_high_risk_country': event.get('from_high_risk_country', 0),
72

73
            # Target
74
            'targets_ssh': 1 if event.get('dst_port') == 22 else 0,
75
            'targets_web': 1 if event.get('dst_port') in [80, 443] else 0,
76
            'targets_database': 1 if event.get('dst_port') in [3306, 5432] else 0,
77
        }
78

79
        # If we have a specific feature list, use only those features
80
        if self.feature_list:
81
            feature_vector = np.array([features.get(f, 0) for f in self.feature_list])
82
        else:
83
            feature_vector = np.array(list(features.values()))
84

85
        feature_vector = feature_vector.reshape(1, -1)
86

87
        # Apply scaling if scaler is available
88
        if self.scaler:
89
            feature_vector = self.scaler.transform(feature_vector)
90

91
        return feature_vector
92

93
    def predict_threat(self, event):
94
        """
95
        Predict threat level for a security event
96

97
        Args:
98
            event: Security event dictionary
99

100
        Returns:
101
            Threat assessment dictionary
102
        """
103
        features = self.preprocess_event(event)
104

105
        # Get prediction probability
106
        if hasattr(self.model, 'predict_proba'):
107
            threat_probability = self.model.predict_proba(features)[0][1]
108
        else:
109
            # For models without predict_proba (like Isolation Forest)
110
            prediction = self.model.predict(features)[0]
111
            threat_probability = 1.0 if prediction == -1 else 0.0
112

113
        # Determine risk level
114
        if threat_probability > 0.8:
115
            risk_level = 'critical'
116
        elif threat_probability > 0.6:
117
            risk_level = 'high'
118
        elif threat_probability > 0.4:
119
            risk_level = 'medium'
120
        else:
121
            risk_level = 'low'
122

123
        return {
124
            'threat_probability': float(threat_probability),
125
            'is_threat': threat_probability > 0.5,
126
            'risk_level': risk_level,
127
            'timestamp': datetime.now().isoformat(),
128
            'src_ip': event.get('src_ip', 'unknown'),
129
            'dst_port': event.get('dst_port', 0)
130
        }
131

132
    def batch_predict(self, events):
133
        """
134
        Predict threats for multiple events
135

136
        Args:
137
            events: List of security event dictionaries
138

139
        Returns:
140
            List of threat assessments
141
        """
142
        return [self.predict_threat(event) for event in events]