1
#!/bin/bash
2

3
# Network configuration for production Firecracker deployment
4
echo "=== Production Network Setup ==="
5

6
# VLAN configuration for multi-tenant isolation
7
configure_vlans() {
8
    # Management VLAN (VLAN 100)
9
    sudo vconfig add eth0 100
10
    sudo ip addr add 10.0.100.10/24 dev eth0.100
11
    sudo ip link set eth0.100 up
12

13
    # Compute VLAN (VLAN 200)
14
    sudo vconfig add eth0 200
15
    sudo ip addr add 10.0.200.10/24 dev eth0.200
16
    sudo ip link set eth0.200 up
17

18
    # Storage VLAN (VLAN 300)
19
    sudo vconfig add eth0 300
20
    sudo ip addr add 10.0.300.10/24 dev eth0.300
21
    sudo ip link set eth0.300 up
22

23
    # Tenant VLANs (VLAN 400-499)
24
    for vlan in {400..410}; do
25
        sudo vconfig add eth1 $vlan
26
        sudo ip addr add 10.0.$((vlan-300)).10/24 dev eth1.$vlan
27
        sudo ip link set eth1.$vlan up
28
    done
29

30
    echo "✓ VLAN configuration complete"
31
}
32

33
# Configure OpenVSwitch for advanced networking
34
setup_ovs() {
35
    # Install OpenVSwitch
36
    sudo apt update
37
    sudo apt install -y openvswitch-switch
38

39
    # Create management bridge
40
    sudo ovs-vsctl add-br br-mgmt
41
    sudo ovs-vsctl add-port br-mgmt eth0.100
42

43
    # Create compute bridge with VXLAN support
44
    sudo ovs-vsctl add-br br-compute
45
    sudo ovs-vsctl add-port br-compute eth0.200
46

47
    # Configure VXLAN for overlay networking
48
    sudo ovs-vsctl add-port br-compute vxlan1 -- set interface vxlan1 type=vxlan options:remote_ip=10.0.200.11
49

50
    # Create tenant bridges
51
    for tenant in {1..10}; do
52
        bridge_name="br-tenant-$tenant"
53
        sudo ovs-vsctl add-br $bridge_name
54
        sudo ovs-vsctl set bridge $bridge_name other_config:hwaddr=02:00:00:00:00:$(printf "%02x" $tenant)
55
    done
56

57
    echo "✓ OpenVSwitch configuration complete"
58
}
59

60
# Configure SR-IOV for high-performance networking
61
setup_sriov() {
62
    echo "Setting up SR-IOV..."
63

64
    # Enable SR-IOV (requires compatible NIC)
65
    echo 8 | sudo tee /sys/class/net/eth2/device/sriov_numvfs
66

67
    # Configure virtual functions
68
    for vf in {0..7}; do
69
        sudo ip link set eth2 vf $vf mac 02:00:00:00:01:$(printf "%02x" $vf)
70
        sudo ip link set eth2 vf $vf vlan $((400 + vf))
71
        sudo ip link set eth2 vf $vf spoofchk on
72
        sudo ip link set eth2 vf $vf trust off
73
    done
74

75
    echo "✓ SR-IOV configuration complete"
76
}
77

78
# Main network setup
79
configure_vlans
80
setup_ovs
81
setup_sriov
82

83
echo "Production network setup complete!"

1
# main.tf - Firecracker Infrastructure
2
terraform {
3
  required_version = ">= 1.0"
4
  required_providers {
5
    aws = {
6
      source  = "hashicorp/aws"
7
      version = "~> 5.0"
8
    }
9
  }
10
}
11

12
variable "environment" {
13
  description = "Environment name"
14
  type        = string
15
  default     = "production"
16
}
17

18
variable "cluster_name" {
19
  description = "Firecracker cluster name"
20
  type        = string
21
  default     = "firecracker-prod"
22
}
23

24
variable "compute_node_count" {
25
  description = "Number of compute nodes"
26
  type        = number
27
  default     = 9
28
}
29

30
# VPC and networking
31
resource "aws_vpc" "firecracker_vpc" {
32
  cidr_block           = "10.0.0.0/16"
33
  enable_dns_hostnames = true
34
  enable_dns_support   = true
35

36
  tags = {
37
    Name        = "${var.cluster_name}-vpc"
38
    Environment = var.environment
39
  }
40
}
41

42
resource "aws_subnet" "compute_subnets" {
43
  count = 3
44

45
  vpc_id                  = aws_vpc.firecracker_vpc.id
46
  cidr_block              = "10.0.${count.index + 1}.0/24"
47
  availability_zone       = data.aws_availability_zones.available.names[count.index]
48
  map_public_ip_on_launch = false
49

50
  tags = {
51
    Name        = "${var.cluster_name}-compute-subnet-${count.index + 1}"
52
    Environment = var.environment
53
    Type        = "compute"
54
  }
55
}
56

57
resource "aws_subnet" "control_subnets" {
58
  count = 3
59

60
  vpc_id                  = aws_vpc.firecracker_vpc.id
61
  cidr_block              = "10.0.${count.index + 10}.0/24"
62
  availability_zone       = data.aws_availability_zones.available.names[count.index]
63
  map_public_ip_on_launch = true
64

65
  tags = {
66
    Name        = "${var.cluster_name}-control-subnet-${count.index + 1}"
67
    Environment = var.environment
68
    Type        = "control"
69
  }
70
}
71

72
# Security Groups
73
resource "aws_security_group" "compute_nodes" {
74
  name_prefix = "${var.cluster_name}-compute-"
75
  vpc_id      = aws_vpc.firecracker_vpc.id
76

77
  # Firecracker API access
78
  ingress {
79
    from_port   = 8080
80
    to_port     = 8099
81
    protocol    = "tcp"
82
    cidr_blocks = [aws_vpc.firecracker_vpc.cidr_block]
83
  }
84

85
  # SSH access
86
  ingress {
87
    from_port   = 22
88
    to_port     = 22
89
    protocol    = "tcp"
90
    cidr_blocks = ["10.0.0.0/16"]
91
  }
92

93
  # VM networking
94
  ingress {
95
    from_port   = 0
96
    to_port     = 65535
97
    protocol    = "tcp"
98
    cidr_blocks = ["10.0.0.0/16"]
99
  }
100

101
  egress {
102
    from_port   = 0
103
    to_port     = 0
104
    protocol    = "-1"
105
    cidr_blocks = ["0.0.0.0/0"]
106
  }
107

108
  tags = {
109
    Name        = "${var.cluster_name}-compute-sg"
110
    Environment = var.environment
111
  }
112
}
113

114
# Launch template for compute nodes
115
resource "aws_launch_template" "compute_nodes" {
116
  name_prefix   = "${var.cluster_name}-compute-"
117
  image_id      = data.aws_ami.ubuntu.id
118
  instance_type = "m6i.4xlarge"
119
  key_name      = aws_key_pair.cluster_key.key_name
120

121
  vpc_security_group_ids = [aws_security_group.compute_nodes.id]
122

123
  user_data = base64encode(templatefile("${path.module}/user_data/compute_node.sh", {
124
    cluster_name = var.cluster_name
125
    environment  = var.environment
126
  }))
127

128
  block_device_mappings {
129
    device_name = "/dev/sda1"
130
    ebs {
131
      volume_size = 500
132
      volume_type = "gp3"
133
      iops        = 12000
134
      throughput  = 1000
135
      encrypted   = true
136
    }
137
  }
138

139
  # Additional EBS volume for VM storage
140
  block_device_mappings {
141
    device_name = "/dev/sdf"
142
    ebs {
143
      volume_size = 2000
144
      volume_type = "gp3"
145
      iops        = 16000
146
      throughput  = 1000
147
      encrypted   = true
148
    }
149
  }
150

151
  tag_specifications {
152
    resource_type = "instance"
153
    tags = {
154
      Name        = "${var.cluster_name}-compute"
155
      Environment = var.environment
156
      Role        = "compute"
157
    }
158
  }
159

160
  tags = {
161
    Name        = "${var.cluster_name}-compute-template"
162
    Environment = var.environment
163
  }
164
}
165

166
# Auto Scaling Group for compute nodes
167
resource "aws_autoscaling_group" "compute_nodes" {
168
  name                = "${var.cluster_name}-compute-asg"
169
  vpc_zone_identifier = aws_subnet.compute_subnets[*].id
170
  target_group_arns   = []
171
  health_check_type   = "EC2"
172

173
  min_size         = var.compute_node_count
174
  max_size         = var.compute_node_count * 2
175
  desired_capacity = var.compute_node_count
176

177
  launch_template {
178
    id      = aws_launch_template.compute_nodes.id
179
    version = "$Latest"
180
  }
181

182
  tag {
183
    key                 = "Name"
184
    value               = "${var.cluster_name}-compute-node"
185
    propagate_at_launch = true
186
  }
187

188
  tag {
189
    key                 = "Environment"
190
    value               = var.environment
191
    propagate_at_launch = true
192
  }
193

194
  tag {
195
    key                 = "Role"
196
    value               = "compute"
197
    propagate_at_launch = true
198
  }
199
}
200

201
# Application Load Balancer for API access
202
resource "aws_lb" "api_lb" {
203
  name               = "${var.cluster_name}-api-alb"
204
  internal           = true
205
  load_balancer_type = "application"
206
  security_groups    = [aws_security_group.api_lb.id]
207
  subnets            = aws_subnet.control_subnets[*].id
208

209
  enable_deletion_protection = false
210

211
  tags = {
212
    Name        = "${var.cluster_name}-api-alb"
213
    Environment = var.environment
214
  }
215
}
216

217
# RDS for metadata storage
218
resource "aws_rds_cluster" "metadata_db" {
219
  cluster_identifier      = "${var.cluster_name}-metadata"
220
  engine                  = "aurora-postgresql"
221
  engine_version          = "13.7"
222
  availability_zones      = data.aws_availability_zones.available.names
223
  database_name           = "firecracker_metadata"
224
  master_username         = "fcadmin"
225
  manage_master_user_password = true
226

227
  backup_retention_period = 30
228
  preferred_backup_window = "03:00-05:00"
229

230
  vpc_security_group_ids = [aws_security_group.database.id]
231
  db_subnet_group_name   = aws_db_subnet_group.metadata.name
232

233
  tags = {
234
    Name        = "${var.cluster_name}-metadata-db"
235
    Environment = var.environment
236
  }
237
}
238

239
# Outputs
240
output "vpc_id" {
241
  value = aws_vpc.firecracker_vpc.id
242
}
243

244
output "compute_subnets" {
245
  value = aws_subnet.compute_subnets[*].id
246
}
247

248
output "api_load_balancer_dns" {
249
  value = aws_lb.api_lb.dns_name
250
}
251

252
output "database_endpoint" {
253
  value = aws_rds_cluster.metadata_db.endpoint
254
}

1
#!/usr/bin/env python3
2
"""
3
Firecracker VM Lifecycle Manager
4
Manages VM creation, monitoring, and cleanup in production environments
5
"""
6

7
import json
8
import time
9
import uuid
10
import logging
11
import threading
12
import subprocess
13
from pathlib import Path
14
from datetime import datetime, timedelta
15
from typing import Dict, List, Optional, Tuple
16
from dataclasses import dataclass, asdict
17
from enum import Enum
18

19
import psutil
20
import requests
21

22
# Configure logging
23
logging.basicConfig(
24
    level=logging.INFO,
25
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
26
    handlers=[
27
        logging.FileHandler('/var/log/firecracker/vm-manager.log'),
28
        logging.StreamHandler()
29
    ]
30
)
31
logger = logging.getLogger('vm-manager')
32

33
class VMState(Enum):
34
    CREATING = "creating"
35
    RUNNING = "running"
36
    STOPPING = "stopping"
37
    STOPPED = "stopped"
38
    ERROR = "error"
39

40
@dataclass
41
class VMConfiguration:
42
    vm_id: str
43
    vcpus: int
44
    memory_mb: int
45
    kernel_path: str
46
    rootfs_path: str
47
    network_config: Dict
48
    storage_config: List[Dict]
49
    metadata: Dict
50
    created_at: datetime
51
    ttl_hours: Optional[int] = None
52

53
@dataclass
54
class VMInstance:
55
    config: VMConfiguration
56
    state: VMState
57
    pid: Optional[int]
58
    api_socket: str
59
    metrics_path: str
60
    log_path: str
61
    last_health_check: datetime
62
    resource_usage: Dict = None
63

64
class FirecrackerVMManager:
65
    """Production-grade Firecracker VM lifecycle manager"""
66

67
    def __init__(self, config_path: str = '/etc/firecracker/manager.conf'):
68
        self.config = self._load_config(config_path)
69
        self.vms: Dict[str, VMInstance] = {}
70
        self.running = False
71
        self.health_check_interval = 30
72
        self.cleanup_interval = 300
73

74
        # Initialize directories
75
        self.base_dir = Path(self.config['base_directory'])
76
        self.vm_dir = self.base_dir / 'vms'
77
        self.image_dir = self.base_dir / 'images'
78
        self.kernel_dir = self.base_dir / 'kernels'
79
        self.log_dir = Path('/var/log/firecracker')
80

81
        for directory in [self.vm_dir, self.image_dir, self.kernel_dir, self.log_dir]:
82
            directory.mkdir(parents=True, exist_ok=True)
83

84
        # Load existing VMs
85
        self._discover_existing_vms()
86

87
    def _load_config(self, config_path: str) -> Dict:
88
        """Load manager configuration"""
89
        default_config = {
90
            'base_directory': '/var/lib/firecracker',
91
            'max_vms_per_node': 50,
92
            'default_vm_ttl_hours': 24,
93
            'health_check_enabled': True,
94
            'metrics_enabled': True,
95
            'auto_cleanup_enabled': True,
96
            'resource_limits': {
97
                'max_memory_mb': 8192,
98
                'max_vcpus': 8
99
            },
100
            'network': {
101
                'bridge_name': 'br0',
102
                'subnet': '172.16.0.0/16',
103
                'dhcp_range_start': '172.16.1.100',
104
                'dhcp_range_end': '172.16.1.200'
105
            }
106
        }
107

108
        try:
109
            with open(config_path, 'r') as f:
110
                user_config = json.load(f)
111
                default_config.update(user_config)
112
        except FileNotFoundError:
113
            logger.warning(f"Config file {config_path} not found, using defaults")
114

115
        return default_config
116

117
    def _discover_existing_vms(self):
118
        """Discover VMs that are already running"""
119
        logger.info("Discovering existing VMs...")
120

121
        for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
122
            try:
123
                if proc.info['name'] == 'firecracker':
124
                    vm_id = self._extract_vm_id_from_cmdline(proc.info['cmdline'])
125
                    if vm_id:
126
                        vm_dir = self.vm_dir / vm_id
127
                        config_file = vm_dir / 'config.json'
128

129
                        if config_file.exists():
130
                            config = self._load_vm_config(config_file)
131
                            instance = VMInstance(
132
                                config=config,
133
                                state=VMState.RUNNING,
134
                                pid=proc.info['pid'],
135
                                api_socket=str(vm_dir / 'api.sock'),
136
                                metrics_path=str(vm_dir / 'metrics.json'),
137
                                log_path=str(self.log_dir / f'{vm_id}.log'),
138
                                last_health_check=datetime.now()
139
                            )
140
                            self.vms[vm_id] = instance
141
                            logger.info(f"Discovered existing VM: {vm_id}")
142
            except (psutil.NoSuchProcess, psutil.AccessDenied):
143
                continue
144

145
        logger.info(f"Discovered {len(self.vms)} existing VMs")
146

147
    def _extract_vm_id_from_cmdline(self, cmdline: List[str]) -> Optional[str]:
148
        """Extract VM ID from Firecracker command line"""
149
        for i, arg in enumerate(cmdline):
150
            if '--api-sock' in arg and i + 1 < len(cmdline):
151
                socket_path = cmdline[i + 1]
152
                return Path(socket_path).parent.name
153
        return None
154

155
    def _load_vm_config(self, config_file: Path) -> VMConfiguration:
156
        """Load VM configuration from file"""
157
        with open(config_file, 'r') as f:
158
            data = json.load(f)
159
            return VMConfiguration(**data)
160

161
    def create_vm(self, vm_spec: Dict) -> Tuple[str, bool]:
162
        """Create a new VM instance"""
163
        vm_id = vm_spec.get('vm_id', str(uuid.uuid4())[:8])
164

165
        # Validate resource limits
166
        if not self._validate_resources(vm_spec):
167
            return vm_id, False
168

169
        # Check capacity
170
        if len(self.vms) >= self.config['max_vms_per_node']:
171
            logger.error(f"Maximum VM capacity reached: {self.config['max_vms_per_node']}")
172
            return vm_id, False
173

174
        try:
175
            # Create VM configuration
176
            config = VMConfiguration(
177
                vm_id=vm_id,
178
                vcpus=vm_spec.get('vcpus', 1),
179
                memory_mb=vm_spec.get('memory_mb', 512),
180
                kernel_path=vm_spec.get('kernel_path', str(self.kernel_dir / 'vmlinux.bin')),
181
                rootfs_path=vm_spec.get('rootfs_path', str(self.image_dir / 'rootfs.ext4')),
182
                network_config=vm_spec.get('network_config', {}),
183
                storage_config=vm_spec.get('storage_config', []),
184
                metadata=vm_spec.get('metadata', {}),
185
                created_at=datetime.now(),
186
                ttl_hours=vm_spec.get('ttl_hours', self.config['default_vm_ttl_hours'])
187
            )
188

189
            # Create VM directory structure
190
            vm_dir = self.vm_dir / vm_id
191
            vm_dir.mkdir(exist_ok=True)
192

193
            # Save configuration
194
            config_file = vm_dir / 'config.json'
195
            with open(config_file, 'w') as f:
196
                json.dump(asdict(config), f, indent=2, default=str)
197

198
            # Prepare VM files
199
            if not self._prepare_vm_files(config, vm_dir):
200
                return vm_id, False
201

202
            # Start Firecracker process
203
            if not self._start_firecracker(config, vm_dir):
204
                return vm_id, False
205

206
            # Create VM instance
207
            instance = VMInstance(
208
                config=config,
209
                state=VMState.CREATING,
210
                pid=None,
211
                api_socket=str(vm_dir / 'api.sock'),
212
                metrics_path=str(vm_dir / 'metrics.json'),
213
                log_path=str(self.log_dir / f'{vm_id}.log'),
214
                last_health_check=datetime.now()
215
            )
216

217
            # Wait for Firecracker to start
218
            if self._wait_for_api(instance.api_socket, timeout=30):
219
                # Configure and start VM
220
                if self._configure_and_start_vm(instance):
221
                    instance.state = VMState.RUNNING
222
                    self.vms[vm_id] = instance
223
                    logger.info(f"Successfully created VM: {vm_id}")
224
                    return vm_id, True
225

226
            instance.state = VMState.ERROR
227
            logger.error(f"Failed to start VM: {vm_id}")
228
            return vm_id, False
229

230
        except Exception as e:
231
            logger.error(f"Error creating VM {vm_id}: {e}")
232
            return vm_id, False
233

234
    def _validate_resources(self, vm_spec: Dict) -> bool:
235
        """Validate VM resource requirements"""
236
        limits = self.config['resource_limits']
237

238
        vcpus = vm_spec.get('vcpus', 1)
239
        memory_mb = vm_spec.get('memory_mb', 512)
240

241
        if vcpus > limits['max_vcpus']:
242
            logger.error(f"vCPU count {vcpus} exceeds limit {limits['max_vcpus']}")
243
            return False
244

245
        if memory_mb > limits['max_memory_mb']:
246
            logger.error(f"Memory {memory_mb}MB exceeds limit {limits['max_memory_mb']}MB")
247
            return False
248

249
        # Check available system resources
250
        system_memory = psutil.virtual_memory()
251
        used_memory = sum(vm.config.memory_mb for vm in self.vms.values()
252
                         if vm.state == VMState.RUNNING)
253

254
        if used_memory + memory_mb > system_memory.available // (1024 * 1024) * 0.8:
255
            logger.error("Insufficient system memory for new VM")
256
            return False
257

258
        return True
259

260
    def _prepare_vm_files(self, config: VMConfiguration, vm_dir: Path) -> bool:
261
        """Prepare VM filesystem and kernel images"""
262
        try:
263
            # Copy/create rootfs if needed
264
            rootfs_source = Path(config.rootfs_path)
265
            rootfs_dest = vm_dir / 'rootfs.ext4'
266

267
            if not rootfs_dest.exists() and rootfs_source.exists():
268
                subprocess.run(['cp', str(rootfs_source), str(rootfs_dest)], check=True)
269
                logger.info(f"Copied rootfs for VM {config.vm_id}")
270

271
            # Kernel should already exist
272
            if not Path(config.kernel_path).exists():
273
                logger.error(f"Kernel not found: {config.kernel_path}")
274
                return False
275

276
            return True
277

278
        except Exception as e:
279
            logger.error(f"Error preparing VM files: {e}")
280
            return False
281

282
    def _start_firecracker(self, config: VMConfiguration, vm_dir: Path) -> bool:
283
        """Start Firecracker process"""
284
        try:
285
            api_socket = vm_dir / 'api.sock'
286
            log_file = self.log_dir / f'{config.vm_id}.log'
287

288
            cmd = [
289
                'firecracker',
290
                '--api-sock', str(api_socket),
291
                '--config-file', str(vm_dir / 'fc_config.json')
292
            ]
293

294
            # Create Firecracker configuration
295
            fc_config = self._generate_firecracker_config(config, vm_dir)
296
            with open(vm_dir / 'fc_config.json', 'w') as f:
297
                json.dump(fc_config, f, indent=2)
298

299
            # Start process
300
            with open(log_file, 'w') as log:
301
                process = subprocess.Popen(
302
                    cmd,
303
                    stdout=log,
304
                    stderr=subprocess.STDOUT,
305
                    cwd=str(vm_dir)
306
                )
307

308
            # Update VM with PID
309
            if config.vm_id in self.vms:
310
                self.vms[config.vm_id].pid = process.pid
311

312
            logger.info(f"Started Firecracker process for VM {config.vm_id} (PID: {process.pid})")
313
            return True
314

315
        except Exception as e:
316
            logger.error(f"Error starting Firecracker: {e}")
317
            return False
318

319
    def _generate_firecracker_config(self, config: VMConfiguration, vm_dir: Path) -> Dict:
320
        """Generate Firecracker configuration file"""
321

322
        rootfs_path = vm_dir / 'rootfs.ext4'
323

324
        fc_config = {
325
            "boot-source": {
326
                "kernel_image_path": config.kernel_path,
327
                "boot_args": "console=ttyS0 reboot=k panic=1 pci=off nomodules ro"
328
            },
329
            "drives": [
330
                {
331
                    "drive_id": "rootfs",
332
                    "path_on_host": str(rootfs_path),
333
                    "is_root_device": True,
334
                    "is_read_only": False
335
                }
336
            ],
337
            "machine-config": {
338
                "vcpu_count": config.vcpus,
339
                "mem_size_mib": config.memory_mb
340
            },
341
            "logger": {
342
                "level": "Info",
343
                "log_path": str(self.log_dir / f'{config.vm_id}-vmm.log')
344
            },
345
            "metrics": {
346
                "metrics_path": str(vm_dir / 'metrics.json')
347
            }
348
        }
349

350
        # Add network configuration if provided
351
        if config.network_config:
352
            fc_config["network-interfaces"] = [config.network_config]
353

354
        # Add additional storage if provided
355
        for i, storage in enumerate(config.storage_config):
356
            drive_config = {
357
                "drive_id": f"storage_{i}",
358
                "path_on_host": storage["path"],
359
                "is_root_device": False,
360
                "is_read_only": storage.get("read_only", False)
361
            }
362
            fc_config["drives"].append(drive_config)
363

364
        return fc_config
365

366
    def _wait_for_api(self, api_socket: str, timeout: int = 30) -> bool:
367
        """Wait for Firecracker API to become available"""
368
        start_time = time.time()
369

370
        while time.time() - start_time < timeout:
371
            if Path(api_socket).exists():
372
                try:
373
                    import requests_unixsocket
374
                    session = requests_unixsocket.Session()
375
                    base_url = f'http+unix://{api_socket.replace("/", "%2F")}'
376

377
                    response = session.get(f'{base_url}/', timeout=5)
378
                    if response.status_code == 200:
379
                        return True
380
                except Exception:
381
                    pass
382

383
            time.sleep(1)
384

385
        return False
386

387
    def _configure_and_start_vm(self, instance: VMInstance) -> bool:
388
        """Configure and start the VM via Firecracker API"""
389
        try:
390
            import requests_unixsocket
391
            session = requests_unixsocket.Session()
392
            base_url = f'http+unix://{instance.api_socket.replace("/", "%2F")}'
393

394
            # Start the VM
395
            response = session.put(
396
                f'{base_url}/actions',
397
                json={'action_type': 'InstanceStart'},
398
                timeout=10
399
            )
400

401
            if response.status_code == 204:
402
                logger.info(f"Successfully started VM {instance.config.vm_id}")
403
                return True
404
            else:
405
                logger.error(f"Failed to start VM {instance.config.vm_id}: {response.status_code}")
406
                return False
407

408
        except Exception as e:
409
            logger.error(f"Error configuring VM {instance.config.vm_id}: {e}")
410
            return False
411

412
    def stop_vm(self, vm_id: str, force: bool = False) -> bool:
413
        """Stop a running VM"""
414
        if vm_id not in self.vms:
415
            logger.error(f"VM not found: {vm_id}")
416
            return False
417

418
        instance = self.vms[vm_id]
419

420
        try:
421
            if not force:
422
                # Try graceful shutdown first
423
                if self._graceful_shutdown(instance):
424
                    instance.state = VMState.STOPPED
425
                    logger.info(f"Gracefully stopped VM: {vm_id}")
426
                    return True
427

428
            # Force stop
429
            if instance.pid and psutil.pid_exists(instance.pid):
430
                proc = psutil.Process(instance.pid)
431
                proc.terminate()
432

433
                # Wait for process to exit
434
                try:
435
                    proc.wait(timeout=10)
436
                except psutil.TimeoutExpired:
437
                    proc.kill()
438
                    proc.wait(timeout=5)
439

440
                instance.state = VMState.STOPPED
441
                logger.info(f"Force stopped VM: {vm_id}")
442
                return True
443

444
            return False
445

446
        except Exception as e:
447
            logger.error(f"Error stopping VM {vm_id}: {e}")
448
            return False
449

450
    def _graceful_shutdown(self, instance: VMInstance) -> bool:
451
        """Attempt graceful VM shutdown"""
452
        try:
453
            import requests_unixsocket
454
            session = requests_unixsocket.Session()
455
            base_url = f'http+unix://{instance.api_socket.replace("/", "%2F")}'
456

457
            # Send shutdown action
458
            response = session.put(
459
                f'{base_url}/actions',
460
                json={'action_type': 'SendCtrlAltDel'},
461
                timeout=5
462
            )
463

464
            if response.status_code == 204:
465
                # Wait for shutdown
466
                time.sleep(5)
467
                return not psutil.pid_exists(instance.pid)
468

469
            return False
470

471
        except Exception:
472
            return False
473

474
    def delete_vm(self, vm_id: str) -> bool:
475
        """Delete a VM and clean up its resources"""
476
        if vm_id not in self.vms:
477
            logger.error(f"VM not found: {vm_id}")
478
            return False
479

480
        instance = self.vms[vm_id]
481

482
        # Stop VM first
483
        if instance.state == VMState.RUNNING:
484
            if not self.stop_vm(vm_id, force=True):
485
                logger.error(f"Failed to stop VM before deletion: {vm_id}")
486
                return False
487

488
        try:
489
            # Clean up VM directory
490
            vm_dir = self.vm_dir / vm_id
491
            if vm_dir.exists():
492
                subprocess.run(['rm', '-rf', str(vm_dir)], check=True)
493

494
            # Clean up logs
495
            log_file = Path(instance.log_path)
496
            if log_file.exists():
497
                log_file.unlink()
498

499
            # Remove from tracking
500
            del self.vms[vm_id]
501

502
            logger.info(f"Deleted VM: {vm_id}")
503
            return True
504

505
        except Exception as e:
506
            logger.error(f"Error deleting VM {vm_id}: {e}")
507
            return False
508

509
    def get_vm_status(self, vm_id: str) -> Optional[Dict]:
510
        """Get VM status and metrics"""
511
        if vm_id not in self.vms:
512
            return None
513

514
        instance = self.vms[vm_id]
515

516
        status = {
517
            'vm_id': vm_id,
518
            'state': instance.state.value,
519
            'config': asdict(instance.config),
520
            'pid': instance.pid,
521
            'uptime_seconds': (datetime.now() - instance.config.created_at).total_seconds(),
522
            'last_health_check': instance.last_health_check.isoformat()
523
        }
524

525
        # Add resource usage if available
526
        if instance.resource_usage:
527
            status['resource_usage'] = instance.resource_usage
528

529
        # Add Firecracker metrics if available
530
        metrics_file = Path(instance.metrics_path)
531
        if metrics_file.exists():
532
            try:
533
                with open(metrics_file, 'r') as f:
534
                    status['firecracker_metrics'] = json.load(f)
535
            except Exception:
536
                pass
537

538
        return status
539

540
    def list_vms(self) -> List[Dict]:
541
        """List all VMs"""
542
        return [self.get_vm_status(vm_id) for vm_id in self.vms.keys()]
543

544
    def health_check(self):
545
        """Perform health checks on all VMs"""
546
        logger.debug("Performing health checks...")
547

548
        for vm_id, instance in list(self.vms.items()):
549
            try:
550
                if instance.state == VMState.RUNNING:
551
                    # Check if process is still running
552
                    if instance.pid and not psutil.pid_exists(instance.pid):
553
                        logger.warning(f"VM process died: {vm_id}")
554
                        instance.state = VMState.ERROR
555
                        continue
556

557
                    # Update resource usage
558
                    if instance.pid:
559
                        proc = psutil.Process(instance.pid)
560
                        instance.resource_usage = {
561
                            'cpu_percent': proc.cpu_percent(),
562
                            'memory_info': proc.memory_info()._asdict(),
563
                            'io_counters': proc.io_counters()._asdict() if hasattr(proc, 'io_counters') else {}
564
                        }
565

566
                    instance.last_health_check = datetime.now()
567

568
            except Exception as e:
569
                logger.error(f"Health check failed for VM {vm_id}: {e}")
570
                instance.state = VMState.ERROR
571

572
    def cleanup_expired_vms(self):
573
        """Clean up expired VMs based on TTL"""
574
        if not self.config['auto_cleanup_enabled']:
575
            return
576

577
        logger.debug("Checking for expired VMs...")
578

579
        current_time = datetime.now()
580
        expired_vms = []
581

582
        for vm_id, instance in self.vms.items():
583
            if instance.config.ttl_hours:
584
                expiry_time = instance.config.created_at + timedelta(hours=instance.config.ttl_hours)
585
                if current_time > expiry_time:
586
                    expired_vms.append(vm_id)
587

588
        for vm_id in expired_vms:
589
            logger.info(f"Cleaning up expired VM: {vm_id}")
590
            self.delete_vm(vm_id)
591

592
    def start_background_tasks(self):
593
        """Start background maintenance tasks"""
594
        self.running = True
595

596
        def health_check_loop():
597
            while self.running:
598
                try:
599
                    self.health_check()
600
                    time.sleep(self.health_check_interval)
601
                except Exception as e:
602
                    logger.error(f"Error in health check loop: {e}")
603
                    time.sleep(self.health_check_interval)
604

605
        def cleanup_loop():
606
            while self.running:
607
                try:
608
                    self.cleanup_expired_vms()
609
                    time.sleep(self.cleanup_interval)
610
                except Exception as e:
611
                    logger.error(f"Error in cleanup loop: {e}")
612
                    time.sleep(self.cleanup_interval)
613

614
        # Start background threads
615
        self.health_check_thread = threading.Thread(target=health_check_loop, daemon=True)
616
        self.cleanup_thread = threading.Thread(target=cleanup_loop, daemon=True)
617

618
        self.health_check_thread.start()
619
        self.cleanup_thread.start()
620

621
        logger.info("Background tasks started")
622

623
    def stop_background_tasks(self):
624
        """Stop background maintenance tasks"""
625
        self.running = False
626
        logger.info("Background tasks stopped")
627

628
# REST API server for VM management
629
from flask import Flask, request, jsonify
630
from flask_limiter import Limiter
631
from flask_limiter.util import get_remote_address
632

633
app = Flask(__name__)
634
limiter = Limiter(
635
    app,
636
    key_func=get_remote_address,
637
    default_limits=["100 per hour"]
638
)
639

640
vm_manager = FirecrackerVMManager()
641

642
@app.route('/health', methods=['GET'])
643
def health_check():
644
    return jsonify({'status': 'healthy', 'timestamp': datetime.now().isoformat()})
645

646
@app.route('/vms', methods=['GET'])
647
@limiter.limit("10 per minute")
648
def list_vms():
649
    return jsonify({'vms': vm_manager.list_vms()})
650

651
@app.route('/vms', methods=['POST'])
652
@limiter.limit("5 per minute")
653
def create_vm():
654
    vm_spec = request.get_json()
655
    if not vm_spec:
656
        return jsonify({'error': 'Invalid JSON'}), 400
657

658
    vm_id, success = vm_manager.create_vm(vm_spec)
659

660
    if success:
661
        return jsonify({'vm_id': vm_id, 'status': 'created'}), 201
662
    else:
663
        return jsonify({'vm_id': vm_id, 'status': 'failed'}), 500
664

665
@app.route('/vms/<vm_id>', methods=['GET'])
666
def get_vm_status(vm_id):
667
    status = vm_manager.get_vm_status(vm_id)
668
    if status:
669
        return jsonify(status)
670
    else:
671
        return jsonify({'error': 'VM not found'}), 404
672

673
@app.route('/vms/<vm_id>', methods=['DELETE'])
674
@limiter.limit("5 per minute")
675
def delete_vm(vm_id):
676
    if vm_manager.delete_vm(vm_id):
677
        return jsonify({'status': 'deleted'})
678
    else:
679
        return jsonify({'error': 'Failed to delete VM'}), 500
680

681
@app.route('/vms/<vm_id>/stop', methods=['POST'])
682
@limiter.limit("5 per minute")
683
def stop_vm(vm_id):
684
    force = request.get_json().get('force', False) if request.get_json() else False
685

686
    if vm_manager.stop_vm(vm_id, force=force):
687
        return jsonify({'status': 'stopped'})
688
    else:
689
        return jsonify({'error': 'Failed to stop VM'}), 500
690

691
if __name__ == '__main__':
692
    # Start background tasks
693
    vm_manager.start_background_tasks()
694

695
    try:
696
        # Start API server
697
        app.run(host='0.0.0.0', port=8080, debug=False)
698
    except KeyboardInterrupt:
699
        logger.info("Shutting down VM manager...")
700
        vm_manager.stop_background_tasks()