feat: Add comprehensive enterprise Linux infrastructure portfolio with Ansible, Python, and ELK stack

migration-validation-framework/collectors/disk_usage.py

@@ -0,0 +1,207 @@
+"""
+Disk Usage Data Collector
+
+Collects disk usage statistics including directory sizes,
+file system usage, and largest files information.
+"""
+
+import json
+import logging
+import subprocess
+from typing import Dict, Any, List
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class DiskUsageCollector:
+    """Collector for disk usage statistics."""
+
+    def __init__(self):
+        self.max_depth = 3
+        self.exclude_paths = [
+            "/proc",
+            "/sys",
+            "/dev",
+            "/run",
+            "/tmp",
+            "/var/log"
+        ]
+
+    def collect_disk_usage(self, system: str) -> Dict[str, Any]:
+        """Collect disk usage information from target system."""
+        logger.info(f"Collecting disk usage data from {system}")
+
+        try:
+            # Collect filesystem usage
+            filesystem_usage = self.collect_filesystem_usage(system)
+
+            # Collect directory sizes
+            directory_sizes = self.collect_directory_sizes(system)
+
+            # Collect largest files
+            largest_files = self.collect_largest_files(system)
+
+            return {
+                "filesystem_usage": filesystem_usage,
+                "directory_sizes": directory_sizes,
+                "largest_files": largest_files,
+                "timestamp": self.get_timestamp(system)
+            }
+
+        except Exception as e:
+            logger.error(f"Failed to collect disk usage from {system}: {e}")
+            raise
+
+    def collect_filesystem_usage(self, system: str) -> List[Dict[str, Any]]:
+        """Collect filesystem usage statistics."""
+        usage_stats = []
+
+        try:
+            # Run df command
+            result = subprocess.run(
+                ["ssh", system, "df -h --output=source,fstype,size,used,avail,pcent,target"],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(f"df command failed: {result.stderr}")
+
+            # Parse output
+            lines = result.stdout.strip().split('\n')
+            if len(lines) < 2:
+                return usage_stats
+
+            for line in lines[1:]:  # Skip header
+                parts = line.split()
+                if len(parts) >= 7:
+                    usage_stat = {
+                        "filesystem": parts[0],
+                        "type": parts[1],
+                        "size": parts[2],
+                        "used": parts[3],
+                        "available": parts[4],
+                        "use_percent": parts[5],
+                        "mountpoint": parts[6]
+                    }
+                    usage_stats.append(usage_stat)
+
+        except subprocess.TimeoutExpired:
+            logger.error(f"Timeout collecting filesystem usage from {system}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to collect filesystem usage from {system}: {e}")
+            raise
+
+        return usage_stats
+
+    def collect_directory_sizes(self, system: str) -> List[Dict[str, Any]]:
+        """Collect sizes of top-level directories."""
+        directory_sizes = []
+
+        try:
+            # Get top-level directories
+            dirs_to_check = ["/", "/home", "/var", "/usr", "/opt", "/etc"]
+
+            for directory in dirs_to_check:
+                if directory in self.exclude_paths:
+                    continue
+
+                try:
+                    # Run du command for directory size
+                    result = subprocess.run(
+                        ["ssh", system, f"du -sh {directory} 2>/dev/null"],
+                        capture_output=True,
+                        text=True,
+                        timeout=60
+                    )
+
+                    if result.returncode == 0:
+                        size, path = result.stdout.strip().split('\t', 1)
+                        directory_sizes.append({
+                            "path": path,
+                            "size": size
+                        })
+
+                except subprocess.TimeoutExpired:
+                    logger.warning(f"Timeout getting size for {directory} on {system}")
+                    continue
+                except Exception as e:
+                    logger.warning(f"Failed to get size for {directory} on {system}: {e}")
+                    continue
+
+        except Exception as e:
+            logger.error(f"Failed to collect directory sizes from {system}: {e}")
+            raise
+
+        return directory_sizes
+
+    def collect_largest_files(self, system: str) -> List[Dict[str, Any]]:
+        """Collect information about largest files in the system."""
+        largest_files = []
+
+        try:
+            # Find largest files (excluding certain paths)
+            exclude_expr = " ".join(f"-not -path '{path}/*'" for path in self.exclude_paths)
+
+            cmd = f"find / {exclude_expr} -type f -exec ls -lh {{}} \\; 2>/dev/null | sort -k5 -hr | head -20"
+
+            result = subprocess.run(
+                ["ssh", system, cmd],
+                capture_output=True,
+                text=True,
+                timeout=120
+            )
+
+            if result.returncode == 0:
+                for line in result.stdout.strip().split('\n'):
+                    if not line.strip():
+                        continue
+
+                    parts = line.split()
+                    if len(parts) >= 9:
+                        file_info = {
+                            "permissions": parts[0],
+                            "links": parts[1],
+                            "owner": parts[2],
+                            "group": parts[3],
+                            "size": parts[4],
+                            "month": parts[5],
+                            "day": parts[6],
+                            "time": parts[7],
+                            "path": " ".join(parts[8:])
+                        }
+                        largest_files.append(file_info)
+
+        except subprocess.TimeoutExpired:
+            logger.error(f"Timeout collecting largest files from {system}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to collect largest files from {system}: {e}")
+            raise
+
+        return largest_files
+
+    def get_timestamp(self, system: str) -> str:
+        """Get current timestamp from target system."""
+        try:
+            result = subprocess.run(
+                ["ssh", system, "date -Iseconds"],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+
+            if result.returncode == 0:
+                return result.stdout.strip()
+            else:
+                return "unknown"
+
+        except Exception:
+            return "unknown"
+
+def collect(system: str) -> Dict[str, Any]:
+    """Main collection function for disk usage data."""
+    collector = DiskUsageCollector()
+    return collector.collect_disk_usage(system)

migration-validation-framework/collectors/mounts.py

@@ -0,0 +1,173 @@
+"""
+Mounts Data Collector
+
+Collects filesystem mount information including mount points, devices,
+filesystem types, and usage statistics.
+"""
+
+import json
+import logging
+import subprocess
+from typing import Dict, Any, List
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+class MountsCollector:
+    """Collector for filesystem mount information."""
+
+    def __init__(self):
+        self.exclude_patterns = [
+            "/proc/*",
+            "/sys/*",
+            "/dev/*",
+            "/run/*"
+        ]
+
+    def collect_mounts(self, system: str) -> Dict[str, Any]:
+        """Collect mount information from target system."""
+        logger.info(f"Collecting mounts data from {system}")
+
+        try:
+            # Run mount command
+            result = subprocess.run(
+                ["ssh", system, "mount"],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(f"Mount command failed: {result.stderr}")
+
+            mounts = self.parse_mount_output(result.stdout)
+            filtered_mounts = self.filter_mounts(mounts)
+
+            # Get usage statistics
+            usage_stats = self.collect_usage_stats(system, filtered_mounts)
+
+            return {
+                "mounts": filtered_mounts,
+                "usage": usage_stats,
+                "timestamp": self.get_timestamp(system)
+            }
+
+        except subprocess.TimeoutExpired:
+            logger.error(f"Timeout collecting mounts from {system}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to collect mounts from {system}: {e}")
+            raise
+
+    def parse_mount_output(self, output: str) -> List[Dict[str, str]]:
+        """Parse mount command output."""
+        mounts = []
+
+        for line in output.strip().split('\n'):
+            if not line.strip():
+                continue
+
+            # Parse mount output format: device on mountpoint type fstype (options)
+            parts = line.split()
+            if len(parts) >= 6 and parts[1] == 'on' and parts[3] == 'type':
+                mount_info = {
+                    "device": parts[0],
+                    "mountpoint": parts[2],
+                    "fstype": parts[4],
+                    "options": parts[5].strip('()')
+                }
+                mounts.append(mount_info)
+
+        return mounts
+
+    def filter_mounts(self, mounts: List[Dict[str, str]]) -> List[Dict[str, str]]:
+        """Filter out unwanted mount points."""
+        filtered = []
+
+        for mount in mounts:
+            mountpoint = mount["mountpoint"]
+            if not any(Path(mountpoint).match(pattern.rstrip('*')) for pattern in self.exclude_patterns):
+                filtered.append(mount)
+
+        return filtered
+
+    def collect_usage_stats(self, system: str, mounts: List[Dict[str, str]]) -> Dict[str, Any]:
+        """Collect disk usage statistics for mount points."""
+        usage_stats = {}
+
+        for mount in mounts:
+            mountpoint = mount["mountpoint"]
+
+            try:
+                # Run df command for usage statistics
+                result = subprocess.run(
+                    ["ssh", system, f"df -BG {mountpoint}"],
+                    capture_output=True,
+                    text=True,
+                    timeout=15
+                )
+
+                if result.returncode == 0:
+                    usage_stats[mountpoint] = self.parse_df_output(result.stdout)
+
+            except subprocess.TimeoutExpired:
+                logger.warning(f"Timeout getting usage for {mountpoint} on {system}")
+                usage_stats[mountpoint] = {"error": "timeout"}
+            except Exception as e:
+                logger.warning(f"Failed to get usage for {mountpoint} on {system}: {e}")
+                usage_stats[mountpoint] = {"error": str(e)}
+
+        return usage_stats
+
+    def parse_df_output(self, output: str) -> Dict[str, Any]:
+        """Parse df command output."""
+        lines = output.strip().split('\n')
+        if len(lines) < 2:
+            return {"error": "invalid df output"}
+
+        # Parse header and data
+        header = lines[0].split()
+        data = lines[1].split()
+
+        if len(header) != len(data):
+            return {"error": "header/data mismatch"}
+
+        stats = {}
+        for i, field in enumerate(header):
+            if i < len(data):
+                if field in ['1G-blocks', 'Used', 'Available']:
+                    # Convert to GB
+                    value = data[i]
+                    if value.endswith('G'):
+                        stats[field.lower()] = float(value.rstrip('G'))
+                    else:
+                        stats[field.lower()] = float(value) / (1024**3)  # Assume bytes
+                elif field == 'Use%':
+                    stats['use_percent'] = int(data[i].rstrip('%'))
+                else:
+                    stats[field.lower()] = data[i]
+
+        return stats
+
+    def get_timestamp(self, system: str) -> str:
+        """Get current timestamp from target system."""
+        try:
+            result = subprocess.run(
+                ["ssh", system, "date -Iseconds"],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+
+            if result.returncode == 0:
+                return result.stdout.strip()
+            else:
+                return "unknown"
+
+        except Exception:
+            return "unknown"
+
+def collect(system: str) -> Dict[str, Any]:
+    """Main collection function for mounts data."""
+    collector = MountsCollector()
+    return collector.collect_mounts(system)

migration-validation-framework/collectors/services.py

@@ -0,0 +1,223 @@
+"""
+Services Data Collector
+
+Collects system service information including running services,
+their states, startup configuration, and dependencies.
+"""
+
+import json
+import logging
+import subprocess
+from typing import Dict, Any, List
+
+logger = logging.getLogger(__name__)
+
+class ServicesCollector:
+    """Collector for system service information."""
+
+    def __init__(self):
+        self.service_manager = "systemd"  # Default to systemd
+        self.include_disabled = False
+
+    def collect_services(self, system: str) -> Dict[str, Any]:
+        """Collect service information from target system."""
+        logger.info(f"Collecting services data from {system}")
+
+        try:
+            # Detect service manager
+            service_manager = self.detect_service_manager(system)
+
+            if service_manager == "systemd":
+                services = self.collect_systemd_services(system)
+            elif service_manager == "sysv":
+                services = self.collect_sysv_services(system)
+            else:
+                raise RuntimeError(f"Unsupported service manager: {service_manager}")
+
+            return {
+                "service_manager": service_manager,
+                "services": services,
+                "timestamp": self.get_timestamp(system)
+            }
+
+        except Exception as e:
+            logger.error(f"Failed to collect services from {system}: {e}")
+            raise
+
+    def detect_service_manager(self, system: str) -> str:
+        """Detect which service manager is running on the system."""
+        try:
+            # Check for systemd
+            result = subprocess.run(
+                ["ssh", system, "ps -p 1 -o comm="],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+
+            if result.returncode == 0:
+                if "systemd" in result.stdout.strip():
+                    return "systemd"
+                elif "init" in result.stdout.strip():
+                    return "sysv"
+
+            # Fallback check
+            result = subprocess.run(
+                ["ssh", system, "which systemctl"],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+
+            if result.returncode == 0:
+                return "systemd"
+
+            return "sysv"
+
+        except Exception:
+            return "unknown"
+
+    def collect_systemd_services(self, system: str) -> List[Dict[str, Any]]:
+        """Collect systemd service information."""
+        services = []
+
+        try:
+            # Get all services
+            result = subprocess.run(
+                ["ssh", system, "systemctl list-units --type=service --all --no-pager --no-legend"],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(f"systemctl list-units failed: {result.stderr}")
+
+            # Parse service list
+            for line in result.stdout.strip().split('\n'):
+                if not line.strip():
+                    continue
+
+                parts = line.split()
+                if len(parts) >= 4:
+                    service_name = parts[0]
+                    load_state = parts[1]
+                    active_state = parts[2]
+                    sub_state = parts[3]
+
+                    # Skip if disabled and not including disabled
+                    if not self.include_disabled and load_state == "not-found":
+                        continue
+
+                    # Get detailed service info
+                    service_info = self.get_systemd_service_details(system, service_name)
+
+                    services.append({
+                        "name": service_name,
+                        "load_state": load_state,
+                        "active_state": active_state,
+                        "sub_state": sub_state,
+                        **service_info
+                    })
+
+        except subprocess.TimeoutExpired:
+            logger.error(f"Timeout collecting systemd services from {system}")
+            raise
+        except Exception as e:
+            logger.error(f"Failed to collect systemd services from {system}: {e}")
+            raise
+
+        return services
+
+    def get_systemd_service_details(self, system: str, service_name: str) -> Dict[str, Any]:
+        """Get detailed information for a systemd service."""
+        details = {}
+
+        try:
+            # Get service status
+            result = subprocess.run(
+                ["ssh", system, f"systemctl show {service_name} --no-pager"],
+                capture_output=True,
+                text=True,
+                timeout=15
+            )
+
+            if result.returncode == 0:
+                for line in result.stdout.strip().split('\n'):
+                    if '=' in line:
+                        key, value = line.split('=', 1)
+                        details[key.lower()] = value
+
+        except Exception as e:
+            logger.warning(f"Failed to get details for {service_name}: {e}")
+
+        return details
+
+    def collect_sysv_services(self, system: str) -> List[Dict[str, Any]]:
+        """Collect SysV init service information."""
+        services = []
+
+        try:
+            # Get service list from /etc/init.d/
+            result = subprocess.run(
+                ["ssh", system, "ls -1 /etc/init.d/"],
+                capture_output=True,
+                text=True,
+                timeout=15
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(f"Failed to list init.d services: {result.stderr}")
+
+            for service_name in result.stdout.strip().split('\n'):
+                if not service_name.strip():
+                    continue
+
+                # Get service status
+                status_result = subprocess.run(
+                    ["ssh", system, f"/etc/init.d/{service_name} status"],
+                    capture_output=True,
+                    text=True,
+                    timeout=10
+                )
+
+                status = "unknown"
+                if status_result.returncode == 0:
+                    status = "running"
+                elif "not running" in status_result.stdout.lower():
+                    status = "stopped"
+
+                services.append({
+                    "name": service_name,
+                    "status": status,
+                    "type": "sysv"
+                })
+
+        except Exception as e:
+            logger.error(f"Failed to collect SysV services from {system}: {e}")
+            raise
+
+        return services
+
+    def get_timestamp(self, system: str) -> str:
+        """Get current timestamp from target system."""
+        try:
+            result = subprocess.run(
+                ["ssh", system, "date -Iseconds"],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+
+            if result.returncode == 0:
+                return result.stdout.strip()
+            else:
+                return "unknown"
+
+        except Exception:
+            return "unknown"
+
+def collect(system: str) -> Dict[str, Any]:
+    """Main collection function for services data."""
+    collector = ServicesCollector()
+    return collector.collect_services(system)