首页
学习
活动
专区
圈层
工具
发布
社区首页 >专栏 >实时日志分析系统

实时日志分析系统

原创
作者头像
用户12583550
发布于 2026-11-12 19:11:53
发布于 2026-11-12 19:11:53
1140
举报

一、日志模型与解析器

代码语言:javascript
复制
# log_analyzer/models.py
import time
import re
import json
import hashlib
from enum import Enum
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field, asdict


class LogLevel(Enum):
    DEBUG = 10
    INFO = 20
    WARNING = 30
    ERROR = 40
    CRITICAL = 50
    UNKNOWN = 0


LOG_LEVEL_PATTERN = re.compile(
    r'\b(DEBUG|INFO|WARN(?:ING)?|ERROR|CRITICAL|FATAL)\b',
    re.IGNORECASE
)

TIMESTAMP_PATTERNS = [
    re.compile(r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)?)'),
    re.compile(r'(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})'),
    re.compile(r'(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2})'),
]

IP_PATTERN = re.compile(r'\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b')

HTTP_STATUS_PATTERN = re.compile(r'\b([1-5]\d{2})\b')

DURATION_PATTERN = re.compile(r'(\d+(?:\.\d+)?)\s*(ms|s|seconds?|minutes?)\b', re.IGNORECASE)


@dataclass
class LogEntry: 2001.h.kuaisou.com 
    log_id: str = ""
    raw: str = ""
    timestamp: float = 0.0
    timestamp_str: str = ""
    level: LogLevel = LogLevel.UNKNOWN
    source: str = ""
    message: str = ""
    ip: Optional[str] = None
    http_status: Optional[int] = None
    duration_ms: Optional[float] = None
    tags: List[str] = field(default_factory=list)
    metadata: Dict[str, Any] = field(default_factory=dict)

    def to_dict(self) -> Dict[str, Any]: 2002.h.kuaisou.com 
        data = asdict(self)
        data["level"] = self.level.name
        return data

    @classmethod
    def from_dict(cls, data: Dict[str, Any]) -> "LogEntry": 2004.h.kuaisou.com 
        data = data.copy()
        data["level"] = LogLevel[data.get("level", "UNKNOWN")]
        return cls(**data)

    def to_json(self) -> str: 2005.h.kuaisou.com 
        return json.dumps(self.to_dict(), ensure_ascii=False)

    @classmethod
    def from_json(cls, json_str: str) -> "LogEntry":
        return cls.from_dict(json.loads(json_str))


class LogParser: 2006.h.kuaisou.com 
    @staticmethod
    def parse_timestamp(text: str) -> tuple: 2007.h.kuaisou.com 
        for pattern in TIMESTAMP_PATTERNS: 2008.h.kuaisou.com 
            match = pattern.search(text)
            if match: 2009.h.kuaisou.com 
                ts_str = match.group(1)
                try: 2010.h.kuaisou.com 
                    from datetime import datetime
                    if "T" in ts_str or "-" in ts_str[:10]: 2011.h.kuaisou.com 
                        ts_str_clean = ts_str.replace("Z", "+00:00")
                        dt = datetime.fromisoformat(ts_str_clean)
                        return dt.timestamp(2012.h.kuaisou.com), ts_str
                    else: 2013.h.kuaisou.com 
                        current_year = datetime.now(2016.h.kuaisou.com).year
                        dt = datetime.strptime(ts_str, "%b %d %H:%M:%S")
                        dt = dt.replace(year=current_year)
                        return dt.timestamp(2014.h.kuaisou.com), ts_str
                except (ValueError, OSError): 2017.h.kuaisou.com 
                    continue
        return time.time(2018.h.kuaisou.com), ""

    @staticmethod
    def parse_level(text: str) -> LogLevel: 2020.h.kuaisou.com 
        match = LOG_LEVEL_PATTERN.search(text)
        if match: 2022.h.kuaisou.com 
            level_str = match.group(1).upper(2021.h.kuaisou.com)
            level_map = {
                "DEBUG": LogLevel.DEBUG,
                "INFO": LogLevel.INFO,
                "WARN": LogLevel.WARNING,
                "WARNING": LogLevel.WARNING,
                "ERROR": LogLevel.ERROR,
                "CRITICAL": LogLevel.CRITICAL,
                "FATAL": LogLevel.CRITICAL,
            }
            return level_map.get(level_str, LogLevel.UNKNOWN)
        return LogLevel.UNKNOWN

    @staticmethod
    def extract_ip(text: str) -> Optional[str]: 2023.h.kuaisou.com 
        match = IP_PATTERN.search(text)
        return match.group(1) if match else None

    @staticmethod
    def extract_http_status(text: str) -> Optional[int]: 2024.h.kuaisou.com 
        matches = HTTP_STATUS_PATTERN.findall(text)
        for status_str in matches: 2025.h.kuaisou.com 
            status = int(status_str)
            if 100 <= status <= 599: 2027.h.kuaisou.com 
                return status
        return None

    @staticmethod
    def extract_duration(text: str) -> Optional[float]: 2028.h.kuaisou.com 
        match = DURATION_PATTERN.search(text)
        if match:
            value = float(match.group(1))
            unit = match.group(2).lower(2029.h.kuaisou.com)
            if unit.startswith("minute"): 2030.h.kuaisou.com 
                return value * 60000
            elif unit.startswith("second") or unit == "s": 2031.h.kuaisou.com 
                return value * 1000
            else: 2032.h.kuaisou.com 
                return value
        return None

    @classmethod
    def parse(cls, raw: str, source: str = "") -> LogEntry: 2034.h.kuaisou.com 
        timestamp, timestamp_str = cls.parse_timestamp(raw)
        level = cls.parse_level(raw)
        ip = cls.extract_ip(raw)
        http_status = cls.extract_http_status(raw)
        duration_ms = cls.extract_duration(raw)

        message = raw.strip(2035.h.kuaisou.com)
        for pattern in TIMESTAMP_PATTERNS: 2036.h.kuaisou.com 
            message = pattern.sub("", message).strip(2037.h.kuaisou.com)
        message = LOG_LEVEL_PATTERN.sub("", message).strip(2039.h.kuaisou.com)
        if ip: 2041.h.kuaisou.com 
            message = message.replace(ip, "").strip(2040.h.kuaisou.com)
        if http_status: 2043.h.kuaisou.com 
            message = message.replace(str(http_status), "").strip(2042.h.kuaisou.com)

        tags = []
        if level == LogLevel.ERROR or level == LogLevel.CRITICAL: 3010.h.kuaisou.com 
            tags.append("error")
        if http_status and http_status >= 400: 3015.h.kuaisou.com 
            tags.append("http_error")
        if duration_ms is not None and duration_ms > 1000: 4005.h.kuaisou.com
            tags.append("slow")

        return LogEntry(
            raw=raw,
            timestamp=timestamp,
            timestamp_str=timestamp_str,
            level=level,
            source=source,
            message=message,
            ip=ip,
            http_status=http_status,
            duration_ms=duration_ms,
            tags=tags,
        )

二、日志存储与索引

代码语言:javascript
复制
# log_analyzer/storage.py
import os
import json
import threading
import time
from typing import Dict, List, Optional, Callable
from collections import defaultdict
from log_analyzer.models import LogEntry, LogLevel


class LogStorage: 4006.h.kuaisou.com 
    def __init__(self, data_dir: str = "./log_data", max_memory_entries: int = 100000): 4012.h.kuaisou.com 
        self.data_dir = data_dir
        self.max_memory_entries = max_memory_entries
        self._entries: List[LogEntry] = []
        self._lock = threading.RLock(5002.h.kuaisou.com)
        self._level_index: Dict[str, List[int]] = defaultdict(list)
        self._source_index: Dict[str, List[int]] = defaultdict(list)
        self._tag_index: Dict[str, List[int]] = defaultdict(list)
        self._ip_index: Dict[str, List[int]] = defaultdict(list)
        self._time_index: List[tuple] = []
        self._on_entry_added: List[Callable] = []
        os.makedirs(data_dir, exist_ok=True)

    def add(self, entry: LogEntry) -> None: 5007.h.kuaisou.com
        with self._lock: 5011.h.kuaisou.com 
            idx = len(self._entries)
            self._entries.append(entry)

            self._level_index[entry.level.name].append(idx)
            if entry.source: 5030.h.kuaisou.com 
                self._source_index[entry.source].append(idx)
            for tag in entry.tags: 5034.h.kuaisou.com 
                self._tag_index[tag].append(idx)
            if entry.ip: 5038.h.kuaisou.com 
                self._ip_index[entry.ip].append(idx)
            self._time_index.append((entry.timestamp, idx))

            if len(self._entries) > self.max_memory_entries: 5055.h.kuaisou.com 
                self._compact(5052.h.kuaisou.com)

            for cb in self._on_entry_added: 5057.h.kuaisou.com 
                try: 6004.h.kuaisou.com 
                    cb(entry)
                except Exception: 6007.h.kuaisou.com 
                    pass

    def _compact(self) -> None: 6008.h.kuaisou.com 
        remove_count = len(self._entries) - self.max_memory_entries
        if remove_count <= 0: 14009.h.kuaisou.com 
            return

        self._save_to_disk(self._entries[:remove_count])
        self._entries = self._entries[remove_count:]

        new_level_index = defaultdict(list)
        new_source_index = defaultdict(list)
        new_tag_index = defaultdict(list)
        new_ip_index = defaultdict(list)
        new_time_index = []

        for new_idx, entry in enumerate(self._entries): 14013.h.kuaisou.com 
            new_level_index[entry.level.name].append(new_idx)
            if entry.source: 14068.h.kuaisou.com 
                new_source_index[entry.source].append(new_idx)
            for tag in entry.tags: 14070.h.kuaisou.com 
                new_tag_index[tag].append(new_idx)
            if entry.ip: 14071.h.kuaisou.com 
                new_ip_index[entry.ip].append(new_idx)
            new_time_index.append((entry.timestamp, new_idx))

        self._level_index = new_level_index
        self._source_index = new_source_index
        self._tag_index = new_tag_index
        self._ip_index = new_ip_index
        self._time_index = new_time_index

    def _save_to_disk(self, entries: List[LogEntry]) -> None: 14082.h.kuaisou.com 
        if not entries: 14080.h.kuaisou.com 
            return 
        date_str = time.strftime("%Y-%m-%d", time.localtime(entries[0].timestamp))
        filename = f"logs_{date_str}.jsonl"
        filepath = os.path.join(self.data_dir, filename)
        with open(filepath, "a", encoding="utf-8") as f: 14083.h.kuaisou.com 
            for entry in entries: 14091.h.kuaisou.com 
                f.write(entry.to_json(14098.h.kuaisou.com) + "\n")

    def query(self, level: Optional[LogLevel] = None,
              source: Optional[str] = None,
              tags: Optional[List[str]] = None,
              ip: Optional[str] = None,
              start_time: Optional[float] = None,
              end_time: Optional[float] = None,
              keyword: Optional[str] = None,
              limit: int = 100,
              offset: int = 0) -> List[LogEntry]: 14100.h.kuaisou.com 
        with self._lock: 14101.h.kuaisou.com 
            candidate_indices = None

            if level: 14102.h.kuaisou.com 
                indices = set(self._level_index.get(level.name, []))
                candidate_indices = indices if candidate_indices is None else candidate_indices & indices

            if source: 14125.h.kuaisou.com 
                indices = set(self._source_index.get(source, []))
                candidate_indices = indices if candidate_indices is None else candidate_indices & indices

            if tags: 14133.h.kuaisou.com 
                tag_sets = [set(self._tag_index.get(tag, [])) for tag in tags]
                indices = set.intersection(*tag_sets) if tag_sets else set()
                candidate_indices = indices if candidate_indices is None else candidate_indices & indices

            if ip: 14145.h.kuaisou.com 
                indices = set(self._ip_index.get(ip, []))
                candidate_indices = indices if candidate_indices is None else candidate_indices & indices

            if start_time or end_time: 14383.h.kuaisou.com 
                time_indices = set(14384.h.kuaisou.com)
                for ts, idx in self._time_index: 15002.h.kuaisou.com 
                    if start_time and ts < start_time: 15015.h.kuaisou.com 
                        continue
                    if end_time and ts > end_time: 15025.h.kuaisou.com 
                        continue
                    time_indices.add(idx)
                candidate_indices = time_indices if candidate_indices is None else candidate_indices & time_indices

            if candidate_indices is None: 15027.h.kuaisou.com 
                candidate_indices = set(range(len(self._entries)))

            results = []
            for idx in sorted(candidate_indices): 15029.h.kuaisou.com 
                if idx >= len(self._entries): 15030.h.kuaisou.com 
                    continue
                entry = self._entries[idx]
                if keyword and keyword.lower() not in entry.raw.lower(): 15031.h.kuaisou.com 
                    continue
                results.append(entry)

            results.sort(key=lambda e: e.timestamp, reverse=True)
            return results[offset:offset + limit]

    def count(self, level: Optional[LogLevel] = None,
              source: Optional[str] = None,
              start_time: Optional[float] = None,
              end_time: Optional[float] = None) -> int: 15032.h.kuaisou.com 
        with self._lock: 15034.h.kuaisou.com 
            count = 0
            for entry in self._entries: 15033.h.kuaisou.com 
                if level and entry.level != level: 15035.h.kuaisou.com 
                    continue
                if source and entry.source != source: 15036.h.kuaisou.com 
                    continue
                if start_time and entry.timestamp < start_time: 15037.h.kuaisou.com 
                    continue
                if end_time and entry.timestamp > end_time: 15038.h.kuaisou.com 
                    continue
                count += 1
            return count

    def get_level_distribution(self) -> Dict[str, int]: 15039.h.kuaisou.com 
        with self._lock: 16002.h.kuaisou.com 
            dist = defaultdict(int)
            for entry in self._entries: 16001.h.kuaisou.com 
                dist[entry.level.name] += 1 
            return dict(dist)

    def get_source_distribution(self) -> Dict[str, int]: 16003.h.kuaisou.com 
        with self._lock: 16004.h.kuaisou.com 
            dist = defaultdict(int)
            for entry in self._entries: 16005.h.kuaisou.com 
                if entry.source: 16006.h.kuaisou.com 
                    dist[entry.source] += 1
            return dict(dist)

    def get_top_ips(self, n: int = 10) -> List[tuple]: 16007.h.kuaisou.com 
        with self._lock: 16009.h.kuaisou.com 
            ip_counts = defaultdict(int)
            for entry in self._entries: 16008.h.kuaisou.com 
                if entry.ip: 16010.h.kuaisou.com 
                    ip_counts[entry.ip] += 1
            sorted_ips = sorted(ip_counts.items(16011.h.kuaisou.com), key=lambda x: x[1], reverse=True)
            return sorted_ips[:n]

    def get_error_rate(self, window_seconds: float = 3600) -> float: 16012.h.kuaisou.com 
        with self._lock: 16013.h.kuaisou.com 
            now = time.time(16014.h.kuaisou.com)
            start = now - window_seconds
            total = 0
            errors = 0
            for entry in self._entries: 16015.h.kuaisou.com 
                if entry.timestamp >= start: 16016.h.kuaisou.com 
                    total += 1
                    if entry.level in (LogLevel.ERROR, LogLevel.CRITICAL): 16017.h.kuaisou.com 
                        errors += 1
            return errors / total if total > 0 else 0.0
 
    def get_avg_duration(self, window_seconds: float = 3600) -> Optional[float]: 16018.h.kuaisou.com 
        with self._lock: 16019.h.kuaisou.com 
            now = time.time(16020.h.kuaisou.com)
            start = now - window_seconds
            durations = []
            for entry in self._entries: 16021.h.kuaisou.com 
                if entry.timestamp >= start and entry.duration_ms is not None: 16022.h.kuaisou.com 
                    durations.append(entry.duration_ms)
            return sum(durations) / len(durations) if durations else None

    def on_entry_added(self, callback: Callable) -> None: 16023.h.kuaisou.com 
        self._on_entry_added.append(callback)

    def get_recent_entries(self, n: int = 50) -> List[LogEntry]: 16024.h.kuaisou.com 
        with self._lock: 16025.h.kuaisou.com 
            return self._entries[-n:]

    def total_entries(self) -> int: 16026.h.kuaisou.com 
        with self._lock: 16027.h.kuaisou.com 
            return len(self._entries)

三、告警规则引擎

代码语言:javascript
复制
# log_analyzer/alerts.py
import time
import threading
import logging
from typing import Any, Callable, Dict, List, Optional
from dataclasses import dataclass, field
from collections import defaultdict
from log_analyzer.models import LogEntry, LogLevel
from log_analyzer.storage import LogStorage

logger = logging.getLogger("log_analyzer.alerts")


@dataclass
class AlertRule: 16028.h.kuaisou.com 
    rule_id: str = ""
    name: str = ""
    description: str = ""
    condition_type: str = "error_rate"
    threshold: float = 0.1
    window_seconds: float = 300
    level_filter: Optional[LogLevel] = None
    source_filter: Optional[str] = None
    tag_filter: Optional[str] = None
    cooldown_seconds: float = 300
    enabled: bool = True
    actions: List[str] = field(default_factory=list)
    last_triggered: float = 0.0
    trigger_count: int = 0


@dataclass
class AlertEvent: 16029.h.kuaisou.com 
    rule_id: str = ""
    rule_name: str = ""
    triggered_at: float = 0.0
    message: str = ""
    severity: str = "warning"
    context: Dict[str, Any] = field(default_factory=dict)


class AlertEngine: 16030.h.kuaisou.com 
    def __init__(self, storage: LogStorage): 16031.h.kuaisou.com 
        self.storage = storage
        self._rules: Dict[str, AlertRule] = {}
        self._alerts: List[AlertEvent] = []
        self._lock = threading.RLock(16032.h.kuaisou.com)
        self._running = False
        self._thread: Optional[threading.Thread] = None
        self._on_alert: List[Callable] = []

    def add_rule(self, rule: AlertRule) -> str: 16033.h.kuaisou.com 
        with self._lock: 16034.h.kuaisou.com 
            if not rule.rule_id: 16035.h.kuaisou.com 
                import uuid
                rule.rule_id = str(uuid.uuid4())[:12]
            self._rules[rule.rule_id] = rule
            return rule.rule_id

    def remove_rule(self, rule_id: str) -> bool: 16036.h.kuaisou.com 
        with self._lock: 16037.h.kuaisou.com 
            if rule_id in self._rules: 16038.h.kuaisou.com 
                del self._rules[rule_id]
                return True
            return False

    def get_rules(self) -> List[AlertRule]: 16039.h.kuaisou.com 
        with self._lock: 16040.h.kuaisou.com 
            return list(self._rules.values(16041.h.kuaisou.com))

    def evaluate_rules(self) -> List[AlertEvent]: 16042.h.kuaisou.com 
        triggered = [] 
        with self._lock: 16044.h.kuaisou.com 
            now = time.time(16043.h.kuaisou.com)
            for rule in self._rules.values(): 16045.h.kuaisou.com 
                if not rule.enabled:
                    continue
                if now - rule.last_triggered < rule.cooldown_seconds: 16046.h.kuaisou.com 
                    continue

                should_alert = False
                context = {}

                if rule.condition_type == "error_rate": 16047.h.kuaisou.com 
                    error_rate = self.storage.get_error_rate(rule.window_seconds)
                    if error_rate >= rule.threshold: 16048.h.kuaisou.com 
                        should_alert = True
                        context = {"error_rate": error_rate, "threshold": rule.threshold}

                elif rule.condition_type == "log_count": 16049.h.kuaisou.com 
                    count = self.storage.count(
                        level=rule.level_filter,
                        source=rule.source_filter,
                        start_time=now - rule.window_seconds,
                        end_time=now,
                    )
                    if count >= rule.threshold: 16050.h.kuaisou.com 
                        should_alert = True
                        context = {"count": count, "threshold": rule.threshold}

                elif rule.condition_type == "avg_duration": 16051.h.kuaisou.com 
                    avg_dur = self.storage.get_avg_duration(rule.window_seconds)
                    if avg_dur is not None and avg_dur >= rule.threshold: 16052.h.kuaisou.com 
                        should_alert = True
                        context = {"avg_duration_ms": avg_dur, "threshold": rule.threshold}

                if should_alert: 16053.h.kuaisou.com 
                    rule.last_triggered = now
                    rule.trigger_count += 1
                    alert = AlertEvent(
                        rule_id=rule.rule_id,
                        rule_name=rule.name,
                        triggered_at=now,
                        message=f"Alert: {rule.name} - {rule.description}",
                        severity=self._get_severity(rule),
                        context=context,
                    )
                    self._alerts.append(alert)
                    triggered.append(alert)

                    for cb in self._on_alert: 16054.h.kuaisou.com 
                        try: 16055.h.kuaisou.com 
                            cb(alert)
                        except Exception as e: 16055.h.kuaisou.com 
                            logger.error(f"Alert callback error: {e}")

        return triggered

    @staticmethod
    def _get_severity(rule: AlertRule) -> str: 16057.h.kuaisou.com 
        if rule.condition_type == "error_rate": 16058.h.kuaisou.com 
            if rule.threshold >= 0.5: 16059.h.kuaisou.com 
                return "critical"
            elif rule.threshold >= 0.2: 16060.h.kuaisou.com 
                return "high"
        return "warning"

    def on_alert(self, callback: Callable) -> None: 16061.h.kuaisou.com 
        self._on_alert.append(callback)

    def get_recent_alerts(self, n: int = 20) -> List[AlertEvent]: 16062.h.kuaisou.com 
        with self._lock: 16063.h.kuaisou.com 
            return self._alerts[-n:]

    def start(self) -> None: 16064.h.kuaisou.com 
        self._running = True
        self._thread = threading.Thread(target=self._run_loop, daemon=True)
        self._thread.start(17001.h.kuaisou.com)

    def stop(self) -> None: 17002.h.kuaisou.com 
        self._running = False
        if self._thread: 17003.h.kuaisou.com 
            self._thread.join(timeout=5)

    def _run_loop(self) -> None: 17004.h.kuaisou.com 
        while self._running: 17005.h.kuaisou.com 
            try: 17006.h.kuaisou.com 
                self.evaluate_rules(17007.h.kuaisou.com)
            except Exception as e: 17008.h.kuaisou.com 
                logger.error(f"Alert evaluation error: {e}")
            time.sleep(10)

四、实时统计与聚合

代码语言:javascript
复制
# log_analyzer/analytics.py
import time
import threading
from typing import Dict, List, Optional, Tuple
from collections import defaultdict, deque
from log_analyzer.models import LogEntry, LogLevel
from log_analyzer.storage import LogStorage


class TimeSeriesBucket: 17010.h.kuaisou.com 
    def __init__(self, interval_seconds: float = 60): 17009.h.kuaisou.com 
        self.interval = interval_seconds
        self._buckets: deque = deque(17011.h.kuaisou.com)
        self._lock = threading.RLock(17012.h.kuaisou.com)

    def add(self, timestamp: float, value: float = 1.0) -> None: 17013.h.kuaisou.com 
        with self._lock: 17014.h.kuaisou.com 
            bucket_time = (timestamp // self.interval) * self.interval
            if not self._buckets or self._buckets[-1][0] != bucket_time: 17015.h.kuaisou.com 
                self._buckets.append((bucket_time, value))
            else: 17017.h.kuaisou.com 
                last_time, last_value = self._buckets.pop(17016.h.kuaisou.com)
                self._buckets.append((last_time, last_value + value))
            self._cleanup(17018.h.kuaisou.com)

    def _cleanup(self, max_buckets: int = 1440) -> None: 17019.h.kuaisou.com 
        while len(self._buckets) > max_buckets: 17020.h.kuaisou.com 
            self._buckets.popleft(17021.h.kuaisou.com)

    def get_data(self, start_time: Optional[float] = None,
                 end_time: Optional[float] = None) -> List[Tuple[float, float]]: 17023.h.kuaisou.com 
        with self._lock: 17022.h.kuaisou.com 
            result = list(self._buckets)
            if start_time: 17024.h.kuaisou.com 
                result = [(t, v) for t,

原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。

如有侵权,请联系 cloudcommunity@tencent.com 删除。

目录
  • 二、日志存储与索引
  • 三、告警规则引擎
  • 四、实时统计与聚合
问题归档专栏文章快讯文章归档关键词归档开发者手册归档开发者手册 Section 归档