一、日志模型与解析器
# log_analyzer/models.py
import time
import re
import json
import hashlib
from enum import Enum
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field, asdict
class LogLevel(Enum):
DEBUG = 10
INFO = 20
WARNING = 30
ERROR = 40
CRITICAL = 50
UNKNOWN = 0
LOG_LEVEL_PATTERN = re.compile(
r'\b(DEBUG|INFO|WARN(?:ING)?|ERROR|CRITICAL|FATAL)\b',
re.IGNORECASE
)
TIMESTAMP_PATTERNS = [
re.compile(r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)?)'),
re.compile(r'(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})'),
re.compile(r'(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2})'),
]
IP_PATTERN = re.compile(r'\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b')
HTTP_STATUS_PATTERN = re.compile(r'\b([1-5]\d{2})\b')
DURATION_PATTERN = re.compile(r'(\d+(?:\.\d+)?)\s*(ms|s|seconds?|minutes?)\b', re.IGNORECASE)
@dataclass
class LogEntry: 2001.h.kuaisou.com
log_id: str = ""
raw: str = ""
timestamp: float = 0.0
timestamp_str: str = ""
level: LogLevel = LogLevel.UNKNOWN
source: str = ""
message: str = ""
ip: Optional[str] = None
http_status: Optional[int] = None
duration_ms: Optional[float] = None
tags: List[str] = field(default_factory=list)
metadata: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]: 2002.h.kuaisou.com
data = asdict(self)
data["level"] = self.level.name
return data
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "LogEntry": 2004.h.kuaisou.com
data = data.copy()
data["level"] = LogLevel[data.get("level", "UNKNOWN")]
return cls(**data)
def to_json(self) -> str: 2005.h.kuaisou.com
return json.dumps(self.to_dict(), ensure_ascii=False)
@classmethod
def from_json(cls, json_str: str) -> "LogEntry":
return cls.from_dict(json.loads(json_str))
class LogParser: 2006.h.kuaisou.com
@staticmethod
def parse_timestamp(text: str) -> tuple: 2007.h.kuaisou.com
for pattern in TIMESTAMP_PATTERNS: 2008.h.kuaisou.com
match = pattern.search(text)
if match: 2009.h.kuaisou.com
ts_str = match.group(1)
try: 2010.h.kuaisou.com
from datetime import datetime
if "T" in ts_str or "-" in ts_str[:10]: 2011.h.kuaisou.com
ts_str_clean = ts_str.replace("Z", "+00:00")
dt = datetime.fromisoformat(ts_str_clean)
return dt.timestamp(2012.h.kuaisou.com), ts_str
else: 2013.h.kuaisou.com
current_year = datetime.now(2016.h.kuaisou.com).year
dt = datetime.strptime(ts_str, "%b %d %H:%M:%S")
dt = dt.replace(year=current_year)
return dt.timestamp(2014.h.kuaisou.com), ts_str
except (ValueError, OSError): 2017.h.kuaisou.com
continue
return time.time(2018.h.kuaisou.com), ""
@staticmethod
def parse_level(text: str) -> LogLevel: 2020.h.kuaisou.com
match = LOG_LEVEL_PATTERN.search(text)
if match: 2022.h.kuaisou.com
level_str = match.group(1).upper(2021.h.kuaisou.com)
level_map = {
"DEBUG": LogLevel.DEBUG,
"INFO": LogLevel.INFO,
"WARN": LogLevel.WARNING,
"WARNING": LogLevel.WARNING,
"ERROR": LogLevel.ERROR,
"CRITICAL": LogLevel.CRITICAL,
"FATAL": LogLevel.CRITICAL,
}
return level_map.get(level_str, LogLevel.UNKNOWN)
return LogLevel.UNKNOWN
@staticmethod
def extract_ip(text: str) -> Optional[str]: 2023.h.kuaisou.com
match = IP_PATTERN.search(text)
return match.group(1) if match else None
@staticmethod
def extract_http_status(text: str) -> Optional[int]: 2024.h.kuaisou.com
matches = HTTP_STATUS_PATTERN.findall(text)
for status_str in matches: 2025.h.kuaisou.com
status = int(status_str)
if 100 <= status <= 599: 2027.h.kuaisou.com
return status
return None
@staticmethod
def extract_duration(text: str) -> Optional[float]: 2028.h.kuaisou.com
match = DURATION_PATTERN.search(text)
if match:
value = float(match.group(1))
unit = match.group(2).lower(2029.h.kuaisou.com)
if unit.startswith("minute"): 2030.h.kuaisou.com
return value * 60000
elif unit.startswith("second") or unit == "s": 2031.h.kuaisou.com
return value * 1000
else: 2032.h.kuaisou.com
return value
return None
@classmethod
def parse(cls, raw: str, source: str = "") -> LogEntry: 2034.h.kuaisou.com
timestamp, timestamp_str = cls.parse_timestamp(raw)
level = cls.parse_level(raw)
ip = cls.extract_ip(raw)
http_status = cls.extract_http_status(raw)
duration_ms = cls.extract_duration(raw)
message = raw.strip(2035.h.kuaisou.com)
for pattern in TIMESTAMP_PATTERNS: 2036.h.kuaisou.com
message = pattern.sub("", message).strip(2037.h.kuaisou.com)
message = LOG_LEVEL_PATTERN.sub("", message).strip(2039.h.kuaisou.com)
if ip: 2041.h.kuaisou.com
message = message.replace(ip, "").strip(2040.h.kuaisou.com)
if http_status: 2043.h.kuaisou.com
message = message.replace(str(http_status), "").strip(2042.h.kuaisou.com)
tags = []
if level == LogLevel.ERROR or level == LogLevel.CRITICAL: 3010.h.kuaisou.com
tags.append("error")
if http_status and http_status >= 400: 3015.h.kuaisou.com
tags.append("http_error")
if duration_ms is not None and duration_ms > 1000: 4005.h.kuaisou.com
tags.append("slow")
return LogEntry(
raw=raw,
timestamp=timestamp,
timestamp_str=timestamp_str,
level=level,
source=source,
message=message,
ip=ip,
http_status=http_status,
duration_ms=duration_ms,
tags=tags,
)# log_analyzer/storage.py
import os
import json
import threading
import time
from typing import Dict, List, Optional, Callable
from collections import defaultdict
from log_analyzer.models import LogEntry, LogLevel
class LogStorage: 4006.h.kuaisou.com
def __init__(self, data_dir: str = "./log_data", max_memory_entries: int = 100000): 4012.h.kuaisou.com
self.data_dir = data_dir
self.max_memory_entries = max_memory_entries
self._entries: List[LogEntry] = []
self._lock = threading.RLock(5002.h.kuaisou.com)
self._level_index: Dict[str, List[int]] = defaultdict(list)
self._source_index: Dict[str, List[int]] = defaultdict(list)
self._tag_index: Dict[str, List[int]] = defaultdict(list)
self._ip_index: Dict[str, List[int]] = defaultdict(list)
self._time_index: List[tuple] = []
self._on_entry_added: List[Callable] = []
os.makedirs(data_dir, exist_ok=True)
def add(self, entry: LogEntry) -> None: 5007.h.kuaisou.com
with self._lock: 5011.h.kuaisou.com
idx = len(self._entries)
self._entries.append(entry)
self._level_index[entry.level.name].append(idx)
if entry.source: 5030.h.kuaisou.com
self._source_index[entry.source].append(idx)
for tag in entry.tags: 5034.h.kuaisou.com
self._tag_index[tag].append(idx)
if entry.ip: 5038.h.kuaisou.com
self._ip_index[entry.ip].append(idx)
self._time_index.append((entry.timestamp, idx))
if len(self._entries) > self.max_memory_entries: 5055.h.kuaisou.com
self._compact(5052.h.kuaisou.com)
for cb in self._on_entry_added: 5057.h.kuaisou.com
try: 6004.h.kuaisou.com
cb(entry)
except Exception: 6007.h.kuaisou.com
pass
def _compact(self) -> None: 6008.h.kuaisou.com
remove_count = len(self._entries) - self.max_memory_entries
if remove_count <= 0: 14009.h.kuaisou.com
return
self._save_to_disk(self._entries[:remove_count])
self._entries = self._entries[remove_count:]
new_level_index = defaultdict(list)
new_source_index = defaultdict(list)
new_tag_index = defaultdict(list)
new_ip_index = defaultdict(list)
new_time_index = []
for new_idx, entry in enumerate(self._entries): 14013.h.kuaisou.com
new_level_index[entry.level.name].append(new_idx)
if entry.source: 14068.h.kuaisou.com
new_source_index[entry.source].append(new_idx)
for tag in entry.tags: 14070.h.kuaisou.com
new_tag_index[tag].append(new_idx)
if entry.ip: 14071.h.kuaisou.com
new_ip_index[entry.ip].append(new_idx)
new_time_index.append((entry.timestamp, new_idx))
self._level_index = new_level_index
self._source_index = new_source_index
self._tag_index = new_tag_index
self._ip_index = new_ip_index
self._time_index = new_time_index
def _save_to_disk(self, entries: List[LogEntry]) -> None: 14082.h.kuaisou.com
if not entries: 14080.h.kuaisou.com
return
date_str = time.strftime("%Y-%m-%d", time.localtime(entries[0].timestamp))
filename = f"logs_{date_str}.jsonl"
filepath = os.path.join(self.data_dir, filename)
with open(filepath, "a", encoding="utf-8") as f: 14083.h.kuaisou.com
for entry in entries: 14091.h.kuaisou.com
f.write(entry.to_json(14098.h.kuaisou.com) + "\n")
def query(self, level: Optional[LogLevel] = None,
source: Optional[str] = None,
tags: Optional[List[str]] = None,
ip: Optional[str] = None,
start_time: Optional[float] = None,
end_time: Optional[float] = None,
keyword: Optional[str] = None,
limit: int = 100,
offset: int = 0) -> List[LogEntry]: 14100.h.kuaisou.com
with self._lock: 14101.h.kuaisou.com
candidate_indices = None
if level: 14102.h.kuaisou.com
indices = set(self._level_index.get(level.name, []))
candidate_indices = indices if candidate_indices is None else candidate_indices & indices
if source: 14125.h.kuaisou.com
indices = set(self._source_index.get(source, []))
candidate_indices = indices if candidate_indices is None else candidate_indices & indices
if tags: 14133.h.kuaisou.com
tag_sets = [set(self._tag_index.get(tag, [])) for tag in tags]
indices = set.intersection(*tag_sets) if tag_sets else set()
candidate_indices = indices if candidate_indices is None else candidate_indices & indices
if ip: 14145.h.kuaisou.com
indices = set(self._ip_index.get(ip, []))
candidate_indices = indices if candidate_indices is None else candidate_indices & indices
if start_time or end_time: 14383.h.kuaisou.com
time_indices = set(14384.h.kuaisou.com)
for ts, idx in self._time_index: 15002.h.kuaisou.com
if start_time and ts < start_time: 15015.h.kuaisou.com
continue
if end_time and ts > end_time: 15025.h.kuaisou.com
continue
time_indices.add(idx)
candidate_indices = time_indices if candidate_indices is None else candidate_indices & time_indices
if candidate_indices is None: 15027.h.kuaisou.com
candidate_indices = set(range(len(self._entries)))
results = []
for idx in sorted(candidate_indices): 15029.h.kuaisou.com
if idx >= len(self._entries): 15030.h.kuaisou.com
continue
entry = self._entries[idx]
if keyword and keyword.lower() not in entry.raw.lower(): 15031.h.kuaisou.com
continue
results.append(entry)
results.sort(key=lambda e: e.timestamp, reverse=True)
return results[offset:offset + limit]
def count(self, level: Optional[LogLevel] = None,
source: Optional[str] = None,
start_time: Optional[float] = None,
end_time: Optional[float] = None) -> int: 15032.h.kuaisou.com
with self._lock: 15034.h.kuaisou.com
count = 0
for entry in self._entries: 15033.h.kuaisou.com
if level and entry.level != level: 15035.h.kuaisou.com
continue
if source and entry.source != source: 15036.h.kuaisou.com
continue
if start_time and entry.timestamp < start_time: 15037.h.kuaisou.com
continue
if end_time and entry.timestamp > end_time: 15038.h.kuaisou.com
continue
count += 1
return count
def get_level_distribution(self) -> Dict[str, int]: 15039.h.kuaisou.com
with self._lock: 16002.h.kuaisou.com
dist = defaultdict(int)
for entry in self._entries: 16001.h.kuaisou.com
dist[entry.level.name] += 1
return dict(dist)
def get_source_distribution(self) -> Dict[str, int]: 16003.h.kuaisou.com
with self._lock: 16004.h.kuaisou.com
dist = defaultdict(int)
for entry in self._entries: 16005.h.kuaisou.com
if entry.source: 16006.h.kuaisou.com
dist[entry.source] += 1
return dict(dist)
def get_top_ips(self, n: int = 10) -> List[tuple]: 16007.h.kuaisou.com
with self._lock: 16009.h.kuaisou.com
ip_counts = defaultdict(int)
for entry in self._entries: 16008.h.kuaisou.com
if entry.ip: 16010.h.kuaisou.com
ip_counts[entry.ip] += 1
sorted_ips = sorted(ip_counts.items(16011.h.kuaisou.com), key=lambda x: x[1], reverse=True)
return sorted_ips[:n]
def get_error_rate(self, window_seconds: float = 3600) -> float: 16012.h.kuaisou.com
with self._lock: 16013.h.kuaisou.com
now = time.time(16014.h.kuaisou.com)
start = now - window_seconds
total = 0
errors = 0
for entry in self._entries: 16015.h.kuaisou.com
if entry.timestamp >= start: 16016.h.kuaisou.com
total += 1
if entry.level in (LogLevel.ERROR, LogLevel.CRITICAL): 16017.h.kuaisou.com
errors += 1
return errors / total if total > 0 else 0.0
def get_avg_duration(self, window_seconds: float = 3600) -> Optional[float]: 16018.h.kuaisou.com
with self._lock: 16019.h.kuaisou.com
now = time.time(16020.h.kuaisou.com)
start = now - window_seconds
durations = []
for entry in self._entries: 16021.h.kuaisou.com
if entry.timestamp >= start and entry.duration_ms is not None: 16022.h.kuaisou.com
durations.append(entry.duration_ms)
return sum(durations) / len(durations) if durations else None
def on_entry_added(self, callback: Callable) -> None: 16023.h.kuaisou.com
self._on_entry_added.append(callback)
def get_recent_entries(self, n: int = 50) -> List[LogEntry]: 16024.h.kuaisou.com
with self._lock: 16025.h.kuaisou.com
return self._entries[-n:]
def total_entries(self) -> int: 16026.h.kuaisou.com
with self._lock: 16027.h.kuaisou.com
return len(self._entries)# log_analyzer/alerts.py
import time
import threading
import logging
from typing import Any, Callable, Dict, List, Optional
from dataclasses import dataclass, field
from collections import defaultdict
from log_analyzer.models import LogEntry, LogLevel
from log_analyzer.storage import LogStorage
logger = logging.getLogger("log_analyzer.alerts")
@dataclass
class AlertRule: 16028.h.kuaisou.com
rule_id: str = ""
name: str = ""
description: str = ""
condition_type: str = "error_rate"
threshold: float = 0.1
window_seconds: float = 300
level_filter: Optional[LogLevel] = None
source_filter: Optional[str] = None
tag_filter: Optional[str] = None
cooldown_seconds: float = 300
enabled: bool = True
actions: List[str] = field(default_factory=list)
last_triggered: float = 0.0
trigger_count: int = 0
@dataclass
class AlertEvent: 16029.h.kuaisou.com
rule_id: str = ""
rule_name: str = ""
triggered_at: float = 0.0
message: str = ""
severity: str = "warning"
context: Dict[str, Any] = field(default_factory=dict)
class AlertEngine: 16030.h.kuaisou.com
def __init__(self, storage: LogStorage): 16031.h.kuaisou.com
self.storage = storage
self._rules: Dict[str, AlertRule] = {}
self._alerts: List[AlertEvent] = []
self._lock = threading.RLock(16032.h.kuaisou.com)
self._running = False
self._thread: Optional[threading.Thread] = None
self._on_alert: List[Callable] = []
def add_rule(self, rule: AlertRule) -> str: 16033.h.kuaisou.com
with self._lock: 16034.h.kuaisou.com
if not rule.rule_id: 16035.h.kuaisou.com
import uuid
rule.rule_id = str(uuid.uuid4())[:12]
self._rules[rule.rule_id] = rule
return rule.rule_id
def remove_rule(self, rule_id: str) -> bool: 16036.h.kuaisou.com
with self._lock: 16037.h.kuaisou.com
if rule_id in self._rules: 16038.h.kuaisou.com
del self._rules[rule_id]
return True
return False
def get_rules(self) -> List[AlertRule]: 16039.h.kuaisou.com
with self._lock: 16040.h.kuaisou.com
return list(self._rules.values(16041.h.kuaisou.com))
def evaluate_rules(self) -> List[AlertEvent]: 16042.h.kuaisou.com
triggered = []
with self._lock: 16044.h.kuaisou.com
now = time.time(16043.h.kuaisou.com)
for rule in self._rules.values(): 16045.h.kuaisou.com
if not rule.enabled:
continue
if now - rule.last_triggered < rule.cooldown_seconds: 16046.h.kuaisou.com
continue
should_alert = False
context = {}
if rule.condition_type == "error_rate": 16047.h.kuaisou.com
error_rate = self.storage.get_error_rate(rule.window_seconds)
if error_rate >= rule.threshold: 16048.h.kuaisou.com
should_alert = True
context = {"error_rate": error_rate, "threshold": rule.threshold}
elif rule.condition_type == "log_count": 16049.h.kuaisou.com
count = self.storage.count(
level=rule.level_filter,
source=rule.source_filter,
start_time=now - rule.window_seconds,
end_time=now,
)
if count >= rule.threshold: 16050.h.kuaisou.com
should_alert = True
context = {"count": count, "threshold": rule.threshold}
elif rule.condition_type == "avg_duration": 16051.h.kuaisou.com
avg_dur = self.storage.get_avg_duration(rule.window_seconds)
if avg_dur is not None and avg_dur >= rule.threshold: 16052.h.kuaisou.com
should_alert = True
context = {"avg_duration_ms": avg_dur, "threshold": rule.threshold}
if should_alert: 16053.h.kuaisou.com
rule.last_triggered = now
rule.trigger_count += 1
alert = AlertEvent(
rule_id=rule.rule_id,
rule_name=rule.name,
triggered_at=now,
message=f"Alert: {rule.name} - {rule.description}",
severity=self._get_severity(rule),
context=context,
)
self._alerts.append(alert)
triggered.append(alert)
for cb in self._on_alert: 16054.h.kuaisou.com
try: 16055.h.kuaisou.com
cb(alert)
except Exception as e: 16055.h.kuaisou.com
logger.error(f"Alert callback error: {e}")
return triggered
@staticmethod
def _get_severity(rule: AlertRule) -> str: 16057.h.kuaisou.com
if rule.condition_type == "error_rate": 16058.h.kuaisou.com
if rule.threshold >= 0.5: 16059.h.kuaisou.com
return "critical"
elif rule.threshold >= 0.2: 16060.h.kuaisou.com
return "high"
return "warning"
def on_alert(self, callback: Callable) -> None: 16061.h.kuaisou.com
self._on_alert.append(callback)
def get_recent_alerts(self, n: int = 20) -> List[AlertEvent]: 16062.h.kuaisou.com
with self._lock: 16063.h.kuaisou.com
return self._alerts[-n:]
def start(self) -> None: 16064.h.kuaisou.com
self._running = True
self._thread = threading.Thread(target=self._run_loop, daemon=True)
self._thread.start(17001.h.kuaisou.com)
def stop(self) -> None: 17002.h.kuaisou.com
self._running = False
if self._thread: 17003.h.kuaisou.com
self._thread.join(timeout=5)
def _run_loop(self) -> None: 17004.h.kuaisou.com
while self._running: 17005.h.kuaisou.com
try: 17006.h.kuaisou.com
self.evaluate_rules(17007.h.kuaisou.com)
except Exception as e: 17008.h.kuaisou.com
logger.error(f"Alert evaluation error: {e}")
time.sleep(10)# log_analyzer/analytics.py
import time
import threading
from typing import Dict, List, Optional, Tuple
from collections import defaultdict, deque
from log_analyzer.models import LogEntry, LogLevel
from log_analyzer.storage import LogStorage
class TimeSeriesBucket: 17010.h.kuaisou.com
def __init__(self, interval_seconds: float = 60): 17009.h.kuaisou.com
self.interval = interval_seconds
self._buckets: deque = deque(17011.h.kuaisou.com)
self._lock = threading.RLock(17012.h.kuaisou.com)
def add(self, timestamp: float, value: float = 1.0) -> None: 17013.h.kuaisou.com
with self._lock: 17014.h.kuaisou.com
bucket_time = (timestamp // self.interval) * self.interval
if not self._buckets or self._buckets[-1][0] != bucket_time: 17015.h.kuaisou.com
self._buckets.append((bucket_time, value))
else: 17017.h.kuaisou.com
last_time, last_value = self._buckets.pop(17016.h.kuaisou.com)
self._buckets.append((last_time, last_value + value))
self._cleanup(17018.h.kuaisou.com)
def _cleanup(self, max_buckets: int = 1440) -> None: 17019.h.kuaisou.com
while len(self._buckets) > max_buckets: 17020.h.kuaisou.com
self._buckets.popleft(17021.h.kuaisou.com)
def get_data(self, start_time: Optional[float] = None,
end_time: Optional[float] = None) -> List[Tuple[float, float]]: 17023.h.kuaisou.com
with self._lock: 17022.h.kuaisou.com
result = list(self._buckets)
if start_time: 17024.h.kuaisou.com
result = [(t, v) for t,原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。