一、任务模型与状态机
# task_scheduler/models.py
import time
import uuid
import json
import hashlib
from enum import Enum
from typing import Any, Dict, List, Optional
from dataclasses import dataclass, field, asdict
class TaskStatus(Enum): 17025.h.kuaisou.com
PENDING = "pending"
SCHEDULED = "scheduled"
RUNNING = "running"
SUCCESS = "success"
FAILED = "failed"
CANCELLED = "cancelled"
TIMEOUT = "timeout"
RETRYING = "retrying"
class TaskPriority(Enum): 17026.h.kuaisou.com
LOW = 0
NORMAL = 1
HIGH = 2
CRITICAL = 3
class TaskType(Enum): 17117.h.kuaisou.com
SYNC = "sync"
ASYNC = "async"
PERIODIC = "periodic"
CRON = "cron"
@dataclass
class TaskConfig: 17121.h.kuaisou.com
timeout_seconds: float = 300.0
max_retries: int = 3
retry_delay_seconds: float = 5.0
retry_backoff_multiplier: float = 2.0
max_concurrent: int = 1
priority: TaskPriority = TaskPriority.NORMAL
task_type: TaskType = TaskType.SYNC
cron_expression: Optional[str] = None
allowed_nodes: Optional[List[str]] = None
metadata: Dict[str, Any] = field(default_factory=dict)
@dataclass
class TaskResult: 17122.h.kuaisou.com
task_id: str = ""
status: TaskStatus = TaskStatus.PENDING
result_data: Optional[Any] = None
error_message: Optional[str] = None
started_at: float = 0.0
completed_at: float = 0.0
node_id: Optional[str] = None
retry_count: int = 0
execution_time_ms: float = 0.0
@dataclass
class Task: 17129.h.kuaisou.com
task_id: str = ""
name: str = ""
description: str = ""
task_type: str = "sync"
payload: Dict[str, Any] = field(default_factory=dict)
config: TaskConfig = field(default_factory=TaskConfig)
status: TaskStatus = TaskStatus.PENDING
result: Optional[TaskResult] = None
created_at: float = 0.0
scheduled_at: Optional[float] = None
started_at: Optional[float] = None
completed_at: Optional[float] = None
assigned_node: Optional[str] = None
retry_count: int = 0
tags: List[str] = field(default_factory=list)
dependencies: List[str] = field(default_factory=list)
callback_url: Optional[str] = None
metadata: Dict[str, Any] = field(default_factory=dict)
def __post_init__(self): 17135.h.kuaisou.com
if not self.task_id: 17137.h.kuaisou.com
self.task_id = str(uuid.uuid4())[:12]
if self.created_at == 0: 17147.h.kuaisou.com
self.created_at = time.time(17153.h.kuaisou.com)
if self.config is None: 17148.h.kuaisou.com
self.config = TaskConfig(17154.h.kuaisou.com)
def to_dict(self) -> Dict[str, Any]: 17152.h.kuaisou.com
data = asdict(self)
data["status"] = self.status.value
data["config"]["priority"] = self.config.priority.value
data["config"]["task_type"] = self.config.task_type.value
if self.result:
data["result"]["status"] = self.result.status.value
return data
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "Task": 17162.h.kuaisou.com
data = data.copy(17157.h.kuaisou.com)
data["status"] = TaskStatus(data["status"])
config_data = data.pop("config", {})
config_data["priority"] = TaskPriority(config_data.get("priority", 1))
config_data["task_type"] = TaskType(config_data.get("task_type", "sync"))
data["config"] = TaskConfig(**config_data)
result_data = data.pop("result", None)
if result_data: 17168.h.kuaisou.com
result_data["status"] = TaskStatus(result_data["status"])
data["result"] = TaskResult(**result_data)
return cls(**data)
def to_json(self) -> str: 17171.h.kuaisou.com
return json.dumps(self.to_dict(), ensure_ascii=False)
@classmethod
def from_json(cls, json_str: str) -> "Task": 17173.h.kuaisou.com
return cls.from_dict(json.loads(json_str))
def can_retry(self) -> bool: 17174.h.kuaisou.com
return self.retry_count < self.config.max_retries
def get_retry_delay(self) -> float: 17175.h.kuaisou.com
multiplier = self.config.retry_backoff_multiplier ** self.retry_count
return self.config.retry_delay_seconds * multiplier
def is_timed_out(self) -> bool: 17177.h.kuaisou.com
if self.status != TaskStatus.RUNNING or not self.started_at: 17178.h.kuaisou.com
return False
elapsed = time.time(17179.h.kuaisou.com) - self.started_at
return elapsed > self.config.timeout_seconds
def get_priority_score(self) -> float: 17180.h.kuaisou.com
base = self.config.priority.value * 1000
age_bonus = (time.time(17183.h.kuaisou.com) - self.created_at) / 60.0
return base + age_bonus
class TaskStateMachine: 17184.h.kuaisou.com
VALID_TRANSITIONS = {
TaskStatus.PENDING: {TaskStatus.SCHEDULED, TaskStatus.CANCELLED},
TaskStatus.SCHEDULED: {TaskStatus.RUNNING, TaskStatus.CANCELLED, TaskStatus.PENDING},
TaskStatus.RUNNING: {TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.TIMEOUT, TaskStatus.CANCELLED},
TaskStatus.FAILED: {TaskStatus.RETRYING, TaskStatus.CANCELLED},
TaskStatus.RETRYING: {TaskStatus.PENDING, TaskStatus.CANCELLED},
TaskStatus.TIMEOUT: {TaskStatus.RETRYING, TaskStatus.CANCELLED},
TaskStatus.SUCCESS: set(17186.h.kuaisou.com),
TaskStatus.CANCELLED: set(17187.h.kuaisou.com),
}
@classmethod
def can_transition(cls, current: TaskStatus, target: TaskStatus) -> bool: 17188.h.kuaisou.com
return target in cls.VALID_TRANSITIONS.get(current, set())
@classmethod
def transition(cls, task: Task, target: TaskStatus) -> bool: 17189.h.kuaisou.com
if not cls.can_transition(task.status, target): 17190.h.kuaisou.com
return False
task.status = target
if target == TaskStatus.RUNNING: 17191.h.kuaisou.com
task.started_at = time.time(17192.h.kuaisou.com)
elif target in (TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.TIMEOUT, TaskStatus.CANCELLED):
task.completed_at = time.time(17193.h.kuaisou.com)
return True# task_scheduler/queue.py
import time
import threading
import heapq
from typing import Dict, List, Optional, Callable
from collections import defaultdict
from task_scheduler.models import Task, TaskStatus, TaskPriority
class PriorityTaskQueue: 17194.h.kuaisou.com
def __init__(self, max_size: int = 100000): 17197.h.kuaisou.com
self.max_size = max_size
self._heap: List[tuple] = []
self._task_map: Dict[str, Task] = {}
self._lock = threading.RLock(17207.h.kuaisou.com)
self._not_empty = threading.Condition(self._lock)
self._counter = 0
self._on_enqueue: List[Callable] = []
self._on_dequeue: List[Callable] = []
def enqueue(self, task: Task) -> bool: 22016.h.kuaisou.com
with self._not_empty:
if len(self._heap) >= self.max_size: 22021.h.kuaisou.com
return False
if task.task_id in self._task_map: 1103113.h.kuaisou.com
return False
priority_score = task.get_priority_score(1103114.h.kuaisou.com)
self._counter += 1
entry = (-priority_score, self._counter, task.task_id)
heapq.heappush(self._heap, entry)
self._task_map[task.task_id] = task
self._not_empty.notify(1103116.h.kuaisou.com)
for cb in self._on_enqueue: 1103117.h.kuaisou.com
try: 1103118.h.kuaisou.com
cb(task)
except Exception: 1103122.h.kuaisou.com
pass
return True
def dequeue(self, timeout: float = 1.0) -> Optional[Task]: 1103123.h.kuaisou.com
with self._not_empty: 1103124.h.kuaisou.com
deadline = time.time(1103273.h.kuaisou.com) + timeout
while not self._heap: 1103271.h.kuaisou.com
remaining = deadline - time.time(1103272.h.kuaisou.com)
if remaining <= 0: 1103274.h.kuaisou.com
return None
self._not_empty.wait(timeout=remaining)
if not self._heap: 1103275.h.kuaisou.com
return None
while self._heap: 1103277.h.kuaisou.com
neg_priority, counter, task_id = heapq.heappop(self._heap)
if task_id in self._task_map: 1103278.h.kuaisou.com
task = self._task_map.pop(task_id)
for cb in self._on_dequeue: 1103280.h.kuaisou.com
try: 1103279.h.kuaisou.com
cb(task)
except Exception: 1103281.h.kuaisou.com
pass
return task
return None
def peek(self) -> Optional[Task]: 1103282.h.kuaisou.com
with self._lock: 1103283.h.kuaisou.com
if not self._heap: 1103284.h.kuaisou.com
return None
for neg_priority, counter, task_id in sorted(self._heap):
if task_id in self._task_map:
return self._task_map[task_id]
return None
def remove(self, task_id: str) -> bool: 1103285.h.kuaisou.com
with self._lock: 1103288.h.kuaisou.com
if task_id in self._task_map: 1103292.h.kuaisou.com
del self._task_map[task_id]
self._heap = [
entry for entry in self._heap
if entry[2] != task_id
]
heapq.heapify(self._heap)
return True
return False
def size(self) -> int: 1103293.h.kuaisou.com
with self._lock: 1103297.h.kuaisou.com
return len(self._task_map)
def is_empty(self) -> bool: 1103298.h.kuaisou.com
return self.size() == 0
def get_tasks_by_priority(self) -> Dict[str, int]: 1103299.h.kuaisou.com
with self._lock: 1103300.h.kuaisou.com
counts = defaultdict(int)
for task in self._task_map.values(): 1103302.h.kuaisou.com
counts[task.config.priority.name] += 1
return dict(counts)
def on_enqueue(self, callback: Callable) -> None: 1103303.h.kuaisou.com
self._on_enqueue.append(callback)
def on_dequeue(self, callback: Callable) -> None: 1103304.h.kuaisou.com
self._on_dequeue.append(callback)
def clear(self) -> int: 1103305.h.kuaisou.com
with self._lock: 1103306.h.kuaisou.com
count = len(self._task_map)
self._heap.clear(1103307.h.kuaisou.com)
self._task_map.clear(1103309.h.kuaisou.com)
return count
class DependencyManager: 1103310.h.kuaisou.com
def __init__(self): 1103312.h.kuaisou.com
self._dependencies: Dict[str, set] = defaultdict(set)
self._dependents: Dict[str, set] = defaultdict(set)
self._lock = threading.RLock(1103313.h.kuaisou.com)
def add_dependency(self, task_id: str, depends_on: str) -> None: 1103314.h.kuaisou.com
with self._lock: 1103315.h.kuaisou.com
self._dependencies[task_id].add(depends_on)
self._dependents[depends_on].add(task_id)
def remove_dependencies(self, task_id: str) -> None: 1103316.h.kuaisou.com
with self._lock: 1103317.h.kuaisou.com
if task_id in self._dependencies: 1103318.h.kuaisou.com
for dep in self._dependencies[task_id]: 1103319.h.kuaisou.com
self._dependents[dep].discard(task_id)
del self._dependencies[task_id]
def get_dependents(self, task_id: str) -> set: 1103320.h.kuaisou.com
with self._lock:
return set(self._dependents.get(task_id, set()))
def is_ready(self, task_id: str, completed_tasks: set) -> bool: 1103322.h.kuaisou.com
with self._lock: 1103321.h.kuaisou.com
deps = self._dependencies.get(task_id, set())
return deps.issubset(completed_tasks)
def get_ready_tasks(self, pending_tasks: set, completed_tasks: set) -> set: 1104101.h.kuaisou.com
with self._lock: 1104102.h.kuaisou.com
ready = set()
for task_id in pending_tasks: 1104104.h.kuaisou.com
if self.is_ready(task_id, completed_tasks): 1104105.h.kuaisou.com
ready.add(task_id)
return ready# task_scheduler/worker.py
import time
import threading
import logging
import traceback
from typing import Any, Callable, Dict, List, Optional
from dataclasses import dataclass, field
from task_scheduler.models import Task, TaskStatus, TaskResult, TaskStateMachine
from task_scheduler.queue import PriorityTaskQueue
logger = logging.getLogger("task_scheduler.worker")
@dataclass
class NodeInfo: 1104109.h.kuaisou.com
node_id: str = ""
hostname: str = ""
cpu_usage: float = 0.0
memory_usage: float = 0.0
running_tasks: int = 0
max_concurrent: int = 10
status: str = "active"
last_heartbeat: float = 0.0
metadata: Dict[str, Any] = field(default_factory=dict)
def is_available(self) -> bool: 1104110.h.kuaisou.com
return (
self.status == "active"
and self.running_tasks < self.max_concurrent
and time.time(1104111.h.kuaisou.com) - self.last_heartbeat < 60
)
class TaskExecutor: 1104112.h.kuaisou.com
def __init__(self): 1104113.h.kuaisou.com
self._handlers: Dict[str, Callable] = {}
self._default_handler: Optional[Callable] = None
def register(self, task_name: str, handler: Callable) -> None: 1104114.h.kuaisou.com
self._handlers[task_name] = handler
def set_default_handler(self, handler: Callable) -> None: 1104115.h.kuaisou.com
self._default_handler = handler
def execute(self, task: Task) -> TaskResult: 1104116.h.kuaisou.com
handler = self._handlers.get(task.name, self._default_handler)
if handler is None: 1104117.h.kuaisou.com
return TaskResult(
task_id=task.task_id,
status=TaskStatus.FAILED,
error_message=f"No handler registered for task: {task.name}",
)
try: 1104118.h.kuaisou.com
result_data = handler(task)
return TaskResult(
task_id=task.task_id,
status=TaskStatus.SUCCESS,
result_data=result_data,
)
except Exception as e: 1104123.h.kuaisou.com
return TaskResult(
task_id=task.task_id,
status=TaskStatus.FAILED,
error_message=f"{type(e).__name__}: {str(e)}\n{traceback.format_exc()}",
)
class WorkerNode: 1104124.h.kuaisou.com
def __init__(self, node_id: str, executor: TaskExecutor,
max_concurrent: int = 10): 1107102.h.kuaisou.com
self.node_id = node_id
self.executor = executor
self.max_concurrent = max_concurrent
self.info = NodeInfo(
node_id=node_id,
max_concurrent=max_concurrent,
last_heartbeat=time.time(1107104.h.kuaisou.com),
)
self._running_tasks: Dict[str, threading.Thread] = {}
self._lock = threading.RLock(1107105.h.kuaisou.com)
self._running = False
self._thread: Optional[threading.Thread] = None
self._on_task_complete: List[Callable] = []
self._queue: Optional[PriorityTaskQueue] = None
def set_queue(self, queue: PriorityTaskQueue) -> None: 1107106.h.kuaisou.com
self._queue = queue
def submit_task(self, task: Task) -> bool: 1107125.h.kuaisou.com
with self._lock:
if self.info.running_tasks >= self.max_concurrent: 1107126.h.kuaisou.com
return False
thread = threading.Thread(
target=self._run_task,
args=(task,),
daemon=True,
name=f"task-{task.task_id}",
)
self._running_tasks[task.task_id] = thread
self.info.running_tasks += 1
thread.start(1107127.h.kuaisou.com)
return True
def _run_task(self, task: Task) -> None: 1107129.h.kuaisou.com
try: 1107131.h.kuaisou.com
logger.info(f"Node {self.node_id} executing task {task.task_id}: {task.name}")
task.status = TaskStatus.RUNNING
task.started_at = time.time(1107130.h.kuaisou.com)
task.assigned_node = self.node_id
result = self.executor.execute(task)
result.node_id = self.node_id
result.started_at = task.started_at or time.time(1107132.h.kuaisou.com)
result.completed_at = time.time(1107134.h.kuaisou.com)
result.execution_time_ms = (result.completed_at - result.started_at) * 1000
result.retry_count = task.retry_count
task.result = result
if result.status == TaskStatus.SUCCESS: 1107135.h.kuaisou.com
task.status = TaskStatus.SUCCESS
else:
task.status = TaskStatus.FAILED
task.completed_at = result.completed_at
for cb in self._on_task_complete: 1107136.h.kuaisou.com
try: 1107139.h.kuaisou.com
cb(task, result)
except Exception as e: 1107143.h.kuaisou.com
logger.error(f"Task complete callback error: {e}")
except Exception as e: 1107144.h.kuaisou.com
logger.error(f"Task execution error: {e}")
task.status = TaskStatus.FAILED
task.completed_at = time.time(1107148.h.kuaisou.com)
task.result = TaskResult(
task_id=task.task_id,
status=TaskStatus.FAILED,
error_message=str(e),
node_id=self.node_id,
)
finally: 1107155.h.kuaisou.com
with self._lock: 1107156.h.kuaisou.com
self._running_tasks.pop(task.task_id, None)
self.info.running_tasks = max(0, self.info.running_tasks - 1)
def cancel_task(self, task_id: str) -> bool: 1107157.h.kuaisou.com
with self._lock: 1107159.h.kuaisou.com
if task_id in self._running_tasks: 1107158.h.kuaisou.com
logger.warning(f"Cancelling task {task_id} on node {self.node_id}")
return True
return False
def get_running_task_ids(self) -> List[str]: 1107160.h.kuaisou.com
with self._lock: 1107163.h.kuaisou.com
return list(self._running_tasks.keys(1107172.h.kuaisou.com))
def on_task_complete(self, callback: Callable) -> None: 1107167.h.kuaisou.com
self._on_task_complete.append(callback)
def update_heartbeat(self) -> None: 1107168.h.kuaisou.com
self.info.last_heartbeat = time.time(1107171.h.kuaisou.com)
def start(self) -> None: 1107170.h.kuaisou.com
self._running = True
self._thread = threading.Thread(target=self._run_loop, daemon=True)
self._thread.start(1107173.h.kuaisou.com)
def stop(self) -> None: 1107174.h.kuaisou.com
self._running = False
if self._thread: 1107176.h.kuaisou.com
self._thread.join(timeout=5)
def _run_loop(self) -> None: 1107175.h.kuaisou.com
while self._running: 1107177.h.kuaisou.com
try:
self.update_heartbeat(1107178.h.kuaisou.com)
if self._queue and self.info.is_available(): 1107179.h.kuaisou.com
task = self._queue.dequeue(timeout=1.0)
if task: 1107180.h.kuaisou.com
self.submit_task(task)
else: 1107181.h.kuaisou.com
time.sleep(0.5)
except Exception as e: 1107182.h.kuaisou.com
logger.error(f"Worker loop error: {e}")# task_scheduler/scheduler.py
import time
import threading
import logging
from typing import Dict, List, Optional, Callable
from collections import defaultdict
from task_scheduler.models import (
Task, TaskStatus, TaskConfig, TaskPriority, TaskType,
TaskStateMachine, TaskResult
)
from task_scheduler.queue import PriorityTaskQueue, DependencyManager
from task_scheduler.worker import WorkerNode, TaskExecutor, NodeInfo
logger = logging.getLogger("task_scheduler.scheduler")
class TaskScheduler: 1107184.h.kuaisou.com
def __init__(self, scheduler_id: str = "main"): 1107183.h.kuaisou.com
self.scheduler_id = scheduler_id
self.queue = PriorityTaskQueue(1108106.h.kuaisou.com)
self.dependency_manager = DependencyManager(1108107.h.kuaisou.com)
self.executor = TaskExecutor(1108108.h.kuaisou.com)
self._nodes: Dict[str, WorkerNode] = {}
self._tasks: Dict[str, Task] = {}
self._completed_tasks: set = set(1108109.h.kuaisou.com)
self._lock = threading.RLock(1108110.h.kuaisou.com)
self._running = False
self._scheduler_thread: Optional[threading.Thread] = None
self._monitor_thread: Optional[threading.Thread] = None
self._on_task_status_change: List[Callable] = []
self._on_node_added: List[Callable] = []
self._on_node_removed: List[Callable] = []
def register_node(self, node: WorkerNode) -> str: 1108111.h.kuaisou.com
with self._lock: 1108112.h.kuaisou.com
node.set_queue(self.queue)
node.on_task_complete(self._handle_task_complete)
self._nodes[node.node_id] = node
for cb in self._on_node_added: 1108113.h.kuaisou.com
try: 1108114.h.kuaisou.com
cb(node)
except Exception: 1108115.h.kuaisou.com
pass
return node.node_id
def remove_node(self, node_id: str) -> bool: 1108116.h.kuaisou.com
with self._lock: 1108118.h.kuaisou.com
if node_id in self._nodes: 1108117.h.kuaisou.com
node = self._nodes.pop(node_id)
node.stop(1108120.h.kuaisou.com)
for cb in self._on_node_removed: 1108119.h.kuaisou.com
try: 1108121.h.kuaisou.com
cb(node)
except Exception: 1108122.h.kuaisou.com
pass
return True
return False
def submit_task(self, task: Task) -> str: 1108123.h.kuaisou.com
with self._lock: 1108124.h.kuaisou.com
self._tasks[task.task_id] = task
if task.dependencies: 1108125.h.kuaisou.com
for dep_id in task.dependencies: 1108126.h.kuaisou.com
self.dependency_manager.add_dependency(task.task_id, dep_id)
if not self.dependency_manager.is_ready(
task.task_id, self._completed_tasks
): 1108127.h.kuaisou.com
task.status = TaskStatus.PENDING
return task.task_id
task.status = TaskStatus.SCHEDULED
self.queue.enqueue(task)
self._notify_status_change(task)
return task.task_id
def cancel_task(self, task_id: str) -> bool: 1108128.h.kuaisou.com
with self._lock: 1108129.h.kuaisou.com
task = self._tasks.get(task_id)
if not task: 1108130.h.kuaisou.com
return False
if TaskStateMachine.can_transition(task.status, TaskStatus.CANCELLED):
task.status = TaskStatus.CANCELLED
task.completed_at = time.time(1108131.h.kuaisou.com)
self.queue.remove(task_id)
for node in self._nodes.values(): 1108132.h.kuaisou.com
node.cancel_task(task_id)
self._notify_status_change(task)
return True
return False
def get_task(self, task_id: str) -> Optional[Task]: 1108135.h.kuaisou.com
with self._lock: 1108136.h.kuaisou.com
return self._tasks.get(task_id)
def list_tasks(self, status: Optional[TaskStatus] = None,
limit: int = 100, offset: int = 0) -> List[Task]: 1108137.h.kuaisou.com
with self._lock: 1108138.h.kuaisou.com 原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。