从架构到代码：深入理解 OpenClaw 的双源记忆系统

{"type":"message","message":{"role":"user","content":"帮我写一个 Python 爬虫"}}
{"type":"message","message":{"role":"assistant","content":"好的，我来帮你写..."}}
{"type":"tool_call","tool":"bash","input":{"command":"python crawler.py"}}

export async function buildSessionEntry(absPath: string): Promise<SessionFileEntry | null> {
  // 读取 JSONL 文件
  const raw = await fs.readFile(absPath, "utf-8");
  const lines = raw.split("\n");
  const collected: string[] = [];

  for (const line of lines) {
    const record = JSON.parse(line);
    // 只提取 user 和 assistant 的消息
    if (message.role === "user" || message.role === "assistant") {
      const label = message.role === "user" ? "User" : "Assistant";
      collected.push(`${label}: ${text}`);
    }
  }

  return {
    path: sessionPathForFile(absPath),
    hash: hashText(content),
    content,  // "User: 帮我写... \n Assistant: 好的..."
  };
}

export const DEFAULT_MEMORY_FLUSH_PROMPT = [
  "Pre-compaction memory flush.",
  "Store durable memories now (use memory/YYYY-MM-DD.md; create memory/ if needed).",
  `If nothing to store, reply with ${SILENT_REPLY_TOKEN}.`,
].join(" ");

const MERGE_SUMMARIES_INSTRUCTIONS =
  "Merge these partial summaries into a single cohesive summary. Preserve decisions," +
  " TODOs, open questions, and any constraints.";

-- 文件元数据
CREATE TABLE files (
  path TEXT PRIMARY KEY,      -- 'memory/projects.md'
  source TEXT NOT NULL,       -- 'memory' | 'sessions'
  hash TEXT NOT NULL,         -- SHA256 用于增量更新
  mtime INTEGER NOT NULL,
  size INTEGER NOT NULL
);
 
-- 文本块（带 embedding）
CREATE TABLE chunks (
  id TEXT PRIMARY KEY,        -- UUID
  path TEXT NOT NULL,         -- 来源文件
  source TEXT NOT NULL,       -- 'memory' | 'sessions'
  start_line INTEGER,
  end_line INTEGER,
  hash TEXT NOT NULL,
  model TEXT NOT NULL,        -- 'text-embedding-3-small'
  text TEXT NOT NULL,         -- 原文
  embedding TEXT NOT NULL,    -- JSON 数组 [0.1, 0.2, ...]
  updated_at INTEGER
);
 
-- 向量索引（sqlite-vec 扩展）
CREATE VIRTUAL TABLE chunks_vec USING vec0(...);
 
-- 全文索引（FTS5）
CREATE VIRTUAL TABLE chunks_fts USING fts5(
  path, source, model, text,
  tokenize='porter unicode61'
);

async search(query: string, opts?: { maxResults?: number; minScore?: number }) {
  // 1. 关键词搜索 (BM25)
  const keywordResults = hybrid.enabled
    ? await this.searchKeyword(cleaned, candidates)
    : [];
 
  // 2. 向量搜索
  const queryVec = await this.embedQueryWithTimeout(cleaned);
  const vectorResults = await this.searchVector(queryVec, candidates);
 
  // 3. 混合排序
  if (!hybrid.enabled) {
    return vectorResults.filter(r => r.score >= minScore).slice(0, maxResults);
  }
 
  const merged = this.mergeHybridResults({
    vector: vectorResults,
    keyword: keywordResults,
    vectorWeight: 0.7,  // 向量权重
    textWeight: 0.3,    // 关键词权重
  });
 
  return merged.filter(r => r.score >= minScore).slice(0, maxResults);
}

const merged = Array.from(byId.values()).map((entry) => {
  // 最终得分 = 向量权重 × 向量相似度 + 关键词权重 × BM25 得分
  const score = vectorWeight * entry.vectorScore + textWeight * entry.textScore;
  return { path, startLine, endLine, score, snippet, source };
});
 
return merged.toSorted((a, b) => b.score - a.score);

  return {
    label: "Memory Search",
    name: "memory_search",
    description:
      "Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) before answering questions about prior work, decisions, dates, people, preferences, or todos; returns top snippets with path + lines.",
    parameters: MemorySearchSchema,
    execute: async (_toolCallId, params) => {
      // 搜索查询文本（必填）
      const query = readStringParam(params, "query", { required: true });
      // 返回结果数量（默认 6）
      const maxResults = readNumberParam(params, "maxResults");
      // 最低相关度阈值（默认 0.35）
      const minScore = readNumberParam(params, "minScore");
      const { manager, error } = await getMemorySearchManager({
        cfg,
        agentId,
      });
      // ... 执行搜索
      const results = await manager.search(query, {
        maxResults,
        minScore,
        sessionKey: options.agentSessionKey,
      });
      return jsonResult({
        results,
        provider: status.provider,
        model: status.model,
        fallback: status.fallback,
      });
    },
  };

{
  "results": [
    {
      "path": "memory/2026-01-10.md",
      "startLine": 15,
      "endLine": 20,
      "score": 0.85,
      "snippet": "用户提到喜欢蓝色，特别是天空蓝...",
      "source": "memory"
    },
    {
      "path": "MEMORY.md",
      "startLine": 5,
      "endLine": 8,
      "score": 0.72,
      "snippet": "颜色偏好：蓝色系...",
      "source": "memory"
    }
  ],
  "provider": "openai",
  "model": "text-embedding-3-small"
}

  return {
    label: "Memory Get",
    name: "memory_get",
    description:
      "Safe snippet read from MEMORY.md, memory/*.md, or configured memorySearch.extraPaths with optional from/lines; use after memory_search to pull only the needed lines and keep context small.",
    parameters: MemoryGetSchema,
    execute: async (_toolCallId, params) => {
      // 文件相对路径（如 memory/2026-01-10.md）
      const relPath = readStringParam(params, "path", { required: true });
      // 起始行号
      const from = readNumberParam(params, "from", { integer: true });
      // 读取行数
      const lines = readNumberParam(params, "lines", { integer: true });
      // ... 读取文件
      const result = await manager.readFile({
        relPath,
        from: from ?? undefined,
        lines: lines ?? undefined,
      });
      return jsonResult(result);
    },
  };

{
  "text": "用户提到喜欢蓝色，特别是天空蓝。\n在选择UI时偏好冷色调。\n...",
  "path": "memory/2026-01-10.md"
}

  return [
    "## Memory Recall",
    "Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.",
    "",
  ];

exec(command="echo '## 用户偏好\n- 喜欢蓝色' >> memory/...")
write(path="memory/2026-02-05.md", content="用户喜欢蓝色")

  async readFile(params: {
    relPath: string;
    from?: number;
    lines?: number;
  }): Promise<{ text: string; path: string }> {
    // ... 路径验证
    const inWorkspace =
      relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
    const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
    // 只允许读取：
    // 1. MEMORY.md / memory.md
    // 2. memory/*.md
    // 3. 配置的 extraPaths 中的 .md 文件
    if (!allowedWorkspace && !allowedAdditional) {
      throw new Error("path required");  // 安全限制
    }
    if (!absPath.endsWith(".md")) {
      throw new Error("path required");  // 只允许 .md 文件
    }
    // ...
  }

~/.openclaw/memory/2026-01-10-reminders.md

# Session: 2026-01-10 08:00:15 UTC
 
## Summary
用户请求设置每日提醒，讨论了健身计划安排。
 
## Key Points
- 用户想要每天下午3点收到健身提醒
- 健身计划包括：周一胸肌、周三背部、周五腿部
- 用户偏好使用 Telegram 接收提醒通知
 
## Conversation Highlights
User: 帮我设置一个每日提醒，下午3点提醒我去健身
Assistant: 好的，我已经为你设置了每日下午3点的健身提醒。
 
User: 对了，我的健身计划是周一练胸，周三练背，周五练腿
Assistant: 明白了，我已记录你的健身计划安排。
 
## Action Items
- [x] 设置每日 15:00 健身提醒
- [ ] 后续可添加具体训练动作细节 

const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
// 结果: ["~/.openclaw/MEMORY.md", "~/.openclaw/memory/2026-01-10-reminders.md", ...]

// buildFileEntry 实现
const stat = await fs.stat(absPath);
const content = await fs.readFile(absPath, "utf-8");  // 读取整个文件
const hash = hashText(content);                        // 计算整个文件的 hash
return {
  path: "memory/2026-01-10-reminders.md",
  absPath: "/Users/xxx/.openclaw/memory/2026-01-10-reminders.md",
  mtimeMs: 1736496015000,
  size: 847,
  hash: "e4d909c290d0fb1ca068ffaddf22cbd0"  // 整个文件内容的 MD5
};

const record = this.db
  .prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
  .get(entry.path, "memory");
 
// 情况A: 数据库中无记录，或 hash 不匹配 → 需要索引
// 情况B: hash 匹配 → 跳过，无需重新索引
if (!params.needsFullReindex && record?.hash === entry.hash) {
  return;  // 跳过
}
await this.indexFile(entry, { source: "memory" });  // 需要索引

[需要索引] memory/2026-01-10-reminders.md (hash: e4d909c290d0fb1ca068ffaddf22cbd0)

const content = await fs.readFile(entry.absPath, "utf-8");
const chunks = chunkMarkdown(content, this.settings.chunking);
// 默认配置: { tokens: 400, overlap: 80 }
 
export function chunkMarkdown(
  content: string,
  chunking: { tokens: number; overlap: number },
): MemoryChunk[] {
  const lines = content.split("\n");
  const maxChars = Math.max(32, chunking.tokens * 4);  // 1600 chars
  const overlapChars = Math.max(0, chunking.overlap * 4);  // 320 chars
  const chunks: MemoryChunk[] = [];
 
  let current: Array<{ line: string; lineNo: number }> = [];
  let currentChars = 0;
 
  const flush = () => {
    // 将当前累积的行打包成一个 chunk
    const text = current.map((entry) => entry.line).join("\n");
    const startLine = firstEntry.lineNo;
    const endLine = lastEntry.lineNo;
    chunks.push({
      startLine,
      endLine,
      text,
      hash: hashText(text),  // SHA-256(chunk文本)
    });
  };
 
  const carryOverlap = () => {
    // 保留最后 overlapChars 个字符到下一个 chunk（滑动窗口）
    // ...
  };
 
  for (let i = 0; i < lines.length; i += 1) {
    // 逐行累积，超过 maxChars 就 flush
    if (currentChars + lineSize > maxChars && current.length > 0) {
      flush();
      carryOverlap();  // 保留重叠部分
    }
    current.push({ line: segment, lineNo });
    currentChars += lineSize;
  }
  flush();  // 最后一批
  return chunks;
}

const cached = this.loadEmbeddingCache(chunks.map(chunk => chunk.hash));
// 如果 chunk.hash 在缓存中命中，直接复用向量，跳过 API 调用

// 批量请求 embedding
const batchEmbeddings = await this.provider.embedBatch([
  "# Session: 2026-01-10 08:00:15 UTC\n\n## Summary\n用户请求设置每日提醒...",
  "- 用户偏好使用 Telegram...\n\n## Conversation Highlights..."
]);
 
// 返回结果（1536维向量，text-embedding-3-small）
// Chunk 0: [0.0234, -0.0567, 0.0891, ..., -0.0123]  (1536个浮点数)
// Chunk 1: [0.0345, -0.0678, 0.0902, ..., -0.0234]  (1536个浮点数)

this.upsertEmbeddingCache([
  { hash: "7a8b9c...", embedding: [0.0234, -0.0567, ...] },
  { hash: "3d4e5f...", embedding: [0.0345, -0.0678, ...] }
]); 

const id = hashText(
  `${source}:${path}:${startLine}:${endLine}:${chunkHash}:${model}`
);
 
// Chunk 0 的 id 计算:
// hashText("memory:memory/2026-01-10-reminders.md:1:12:7a8b9c...:text-embedding-3-small")
// → "a1b2c3d4e5f6..."
 
// Chunk 1 的 id 计算:
// hashText("memory:memory/2026-01-10-reminders.md:10:22:3d4e5f...:text-embedding-3-small")
// → "g7h8i9j0k1l2..."

this.db.prepare(`
  INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(
  "a1b2c3d4e5f6...",                    // id
  "memory/2026-01-10-reminders.md",     // path
  "memory",                              // source
  1,                                     // start_line
  12,                                    // end_line
  "7a8b9c...",                          // chunk hash
  "text-embedding-3-small",             // model
  "# Session: 2026-01-10...",           // 原始文本
  "[0.0234, -0.0567, ...]",             // 向量 (JSON 字符串)
  1736496100000                          // updated_at
);

this.db.prepare(`INSERT INTO chunks_vec (id, embedding) VALUES (?, ?)`)
  .run(
    "a1b2c3d4e5f6...",           // 与 chunks 表相同的 id
    vectorToBlob([0.0234, ...])  // 转换为二进制 BLOB 格式
  );

this.db.prepare(`
  INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line)
  VALUES (?, ?, ?, ?, ?, ?, ?)
`).run(
  "# Session: 2026-01-10...",           // text (FTS5 自动分词)
  "a1b2c3d4e5f6...",                    // id (UNINDEXED，不分词)
  "memory/2026-01-10-reminders.md",     // path (UNINDEXED)
  "memory",                              // source (UNINDEXED)
  "text-embedding-3-small",             // model (UNINDEXED)
  1,                                     // start_line (UNINDEXED)
  12                                     // end_line (UNINDEXED)
);

this.db.prepare(`
  INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
  ON CONFLICT(path) DO UPDATE SET hash=excluded.hash, mtime=excluded.mtime, size=excluded.size
`).run(
  "memory/2026-01-10-reminders.md",
  "memory",
  "e4d909c290d0fb1ca068ffaddf22cbd0",  // 整个文件的 hash
  1736496015000,
  847
);

> - "健身" → [a1b2c3d4e5f6..., g7h8i9j0k1l2...]
> - "提醒" → [a1b2c3d4e5f6...]
> - "Telegram" → [g7h8i9j0k1l2...]

请求 1：System Prompt + 用户消息 1 + 回复 1
请求 2：System Prompt + 用户消息 1 + 回复 1 + 用户消息 2 + 回复 2
请求 3：System Prompt + 历史累积... + 用户消息 3
...
请求 N：触发 Compaction → 摘要替换历史

export async function runMemoryFlushIfNeeded(params: {
  cfg: OpenClawConfig;
  followupRun: FollowupRun;
  // ...
}): Promise<SessionEntry | undefined> {
  // 触发一个完整的 Agent 回合
  // 包含 System Prompt + 工具 + 指令
  await runEmbeddedPiAgent({
    // ...
    prompt: memoryFlushSettings.prompt,
    // ...
  });
}

用户: "我之前说过喜欢什么颜色？"
         ↓
Agent 调用: memory_search(query="喜欢的颜色")
         ↓
返回结果: 
  - memory/2026-01-10.md:15-20 (score: 0.85)
    "用户提到喜欢蓝色，特别是天空蓝..."
  - MEMORY.md:5-8 (score: 0.72)
    "颜色偏好：蓝色系..."
         ↓
这些结果被加入上下文 → 消耗 token

用户: "帮我查一下天气并发到 Telegram"
 
调用链:
1. web_search("天气") → 返回结果 (+500 tokens)
2. memory_search("用户位置") → 返回结果 (+200 tokens)  
3. message(channel="telegram", ...) → 返回确认 (+100 tokens)