使用Python实现基于特定模式的标记化

import re

def tokenize_text(text):
    patterns = {
        'DATE': r'\d{2}/\d{2}/\d{4}',  # 匹配日期格式（dd/mm/yyyy）
        'EMAIL': r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',  # 匹配电子邮件地址
        'URL': r'https?://[^\s]+',  # 匹配网址
    }

    tokens = []
    for key, pattern in patterns.items():
        matches = re.findall(pattern, text)
        for match in matches:
            tokens.append((match, key))

    return tokens

text = 'Please contact us at support@example.com or visit our website at https://www.example.com. The event will be held on 12/31/2022.'

tokens = tokenize_text(text)
for token, label in tokens:
    print(f'{token} ({label})')

# 输出结果:
# support@example.com (EMAIL)
# https://www.example.com (URL)
# 12/31/2022 (DATE)