一文速学-知识图谱从零开始构建实战:知识图谱搭建构架实践-信息抽取

! pip install --upgrade paddlenlp
! pip show paddlenlp

schema = ['时间', '城市'] # Define the schema for entity extraction
ie = Taskflow('information_extraction', schema=schema, model='uie-base')
ie_en = Taskflow('information_extraction', schema=schema, model='uie-base-en')
pprint(ie("凡我市（含县（区）、开发区（新区））国有资金投资的房屋建筑和市政基础设施工程自2017年6月3日起发布招标文件的施工招标项目，应当执行《南昌市人民政府办公厅关于印发南昌市国有资金投资建设工程项目招标实施年度投标保证金制度（试行）的通知》（洪府厅发〔2016〕74号）的规定。")) # Better print results using pprint

[{'城市': [{'end': 71,
          'probability': 0.9465240012183784,
          'start': 68,
          'text': '南昌市'},
         {'end': 85,
          'probability': 0.9736215907851147,
          'start': 82,
          'text': '南昌市'}],
  '时间': [{'end': 48,
          'probability': 0.5193512646392762,
          'start': 39,
          'text': '2017年6月3日'}]}]

schema = {'部门':['指导','活动名称']}
ie.set_schema(schema)
txt_file='国务院发展改革部门指导和协调全国招标投标工作，对国家重大建设项目的工程招标投标活动实施监督检查。'
pprint(ie(txt_file))

[{'部门': [{'end': 9,
          'probability': 0.9600018973785609,
          'relations': {'指导': [{'end': 22,
                                'probability': 0.9368320465242093,
                                'start': 14,
                                'text': '全国招标投标工作'}]},
          'start': 0,
          'text': '国务院发展改革部门'}]}]
​

schema = {'公司':['创始人','成立时间','地点']}
ie.set_schema(schema)
txt_file='阿里巴巴集团控股有限公司(简称：阿里巴巴集团)是马云带领下的18位创始人于1999年在浙江省杭州市创立的公司。'
pprint(ie(txt_file))

[{'公司': [{'end': 12,
          'probability': 0.8997031190817637,
          'relations': {'创始人': [{'end': 26,
                                 'probability': 0.7737946618519764,
                                 'start': 24,
                                 'text': '马云'}],
                        '地点': [{'end': 49,
                                'probability': 0.3552619337063625,
                                'start': 43,
                                'text': '浙江省杭州市'}],
                        '成立时间': [{'end': 42,
                                  'probability': 0.8343506849196594,
                                  'start': 37,
                                  'text': '1999年'}]},
          'start': 0,
          'text': '阿里巴巴集团控股有限公司'}]}]
​

schema = {'启动':['时间','对象']}
ie.set_schema(schema)
txt_file='2020年，阿里巴巴宣布启动新零售计划'
pprint(ie(txt_file))

[{'启动': [{'end': 19,
          'probability': 0.975417622363075,
          'relations': {'时间': [{'end': 5,
                                'probability': 0.9598323070613546,
                                'start': 0,
                                'text': '2020年'}]},
          'start': 14,
          'text': '新零售计划'}]}]
​

import paddlehub as hub
import json
from collections import defaultdict
from docx import Document
import jieba.posseg as pseg
from paddlenlp import Taskflow
​
def extract_text_from_docx(docx_path):
    document = Document(docx_path)
    return [paragraph.text.strip() for paragraph in document.paragraphs if paragraph.text.strip()]
​
def split_paragraphs_into_sections(paragraphs):
    sections = defaultdict(list)
    current_section = "其他"
    for paragraph in paragraphs:
        if "招标类型" in paragraph:
            current_section = "招标类型"
        elif "招标组织方式" in paragraph:
            current_section = "招标组织方式"
        elif "项目要求" in paragraph:
            current_section = "项目要求"
        sections[current_section].append(paragraph)
    return sections
​
def generate_schema_from_sections(sections):
    section_schemas = {}
    for section, paragraphs in sections.items():
        entity_candidates = defaultdict(int)
        relation_candidates = defaultdict(int)
        for paragraph in paragraphs:
            words = pseg.cut(paragraph)
            for word, flag in words:
                if flag in ['n', 'nr', 'ns', 'nt', 'nz']: 
                    entity_candidates[word] += 1
                elif flag in ['v']:  
                    relation_candidates[word] += 1
        top_relations = sorted(relation_candidates, key=relation_candidates.get, reverse=True)[:10]
        schema = {}
        for entity in entity_candidates:
            schema[entity] = top_relations
        section_schemas[section] = schema
    return section_schemas
​
def process_text_with_uie_by_sections(sections, ie_model, section_schemas):
​
    extracted_info = {}
    for section, paragraphs in sections.items():
        schema = section_schemas.get(section, {})
        ie_model.set_schema(schema)
        section_info = []
        for paragraph in paragraphs:
            extraction_result = ie_model(paragraph)
            section_info.append(extraction_result)
        extracted_info[section] = section_info
    return extracted_info
​
def save_extracted_info_to_json(extracted_info, output_path):
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(extracted_info, f, ensure_ascii=False, indent=4)
​
def main():
    docx_path = 'data/招标采购基本知识.docx'  # Replace with your actual file path
    output_json_path = 'data/extracted_info.json'  # Replace with your output path
​
    paragraphs = extract_text_from_docx(docx_path)
    
    sections = split_paragraphs_into_sections(paragraphs)
    
    section_schemas = generate_schema_from_sections(sections)
    print(f"Generated Schemas: {section_schemas}")
    
    ie_model = Taskflow('information_extraction', model='uie-base')  # Load PaddleHub UIE model
    extracted_info = process_text_with_uie_by_sections(sections, ie_model, section_schemas)
    save_extracted_info_to_json(extracted_info, output_json_path)
​
    print(f"Extracted information and saved to {output_json_path}")
​
if __name__ == "__main__":
    main()
​

pip3 install setuptools_scm -i https://pypi.tuna.tsinghua.edu.cn/simple

pip3 install doccano -i https://pypi.tuna.tsinghua.edu.cn/simple

$ doccano init
​
$ doccano createuser --username fanstuck --password xwt353008

$ doccano webserver --port 8000

$ doccano task

! python preprocess.py --input_file ./data/test.jsonl --save_dir ./data/ --negative_ratio 5 --splits 0.2 0.8 0.0 --seed 1000

! python finetune.py --train_path ./data/train.txt --dev_path ./data/dev.txt --save_dir ./checkpoint --model uie-base --learning_rate 1e-5 --batch_size 16 --max_seq_len 512 --num_epochs 50 --seed 1000 --logging_steps 10 --valid_steps 10