spaCy入门工业级NLP管道、实体识别、依存分析一、spaCy概述1.1 为什么选择spaCyimportspacyimportnumpyasnpimportmatplotlib.pyplotaspltfrommatplotlib.patchesimportRectangle,FancyBboxPatchimportwarnings warnings.filterwarnings(ignore)print(*60)print(spaCy工业级NLP工具)print(*60)# spaCy vs NLTK对比fig,axesplt.subplots(1,2,figsize(12,5))# spaCyax1axes[0]ax1.axis(off)ax1.set_title(spaCy - 工业级,fontsize11)spacy_features[✓ 快速Cython实现,✓ 生产就绪,✓ 深度学习模型,✓ 完整管道,✓ 易于部署,]y_pos0.7forfeatinspacy_features:ax1.text(0.1,y_pos,feat,fontsize9,colorgreen)y_pos-0.1# NLTKax2axes[1]ax2.axis(off)ax2.set_title(NLTK - 学术/教育,fontsize11)nltk_features[○ 教学友好,○ 算法丰富,○ 语料库多,○ 速度较慢,○ 适合学习,]y_pos0.7forfeatinnltk_features:ax2.text(0.1,y_pos,feat,fontsize9,colorblue)y_pos-0.1plt.suptitle(spaCy vs NLTK,fontsize12)plt.tight_layout()plt.show()print(\n spaCy特点:)print( - 工业级速度Cython优化)print( - 预训练模型支持多语言)print( - 端到端NLP管道)print( - 易于集成到生产环境)二、spaCy基础2.1 安装与加载模型defspacy_basics():spaCy基础操作print(\n*60)print(spaCy基础操作)print(*60)print( # 安装 pip install spacy # 下载模型 python -m spacy download en_core_web_sm python -m spacy download zh_core_web_sm )code import spacy # 加载模型 nlp spacy.load(en_core_web_sm) # 处理文本 doc nlp(Apple is looking at buying U.K. startup for $1 billion) # 查看模型信息 print(f模型名称: {nlp.meta[name]}) print(f模型版本: {nlp.meta[version]}) print(f支持语言: {nlp.meta[lang]}) print(f管道组件: {nlp.pipe_names}) # 输出: # 模型名称: core_web_sm # 模型版本: 3.7.1 # 支持语言: en # 管道组件: [tok2vec, tagger, parser, ner, attribute_ruler, lemmatizer] print(code)spacy_basics()2.2 Doc对象defdoc_object():Doc对象详解print(\n*60)print(Doc对象NLP结果容器)print(*60)code import spacy nlp spacy.load(en_core_web_sm) text Steve Jobs founded Apple in Cupertino in 1976. doc nlp(text) # 1. 句子分割 print(f句子数量: {len(list(doc.sents))}) for sent in doc.sents: print(f 句子: {sent}) # 2. 分词 print(f\\nToken数量: {len(doc)}) for token in doc[:10]: print(f {token.text}) # 3. 词性标注 print(\\n词性标注:) for token in doc: print(f {token.text:12} → {token.pos_:8} ({token.tag_})) # 4. 依存句法分析 print(\\n依存关系:) for token in doc: print(f {token.text:12} → {token.dep_:10} ← {token.head.text}) # 5. 命名实体识别 print(\\n命名实体:) for ent in doc.ents: print(f {ent.text:20} → {ent.label_}) # 6. 词形还原 print(\\n词形还原:) for token in doc: print(f {token.text:12} → {token.lemma_:12}) # 7. 向量表示 print(f\\n向量维度: {doc.vector.shape}) print(f第一个词向量维度: {doc[0].vector.shape}) print(code)doc_object()三、实体识别NER3.1 内置实体类型defner_demo():命名实体识别print(\n*60)print(命名实体识别NER)print(*60)code import spacy from spacy import displacy nlp spacy.load(en_core_web_sm) # 1. 基本NER text Apple Inc.was founded by Steve Jobs,Steve Wozniak,andRonald WayneinApril1976.The companyisheadquarteredinCupertino,California. doc nlp(text) print(识别到的实体:) for ent in doc.ents: print(f {ent.text:25} → {ent.label_:10} ({spacy.explain(ent.label_)})) # 2. 实体类型说明 entity_types { PERSON: 人名, ORG: 组织/公司, GPE: 地缘政治实体国家、城市, LOC: 地理位置, DATE: 日期, TIME: 时间, MONEY: 金额, PERCENT: 百分比, PRODUCT: 产品, EVENT: 事件, } print(\\n常用实体类型:) for code, name in entity_types.items(): print(f {code}: {name}) # 3. 可视化NER # displacy.render(doc, styleent, jupyterTrue) # 4. 提取特定类型实体 persons [ent.text for ent in doc.ents if ent.label_ PERSON] orgs [ent.text for ent in doc.ents if ent.label_ ORG] dates [ent.text for ent in doc.ents if ent.label_ DATE] print(f\\n人物: {persons}) print(f组织: {orgs}) print(f日期: {dates}) print(code)ner_demo()3.2 可视化NERdefner_visualization():NER可视化print(\n*60)print(NER可视化)print(*60)code import spacy from spacy import displacy nlp spacy.load(en_core_web_sm) # 1. 基本可视化 text Elon Musk founded SpaceX in Hawthorne, California. doc nlp(text) # HTML输出 html displacy.render(doc, styleent, pageTrue) # 2. 自定义颜色 colors { PERSON: #FF6B6B, ORG: #4ECDC4, GPE: #45B7D1, } options { ents: [PERSON, ORG, GPE], colors: colors, } html displacy.render(doc, styleent, optionsoptions, pageTrue) # 3. 批量处理 texts [ Google is headquartered in Mountain View., Microsoft was founded by Bill Gates in Albuquerque., Amazon was founded by Jeff Bezos in Seattle. ] docs list(nlp.pipe(texts)) html displacy.render(docs, styleent, pageTrue) # 4. 保存为文件 with open(ner_visualization.html, w) as f: f.write(html) print(code)ner_visualization()四、依存句法分析4.1 依存关系defdependency_parsing():依存句法分析print(\n*60)print(依存句法分析)print(*60)code import spacy from spacy import displacy nlp spacy.load(en_core_web_sm) text The quick brown fox jumps over the lazy dog. doc nlp(text) # 1. 依存关系 print(依存关系分析:) print(f{Token:12} {POS:8} {Dep:12} {Head:12} {Children}) print(- * 60) for token in doc: children [child.text for child in token.children] print(f{token.text:12} {token.pos_:8} {token.dep_:12} f{token.head.text:12} {children}) # 2. 常用依存关系 dependency_types { nsubj: 名词性主语, dobj: 直接宾语, amod: 形容词修饰语, det: 限定词, prep: 介词修饰语, pobj: 介词宾语, aux: 助动词, conj: 连词连接, } print(\\n常用依存关系:) for dep, meaning in dependency_types.items(): print(f {dep}: {meaning}) # 3. 提取主语-谓语-宾语 def extract_svo(doc): 提取主谓宾结构 triples [] for token in doc: if token.dep_ nsubj: subject token.text verb token.head.text # 找宾语 obj None for child in token.head.children: if child.dep_ dobj: obj child.text break triples.append((subject, verb, obj)) return triples svo extract_svo(doc) print(f\\n主谓宾: {svo}) # 4. 可视化依存关系 # displacy.render(doc, styledep, jupyterTrue) print(code)dependency_parsing()4.2 依存关系可视化defdependency_viz():依存关系可视化print(\n*60)print(依存关系可视化)print(*60)code import spacy from spacy import displacy nlp spacy.load(en_core_web_sm) # 1. 基本依存可视化 text The cat sat on the mat. doc nlp(text) # 2. 自定义选项 options { compact: True, color: #4ECDC4, bg: #2C3E50, font: Arial, } html displacy.render(doc, styledep, optionsoptions, pageTrue) # 3. 设置距离 options {distance: 120} html displacy.render(doc, styledep, optionsoptions, pageTrue) # 4. 批量处理 texts [ I love natural language processing., The quick brown fox jumps over the lazy dog., ] docs list(nlp.pipe(texts)) html displacy.render(docs, styledep, pageTrue) # 5. 保存为SVG from spacy import displacy svg displacy.render(doc, styledep, options{fine_grained: True}) with open(dependency.svg, w) as f: f.write(svg) print(code)dependency_viz()五、spaCy管道5.1 管道组件defspacy_pipeline():spaCy管道print(\n*60)print(spaCy管道组件)print(*60)code import spacy nlp spacy.load(en_core_web_sm) # 1. 查看管道组件 print(f管道组件: {nlp.pipe_names}) print(f管道顺序: {nlp.pipeline}) # 2. 禁用组件加速 with nlp.disable_pipes(parser, ner): doc nlp(This is a fast processing without parser and NER.) print(f禁用后可用组件: {nlp.pipe_names}) # 3. 自定义组件 from spacy.language import Language Language.component(custom_component) def custom_component(doc): # 添加自定义处理逻辑 print(f处理文档: {doc.text[:50]}...) return doc # 添加组件 nlp.add_pipe(custom_component, beforener) # 4. 移除组件 nlp.remove_pipe(custom_component) # 5. 替换组件 nlp.replace_pipe(ner, custom_component) # 6. 组件顺序调整 nlp.move_pipe(ner, lastTrue) print(code)spacy_pipeline()5.2 自定义管道组件defcustom_pipeline_component():自定义管道组件print(\n*60)print(自定义管道组件)print(*60)code import spacy from spacy.language import Language # 1. 简单自定义组件 Language.component(entity_extractor) def entity_extractor(doc): # 提取特定模式 entities [] for token in doc: if token.like_email: entities.append((token.text, EMAIL)) elif token.like_url: entities.append((token.text, URL)) # 添加到doc扩展 doc._.custom_entities entities return doc # 添加扩展属性 from spacy.tokens import Doc Doc.set_extension(custom_entities, default[]) # 注册组件 nlp spacy.load(en_core_web_sm) nlp.add_pipe(entity_extractor, afterner) # 2. 带参数的组件 Language.component(keyword_extractor) def keyword_extractor(doc, min_length3, top_k5): # 统计词频 freq {} for token in doc: if (not token.is_stop and not token.is_punct and len(token.text) min_length): freq[token.lemma_] freq.get(token.lemma_, 0) 1 # 排序取top_k keywords sorted(freq.items(), keylambda x: x[1], reverseTrue)[:top_k] doc._.keywords keywords return doc Doc.set_extension(keywords, default[]) # 3. 工厂组件可配置 Language.factory(sentiment_analyzer) class SentimentAnalyzer: def __init__(self, nlp, name, threshold0.5): self.nlp nlp self.threshold threshold def __call__(self, doc): # 简单的情感分析 positive_words set([good, great, excellent, amazing]) negative_words set([bad, terrible, awful, poor]) pos_count sum(1 for token in doc if token.lemma_ in positive_words) neg_count sum(1 for token in doc if token.lemma_ in negative_words) score (pos_count - neg_count) / (len(doc) 1) doc._.sentiment score doc._.sentiment_label positive if score self.threshold else negative return doc # 注册工厂组件 nlp.add_pipe(sentiment_analyzer, afterner) # 使用 doc nlp(This movie is absolutely great and amazing!) print(f情感分数: {doc._.sentiment}) print(f情感标签: {doc._.sentiment_label}) print(code)custom_pipeline_component()六、相似度计算6.1 词向量相似度defsimilarity_demo():相似度计算print(\n*60)print(词向量相似度)print(*60)code import spacy # 加载带词向量的模型需要较大模型 nlp spacy.load(en_core_web_md) # 或 en_core_web_lg # 1. 词语相似度 word1 nlp(apple) word2 nlp(orange) word3 nlp(car) print(fapple vs orange: {word1.similarity(word2):.3f}) print(fapple vs car: {word1.similarity(word3):.3f}) # 2. 句子相似度 text1 nlp(I love programming) text2 nlp(I enjoy coding) text3 nlp(The weather is nice) print(f\\n句子相似度:) print(f I love programming vs I enjoy coding: {text1.similarity(text2):.3f}) print(f I love programming vs The weather is nice: {text1.similarity(text3):.3f}) # 3. 文档相似度 doc1 nlp(Machine learning is fascinating.) doc2 nlp(Deep learning is a subset of machine learning.) doc3 nlp(I like to eat pizza.) print(f\\n文档相似度:) print(f ML vs DL: {doc1.similarity(doc2):.3f}) print(f ML vs Pizza: {doc1.similarity(doc3):.3f}) # 4. 查找最相似的词 def most_similar(word, top_n5): 查找最相似的词 target nlp(word) similarities [] # 需要词表简化示例 vocab [apple, orange, banana, car, truck, bike, dog, cat, bird, happy, sad, angry] for w in vocab: if w ! word: sim target.similarity(nlp(w)) similarities.append((w, sim)) similarities.sort(keylambda x: x[1], reverseTrue) return similarities[:top_n] print(f\\n与apple最相似的词:) for word, sim in most_similar(apple, 5): print(f {word}: {sim:.3f}) print(code)similarity_demo()七、实战信息抽取系统7.1 完整信息抽取definformation_extraction():信息抽取系统print(\n*60)print(信息抽取系统)print(*60)code import spacy from typing import List, Dict, Any class InformationExtractor: def __init__(self, model_nameen_core_web_sm): self.nlp spacy.load(model_name) def extract_entities(self, text: str) - Dict[str, List[str]]: 提取命名实体 doc self.nlp(text) entities { PERSON: [], ORG: [], GPE: [], DATE: [], MONEY: [], } for ent in doc.ents: if ent.label_ in entities: entities[ent.label_].append(ent.text) return entities def extract_relations(self, text: str) - List[Dict]: 提取实体关系 doc self.nlp(text) relations [] for token in doc: # 提取主谓宾关系 if token.dep_ nsubj and token.head.pos_ VERB: subject token.text predicate token.head.text # 找宾语 obj None for child in token.head.children: if child.dep_ dobj: obj child.text break if obj: relations.append({ subject: subject, predicate: predicate, object: obj }) return relations def extract_keywords(self, text: str, top_k: int 10) - List[str]: 提取关键词 doc self.nlp(text) # 统计词频排除停用词和标点 freq {} for token in doc: if not token.is_stop and not token.is_punct and token.is_alpha: lemma token.lemma_.lower() freq[lemma] freq.get(lemma, 0) 1 # 排序取top_k keywords sorted(freq.items(), keylambda x: x[1], reverseTrue) return [word for word, _ in keywords[:top_k]] def process(self, text: str) - Dict[str, Any]: 完整处理 return { entities: self.extract_entities(text), relations: self.extract_relations(text), keywords: self.extract_keywords(text), } # 使用示例 extractor InformationExtractor() text Apple Inc.was founded by Steve JobsinCupertino,CaliforniainApril1976.The companyisvalued at over $2trillion. result extractor.process(text) print(实体识别结果:) for entity_type, entities in result[entities].items(): if entities: print(f {entity_type}: {entities}) print(\\n关系抽取结果:) for relation in result[relations]: print(f {relation[subject]} → {relation[predicate]} → {relation[object]}) print(f\\n关键词: {result[keywords]}) print(code)information_extraction()八、总结功能方法应用分词doc基础处理词性标注token.pos_语法分析依存分析token.dep_句法结构NERdoc.ents信息抽取相似度similarity()语义匹配管道nlp.pipe()批量处理spaCy最佳实践使用nlp.pipe()批量处理提高效率禁用不需要的组件加速使用小模型sm开发大模型lg部署自定义管道组件扩展功能