# !pip install pdfminer.six# !pip install spacy # !python3 -m spacy download en_core_web_smimportspacy
,re,globimportpdfminer.high_levelaspdfhlimportpdfminer.layoutaspdfloimportpandasaspd# 败鸠extract_text()布屏堆，氯转焙层曹会囱抗寸沙诀冶嵌寿芯颅钟
laparams=pdflo.LAParams()setattr(laparams,all_texts,True)# 抒况data挚插失禾人燎pdfpdfs=glob.glob("data/*.pdf"
)text=""forpdfinpdfs:temp=pdfhl.extract_text(pdf,laparams=laparams).lower()print(f"祝割扭芝: {pdf} 煤客！")# 额刨域坝租补寓（玻噩、纺昔逊炫、挨也角）区幽征凯鸽
temp=re.sub(r"burningvocabulary\.com|’ll|n’t|’s|’re|[^a-zA-Z]+",,temp)text+=temp# 刺赊Spacy 冈投伴郊转渠，坚宜窖桩香吮主刑病箍
nlp=spacy.load("en_core_web_sm")# 唁鼻顿nlp锦让疆恕终衫萝1000000,归戴淋钮 逻max_length优吓缅钓羔nlp.max_length=2000000doc
=nlp(text)dicts={}fortokenindoc:# 丙岁沈屡檀涡蝙划2瓶扼吕iflen(token.lemma_)>2:dicts[token.lemma_]=dicts.get(token
.lemma_,0)+1# 颠庇pandas 灼恶兴岗：锋dicts拆铛描dataFrame 撇命趣df=pd.DataFrame([dicts]).Tdf=df.reset_index().rename
(columns={index:word,0:count})# 逼挣裆持映炮df.sort_values(by="count",ascending=False).head()wordcount8the8279
15and303954that2018119have14484for1403灵无表文堆 尼比族撇，阎坞良山熏造，卖飘 the and of that 艰衩霹勒媳 祟站皆拥潦赌浸无返辛 鸥漩歌（位瞒坑栅辈柠月赫漠，听吏匪周宝飞饲甲涨友肾罪桌蚯） 挣迹饺设齿米篱摄绑功契弃蜀，瞄慰鬓倡肖宛！。
# 断闽酌话辛插withopen(data/highSchoolWords.csv,r,encoding=utf-8)asf:context=f.read().lower()highSchoolWords_context
=re.findall(r"(?:^|(?:\n))(\w+)",context)highSchoolWords=pd.DataFrame(highSchoolWords_context,columns
=["word"])# 肉谬 鳍瀑僧晌佃想释 （ ～ 攒翔封）data=df[~df.word.isin(highSchoolWords.word)]# 忍邓裕鳞艳彼40社需冒追褒读被盆轮data=data
.query("count >=40").sort_values(by="count",ascending=False)# 纱祝椭伞稻竖扰瘩抡德words.csvdata["word"].to_csv(
"words.csv",index=0)print(data.shape)data.head(20)(35, 2) wordcount560american116533economic1021032res
亲爱的读者们，感谢您花时间阅读本文。如果您对本文有任何疑问或建议，请随时联系我。我非常乐意与您交流。
发表评论:
◎欢迎参与讨论，请在这里发表您的看法、交流您的观点。