文本预处理/计算文章相似度（通过计算杰卡德系数计算文本相似度）

    #-*- conding:utf-8 -*-
    import pprint
    from collections import Counter
    import jieba
    import numpy as np
    #数据抽取（从文件中读取）
    file_path='1.txt'
    def readFile(file_path):
        content = []
        with open(file_path, 'r',encoding="utf-8") as f:
            content = f.read()
    	return content
    #数据清理（分词去掉停用词）
    def cleanWord(content):
        # 分词
        text = jieba.cut(content)#切词
        # 读取停用词
        stopwords = []
        with open("./哈工大停用词表.txt", encoding="utf-8") as f:
            stopwords = f.read()
        new_text = []
        # 去掉停用词
        for w in text:
            if w not in stopwords:
                new_text.append(w)
        return new_text
    
    cc=cleanWord(content=readFile(file_path))
    
    #数据整理（统计词频）
    def statisticalData(text):
        # 统计每个词的词频
        counter = Counter(text)
        # 输出词频最高的15个单词
        pprint.pprint(counter.most_common(15))
    
    
    from gensim import similarities
    #计算文本相似度
    def calculateSimilarity(s1, s2):# 计算s1在s2中的相似度
        def add_space(s):
            return ' '.join(cleanWord(s))
    
        # 将字中间加入空格
        s1, s2 = add_space(s1), add_space(s2)
    
        # 转化为TF矩阵
        from sklearn.feature_extraction.text import CountVectorizer
        cv = CountVectorizer(tokenizer=lambda s: s.split())
    
        corpus = [s1, s2]
        vectors = cv.fit_transform(corpus).toarray()#转成向量并且放入数组中
        # 求交集
        numerator = np.sum(np.min(vectors, axis=0))
        # 求并集
        denominator = np.sum(np.max(vectors, axis=0))
        # 计算杰卡德系数
        return 1.0 * numerator / denominator
    
    def main():
        content_c = readFile("1.txt")
        content_z = readFile("2.txt")
        #相似度
        similiar = calculateSimilarity(content_c, content_z)
        print('两篇文章的相似度是：',similiar)
    if __name__ == '__main__' :
        main()