(相关资料图)
# 代码12-1 评论去重的代码import pandas as pdimport reimport jieba.posseg as psgimport numpy as np# 去重，去除完全重复的数据reviews = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\reviews.csv")reviews = reviews[["content", "content_type"]].drop_duplicates()content = reviews["content"]# 代码12-2 数据清洗# 去除去除英文、数字等# 由于评论主要为京东美的电热水器的评论，因此去除这些词语strinfo = re.compile("[0-9a-zA-Z]|京东|美的|电热水器|热水器|")content = content.apply(lambda x: strinfo.sub("", x))# 代码12-3 分词、词性标注、去除停用词代码import numpy as np# 分词worker = lambda s: [(x.word, x.flag) for x in psg.cut(s)] # 自定义简单分词函数seg_word = content.apply(worker) # 将词语转为数据框形式，一列是词，一列是词语所在的句子ID，最后一列是词语在该句子的位置n_word = seg_word.apply(lambda x: len(x)) # 每一评论中词的个数n_content = [[x+1]*y for x,y in zip(list(seg_word.index), list(n_word))]index_content = sum(n_content, []) # 将嵌套的列表展开，作为词所在评论的idseg_word = sum(seg_word, [])word = [x[0] for x in seg_word] # 词nature = [x[1] for x in seg_word] # 词性content_type = [[x]*y for x,y in zip(list(reviews["content_type"]), list(n_word))]content_type = sum(content_type, []) # 评论类型result = pd.DataFrame({"index_content":index_content, "word":word, "nature":nature, "content_type":content_type}) # 删除标点符号result = result[result["nature"] != "x"] # x表示标点符号# 删除停用词stop_path = open("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\stoplist.txt", "r",encoding="UTF-8")stop = stop_path.readlines()stop = [x.replace("\n", "") for x in stop]word = list(set(word) - set(stop))result = result[result["word"].isin(word)]# 构造各词在对应评论的位置列n_word = list(result.groupby(by = ["index_content"])["index_content"].count())index_word = [list(np.arange(0, y)) for y in n_word]index_word = sum(index_word, []) # 表示词语在改评论的位置# 合并评论id，评论中词的id，词，词性，评论类型result["index_word"] = index_word# 代码12-4 提取含有名词的评论# 提取含有名词类的评论ind = result[["n" in x for x in result["nature"]]]["index_content"].unique()result = result[[x in ind for x in result["index_content"]]]
# 代码12-5 绘制词云import matplotlib.pyplot as pltfrom wordcloud import WordCloudfrequencies = result.groupby(by = ["word"])["word"].count()frequencies = frequencies.sort_values(ascending = False)backgroud_Image=plt.imread("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\pl.jpg")wordcloud = WordCloud(font_path="C:\\Windows\\Fonts\\STZHONGS.ttf", max_words=100, background_color="white", mask=backgroud_Image)my_wordcloud = wordcloud.fit_words(frequencies)plt.imshow(my_wordcloud)plt.axis("off")plt.rcParams["font.sans-serif"] = ["SimHei"] # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.show()# 将结果写出result.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\word.csv", index = False, encoding = "utf-8")# 代码12-6 匹配情感词import pandas as pdimport numpy as npword = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\word.csv")# 读入正面、负面情感评价词pos_comment = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\正面评价词语（中文）.txt", header=None,sep="\n", encoding = "utf-8", engine="python")neg_comment = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\负面评价词语（中文）.txt", header=None,sep="\n", encoding = "utf-8", engine="python")pos_emotion = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\正面情感词语（中文）.txt", header=None,sep="\n", encoding = "utf-8", engine="python")neg_emotion = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\负面情感词语（中文）.txt", header=None,sep="\n", encoding = "utf-8", engine="python") # 合并情感词与评价词positive = set(pos_comment.iloc[:,0])|set(pos_emotion.iloc[:,0])negative = set(neg_comment.iloc[:,0])|set(neg_emotion.iloc[:,0])intersection = positive&negative # 正负面情感词表中相同的词语positive = list(positive - intersection)negative = list(negative - intersection)positive = pd.DataFrame({"word":positive, "weight":[1]*len(positive)})negative = pd.DataFrame({"word":negative, "weight":[-1]*len(negative)}) posneg = positive.append(negative)# 将分词结果与正负面情感词表合并，定位情感词data_posneg = posneg.merge(word, left_on = "word", right_on = "word", how = "right")data_posneg = data_posneg.sort_values(by = ["index_content","index_word"])# 代码12-7 修正情感倾向# 根据情感词前时候有否定词或双层否定词对情感值进行修正# 载入否定词表notdict = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\not.csv")# 处理否定修饰词data_posneg["amend_weight"] = data_posneg["weight"] # 构造新列，作为经过否定词修正后的情感值data_posneg["id"] = np.arange(0, len(data_posneg))only_inclination = data_posneg.dropna() # 只保留有情感值的词语only_inclination.index = np.arange(0, len(only_inclination))index = only_inclination["id"]for i in np.arange(0, len(only_inclination)): review = data_posneg[data_posneg["index_content"] == only_inclination["index_content"][i]] # 提取第i个情感词所在的评论 review.index = np.arange(0, len(review)) affective = only_inclination["index_word"][i] # 第i个情感值在该文档的位置 if affective == 1: ne = sum([i in notdict["term"] for i in review["word"][affective - 1]]) if ne == 1: data_posneg["amend_weight"][index[i]] = -\ data_posneg["weight"][index[i]] elif affective > 1: ne = sum([i in notdict["term"] for i in review["word"][[affective - 1, affective - 2]]]) if ne == 1: data_posneg["amend_weight"][index[i]] = -\ data_posneg["weight"][index[i]] # 更新只保留情感值的数据only_inclination = only_inclination.dropna()# 计算每条评论的情感值emotional_value = only_inclination.groupby(["index_content"], as_index=False)["amend_weight"].sum()# 去除情感值为0的评论emotional_value = emotional_value[emotional_value["amend_weight"] != 0]# 代码12-8 查看情感分析效果# 给情感值大于0的赋予评论类型（content_type）为pos,小于0的为negemotional_value["a_type"] = ""emotional_value["a_type"][emotional_value["amend_weight"] > 0] = "pos"emotional_value["a_type"][emotional_value["amend_weight"] < 0] = "neg"# 查看情感分析结果result = emotional_value.merge(word, left_on = "index_content", right_on = "index_content", how = "left")result = result[["index_content","content_type", "a_type"]].drop_duplicates() confusion_matrix = pd.crosstab(result["content_type"], result["a_type"], margins=True) # 制作交叉表(confusion_matrix.iat[0,0] + confusion_matrix.iat[1,1])/confusion_matrix.iat[2,2]# 提取正负面评论信息ind_pos = list(emotional_value[emotional_value["a_type"] == "pos"]["index_content"])ind_neg = list(emotional_value[emotional_value["a_type"] == "neg"]["index_content"])posdata = word[[i in ind_pos for i in word["index_content"]]]negdata = word[[i in ind_neg for i in word["index_content"]]]# 绘制词云import matplotlib.pyplot as pltfrom wordcloud import WordCloud# 正面情感词词云freq_pos = posdata.groupby(by = ["word"])["word"].count()freq_pos = freq_pos.sort_values(ascending = False)backgroud_Image=plt.imread("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\pl.jpg")wordcloud = WordCloud(font_path="C:\\Windows\\Fonts\\STZHONGS.ttf", max_words=100, background_color="white", mask=backgroud_Image)pos_wordcloud = wordcloud.fit_words(freq_pos)plt.imshow(pos_wordcloud)plt.rcParams["font.sans-serif"] = ["SimHei"] # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.axis("off") plt.show()# 负面情感词词云freq_neg = negdata.groupby(by = ["word"])["word"].count()freq_neg = freq_neg.sort_values(ascending = False)neg_wordcloud = wordcloud.fit_words(freq_neg)plt.imshow(neg_wordcloud)plt.rcParams["font.sans-serif"] = ["SimHei"] # 添加这条可以让图形显示中文plt.title("This is a Title(学号 3110)")plt.axis("off") plt.show()# 将结果写出,每条评论作为一行posdata.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\posdata.csv", index = False, encoding = "utf-8")negdata.to_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\negdata.csv", index = False, encoding = "utf-8")# 代码12-9 建立词典及语料库import pandas as pdimport numpy as npimport reimport itertoolsimport matplotlib.pyplot as plt# 载入情感分析后的数据posdata = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\posdata.csv", encoding = "utf-8")negdata = pd.read_csv("D:\\360MoveData\\Users\\86130\\Documents\\Tencent Files\\2268756693\\FileRecv\\negdata.csv", encoding = "utf-8")from gensim import corpora, models# 建立词典pos_dict = corpora.Dictionary([[i] for i in posdata["word"]]) # 正面neg_dict = corpora.Dictionary([[i] for i in negdata["word"]]) # 负面# 建立语料库pos_corpus = [pos_dict.doc2bow(j) for j in [[i] for i in posdata["word"]]] # 正面neg_corpus = [neg_dict.doc2bow(j) for j in [[i] for i in negdata["word"]]] # 负面# 代码12-10 主题数寻优# 构造主题数寻优函数def cos(vector1, vector2): # 余弦相似度函数 dot_product = 0.0; normA = 0.0; normB = 0.0; for a,b in zip(vector1, vector2): dot_product += a*b normA += a**2 normB += b**2 if normA == 0.0 or normB==0.0: return(None) else: return(dot_product / ((normA*normB)**0.5)) # 主题数寻优def lda_k(x_corpus, x_dict): # 初始化平均余弦相似度 mean_similarity = [] mean_similarity.append(1) # 循环生成主题并计算主题间相似度 for i in np.arange(2,11): lda = models.LdaModel(x_corpus, num_topics = i, id2word = x_dict) # LDA模型训练 for j in np.arange(i): term = lda.show_topics(num_words = 50) # 提取各主题词 top_word = [] for k in np.arange(i): top_word.append(["".join(re.findall(""(.*)"",i)) \ for i in term[k][1].split("+")]) # 列出所有词 # 构造词频向量 word = sum(top_word,[]) # 列出所有的词 unique_word = set(word) # 去除重复的词 # 构造主题词列表，行表示主题号，列表示各主题词 mat = [] for j in np.arange(i): top_w = top_word[j] mat.append(tuple([top_w.count(k) for k in unique_word])) p = list(itertools.permutations(list(np.arange(i)),2)) l = len(p) top_similarity = [0] for w in np.arange(l): vector1 = mat[p[w][0]] vector2 = mat[p[w][1]] top_similarity.append(cos(vector1, vector2)) # 计算平均余弦相似度 mean_similarity.append(sum(top_similarity)/l) return(mean_similarity) # 计算主题平均余弦相似度pos_k = lda_k(pos_corpus, pos_dict)neg_k = lda_k(neg_corpus, neg_dict) # 绘制主题平均余弦相似度图形from matplotlib.font_manager import FontProperties font = FontProperties(size=14)#解决中文显示问题plt.rcParams["font.sans-serif"]=["SimHei"]plt.rcParams["axes.unicode_minus"] = False fig = plt.figure(figsize=(10,8))ax1 = fig.add_subplot(211)ax1.plot(pos_k)ax1.set_xlabel("（学号 3110）正面评论LDA主题数寻优", fontproperties=font)ax2 = fig.add_subplot(212)ax2.plot(neg_k)ax2.set_xlabel("（学号 3110）负面评论LDA主题数寻优", fontproperties=font)# 代码12-11 LDA主题分析# LDA主题分析pos_lda = models.LdaModel(pos_corpus, num_topics = 3, id2word = pos_dict) neg_lda = models.LdaModel(neg_corpus, num_topics = 3, id2word = neg_dict) pos_lda.print_topics(num_words = 10)neg_lda.print_topics(num_words = 10)
[(0, "0.124*"安装" + 0.031*"垃圾" + 0.031*"师傅" + 0.018*"小时" + 0.017*"不好" + 0.017*"打电话" + 0.016*"加热" + 0.015*"烧水" + 0.015*"太慢" + 0.012*"坑人""), (1, "0.033*"太" + 0.031*"差" + 0.027*"安装费" + 0.020*"收" + 0.016*"漏水" + 0.014*"人员" + 0.014*"真的" + 0.013*"坑" + 0.012*"服务" + 0.012*"高""), (2, "0.030*"售后" + 0.021*"东西" + 0.020*"客服" + 0.019*"装" + 0.017*"收费" + 0.017*"贵" + 0.016*"慢" + 0.011*"产品" + 0.011*"上门" + 0.010*"评"")]
标签：
