diff --git a/chat_function__analysis.py b/chat_function__analysis.py deleted file mode 100644 index 5f1d7ba53ebb8edc7aa686ee49cb9e7b9edf8eca..0000000000000000000000000000000000000000 --- a/chat_function__analysis.py +++ /dev/null @@ -1,334 +0,0 @@ -import pandas as pd -import numpy as np -from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer -import matplotlib.pyplot as plt -from sklearn.manifold import MDS -from sklearn.metrics.pairwise import cosine_similarity -import random -from matplotlib.font_manager import FontProperties -from sklearn.cluster import KMeans -from sklearn import metrics -from collections import Counter -from scipy.cluster.hierarchy import ward, dendrogram -import bottom_function.normalization as normalization -import bottom_function.m_SQL as qb -import json -from flask import Flask -from flask import request -from flask_cors import CORS - - -class Culter: - - def __init__(self, start_time, end_time): - self.start_time = start_time - self.end_time = end_time - - csv_data = pd.DataFrame() - self.chat_data = pd.DataFrame() - tablename = "semantic_data_table" - db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", - port="3306") - csv_data = db.getData(tableName=tablename, startTime=start_time, endTime=end_time) - self.chat_data = csv_data[(csv_data['domain'] == 'chat') & (csv_data['code'] == 0)] - - self.chat_data.drop_duplicates(subset=['query'], inplace=True) - self.chat_data.dropna(subset=['query'], inplace=True) - - self.out_data = '' # 写入分析结果 - self.feature_names = [] - self.f_sse = [] - self.feature_matrix = np.matrix([]) - - def build_feature_matrix(self, documents, feature_type, ngram_range, min_df, max_df): - - feature_type = feature_type.lower().strip() - if feature_type == 'binary': - vectorizer = CountVectorizer(binary=True, - max_df=max_df, ngram_range=ngram_range) - elif feature_type == 'frequency': - vectorizer = CountVectorizer(binary=False, min_df=min_df, - max_df=max_df, ngram_range=ngram_range) - elif feature_type == 'tfidf': - vectorizer = TfidfVectorizer(token_pattern=r"(?u)\b\w+\b", max_df=max_df) - else: - raise Exception("Wrong feature type entered. Possible values: 'binary', 'frequency', 'tfidf'") - - feature_matrix = vectorizer.fit_transform(documents).astype(float) - - return vectorizer, feature_matrix - - def feature_extraction_data(self): - - chat_one = self.chat_data['query'].tolist() - - norm_chat_one = normalization.normalize_corpus(chat_one, pos=False) - - # 提取 tf-idf 特征 - vectorizer, self.feature_matrix = self.build_feature_matrix(norm_chat_one, feature_type='tfidf', min_df=0.2, - max_df=0.90, - ngram_range=(1, 2)) - - # 查看特征数量) - self.out_data = '聚类分析结果：\n' + '**' * 30 - self.out_data = self.out_data + '\n特征数量：\n' + str(self.feature_matrix.shape) - - # 获取特征名字 - self.feature_names = vectorizer.get_feature_names() - - # 打印某些特征 - self.out_data = self.out_data + '\n部分特征：\n' + ', '.join(self.feature_names[:5]) - - def get_cluster_data(self, clustering_obj, m_data, feature_names, num_clusters, topn_features): - cluster_data = {} - - # 获取cluster的center - ordered_centroids = clustering_obj.cluster_centers_.argsort()[:, ::-1] - # 获取每个cluster的关键特征 - # 获取每个cluster的query - for cluster_num in range(num_clusters): - cluster_data[cluster_num] = {} - cluster_data[cluster_num]['cluster_num'] = cluster_num - key_features = [feature_names[index] - for index - in ordered_centroids[cluster_num, :topn_features]] - cluster_data[cluster_num]['key_features'] = key_features - - c_datas = m_data[m_data['Cluster'] == cluster_num]['query'].values.tolist() - cluster_data[cluster_num]['query'] = c_datas - - return cluster_data - - def print_cluster_data(self, cluster_data): - self.out_data = self.out_data + '\n\n聚类详细数据：\n' - - for cluster_num, cluster_details in cluster_data.items(): - self.out_data = self.out_data + '\nCluster {} details:\n'.format(cluster_num) - - self.out_data = self.out_data + '-' * 20 - self.out_data = self.out_data + '\nKey features:\n' - self.out_data = self.out_data + ', '.join(cluster_details['key_features']) - - self.out_data = self.out_data + '\ndata in this cluster:\n' - self.out_data = self.out_data + ', '.join(cluster_details['query']) - self.out_data = self.out_data + '\n' + '=' * 40 - - def plot_clusters(self, feature_matrix, cluster_data, m_data, plot_size): - def generate_random_color(): # generate random color for clusters - color = '#%06x' % random.randint(0, 0xFFFFFF) - return color - - # define markers for clusters - markers = ['o', 'v', '^', '<', '>', '8', 's', 'p', '*', 'h', 'H', 'D', 'd'] - # build cosine distance matrix - cosine_distance = 1 - cosine_similarity(feature_matrix) - # dimensionality reduction using MDS - mds = MDS(n_components=2, dissimilarity="precomputed", - random_state=1) - # get coordinates of clusters in new low-dimensional space - plot_positions = mds.fit_transform(cosine_distance) - x_pos, y_pos = plot_positions[:, 0], plot_positions[:, 1] - # build cluster plotting data - cluster_color_map = {} - cluster_name_map = {} - # print(cluster_data) - for cluster_num, cluster_details in cluster_data.items(): - # assign cluster features to unique label - cluster_color_map[cluster_num] = generate_random_color() - cluster_name_map[cluster_num] = ', '.join(cluster_details['key_features'][:5]).strip() - # map each unique cluster label with its coordinates and books - cluster_plot_frame = pd.DataFrame({'x': x_pos, - 'y': y_pos, - 'label': m_data['Cluster'].values.tolist(), - 'query': m_data['query'].values.tolist() - }) - grouped_plot_frame = cluster_plot_frame.groupby('label') - # set plot figure size and axes - - plt.rcParams['font.sans-serif'] = ['SimHei'] # 用来正常显示中文标签 - plt.rcParams['axes.unicode_minus'] = False # 用来正常显示负号 - fig, ax = plt.subplots(figsize=plot_size) - ax.margins(0.05) - # plot each cluster using co-ordinates and titles - for cluster_num, cluster_frame in grouped_plot_frame: - marker = markers[cluster_num] if cluster_num < len(markers) \ - else np.random.choice(markers, size=1)[0] - ax.plot(cluster_frame['x'], cluster_frame['y'], - marker=marker, linestyle='', ms=12, - label=cluster_name_map[cluster_num], - color=cluster_color_map[cluster_num], mec='none') - ax.set_aspect('auto') - ax.tick_params(axis='x', which='both', bottom=False, top=False, - labelbottom='off') - ax.tick_params(axis='y', which='both', left=False, top=False, - labelleft=False) - fontP = FontProperties() - fontP.set_size(23) - ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.01), fancybox=True, - shadow=True, ncol=5, numpoints=1, prop=fontP) - # add labels as the film titles - for index in range(len(cluster_plot_frame)): - ax.text(cluster_plot_frame.ix[index]['x'], cluster_plot_frame.ix[index]['y'], - cluster_plot_frame.ix[index]['query'], size=20) - # show the plot - plt.title(self.start_time + ' to ' + self.end_time + 'chat data cluster point set') - path = '/roobo/soft/phpmyadmin/cluster_point.jpg' - plt.savefig(path) - return path - - def k_means(self, feature_matrix): - f_sse = [] - num_clusters = [] - for i in range(2, 21): - km = KMeans(n_clusters=i, max_iter=10000) - km.fit(feature_matrix) - clusters = km.labels_ - num_matrix = feature_matrix.todense() - sse = metrics.calinski_harabaz_score(num_matrix, clusters) - num_clusters.append(i) - f_sse.append(sse) - - pd_see = pd.Series(f_sse, index=num_clusters) - pct_see = pd_see.pct_change() - - fig, ax = plt.subplots() - ax.plot(num_clusters, f_sse, 'o-', c='orangered', label='clustering quality') - plt.legend(loc=2) - plt.xticks(num_clusters) - ax.set_xlabel("cluster number") - - ax.set_ylabel("coefficient") - - ax1 = ax.twinx() - ax1.plot(pct_see.values, 'o-', c='blue', label='gradient change') - ax1.set_ylabel("gradient") - plt.legend(loc=1) - - plt.title(self.start_time + " to " + self.end_time + " the analysis of clusters with different numbers") - path = '/roobo/soft/phpmyadmin/choice_num.jpg' - plt.savefig(path) - - # input_num = input('输入最优聚类数目：') - # best_num = int(input_num) - - self.f_sse = f_sse - return path - - def k_means_cluster(self, best_num): - - self.out_data = self.out_data + '\n' + "=" * 20 - self.out_data = self.out_data + "\n\n聚类效果分析：\n" - self.out_data = self.out_data + "\n聚类数目为：" + str(best_num) - - f_sse = self.f_sse - sse = f_sse[best_num] - km = KMeans(n_clusters=best_num, max_iter=10000) - km.fit(self.feature_matrix) - clusters = km.labels_ - self.chat_data['Cluster'] = clusters - - # 获取每个cluster的数量 - c = Counter(clusters) - - sort_c = sorted(c.items(), key=lambda c: c[0], reverse=False) - c.clear() - for key, value in sort_c: - c[key] = value - - self.out_data = self.out_data + '\nCalinski-Harabasz分数:' + str(sse) - self.out_data = self.out_data + '\n每个特征的数据量：\n' - self.out_data = self.out_data + (str(c.items())) - self.out_data = self.out_data + '\n' + "=" * 20 - cluster_data = self.get_cluster_data(clustering_obj=km, - m_data=self.chat_data, - feature_names=self.feature_names, - num_clusters=best_num, - topn_features=5) - - self.print_cluster_data(cluster_data) - - path = self.plot_clusters(feature_matrix=self.feature_matrix, cluster_data=cluster_data, m_data=self.chat_data, - plot_size=(40, 25)) - return path - - def ward_hierarchical_clustering(self, feature_matrix): - cosine_distance = 1 - cosine_similarity(feature_matrix) - linkage_matrix = ward(cosine_distance) - return linkage_matrix - - def plot_hierarchical_clusters(self, linkage_matrix, m_data, figure_size): - # set size - fig, ax = plt.subplots(figsize=figure_size) - m_titles = m_data['query'].values.tolist() - - # plot dendrogram - ax = dendrogram(linkage_matrix, orientation="left", labels=m_titles) - plt.tick_params(axis='x', - which='both', - bottom=False, - top=False, - labelbottom=False) - plt.tight_layout() - plt.title(self.start_time + ' to ' + self.end_time + 'chat data ward hierachical clusters') - path = '/roobo/soft/phpmyadmin/hierachical_clusters.jpg' - plt.savefig(path) - return path - - -app = Flask(__name__) -CORS(app, supports_credentials=True) - -data_cluster = Culter(start_time="2018-12-01 00:00:00", end_time="2018-12-02 00:00:00") - - -@app.route('/SPDAS/chat_function_analysis/choice1', methods=['POST']) -def choice(): - param = ({"time": "2018-12-01 00:00:00/2018-12-02 00:00:00"}) - return json.JSONEncoder().encode(param) - - -@app.route('/SPDAS/chat_function_analysis/choice2', methods=['POST']) -def choice_form(): - # 需要从request对象读取表单内容： - data = request.get_data() - json_re = json.loads(data) - - m_time = json_re['time'] - str_time = str(m_time) - m_time = str_time.split('/') - starttime = m_time[0] - endtime = m_time[1] - data_cluster = Culter(start_time=starttime, end_time=endtime) - data_cluster.feature_extraction_data() - image_path = data_cluster.k_means(data_cluster.feature_matrix) - path = ({"num_image": image_path}) - return json.JSONEncoder().encode(path) - - -@app.route('/SPDAS/chat_function_analysis/chat1', methods=['POST']) -def chat(): - param = ({"best_num": "2"}) - return json.JSONEncoder().encode(param) - - -@app.route('/SPDAS/chat_function_analysis/chat2', methods=['POST']) -def chat_form(): - # 需要从request对象读取表单内容： - data = request.get_data() - json_re = json.loads(data) - bestnum = json_re['best_num'] - image_path1 = data_cluster.k_means_cluster(best_num=bestnum) - - linkage_matrix = data_cluster.ward_hierarchical_clustering(data_cluster.feature_matrix) - - image_path2 = data_cluster.plot_hierarchical_clusters(linkage_matrix=linkage_matrix, m_data=data_cluster.chat_data, - figure_size=(16, 14)) - with open("/roobo/soft/phpmyadmin/chat_function_data.txt", 'w') as file: - file.writelines(data_cluster.out_data) - - path = ({"cluster_point": image_path1, "ward_image": image_path2}) - return json.JSONEncoder().encode(path) - - -if __name__ == '__main__': - app.run(debug=True, host='10.7.19.129', port=5000)