From 9730a1f7464bcb77823fa0ad580e906316336a5f Mon Sep 17 00:00:00 2001 From: limingjie <560193@gree.com.cn> Date: Wed, 15 May 2019 14:19:51 +0800 Subject: [PATCH] jayling --- bottom_function/data_read.py | 53 +++++++++++++++++++++++++--------- bottom_function/m_SQL.py | 12 ++++---- processing_unisound_logfile.py | 2 +- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/bottom_function/data_read.py b/bottom_function/data_read.py index 8a89402..9c3fdb3 100644 --- a/bottom_function/data_read.py +++ b/bottom_function/data_read.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # author：Li Mingjie time:2019/1/27 # Brief: read data import pandas as pd @@ -9,24 +9,46 @@ import re def read_data(datatype, starttime, endtime): csv_data = pd.DataFrame() m_data = pd.DataFrame() - tablename = "semantic_data_table" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") - csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + if datatype == "control": - m_data = csv_data[csv_data['classify'] == 'gree'] + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + m_data = csv_data[csv_data['classify'] == 'control'] elif datatype == "application": - m_data = csv_data[csv_data['classify'] == 'tencent'] + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + m_data = csv_data[csv_data['classify'] == 'application'] elif datatype == "chat": + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) m_data = csv_data[csv_data['domain'] == 'chat'] elif datatype == "all": + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) m_data = csv_data elif datatype == "error_control": - m_data = db.getData(tableName="control_error_data", startTime=starttime, endTime=endtime) + tablename = "control_error_data" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + m_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "error_application": - m_data = db.getData(tableName="application_error_data", startTime=starttime, endTime=endtime) + tablename = "application_error_data" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + m_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) else: print("The datatype you selected is incorrect. Please re-select it.") return -1 + m_data = m_data.reset_index(drop=True) + print(len(m_data)) return m_data @@ -34,20 +56,24 @@ def read_domain_data(datatype, starttime, endtime): csv_data = pd.DataFrame() if datatype == "control": tablename = "control_domain_data" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "application": tablename = "application_domain_data" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "all": - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") gcsv_data = db.getData(tableName="control_domain_data", startTime=starttime, endTime=endtime) tcsv_data = db.getData(tableName="application_domain_data", startTime=starttime, endTime=endtime) tcsv_data = tcsv_data.drop(columns=['datetime']) csv_data = pd.concat([gcsv_data, tcsv_data], axis=1) + csv_data = csv_data.reset_index(drop=True) return csv_data @@ -62,7 +88,8 @@ def read_cost_time_data(datatype, starttime, endtime): time_dict = {} cost_time_dict = {} - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName="cost_time_data", startTime=starttime, endTime=endtime) for col in list(csv_data.columns)[1:]: gree_dict.clear() @@ -102,5 +129,3 @@ def read_cost_time_data(datatype, starttime, endtime): cost_time_dict.update({col: sort_data_list}) return cost_time_dict - - diff --git a/bottom_function/m_SQL.py b/bottom_function/m_SQL.py index 4969b76..42a3c63 100644 --- a/bottom_function/m_SQL.py +++ b/bottom_function/m_SQL.py @@ -14,7 +14,7 @@ class Schema: self.mysqlName = mysqlName self.port = port # 打开数据库连接 - self.db = MySQLdb.connect(host=host, user=user, password=password, db=mysqlName,charset = 'utf8') + self.db = MySQLdb.connect(host=host, user=user, password=password, db=mysqlName, charset='utf8') # 使用 cursor() 方法创建一个游标对象 cursor self.cursor = self.db.cursor() @@ -82,16 +82,17 @@ class Schema: engine_address = "mysql+pymysql://" + self.user + ":" + self.password + "@" + self.host + ":" + self.port + "/" + self.mysqlName engine = create_engine(engine_address, echo=True) sql = 'SELECT * FROM ' + tableName + ' WHERE datetime >= "' + startTime + '" and datetime< "' + endTime + '"' + print (sql) data = pd.read_sql_query(sql, con=engine) + print ("****%d"%(len(data))) return data def delData(self, tableName, keyWord): - word='"'+keyWord+'"' - sql = 'DELETE FROM {tableName} WHERE query={keys}'.format(tableName=tableName,keys=word) + word = '"' + keyWord + '"' + sql = 'DELETE FROM {tableName} WHERE query={keys}'.format(tableName=tableName, keys=word) self.cursor.execute(sql) self.commit() - # 关闭数据库连接 def closeConnect(self): self.db.close() @@ -102,6 +103,3 @@ class Schema: def delete(self, tableName): self.db.cursor("DELETE * FROM {}".format(tableName)) - - - diff --git a/processing_unisound_logfile.py b/processing_unisound_logfile.py index 088c830..4bc5aa0 100644 --- a/processing_unisound_logfile.py +++ b/processing_unisound_logfile.py @@ -77,7 +77,7 @@ class timing_processing: macw = 'null' macv = 'null' datetime = js_data['ctoc query log']['time'] - qu = js_data['ctoc query log']['request']['reqparam']['nluret']['asr_recongize'] + qu = js_data['ctoc query log']['response']['asr_recongize'] qu = re.sub('，', '', qu) if 'status' in js_data['ctoc query log']['response']: cla = 'control' -- GitLab