diff --git a/bottom_function/data_read.py b/bottom_function/data_read.py index 8a8940264bf0ca854a1f60a7f5660fab53093a4b..9c3fdb32ae3bb883dc3a33b70aa204f163e5339e 100644 --- a/bottom_function/data_read.py +++ b/bottom_function/data_read.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- # author:Li Mingjie time:2019/1/27 # Brief: read data import pandas as pd @@ -9,24 +9,46 @@ import re def read_data(datatype, starttime, endtime): csv_data = pd.DataFrame() m_data = pd.DataFrame() - tablename = "semantic_data_table" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") - csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + if datatype == "control": - m_data = csv_data[csv_data['classify'] == 'gree'] + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + m_data = csv_data[csv_data['classify'] == 'control'] elif datatype == "application": - m_data = csv_data[csv_data['classify'] == 'tencent'] + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) + m_data = csv_data[csv_data['classify'] == 'application'] elif datatype == "chat": + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) m_data = csv_data[csv_data['domain'] == 'chat'] elif datatype == "all": + tablename = "semantic_data_table" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) m_data = csv_data elif datatype == "error_control": - m_data = db.getData(tableName="control_error_data", startTime=starttime, endTime=endtime) + tablename = "control_error_data" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + m_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "error_application": - m_data = db.getData(tableName="application_error_data", startTime=starttime, endTime=endtime) + tablename = "application_error_data" + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") + m_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) else: print("The datatype you selected is incorrect. Please re-select it.") return -1 + m_data = m_data.reset_index(drop=True) + print(len(m_data)) return m_data @@ -34,20 +56,24 @@ def read_domain_data(datatype, starttime, endtime): csv_data = pd.DataFrame() if datatype == "control": tablename = "control_domain_data" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "application": tablename = "application_domain_data" - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName=tablename, startTime=starttime, endTime=endtime) elif datatype == "all": - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") gcsv_data = db.getData(tableName="control_domain_data", startTime=starttime, endTime=endtime) tcsv_data = db.getData(tableName="application_domain_data", startTime=starttime, endTime=endtime) tcsv_data = tcsv_data.drop(columns=['datetime']) csv_data = pd.concat([gcsv_data, tcsv_data], axis=1) + csv_data = csv_data.reset_index(drop=True) return csv_data @@ -62,7 +88,8 @@ def read_cost_time_data(datatype, starttime, endtime): time_dict = {} cost_time_dict = {} - db = qb.Schema(host="localhost", user="root", password="560193", mysqlName="semantic_data_schema", port="3306") + db = qb.Schema(host="localhost", user="560193", password="jay560193", mysqlName="semantic_data_schema", + port="3306") csv_data = db.getData(tableName="cost_time_data", startTime=starttime, endTime=endtime) for col in list(csv_data.columns)[1:]: gree_dict.clear() @@ -102,5 +129,3 @@ def read_cost_time_data(datatype, starttime, endtime): cost_time_dict.update({col: sort_data_list}) return cost_time_dict - - diff --git a/bottom_function/m_SQL.py b/bottom_function/m_SQL.py index 4969b762c67ce9650ee267f527d9b38c9a825790..42a3c63276fbf20f589335bcb14e6250f570e270 100644 --- a/bottom_function/m_SQL.py +++ b/bottom_function/m_SQL.py @@ -14,7 +14,7 @@ class Schema: self.mysqlName = mysqlName self.port = port # 打开数据库连接 - self.db = MySQLdb.connect(host=host, user=user, password=password, db=mysqlName,charset = 'utf8') + self.db = MySQLdb.connect(host=host, user=user, password=password, db=mysqlName, charset='utf8') # 使用 cursor() 方法创建一个游标对象 cursor self.cursor = self.db.cursor() @@ -82,16 +82,17 @@ class Schema: engine_address = "mysql+pymysql://" + self.user + ":" + self.password + "@" + self.host + ":" + self.port + "/" + self.mysqlName engine = create_engine(engine_address, echo=True) sql = 'SELECT * FROM ' + tableName + ' WHERE datetime >= "' + startTime + '" and datetime< "' + endTime + '"' + print (sql) data = pd.read_sql_query(sql, con=engine) + print ("****%d"%(len(data))) return data def delData(self, tableName, keyWord): - word='"'+keyWord+'"' - sql = 'DELETE FROM {tableName} WHERE query={keys}'.format(tableName=tableName,keys=word) + word = '"' + keyWord + '"' + sql = 'DELETE FROM {tableName} WHERE query={keys}'.format(tableName=tableName, keys=word) self.cursor.execute(sql) self.commit() - # 关闭数据库连接 def closeConnect(self): self.db.close() @@ -102,6 +103,3 @@ class Schema: def delete(self, tableName): self.db.cursor("DELETE * FROM {}".format(tableName)) - - - diff --git a/processing_unisound_logfile.py b/processing_unisound_logfile.py index 088c830826ae54572c58111005df479cf88eb0ed..4bc5aa0008057d607a63c26a340b197d89210be0 100644 --- a/processing_unisound_logfile.py +++ b/processing_unisound_logfile.py @@ -77,7 +77,7 @@ class timing_processing: macw = 'null' macv = 'null' datetime = js_data['ctoc query log']['time'] - qu = js_data['ctoc query log']['request']['reqparam']['nluret']['asr_recongize'] + qu = js_data['ctoc query log']['response']['asr_recongize'] qu = re.sub(',', '', qu) if 'status' in js_data['ctoc query log']['response']: cla = 'control'