titan-data-sync:

1. 规范参数定义
2. 为exception操作添加邮件通知接口

titan_cloud/titan-data-sync/titan_data_basic.py

@@ -16,7 +16,7 @@ class Demo:
     def __init__(self):
     def __init__(self):
         # 创建请求数据线程
         # 创建请求数据线程
         self._thread_data = Thread(target=self.get_data)
         self._thread_data = Thread(target=self.get_data)
-        self.go = Data(settings.ts_token)
+        self.go = Data(settings.TS_TOKEN)
 
 
     def start(self):
     def start(self):
         """
         """

titan_cloud/titan-data-sync/titan_data_function.py

 import pandas as pd
 import pandas as pd
 import tushare as ts
 import tushare as ts
 import baostock as bs
 import baostock as bs
-import pymysql
-import traceback
+import smtplib
+from email.mime.text import MIMEText
 import time
 import time
 import calendar
 import calendar
 from datetime import datetime
 from datetime import datetime
@@ -29,7 +29,7 @@ class Data:
     def __init__(self, token):
     def __init__(self, token):
         # 创建请求数据线程
         # 创建请求数据线程
         self.token = token
         self.token = token
-        self.engine = create_engine(settings.mysql_url, encoding='utf-8', echo=False)
+        self.engine = create_engine(settings.DB_CONN_URL, encoding='utf-8', echo=False)
 
 
     # delete数据并且append
     # delete数据并且append
     def delete_append(self, data, table_name, info):
     def delete_append(self, data, table_name, info):
@@ -42,6 +42,7 @@ class Data:
                 self.update_log(table_name, conn, info)
                 self.update_log(table_name, conn, info)
         except Exception as ee:
         except Exception as ee:
             logging.error('delete_append failed: '+info, ee)
             logging.error('delete_append failed: '+info, ee)
+            self.sendMail('delete_append failed: '+info, ee.args)
             self.error_log(table_name, self.engine, info, ee.args)
             self.error_log(table_name, self.engine, info, ee.args)
 
 
     # keep数据并且append
     # keep数据并且append
@@ -54,6 +55,7 @@ class Data:
                 self.update_log(table_name, conn, info)
                 self.update_log(table_name, conn, info)
         except Exception as ee:
         except Exception as ee:
             logging.error('keep_append failed: '+info, ee)
             logging.error('keep_append failed: '+info, ee)
+            self.sendMail('delete_append failed: ' + info, ee.args)
             self.error_log(table_name, self.engine, info, ee.args)
             self.error_log(table_name, self.engine, info, ee.args)
 
 
     # 同步更新日志表
     # 同步更新日志表
@@ -170,7 +172,7 @@ class Data:
     # Deprecated
     # Deprecated
     def get_all_stockdata(self, start_date, end_date):
     def get_all_stockdata(self, start_date, end_date):
         # 此方法连接数据库，密码可以输入特殊字符串
         # 此方法连接数据库，密码可以输入特殊字符串
-        engine = create_engine(settings.mysql_url)
+        engine = create_engine(settings.DB_CONN_URL)
         print('数据库连接成功')
         print('数据库连接成功')
         ts.set_token(self.token)
         ts.set_token(self.token)
         pro = ts.pro_api()
         pro = ts.pro_api()
@@ -228,8 +230,27 @@ class Data:
         end_date = '%s-%s-%s' % (year, month, end)
         end_date = '%s-%s-%s' % (year, month, end)
         return start_date, end_date
         return start_date, end_date
 
 
+    # send email
+    def sendMail(self, subject, info):
+
+        content = info
+        msg = MIMEText(content)
+
+        msg['Subject'] = subject
+        msg['From'] = settings.MAIL_FROM
+        msg['To'] = settings.MAIL_TO
+
+        ## smtp ssl port 465
+        smtpServer = smtplib.SMTP_SSL(settings.MAIL_HOST, 465)  # SMTP_SSL
+        smtpServer.login(settings.MAIL_FROM, settings.MAIL_PASS)
+        smtpServer.sendmail(settings.MAIL_FROM, settings.MAIL_TO, msg.as_string())
+        smtpServer.quit()
+        'send success by port 465'
+        print("sendMail pass")
+
+
 if __name__=='__test__':
 if __name__=='__test__':
-    data = Data(settings.ts_token)
+    data = Data(settings.TS_TOKEN)
     start = "2020-02-01"
     start = "2020-02-01"
     for i in range(1):
     for i in range(1):
         date = pd.to_datetime(start) + relativedelta(months=+i)  # 当前日期往后推i个月
         date = pd.to_datetime(start) + relativedelta(months=+i)  # 当前日期往后推i个月
@@ -240,8 +261,13 @@ if __name__=='__test__':
         time.sleep(10)
         time.sleep(10)
     print("finish")
     print("finish")
 
 
-if __name__=='__main__':
-    data = Data(settings.ts_token)
+if __name__=='__test1__':
+    data = Data(settings.TS_TOKEN)
     date = datetime.now()
     date = datetime.now()
     data.get_AShareDescription(date=date, type=0)
     data.get_AShareDescription(date=date, type=0)
     print("finish")
     print("finish")
+
+if __name__=='__main__':
+    data = Data(settings.TS_TOKEN)
+    data.sendMail("Titan-Data-Sync error", "update tbl_AShareDescription failure")
+    print("finish")
\ No newline at end of file

titan_cloud/titan-data-sync/titan_data_settings.py

@@ -5,10 +5,18 @@ import pymysql
 from urllib import parse
 from urllib import parse
 from sqlalchemy import create_engine
 from sqlalchemy import create_engine
 
 
-ts_token  = "33261e14a0f45680d6afdc86b85bc0c4c80ba6b8dc199d7313a30838"
-mysql_url = "mysql+pymysql://sharpe_dev:123456@47.99.110.89:3306/sharpe?charset=utf8"
-ip_address = "47.99.110.89"
-ip_port = 3306
-db_name = "sharpe"
-user_name = "sharpe_dev"
-password ="123456"
+TS_TOKEN = "33261e14a0f45680d6afdc86b85bc0c4c80ba6b8dc199d7313a30838"
+DB_HOST = "47.99.110.89"
+DB_PORT = 3306
+DB_NAME = "sharpe"
+DB_USER = "sharpe_dev"
+DB_PASS = "123456"
+
\ No newline at end of file
+# DB driver: MySQL-python
+DB_CONN_URL = 'mysql+pymysql://%s:%s@%s:%d/%s?charset=utf8' % (DB_USER, DB_PASS, DB_HOST, DB_PORT, DB_NAME)
+
+# Mail setting
+MAIL_HOST = 'smtp.163.com'
+MAIL_FROM = 'titanplus2021@163.com'
+MAIL_PASS = "DWSBWWENHZPEUZKB"
+MAIL_TO = 'jesu@digjob.net'
\ No newline at end of file

titan_cloud/titan-data-sync/titan_table_structure.py

@@ -9,7 +9,7 @@ from sqlalchemy import Column, Integer, String
 import titan_data_settings as settings
 import titan_data_settings as settings
 
 
 # 创建连接
 # 创建连接
-engine = create_engine(settings.mysql_url, encoding='utf-8', echo=False)
+engine = create_engine(settings.DB_CONN_URL, encoding='utf-8', echo=False)
 # 生成orm基类
 # 生成orm基类
 Base = declarative_base()
 Base = declarative_base()