update

1. 添加update_log功能

titan_data/titan_data_function.py

@@ -2,11 +2,14 @@ import pandas as pd
 import tushare as ts
 import baostock as bs
 import pymysql
+import time
 import calendar
+from datetime import datetime
 import titan_data_settings as settings
 from urllib import parse
 from sqlalchemy import create_engine
 from titan_table_structure import Base
+from dateutil.relativedelta import relativedelta
 
 '''
 author: jscat 2021/02/02
@@ -22,20 +25,25 @@ class Data:
     def __init__(self, token):
         # 创建请求数据线程
         self.token = token
-        self.engine = create_engine(settings.mysql_url, encoding='utf-8', echo=True)
+        self.engine = create_engine(settings.mysql_url, encoding='utf-8', echo=False)
 
+    # truncate数据并且append
     def truncate_append(self, data, table_name):
         """删除mysql表所有数据，to_sql追加新数据"""
         conn = self.engine.connect()
         conn.execute('truncate ' + table_name)
         print(data)
         data.to_sql(table_name, self.engine, if_exists='append', index=False)
-        self.engine.dispose()
 
+    # keep数据并且append
     def keep_append(self, data, table_name):
         """保留mysql表所有数据，to_sql追加新数据"""
         data.to_sql(table_name, self.engine, if_exists='append', index=False)
-        self.engine.dispose()
+
+    # 同步更新日志表
+    def update_log(self, table_name, info):
+        data = pd.DataFrame({'TARGET_TABLE': table_name, 'UPDATE_INFO': info, 'CREATE_DT': str(datetime.now())}, index=[0])
+        data.to_sql("tbl_update_log", self.engine, if_exists='append', index=False)
 
     '''
     start_date='20210101', end_date='20311231'
@@ -83,6 +91,8 @@ class Data:
             self.truncate_append(df_all, 'tbl_AShareCalendar')
         else:
             self.keep_append(df_all, 'tbl_AShareCalendar')
+        self.update_log('tbl_AShareCalendar', "update record: "+str(start_date)+"_"+str(end_date))
+        self.engine.dispose();
         # print('{}成功导入数据库'.format(date))
 
     # Deprecated
@@ -137,6 +147,7 @@ class Data:
         :param date:
         :return:本月第一天日期和本月最后一天日期
         """
+        date = str(date)
         if date.count('-') != 2:
             raise ValueError('- is error')
         year, month = str(date).split('-')[0], str(date).split('-')[1]
@@ -148,10 +159,12 @@ class Data:
 if __name__=='__main__':
     data = Data(settings.ts_token)
     Base.metadata.create_all(data.engine)
-    data.get_AShareCalendar("2015-01-01", "2015-12-31", 0)
-    data.get_AShareCalendar("2016-01-01", "2016-12-31", 0)
-    data.get_AShareCalendar("2017-01-01", "2017-12-31", 0)
-    data.get_AShareCalendar("2018-01-01", "2018-12-31", 0)
-    data.get_AShareCalendar("2019-01-01", "2019-12-31", 0)
-    data.get_AShareCalendar("2020-01-01", "2020-12-31", 0)
+    start = "2019-02-01"
+    for i in range(11+2):
+        date = pd.to_datetime(start) + relativedelta(months=+i)  # 当前日期往后推i个月
+        date_str = date.strftime("%Y-%m-%d")
+        start_date, end_date = data.get_current_month_start_and_end(date)
+        print(start_date, end_date)
+        data.get_AShareCalendar(start_date=start_date, end_date=end_date, type=1)
+        time.sleep(10)
 

titan_data/titan_table_structure.py

@@ -9,7 +9,7 @@ from sqlalchemy import Column, Integer, String
 import titan_data_settings as settings
 
 # 创建连接
-engine = create_engine(settings.mysql_url, encoding='utf-8', echo=True)
+engine = create_engine(settings.mysql_url, encoding='utf-8', echo=False)
 # 生成orm基类
 Base = declarative_base()