Skip to content

T
titan
Commit 9279a73a by jscat
Options

nlp crawler: 

从python2转换成python3
parent fa424284
正在显示 包含 38 行增加 和 45 行删除
nlp_strategy/crawler_sina_breaking/abstock.py
...@@ -6,7 +6,6 @@ import collections ...@@ -6,7 +6,6 @@ import collections
import csv import csv
import string import string
import urllib import urllib
import urllib2
import json import json
import time import time
import MySQLdb import MySQLdb
...@@ -14,17 +13,19 @@ import datetime ...@@ -14,17 +13,19 @@ import datetime
import logging import logging
import re import re
import sys, os import sys, os
reload(sys) import pandas as pd
sys.setdefaultencoding('utf8')
import math import math
#str1=r"平安银行(000001)2016-10-11融资融券信息显示，千红制药\*ST常林融资余额3,158,497,830元，龙江交通63,234,872\*ST雪博元，乐视网融资买入额天赐材料苏州固得68,490,198元，宝鹰股份融资偿苏州固锝还额65,808,891元，融资净买额2,681,307元，融券余量6,910,915股，融券卖出量18,000股，融券偿还量100股，融资融券余额3,221,732,702元。平安银行融资融券详细信息如下表" #str1=r"平安银行(000001)2016-10-11融资融券信息显示，千红制药\*ST常林融资余额3,158,497,830元，龙江交通63,234,872\*ST雪博元，乐视网融资买入额天赐材料苏州固得68,490,198元，宝鹰股份融资偿苏州固锝还额65,808,891元，融资净买额2,681,307元，融券余量6,910,915股，融券卖出量18,000股，融券偿还量100股，融资融券余额3,221,732,702元。平安银行融资融券详细信息如下表"
csvfile = open('stocknamelist.csv', "rb") # csvfile = open('stocknamelist.csv', "rb")
reader = csv.reader(csvfile) # reader = csv.reader(csvfile)
stocknamelist= list(reader) # stocknamelist= list(reader)
df=pd.read_csv('stocknamelist.csv', sep=',',header=None)
stocknamelist = df.values.tolist()
namecodedict={} namecodedict={}
restr='' restr=''
for stock in stocknamelist: for stock in stocknamelist:
namecodedict[stock[1]]=unicode(stock[0]) namecodedict[stock[1]]=str(stock[0])
restr+=(stock[1]) restr+=(stock[1])
restr+='|' restr+='|'
restr=restr.lstrip(' ').rstrip('|') restr=restr.lstrip(' ').rstrip('|')
......
nlp_strategy/crawler_sina_breaking/apiUtil.py
...@@ -4,8 +4,6 @@ import requests ...@@ -4,8 +4,6 @@ import requests
import json import json
import urllib import urllib
import sys import sys
reload(sys)
sys.setdefaultencoding('utf8')
def http_post_request(url, params,add_to_headers=None): def http_post_request(url, params,add_to_headers=None):
...@@ -16,11 +14,11 @@ def http_post_request(url, params,add_to_headers=None): ...@@ -16,11 +14,11 @@ def http_post_request(url, params,add_to_headers=None):
if add_to_headers: if add_to_headers:
headers.update(add_to_headers) headers.update(add_to_headers)
postdata = json.dumps(params,ensure_ascii=False) postdata = json.dumps(params,ensure_ascii=False)
print postdata print (postdata)
response = requests.post(url, data=params, headers=headers, timeout=5,verify=False) response = requests.post(url, data=params, headers=headers, timeout=5,verify=False)
try: try:
if response.status_code == 200: if response.status_code == 200:
print json.dumps(response.json(), encoding="UTF-8", ensure_ascii=False) print (json.dumps(response.json(), encoding="UTF-8", ensure_ascii=False))
return response.json() return response.json()
else: else:
return json.dumps({},ensure_ascii=False) return json.dumps({},ensure_ascii=False)
......
nlp_strategy/crawler_sina_breaking/install.txt
pymysql pip install pymysql
requests pip install requests
pip install mysqlclient
nlp_strategy/crawler_sina_breaking/pyProfitStat.py
#coding=utf-8 #coding=utf-8
import urllib import urllib
import urllib2
import json import json
import time import time
import datetime import datetime
...@@ -11,21 +10,20 @@ import logging ...@@ -11,21 +10,20 @@ import logging
import sys,os import sys,os
import csv import csv
import rdcalendar import rdcalendar
reload(sys)
sys.setdefaultencoding('utf8') sys.setdefaultencoding('utf8')
def stat_calcprofit(ndays): def stat_calcprofit(ndays):
# type: (object) -> object # type: (object) -> object
# start = time.time() # start = time.time()
calenarray = rdcalendar.rdcalen() calenarray = rdcalendar.rdcalen()
print calenarray print (calenarray)
timenow = datetime.date.today() timenow = datetime.date.today()
str_time = str(timenow) str_time = str(timenow)
list_time = ''.join(str_time.split('-')) list_time = ''.join(str_time.split('-'))
try: try:
num = calenarray.index(list_time) num = calenarray.index(list_time)
except ValueError, e: except ValueError as e:
print "Today is not a tradeday!" print ("Today is not a tradeday!")
return return
pretime = calenarray[num + ndays] pretime = calenarray[num + ndays]
...@@ -40,10 +38,10 @@ def stat_calcprofit(ndays): ...@@ -40,10 +38,10 @@ def stat_calcprofit(ndays):
charset="utf8" charset="utf8"
) )
cur = conn.cursor() cur = conn.cursor()
except MySQLdb.Error, e: except MySQLdb.Error as e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1]) print ("Mysql Error %d: %s" % (e.args[0], e.args[1]))
strSql = "SELECT SUM(sell_price-buy_price) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime strSql = "SELECT SUM(sell_price-buy_price) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql print (strSql)
cur.execute(strSql) cur.execute(strSql)
conn.commit() conn.commit()
result = cur.fetchall() result = cur.fetchall()
...@@ -51,34 +49,34 @@ def stat_calcprofit(ndays): ...@@ -51,34 +49,34 @@ def stat_calcprofit(ndays):
if profit is None: if profit is None:
profit = 0 profit = 0
print profit print (profit)
strSql = "SELECT COUNT(DISTINCT(news_id)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime strSql = "SELECT COUNT(DISTINCT(news_id)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql print (strSql)
cur.execute(strSql) cur.execute(strSql)
conn.commit() conn.commit()
result = cur.fetchall() result = cur.fetchall()
numNews = list(result)[0][0] numNews = list(result)[0][0]
print numNews print (numNews)
strSql = "SELECT COUNT(DISTINCT(stock_code)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime strSql = "SELECT COUNT(DISTINCT(stock_code)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql print (strSql)
cur.execute(strSql) cur.execute(strSql)
conn.commit() conn.commit()
result = cur.fetchall() result = cur.fetchall()
numStock = list(result)[0][0] numStock = list(result)[0][0]
print numStock print (numStock)
now = time.localtime() now = time.localtime()
str_time = time.strftime('%Y-%m-%d', now) str_time = time.strftime('%Y-%m-%d', now)
sqlCount = "SELECT COUNT(*) FROM tbl_stats WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")" sqlCount = "SELECT COUNT(*) FROM tbl_stats WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")"
print sqlCount print (sqlCount)
#判断是否已存在 #判断是否已存在
cur.execute(sqlCount) cur.execute(sqlCount)
conn.commit() conn.commit()
result = cur.fetchall() result = cur.fetchall()
numCount = list(result)[0][0]+0 numCount = list(result)[0][0]+0
print numCount print (numCount)
# print "matchid:" + str(matchid) # print "matchid:" + str(matchid)
# print "sellprice: " + str(sellprice) # print "sellprice: " + str(sellprice)
...@@ -89,10 +87,10 @@ def stat_calcprofit(ndays): ...@@ -89,10 +87,10 @@ def stat_calcprofit(ndays):
else: else:
strSql = "update tbl_stats set profit=\""+str(profit)+"\",num_news=\""+str(numNews)+"\",num_stocks=\""+str(numStock)+\ strSql = "update tbl_stats set profit=\""+str(profit)+"\",num_news=\""+str(numNews)+"\",num_stocks=\""+str(numStock)+\
"\" WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")" "\" WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")"
print strSql print (strSql)
cur.execute(strSql) cur.execute(strSql)
conn.commit() conn.commit()
print "success" print ("success")
ndays = 2 ndays = 2
stat_calcprofit(ndays) stat_calcprofit(ndays)
......
nlp_strategy/crawler_sina_breaking/searchprice.py
import urllib import urllib
import urllib2
import json import json
import time import time
import MySQLdb import MySQLdb
...@@ -7,9 +6,6 @@ import datetime ...@@ -7,9 +6,6 @@ import datetime
import logging import logging
import sys,os import sys,os
import csv import csv
reload(sys)
sys.setdefaultencoding('utf8')
def getprice(stockcode,N): def getprice(stockcode,N):
code=stockcode code=stockcode
...@@ -29,15 +25,15 @@ def getprice(stockcode,N): ...@@ -29,15 +25,15 @@ def getprice(stockcode,N):
post_data = { post_data = {
} }
data = urllib.urlencode(post_data) data = urllib.urlencode(post_data)
req = urllib2.Request( req = urllib.Request(
url=url, url=url,
data="", data="",
headers=headers headers=headers
) )
try: try:
result = urllib2.urlopen(req) result = urllib.urlopen(req)
except: except:
print "urlopen doesn't work, sina API not work" print("urlopen doesn't work, sina API not work")
exit() exit()
text = result.read() text = result.read()
textlist = text.split('"') textlist = text.split('"')
......
nlp_strategy/crawler_sina_breaking/stocknamelist.csv
300104,乐视网 300104,乐视网
002289,\*ST宇顺 002289,\*ST宇顺
002448,中原内配 002448,中原内配
600421,仰帆控股 600421,仰帆控股
......
nlp_strategy/crawler_sina_breaking/writesell.py
...@@ -5,20 +5,18 @@ import datetime ...@@ -5,20 +5,18 @@ import datetime
import time import time
import searchprice import searchprice
import rdcalendar import rdcalendar
reload(sys)
sys.setdefaultencoding('utf8')
def wr_sellprice(): def wr_sellprice():
# start = time.time() # start = time.time()
calenarray=rdcalendar.rdcalen() calenarray=rdcalendar.rdcalen()
print calenarray print (calenarray)
timenow = datetime.date.today() timenow = datetime.date.today()
str_time = str(timenow) str_time = str(timenow)
list_time = ''.join(str_time.split('-')) list_time = ''.join(str_time.split('-'))
try: try:
num=calenarray.index(list_time) num=calenarray.index(list_time)
except ValueError,e: except ValueError as e:
print "Today is not a tradeday!" print ("Today is not a tradeday!")
return return
pretime=calenarray[num+2] pretime=calenarray[num+2]
...@@ -33,8 +31,8 @@ def wr_sellprice(): ...@@ -33,8 +31,8 @@ def wr_sellprice():
charset="utf8" charset="utf8"
) )
cur = conn.cursor() cur = conn.cursor()
except MySQLdb.Error, e: except MySQLdb.Error as e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1]) print ("Mysql Error %d: %s" % (e.args[0], e.args[1]))
cur.execute( cur.execute(
"select * from tbl_match where news_fk_id in (select id from tbl_news where TO_DAYS(news_time)=TO_DAYS(%s) and newsUptrendMan=1) and buy_price is not null" %pretime) "select * from tbl_match where news_fk_id in (select id from tbl_news where TO_DAYS(news_time)=TO_DAYS(%s) and newsUptrendMan=1) and buy_price is not null" %pretime)
conn.commit() conn.commit()
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论