Skip to content

T
titan
Commit 9279a73a by jscat
Options

nlp crawler: 

从python2转换成python3
parent fa424284
正在显示 包含 38 行增加 和 45 行删除
nlp_strategy/crawler_sina_breaking/abstock.py
......@@ -6,7 +6,6 @@ import collections
import csv
import string
import urllib
import urllib2
import json
import time
import MySQLdb
......@@ -14,17 +13,19 @@ import datetime
import logging
import re
import sys, os
reload(sys)
sys.setdefaultencoding('utf8')
import pandas as pd
import math
#str1=r"平安银行(000001)2016-10-11融资融券信息显示，千红制药\*ST常林融资余额3,158,497,830元，龙江交通63,234,872\*ST雪博元，乐视网融资买入额天赐材料苏州固得68,490,198元，宝鹰股份融资偿苏州固锝还额65,808,891元，融资净买额2,681,307元，融券余量6,910,915股，融券卖出量18,000股，融券偿还量100股，融资融券余额3,221,732,702元。平安银行融资融券详细信息如下表"
csvfile = open('stocknamelist.csv', "rb")
reader = csv.reader(csvfile)
stocknamelist= list(reader)
# csvfile = open('stocknamelist.csv', "rb")
# reader = csv.reader(csvfile)
# stocknamelist= list(reader)
df=pd.read_csv('stocknamelist.csv', sep=',',header=None)
stocknamelist = df.values.tolist()
namecodedict={}
restr=''
for stock in stocknamelist:
namecodedict[stock[1]]=unicode(stock[0])
namecodedict[stock[1]]=str(stock[0])
restr+=(stock[1])
restr+='|'
restr=restr.lstrip(' ').rstrip('|')
......
nlp_strategy/crawler_sina_breaking/apiUtil.py
......@@ -4,8 +4,6 @@ import requests
import json
import urllib
import sys
reload(sys)
sys.setdefaultencoding('utf8')
def http_post_request(url, params,add_to_headers=None):
......@@ -16,11 +14,11 @@ def http_post_request(url, params,add_to_headers=None):
if add_to_headers:
headers.update(add_to_headers)
postdata = json.dumps(params,ensure_ascii=False)
print postdata
print (postdata)
response = requests.post(url, data=params, headers=headers, timeout=5,verify=False)
try:
if response.status_code == 200:
print json.dumps(response.json(), encoding="UTF-8", ensure_ascii=False)
print (json.dumps(response.json(), encoding="UTF-8", ensure_ascii=False))
return response.json()
else:
return json.dumps({},ensure_ascii=False)
......
nlp_strategy/crawler_sina_breaking/install.txt
pymysql
requests
pip install pymysql
pip install requests
pip install mysqlclient
nlp_strategy/crawler_sina_breaking/pyProfitStat.py
#coding=utf-8
import urllib
import urllib2
import json
import time
import datetime
......@@ -11,21 +10,20 @@ import logging
import sys,os
import csv
import rdcalendar
reload(sys)
sys.setdefaultencoding('utf8')
def stat_calcprofit(ndays):
# type: (object) -> object
# start = time.time()
calenarray = rdcalendar.rdcalen()
print calenarray
print (calenarray)
timenow = datetime.date.today()
str_time = str(timenow)
list_time = ''.join(str_time.split('-'))
try:
num = calenarray.index(list_time)
except ValueError, e:
print "Today is not a tradeday!"
except ValueError as e:
print ("Today is not a tradeday!")
return
pretime = calenarray[num + ndays]
......@@ -40,10 +38,10 @@ def stat_calcprofit(ndays):
charset="utf8"
)
cur = conn.cursor()
except MySQLdb.Error, e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
except MySQLdb.Error as e:
print ("Mysql Error %d: %s" % (e.args[0], e.args[1]))
strSql = "SELECT SUM(sell_price-buy_price) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql
print (strSql)
cur.execute(strSql)
conn.commit()
result = cur.fetchall()
......@@ -51,34 +49,34 @@ def stat_calcprofit(ndays):
if profit is None:
profit = 0
print profit
print (profit)
strSql = "SELECT COUNT(DISTINCT(news_id)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql
print (strSql)
cur.execute(strSql)
conn.commit()
result = cur.fetchall()
numNews = list(result)[0][0]
print numNews
print (numNews)
strSql = "SELECT COUNT(DISTINCT(stock_code)) FROM view_match_news_stock WHERE TO_DAYS(%s)=TO_DAYS(news_time) AND newsUptrendMan=1 AND buy_price > 0.0 AND sell_price>0.0" % pretime
print strSql
print (strSql)
cur.execute(strSql)
conn.commit()
result = cur.fetchall()
numStock = list(result)[0][0]
print numStock
print (numStock)
now = time.localtime()
str_time = time.strftime('%Y-%m-%d', now)
sqlCount = "SELECT COUNT(*) FROM tbl_stats WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")"
print sqlCount
print (sqlCount)
#判断是否已存在
cur.execute(sqlCount)
conn.commit()
result = cur.fetchall()
numCount = list(result)[0][0]+0
print numCount
print (numCount)
# print "matchid:" + str(matchid)
# print "sellprice: " + str(sellprice)
......@@ -89,10 +87,10 @@ def stat_calcprofit(ndays):
else:
strSql = "update tbl_stats set profit=\""+str(profit)+"\",num_news=\""+str(numNews)+"\",num_stocks=\""+str(numStock)+\
"\" WHERE DATE_FORMAT(create_datetime, \"%Y-%m-%d\")=DATE_FORMAT(\""+str(str_time)+"\", \"%Y-%m-%d\")"
print strSql
print (strSql)
cur.execute(strSql)
conn.commit()
print "success"
print ("success")
ndays = 2
stat_calcprofit(ndays)
......
nlp_strategy/crawler_sina_breaking/searchprice.py
import urllib
import urllib2
import json
import time
import MySQLdb
......@@ -7,9 +6,6 @@ import datetime
import logging
import sys,os
import csv
reload(sys)
sys.setdefaultencoding('utf8')
def getprice(stockcode,N):
code=stockcode
......@@ -29,15 +25,15 @@ def getprice(stockcode,N):
post_data = {
}
data = urllib.urlencode(post_data)
req = urllib2.Request(
req = urllib.Request(
url=url,
data="",
headers=headers
)
try:
result = urllib2.urlopen(req)
result = urllib.urlopen(req)
except:
print "urlopen doesn't work, sina API not work"
print("urlopen doesn't work, sina API not work")
exit()
text = result.read()
textlist = text.split('"')
......
nlp_strategy/crawler_sina_breaking/stocknamelist.csv
300104,乐视网
300104,乐视网
002289,\*ST宇顺
002448,中原内配
600421,仰帆控股
......
nlp_strategy/crawler_sina_breaking/writesell.py
......@@ -5,20 +5,18 @@ import datetime
import time
import searchprice
import rdcalendar
reload(sys)
sys.setdefaultencoding('utf8')
def wr_sellprice():
# start = time.time()
calenarray=rdcalendar.rdcalen()
print calenarray
print (calenarray)
timenow = datetime.date.today()
str_time = str(timenow)
list_time = ''.join(str_time.split('-'))
try:
num=calenarray.index(list_time)
except ValueError,e:
print "Today is not a tradeday!"
except ValueError as e:
print ("Today is not a tradeday!")
return
pretime=calenarray[num+2]
......@@ -33,8 +31,8 @@ def wr_sellprice():
charset="utf8"
)
cur = conn.cursor()
except MySQLdb.Error, e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
except MySQLdb.Error as e:
print ("Mysql Error %d: %s" % (e.args[0], e.args[1]))
cur.execute(
"select * from tbl_match where news_fk_id in (select id from tbl_news where TO_DAYS(news_time)=TO_DAYS(%s) and newsUptrendMan=1) and buy_price is not null" %pretime)
conn.commit()
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论