content+os+mysql

content+os+mysql

1 #coding:utf-8

2 importrequests3 importos4 importjson5 importre6 importMySQLdb7 importthreading8 #获取数据url

9 gilsUrl='.js'

10 gilsDetailUrl='.html'

11 gilsImgUrl='/%s/%s'

12 executor = threading.BoundedSemaphore(10)13 regex=repile('/([^/]*?.jpg)$')14 regexhead=repile('/([^/]*?).jpg$')15 classMySQL:16 def __init__(self,host,user,pwd,db):17 self.host=host18 self.user=user19 self.db=db20 self.pwd=pwd21 defGetConnect(self):22 if notself.db:23 raise(NameError,'没有目标数据库')t&#t(host=self.host,user=self.user,password=self.pwd,database=self.db,port=3306,charset='utf8')25 cur&#t.cursor()26 if notcur:27 raise(NameError,'数据库访问失败')28 else:29 returncur30 defExecSql(self,sql):31 cur=self.GetConnect()ute(sql)tmit()t.close()35 defExecQuery(self,sql):36 cur=self.GetConnect()ute(sql)38 resList =cur.fetchall()t.close()40 returnresList41

42 defgetGirlsData():43 regex=repile("var ugirlsData=(.+)")44 r&#(gilsUrl)45 jsond=regex.)46 with open('ugirlsdata.json','w+',encoding='utf-8') as f:47 f.write(jsond[0])48 #print('写入json成功')

49 returnjson.loads(jsond[0])50

51

52 defgetImgName(imgurl):53 if(imgurl==''):54 return ''

55 m=regex.findall(imgurl)56 if m isNone:57 return ''

58 else:59 return m[0] if len(m)>0 else ''

60

61 defgetImgNameHead(imgurl):62 if(imgurl==''):63 return ''

64 m=regexhead.findall(imgurl)65 if m isNone:66 return ''

67 else:68 return m[0] if len(m)>0 else ''

69

70 defWriteDB(jsdata):71 ms = MySQL(host="192.168.0.108", user="lin", pwd="12345678", db="grils")72 for data injsdata:73 sql="insert into grilsbase(74 name,height,bwh,title,img_upload,pc_img_upload,resource_id,totals,recommend_id,75 date,headimg_upload,show_datetime,client_show_datetime,video_duration,free_select,trial_time,76 viewtimes,coop_customselect_654,coop_id,tag_class,tag_name,playerid,block_detailid,type,istop)77 values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" %78 (data['name'],data['height'],data['bwh'],data['title'],('img_upload','')),data['pc_img_upload'],data['resource_id'],data["totals"],data["recommend_id"], 79 data['date'],("headimg_upload",'')),data["show_datetime"],data["client_show_datetime"],data["video_duration"],data["free_select"],data["trial_time"], 80 data['viewtimes'],data['coop_customselect_654'],data['coop_id'],('tag_class',''),('tag_name',''),('playerid',''),data['block_detailid'],data['type'],data['istop'])81 #print(sql)

82 ms.ExecSql(sql)83 print('完成'+data['name']+'数据更新...')84 DownImg(data['name'],data["totals"],data['resource_id'],data["headimg_upload"],data["img_upload"])85

86

87

88 defDownImg(name,totals,resource_id,headimg_upload,img_upload):89 path=creatFile(resource_id)90 if headimg_upload.strip()!='':91 #os.remove('./pic/'+resource_id+'/'+getImgName(headimg_upload)+'.jpg')

92 DownImgRun(headimg_upload,path,getImgNameHead(headimg_upload))93 if img_upload.strip()!='':94 #os.remove('./pic/'+resource_id+'/'+getImgName(img_upload)+'.jpg')

95 DownImgRun(img_upload,path,getImgNameHead(img_upload))96 #print('正在下载'+name+'图片')

97

98 for i in range(1,int(totals)+1):99 url=gilsImgUrl%(resource_id,str(i)+'.jpg')100 DownImgRun(url,path,i)101 #t=threading.Thread(target=DownImgRun,args={url,path,i})

102 #t.start()

103 #t.join()

104

105

106

107 defDownImgRun(url,path,i):108 #print(url)

109

110 r&#(url)111 if(r.status_code==200):112 with open(path+'/'+str(i)+'.jpg','wb') as fimg:113 fimg.t)114

115

116

117 defcreatFile(dirname):118 path='./pic/'+dirname119 ists(path):120 returnpath121 else:122 os.makedirs(path)123 returnpath124

125

126 if __name__ == '__main__':127 gri=getGirlsData()128 WriteDB(gri)