Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!
# O m$ |- G) c; X$ f. Q5 o) B3 h( T& X- #!/usr/bin/env python# s5 g. ~: l9 S! Y
- # -*- encoding: utf-8 -*-
8 p5 V8 Z* }2 |. d7 R% e1 ` - # Created on 2019-05-05 21:43:11
; R0 B# t; v' r6 J V, d% I - # Project: XiaoShuo5 }7 c4 D9 }. L$ _8 E) |. q9 I
-
( ~7 [3 F: t, K0 z7 Q; k - from pyspider.libs.base_handler import *; h) W- X: m1 f- K0 H1 d
- import pymysql
0 w0 K/ Q5 \& d% V - import random
0 {$ L- X; m8 B9 p/ [1 e3 ~ - import datetime
4 u6 T. a: l3 S) Z4 A* b4 r - import urllib2,HTMLParser,re
2 R" H/ q5 a) Z1 M5 T2 }, r - import os
$ o9 t: N+ l* A/ S7 q5 N4 h - import sys$ B" }8 J( S) x
- import re. ~6 o' p+ h! ?: x9 Z. n2 M% E7 W. `
- import codecs
9 e1 F3 V( g- r# z; ~" w - import requests
, Q- I/ i2 j! M% J4 s$ G% `/ \ - import json
+ z6 P) p, ]/ U4 o3 U -
) q* r3 \7 p- I - class Handler(BaseHandler):( \8 C; {( ^* H0 K, ^) v
- global Datos- n+ D: @+ m5 y: e3 K! b
- global P_dir
3 O; p- @. c% L" F - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径) h- i. V% A+ p6 e) L$ T1 n3 z
- global Datos3 E6 X; H8 F$ k; n" e4 ~
- Datos = {}
8 N: Q) n* o/ K! \; `7 N - headers= { C7 g- ] n. C, k
- 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
: ~$ l. S: ^) r: _! ~- Q - 'Accept-Encoding':'gzip, deflate, sdch',4 c T2 D" r0 j/ {
- 'Accept-Language':'zh-CN,zh;q=0.8',& `% ~7 r7 \, R' p
- 'Cache-Control':'max-age=0',% ^; Z: l- ] d: u5 F3 z' I
- 'Connection':'keep-alive',9 N: X: N' |- @# p2 ]7 ^
- 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'+ U" }: W& ~4 g; F5 o/ N
- }- H$ {, X0 a5 R
- crawl_config = {
: A5 O$ E" S; X1 z% P - 'headers' : headers,
, n1 ~+ i% E! x) n% J - 'timeout' : 300
( z- p4 e2 N; z |& J1 e - }, \* \, T6 |/ A l# i+ z1 v
- def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):
1 u7 [! i2 y% J; G, ` - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
- ?: m8 H. ?% Q9 Z0 M! V. o( ~# h6 j - try:/ C ^5 J( s T. o8 k
- cursor = db.cursor()3 f: }2 v% e: [" t2 R8 H
- #注意此处字符串的占位符要加双引号"%s"- Y; J+ _" q1 _9 z) h3 X5 H
- sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);: _/ ~3 Y2 k; G
- # print(sql)' a: N3 H8 V: t1 s& g4 t
- cursor.execute(sql)
# I" U0 p& Q7 x9 @: H: \ -
4 B7 |) J; K# m# \2 w - #qid = cursor.lastrowid+ ]9 Y# Y6 j- Y, A
- #print(qid)2 ]4 f3 [! k: `5 q z1 e
-
9 A1 Z# h2 w4 z- P- w* Z0 z - db.commit()
V6 \: K* c. K8 w2 C$ Q$ p - except Exception as err:; a3 l4 L! e) n" P! x, V( z
- print("Error %s for execute sql: %s" % (err, sql))
" Z9 p- f- K! G* C. W8 C - db.rollback() I2 s0 R) Q8 {- s
- def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):& b9 B1 W4 {/ E2 i+ b& o' ]& U
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8"), t1 S4 h" S; c5 g" H/ P$ t
- try: @, e0 K! F& P8 j5 H( f6 E! @
- cursor = db.cursor()4 Y* O" ^9 r, [9 q
- #注意此处字符串的占位符要加双引号"%s"9 J8 m; @/ y2 I' K! n0 A. C
- sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);
# @6 N: m! [' ]1 x& z% ~+ p - # print(sql)
! M, `3 T' r" m, T: ] - cursor.execute(sql)
: a2 h+ W, y" ?1 B+ D9 e& H3 a8 E - * y$ s- H9 y) k; q' ?
- #qid = cursor.lastrowid, S0 f& p8 P- _& r( ^5 ?. n* h( t
- #print(qid)
0 @7 s. X9 S4 v6 E) [: z6 f - : I9 j5 v! ~; J1 F! ^1 @; F
- db.commit(): o1 z' P- Z; w4 a( Y4 `2 ]
- except Exception as err:" K6 Q' q) A+ X6 K9 c- O
- print("Error %s for execute sql: %s" % (err, sql))
- h# U" r% e$ m - db.rollback()
# M/ @$ R3 i) N* o% u' }8 k1 Z - def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):, A: @% t' O3 @6 n% `
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")+ }7 {* o# z1 L8 c: n# c
- try:
! p9 ^ l/ ]# Q' p1 { - cursor = db.cursor()' S- B+ p0 N0 j- c9 Z
- #注意此处字符串的占位符要加双引号"%s"
- b6 n$ R5 s6 w - sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
) P/ U2 h a$ l7 u3 o# s - print(sql)
K9 `4 i, A& d& c0 ]1 P4 z7 z - cursor.execute(sql)
% X3 o! O4 S4 P: z+ U - print(cursor.lastrowid), {. e( m5 P% f; G% ^% B9 w! p
- db.commit()6 |4 _ w- S! m1 q( V- e% W
- except Exception as err:9 n9 c3 d9 e9 c, g% W; r
- # except:6 `! w, v1 P5 \0 e' r: \7 `
- # print('Failed')% H `' ?% U& o7 P. w
- print("Error %s for execute sql: %s" % (err, sql))" r. U/ V/ E/ z' ~, @* s1 ]
- db.rollback()' t$ H+ E$ I# i' D
-
% c, ^( C }$ g& N3 ?- B% B - def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover): & T- H: y2 e0 @4 M- L8 u4 K: H ^
- reload(sys) J" T3 V- @3 n
- sys.setdefaultencoding("gbk")
) P0 [" J5 x5 k, S2 [5 K- s - locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址
8 Z, m8 H* c- K# ~ - locoy_data = {
8 l0 n% K* n) Z% J/ P; j - 'my_u':'用户名', #后台用户名+ j! [" j- P1 x5 n* N4 B
- 'my_p':'密码', #后台密码8 K8 h) g1 Y# K$ p% M* }
- 'subject_669977_net':Bookname.encode('gbk', 'ignore'),7 f6 r) f9 y' r1 T( h/ r, P7 z l
- 'caid':Cater_Name.encode('gbk', 'ignore'),/ [ p/ X- q, b! m
- 'title_669977_net':Booktitle.encode('gbk', 'ignore'),' b4 I" ~) i/ W: Z. T# h
- 'article':BookConte.encode('gbk', 'ignore'),$ G- x k9 \& q' ^4 t7 [5 }) m
- 'author':Book_author.encode('gbk', 'ignore'),
8 D2 A5 r6 L: R& G- P! K: n - 'ready_1':Book_Palabras.encode('gbk', 'ignore'),
7 ^$ W, {" j7 g7 q5 P, M1 m - 'thumb':Book_img,
) L ~( Y0 x1 U. t7 y' k5 w - 'content':Book_Introduction.encode('gbk', 'ignore'),
8 y, N7 v; `8 ?. c! i8 ~ - 'abover':abover.encode('gbk', 'ignore') 2 F2 M6 [/ g+ n7 I
- }7 p5 |% |, S5 R3 W' Y% l
- res = requests.post(locoy_url, data=locoy_data), k. B, T( O/ t: Q% j
- print res.text; }! D0 f a8 E8 K- @" E! P
- print res.content
3 J1 ^# X4 q L+ ]- u. u* ~4 a8 @ - # print Dsd, n4 ?& @% @3 `% S; [
- return res& b! f- ^7 b* o% ?' l/ K& D
- ' l3 j4 S, {8 Y* Q) l
- def __init__(self):' d, M3 T- F* s% j* u6 D& C
- self.base_url1 = 'https://www.****.cc/'
: N) A' ^9 Z) _% g7 ~2 O - self.base_url2 = '/'1 m9 M. ?" b( C' H1 S6 {
- self.CaterId = []
# l: H! O7 z9 W% ~2 c0 \" ?- r - self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']. P/ ]- Y" Y8 c7 F6 @" l$ W- Z
- self.page_num = 18 z1 v" J. w6 H0 s* a5 r# S
- self.total_num = 200
" r! R2 L* N' s) u; \5 y7 ? -
; Z- L( w; Y# F - @every(minutes=8 * 60)+ Q; A/ l8 z: G9 ]
- def on_start(self):+ Q* u- X( T' h/ z3 i6 ~4 i
- global Cater_Name
" A2 W; z7 k5 f! a# h - Cater_Name = []
- R% x( x& L0 { - while self.page_num <= self.total_num: & C/ U& u ~/ X9 t Y
- for self.CaterId in self.CaterIds:
! T7 W) I4 O* | - if self.CaterId == 'xuanhuan':
) P5 K/ Y0 ^) @' }) B9 S' b - Cater_Name = '玄幻'
\6 m6 ^) Q: d$ V4 n/ F V ]$ Q - if self.CaterId == 'wuxia':$ w8 x( l( ~- V. N+ Q$ O3 `: p6 L: W# x
- Cater_Name = '武侠'8 S% a- [: ?1 Y- e
- if self.CaterId == 'lishi':9 R* w n$ Y0 i6 _1 i
- Cater_Name = '历史'
9 Z2 B2 m( E: r+ Y% U6 U - if self.CaterId == 'yanqing':( u. P( K) E$ i$ \4 |
- Cater_Name = '都市'
% S* k# B$ ~2 n# M - if self.CaterId == 'nvsheng':
* v% F* z& k0 ?$ Z6 x - Cater_Name = '都市' / @ C |9 N/ _8 A
- if self.CaterId == 'kehuan':
- r0 e% h- K, V4 {2 k( R - Cater_Name = '科幻' & G" z+ [# N c/ y
- if self.CaterId == 'kongbu':2 J* I4 g% r' L& ^1 |
- Cater_Name = '游戏'
/ L- C' _( N% s ~/ s - print self.CaterId
* X! v+ r7 C E) i - url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/" : }) G8 a9 X1 }3 c% ~
- self.crawl(url, callback=self.list_Caterg,save=Cater_Name)0 c# ]8 Z5 v& g/ M) \! Q
- self.page_num += 1
) _, b/ T9 X7 S' y1 C -
. j' } o: _* Y& q$ \9 R, k - def list_Caterg(self, response):
! U5 w: d4 ?& { {( P' G. e3 { - Cater_Name = response.save
# |2 a' f2 u: l0 a9 h2 b - for each in response.doc('.pic-list a[href^="http"]').items():" b+ M8 V' D9 @/ b6 t
- self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)
* w+ e( a: K. k$ E - * g; `& [. P) k% e8 ^2 p
- def list_Caterg_detail(self, response):
& d- j! {& m; J9 U - Cater_Name = response.save/ f1 C+ P% o. C2 o, b8 n/ R$ d
- # print Cater_Name
" u) h% m5 a8 ~9 U. R1 {% E - Bookname = response.doc('h1').text()
6 ~$ I+ A; B0 ?8 C3 Q$ r# N* S5 b - print Bookname
5 @8 V2 ^4 h9 P7 Z/ E5 j; d R, Y - Book_author = response.doc('.authorname > a').text()' r c0 q3 o& |4 ~! B! O n! m
- # print Book_author
! O) e# T7 i& ?; R7 @8 ~ - Book_Introduction = response.doc('.book-intro > div').text(), ]/ r" p- h1 R6 Q- i
- # print Book_Introduction
* k% E; C; ]) x. U0 p - Book_Synopsis = response.doc('b').eq(1).text()
: u8 R* ?$ O1 Y% \2 U: n2 | - # print Book_Synopsis: w( {' q+ C- L* u, C
- Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]" x# ]4 z- i, U7 n W5 t
- # print Book_Palabras
1 h$ }! M; t3 n) N3 I - BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID
4 G$ g% |3 u0 K) u, x* ?# v - # print BookIDs
5 V! Y# i% F( O: K$ z8 O - Book_Dates = str(datetime.datetime.now()) 6 z+ }. ?4 |' w% A
- for imgs in response.doc('.bigpic > img[src^="http"]').items():
6 C6 b2 z; V" k* ~) u( U - img = imgs.attr.src
3 f! l n3 j& D5 M0 h+ s$ q - print img0 ~ o6 h0 B- P
- #小说封面下载
3 x+ {) H% s$ |% n/ N7 e$ L - extension = self.getExtension(img) a0 }3 a/ s2 U; W
- name = self.getname(img)
# z T) Q$ T: N3 e - file_name = name + "." + extension8 q+ I4 x; o! ~8 D; g8 U% \
- imgDir = P_dir + name
7 C9 R* G, B5 L) O1 m/ R - Locaimg = imgDir + "/" + file_name
$ l, }/ a" I. n5 B, W4 F - print Locaimg
/ D- i2 \7 y7 X: Z8 \* S% _% Y - if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地
) b$ f; `2 }2 G# y6 `" P" D - print('attachment url is ' + img) #
; i3 k) F3 O6 O' V2 i - Datos = {
+ f5 j y) a( D, r3 {0 Y - "Cater_Name":Cater_Name,
( Y/ n: I, ?: M0 [$ O - "Book_author":Book_author,, K* G* w2 g* w! @1 T3 x
- "Book_Introduction":Book_Introduction,) Y$ l3 ^. _: I0 T
- "Book_Synopsis":Book_Synopsis,
/ q& C- a. `& A! Y - "Book_Palabras":Book_Palabras,
- d% j+ P! M' _. ?6 W - "img":img,! l# G8 a# q) M; r# p
- }5 N& Z% B+ l) H+ n, w
- self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
2 O) l# ?; h3 G2 F, }) i5 u0 D1 X6 v - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():
6 P7 o& D9 a" z& p4 U. U2 ]( t - self.crawl(each.attr.href, callback=self.index_page,save=Datos)* B1 u" b& W6 G9 f+ T" c) d M6 A
-
& o7 z! O" v" j1 n - @config(age=8 * 60 * 60) 1 q0 ^* ~. F1 k: h0 m' n7 p# X9 G% t
- def index_page(self, response):
: k7 ]5 ]3 b5 `( e: i - Datos = {
5 e) H/ M$ [! ?; @" y: }% H8 ] - "Cater_Name":response.save['Cater_Name'],* p9 U; H4 q: i3 [ v4 R; Y
- "Book_author":response.save['Book_author'],0 U, ?/ `" |/ l: S, m
- "Book_Introduction":response.save['Book_Introduction'],
3 x: T: g, E# I( a - "Book_Synopsis":response.save['Book_Synopsis'],, c5 s) y/ _% z& ~. m! L1 r
- "Book_Palabras":response.save['Book_Palabras'],) _+ \' |6 y* j
- "img":response.save['img'],* C$ O" u+ _8 d2 o+ G& y
- }
: z( N# [% z) c - for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():
& c, Y, q, _7 y - # for each in response.doc('.chapter-list a[href^="http"]').items():
) c+ @& I: k% a' a. u7 Y- a! p - self.crawl(each.attr.href, callback=self.detail_page,save=Datos)0 c+ D# v% z( w" |
- @config(priority=2)9 b1 z- {8 _* o+ {
- @catch_status_code_error
7 ^. O& w$ l# G, G: D - def detail_page(self, response):
* l0 G# |( C# } - NewRe1 = u'哈书'& e3 d+ k' O7 N& L' t# y5 K
- NewRe2 = u'huhjsd.CC'
7 ?9 `8 }; V2 |9 ~ - NewRe3 = r'^\\n\\n'
$ n- J! C- a# i - NewRe5 = u'小说网'
# p# {1 L6 ], q* a - NewRe6 = u'fgdfgf'! J' L" c$ ^! C5 \) q
- NewRe7 = u'fgfgf') [ M0 f" v- X* b! q7 n% k
- NewRe8 = u'ffhgf'
) G: @" R" r0 f - NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'
; v. O0 O8 O. k G6 D# N4 y - ReC1 = u'静思'/ M$ d3 q5 B$ v# V# E& F3 } \- X
- ReC2 = u'aghgf.com'
* C* I) Y. X) D0 l) A1 z& y2 T+ m0 n - ReC3 = u'aghgfh.com'+ p" ~/ }, R0 E/ B" A
- ReC4 = u''
- o8 q+ V' x; c* o - ReC5 = u'文学网'( h# i- H- e( w- [3 u1 ?
- ReC6 = r'<BR>'
8 _3 t; c4 G1 ^ - Bookname = response.doc('.readlocation a').eq(2).text() #小说名称
7 j/ q" W. \1 ? - print Bookname7 ]5 r( r6 V' u0 }
- Cater_Name = response.save['Cater_Name'] # 小说分类9 ?5 [) v# Y- s+ R: g
- Book_author = response.save['Book_author'] #小说作者4 I$ |/ _% j1 e G L
- Book_Introduction1 = response.save['Book_Introduction'] #小说简介0 d6 n' h7 M, ]' D ]# p/ R
- Book_Synopsis = response.save['Book_Synopsis'] #最近更新8 l& i, Y% @" \* s8 h4 t
- Book_Palabras = response.save['Book_Palabras'] #小说字数2 j: A4 F" l6 A) ?# Y
- Bookurl = response.url #小说网址5 E- x# S3 c$ u. ]5 N9 E9 [
- Booktitle = response.doc('.article-title').text() #章节名称) K: a+ h) m3 M ` O
- BookID = response.doc('.readset-r span').text() #小说ID0 \; ~ w" B4 [* n* M0 s
- BookConte1 = response.doc('.article-con').text() #小说章节内容
6 G4 ?; n: k1 y* B5 n0 y& | - abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)0 E5 O5 j: C! _3 }; A, }- j3 W
- Book_Date = str(datetime.datetime.now()) # 采集时间 {* L+ L3 j, d, n- A
- BookConte2 = BookConte1.replace(NewRe1 , ReC1)
3 O% e3 {3 z- T - BookConte3 = BookConte2.replace(NewRe2 , ReC2)
* S) s* y1 {5 v; Z2 K) q% l - BookConte5 = BookConte3.replace(NewRe5 , ReC5)
8 y' P1 r9 q9 Z0 X$ B2 Z - BookConte6 = BookConte5.replace(NewRe6 , ReC2)1 P9 m+ ]7 r& q
- BookConte7 = BookConte6.replace(NewRe7 , ReC2)9 L! `9 Y3 h1 a, C
- BookConte8 = BookConte7.replace(NewRe3 , ReC6)7 A& G" l) f3 a: a- q( l
- BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)# T1 H* o1 f. i3 ?7 ]1 \! K
- BookConte = BookConte4.replace("\n\n","<br>")
, _2 b& b: ]. N* l - print BookConte
: B0 {3 w4 N. u( P7 X+ [ - Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)1 ~% o( L8 z, W6 O- |) U; d& w! }
- Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
! V! _! ?+ t$ Z0 i1 a0 j - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)
6 P" g2 W7 M% y5 t6 J - Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)# y2 C' p) u" i6 ]' U9 _3 U& [) u
- Titleid = response.url.split(BookID + "/")[-1].split("/")[0]
$ u* e& Y! Y V0 n7 K% A0 V - Book_img = response.save['img'], #小说图片: {* h0 {6 C# r* K3 s) y
- % D; o- ^. J. u6 R/ ^$ H" \, D
- #insert into MySQL 小说入库1 p3 v) n- [, G, q. c* A0 ^2 T$ w6 x) P
- self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布( n& U9 |7 k7 N
- self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
: t$ K w. O" H6 y8 K - #post提交发布
" Z, w1 ?' R7 ] - self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消
/ `' j+ L2 J3 T. B" }0 ~ - Datos = {4 }& h3 J+ m8 k7 l Y6 H
- "Cater_Name":response.save['Cater_Name'],
+ T" O9 f5 t3 [, b6 d - "Book_author":response.save['Book_author'],
6 t4 a9 ?! Y' \7 [6 J2 K0 N6 ]' O - "Book_Introduction":response.save['Book_Introduction'],- t) g3 U& l2 j$ y8 f
- "Book_Synopsis":response.save['Book_Synopsis'],
' K" p5 c: ~( C7 S - "Book_Palabras":response.save['Book_Palabras'],
8 {4 B% ~, |7 y, z1 ~. N* Y- }3 L - "img":response.save['img'],5 o9 D9 w# L5 Y: M! [5 a. h
- }( F1 o2 B( [# w; I$ a& X" N
- for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():9 O/ g" G3 f$ a
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos) 3 ^! |- F0 Z. n$ o2 u
- return {
2 j" c/ f; y$ ]" k - "Cater_Name":Cater_Name,$ C* e- |0 {; X8 n1 I0 z! a
- "Bookname":Bookname,; b5 E) L0 P1 Q1 E
- "Book_author":Book_author,
; u- i9 V- O2 B+ G; v - "Book_Introduction":Book_Introduction,. F+ m. K6 Q& l+ s6 t' Y A
- "Book_Synopsis":Book_Synopsis," b/ s& d( k; \3 i. A* d' s
- "Book_Palabras":Book_Palabras,; ]5 i& P' `4 T, a' o4 M4 O
- "Book_img":Book_img,2 G7 g/ \5 \, e; \ o& ?0 d6 h) j5 s
- "Bookurl": response.url,4 S) t7 J5 k9 ]" z0 G: t
- "Booktitle": Booktitle,0 } k f/ z3 B/ w3 `$ j
- "BookID": BookID,
; u) s1 X2 e# Q3 x, ^ - "BookConte": BookConte,) h+ P0 y1 Z8 o3 F. m
- "Titleid": Titleid,
, W" @% j3 a0 ] p) x% `. W0 d1 l* } - "abover":abover,
( B, W: }2 R4 H/ Q) `: v0 B( R - # "Book_Date" = str(datetime.datetime.now()),
, Z9 B5 p* s" U - }1 I4 w* f9 n2 A* `# D
- def download(self, P_dir, imgDir, file_name, Book_img):
; z, ]% ~. a! _* C) F) d - if not os.path.exists(imgDir):
* @' G1 o; E- J$ e4 I - os.makedirs(imgDir)
9 V, T9 n* U0 V0 H& a - file = imgDir + "/" + file_name
5 j5 v& f+ K2 G - # print file# Z( c9 ~ F: S
- f = open(file, 'wb+')
8 ]. D9 V! V# y. H' D - imag = requests.get(Book_img)
9 y2 q1 m0 ^6 W) P$ k# k! X - f.write(imag.content) D5 N2 N4 \1 x0 B# W+ V" L# k
- f.close()! K3 @8 h5 O5 A
- #保存图片前
# X5 t6 l0 C. V: r - def save_imgs(self,response):
1 Q% }* d$ N# v \9 h - content = response.content+ S- Y. t5 }, L3 L7 X
- file_name = response.save["file_name"]8 g! G! t0 {+ x
- imgDir = response.save["imgDir"]
" O2 W( F9 K1 h& T V - file_path = imgDir + file_name0 x' l* M$ q' c( [: t E
- self.save_img(content,imgDir,file_path)
I) ]5 @ O' \7 o" {( h5 j - #保存图片6 ~6 j2 d" | k1 d$ L" Z5 d
- def save_img(self,content,imgDir,path):
# j. v6 \3 _: U' ^0 n' Z9 e - if not os.path.exists(imgDir):
) L- L* z; F: ]6 C# b# O6 z- |$ X - os.makedirs(imgDir)
: c$ x/ K9 K! d8 b3 l9 r, j* N0 P - f = open(path,"wb" ), O6 i4 N. V" c* L* C3 K# g
- f.write(content)
: {5 w# R" N' t4 G( p - f.close()
% y1 ]5 n; B! ]: K; f7 v - #获取url后缀名
6 Y4 x0 F6 I( I# b - def getExtension(self,url):
) t# C; O& _$ N+ n. D6 O - extension = url.split(".")[-1]+ X' b! m1 s! p5 Q- S- _% {: L6 v
- return extension 2 Y8 |& N, v n& m( ]% G% q Q2 w
-
2 p+ q& r( r- h - #获取图片名
& i7 G% l+ f) x1 _5 G2 J - def getname(self,url):
1 [) F$ @, {& G' P8 W: T - name=url.split("/")[-1].split(".")[0]0 L8 y( f8 n, G$ A: @8 s) Y: R
- return name
复制代码 " }( G* |0 y4 k3 w+ \
9 k* I7 F2 ?9 {7 Y4 S4 I% Y; _ |