Python + pyspider某小说站的爬虫,入数据库,火车头发布,资源下载到本地,另可写爬虫!' e/ R* l. H5 m
- #!/usr/bin/env python
7 q. s9 E8 ~1 ~0 M e o3 }( _ - # -*- encoding: utf-8 -*-9 c2 Z. U: Q- c @6 n% b/ b: N5 w
- # Created on 2019-05-05 21:43:113 s7 @& ~3 a, }8 o! A
- # Project: XiaoShuo0 y9 Z. P2 z+ ]7 X7 Y" V+ q# l
-
2 U. w( e. Q" |# h1 C) R - from pyspider.libs.base_handler import *
. M' d9 B- v5 ^! N0 R( M5 @ - import pymysql
/ M' w! ~. S2 z+ B( V ] - import random
$ h0 l& t4 f: b( Y3 s B6 g4 A- { - import datetime
( y ^. C- P- J6 k- `9 j9 E - import urllib2,HTMLParser,re
7 {! P% M, k" s3 k1 o! n - import os, R/ v6 t7 M0 R$ {6 H, t
- import sys" i, K, G: z6 s, g' W" Z
- import re. L5 K& d- Q# s/ f4 }6 T( Q0 A
- import codecs
& z8 C# Y) ~: a. T - import requests+ c1 k. |! }6 l% q
- import json
% e' g& R. u) a' v1 Y8 c -
0 h9 }' i. I8 j2 B" c - class Handler(BaseHandler):: g; q3 L& Z% i3 C9 D- h. Y5 ~
- global Datos
; U1 ~: Z1 W, Z' ~4 w& ^4 y- e% d0 C - global P_dir
6 }7 \2 m% p& z8 j0 g* N% C7 ]2 a - P_dir = '/Tools/Debug/' #采集时候图片保持到本地的路径
- N# x) y( C! A. X) g( d% }! ?9 d - global Datos
, i8 j* q3 g. J1 T - Datos = {}8 \3 u! {+ J7 P
- headers= {
4 w, ?& H* L3 h7 S; }5 l" p - 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',' b: Y" ]+ \: Z' Y S) s
- 'Accept-Encoding':'gzip, deflate, sdch',
0 G+ c8 J& H5 }# @ - 'Accept-Language':'zh-CN,zh;q=0.8',
+ V: A1 F) M m( j6 `! C - 'Cache-Control':'max-age=0',
6 }1 t% f0 l X+ Y - 'Connection':'keep-alive',
: ^; S' ]9 B) N( B& h* `+ c - 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
. P+ N' r8 M8 q8 U% B( w8 o - }2 [; T+ O( g! G8 k# V$ _
- crawl_config = {) x( ?) J) o# h# b8 w8 d5 M
- 'headers' : headers,& R. n1 i; t5 j+ f! b
- 'timeout' : 300
5 T# Z W& S3 S( O9 k - }5 _$ p; Y. |' a& \' Q
- def add_BookFile(self,Bookname, BookIDs, img, Locaimg, Book_Dates):, k5 x3 ]- U0 K- {( n. W7 {
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")0 j, }7 B( }# G( b. X7 ?
- try:
, k$ @- t# }. s - cursor = db.cursor()- o( x* J8 i: g8 L
- #注意此处字符串的占位符要加双引号"%s"
) k8 G5 y- M! P1 q - sql = 'insert into BookFile(Bookname, BookID, img, Locaimg, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, BookIDs, img, Locaimg, Book_Dates);) C* i) j9 f3 ]! {2 h- R& `
- # print(sql): L. }; \# j$ \( A/ N: c2 L9 }
- cursor.execute(sql)
# k6 E/ P9 O( a y7 { - 1 I) P+ I- z+ [/ Z
- #qid = cursor.lastrowid7 k3 R' n; i, K% @
- #print(qid)) y0 F! Z/ M& u
- ; e5 T' p- g( }
- db.commit()
: U( L; ~+ z2 b" V1 C - except Exception as err:
2 `) y* @" q9 G# O - print("Error %s for execute sql: %s" % (err, sql))7 P! |# b" Z7 t+ S
- db.rollback()
" u9 v: W! A6 T) j1 x - def add_comment(self,Bookname, Booktitle, BookID, Titleid, Book_Date):
0 D' H" D8 K4 I X1 v6 | - db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")
, c% O5 m; g, I( }: j) I7 R7 [ - try:
, H z, F$ b2 G8 f - cursor = db.cursor()
4 Z" c8 D8 e! q2 N: H6 | - #注意此处字符串的占位符要加双引号"%s"
1 A6 E9 }2 |: P: z8 i - sql = 'insert into BookTitle(Bookname, Booktitle, BookID, Titleid, Book_Date) values ("%s","%s","%s","%s","%s")' % (Bookname, Booktitle, BookID, Titleid, Book_Date);0 } g/ |. P9 v! S9 ]0 k' l) {
- # print(sql)# H, ~6 e! j# D2 G& n4 t9 I
- cursor.execute(sql)8 m$ L3 v& J+ I# h+ ]! ~
- & b* ~4 J5 l& \. @! W
- #qid = cursor.lastrowid/ l% t. Q" E3 q" @- s
- #print(qid)
3 W# E2 d& i1 o+ ?3 H7 E, v$ v -
4 Q3 p- Y5 i3 o+ C) \0 x - db.commit()
0 ^1 M8 F* v8 i& y3 R - except Exception as err:
1 _ m" g+ f" H, } J% K/ t - print("Error %s for execute sql: %s" % (err, sql))" D0 Q7 H, ]+ v f, I; G' y
- db.rollback()
' P- U) A; T& L, F5 t - def add_question(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date):5 O' g* I% J: F# z# p9 p
- db = pymysql.connect(host="localhost",user="数据库用户名",password="密码",db="数据库名",charset="utf8")1 _. [0 n% z3 k( j' y. r3 @
- try:6 `9 `8 q/ F. B# R, `, r/ O# o7 ?
- cursor = db.cursor()
, j. L4 V, N6 v( m+ S - #注意此处字符串的占位符要加双引号"%s"
- U' p g! |" `$ l) t; u - sql = ' INSERT INTO BookConte (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) VALUES("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")' % (Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date);
% q2 k- r$ t$ Z0 W - print(sql)0 m$ r6 ?6 D0 g$ @5 z
- cursor.execute(sql)6 d0 j ^* y- Z$ O
- print(cursor.lastrowid): m. O& @6 g" ]2 A$ ]
- db.commit()
3 ~1 Z! b4 _% T" c1 d0 R( | - except Exception as err:
2 |$ b H; V) ^+ I: x* g e) t0 S - # except:
, n5 v2 X5 P% y1 R0 L+ N - # print('Failed'): w' N4 G. F9 f$ I7 C9 j8 N
- print("Error %s for execute sql: %s" % (err, sql))6 m3 m8 l; W+ c. v! R. P* T
- db.rollback()
5 U! x" U# z7 _! I1 L0 D - 0 t+ ~0 K Y6 W, G$ `4 x
- def add_locoy(self,Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover): & S, p/ z' |* U
- reload(sys)
& v4 y i2 h% x# D- `0 M - sys.setdefaultencoding("gbk")$ i7 P0 m/ C" G V$ W) U# ~1 f
- locoy_url = 'http://www.******.net/locoy/?my=book' #697火车头发接口地址
8 p) E, M. C1 I6 O - locoy_data = {
9 [( X" `6 ^7 l2 m. E# i - 'my_u':'用户名', #后台用户名
& `0 ]( _0 t( k5 B! S+ G - 'my_p':'密码', #后台密码4 |9 z$ ?% T. {3 s& F
- 'subject_669977_net':Bookname.encode('gbk', 'ignore'),
; p& m7 U3 z* F; n; z) {' D$ | - 'caid':Cater_Name.encode('gbk', 'ignore'),7 K& a' L9 m }# \9 Q& O% W' D
- 'title_669977_net':Booktitle.encode('gbk', 'ignore'),- m3 G6 Q$ D0 Z3 Q$ v
- 'article':BookConte.encode('gbk', 'ignore'),
4 X. a' g+ f' c8 \0 r- H8 a - 'author':Book_author.encode('gbk', 'ignore'),9 }# g' F* \7 o; n: g. S: C2 i
- 'ready_1':Book_Palabras.encode('gbk', 'ignore'),
! K; r f0 u0 t& M; E/ H: y - 'thumb':Book_img,' q% d: R: j1 c
- 'content':Book_Introduction.encode('gbk', 'ignore'),
% K" |( B1 Q) l" p9 z' k* t - 'abover':abover.encode('gbk', 'ignore') 9 d: ?7 ^5 U4 @/ N9 ^
- }/ Q- v% x- g4 o2 _* R
- res = requests.post(locoy_url, data=locoy_data)$ T1 L, H9 V) W6 T
- print res.text
* c' _- M8 k$ n _8 a+ u, W - print res.content1 D6 h0 ]3 v: |8 w- z! J; y! M. ?/ K
- # print Dsd
& S- X% F, Y; n- F. b% } - return res5 w9 {7 M6 d6 m6 e
- 0 G4 @5 ]( J1 |8 a j/ _
- def __init__(self):
: C0 Q% w3 K& s1 o# c; u - self.base_url1 = 'https://www.****.cc/'
& ~; [/ Z2 u7 ]6 j% Y8 P6 q - self.base_url2 = '/'
7 k2 s7 m' ~ R9 _) K2 H) @# v - self.CaterId = []% W0 O f+ f6 f) A9 a8 R$ b
- self.CaterIds = ['xuanhuan', 'wuxia', 'yanqing', 'lishi', 'kehuan', 'kongbu', 'nvsheng']
M& q6 c1 T( ~% z; L, b& ] - self.page_num = 1+ ?# _$ I4 W3 }4 s. F" T( b1 ]
- self.total_num = 200 7 k( o) X1 i2 Y* e2 f+ s8 X* y
- ' o6 x8 d1 c1 _! g- u }. l
- @every(minutes=8 * 60)% X9 g) F6 V4 ~! U( f. k' L/ A
- def on_start(self): a, Q( N3 F7 w+ g3 {7 E
- global Cater_Name
; O w) _* C5 h5 J( i - Cater_Name = []
/ q4 u* S7 A/ g - while self.page_num <= self.total_num: ( s# ]! \% j' d w; V
- for self.CaterId in self.CaterIds: @* h/ V6 x9 n4 I! n
- if self.CaterId == 'xuanhuan':
9 Z. F ?% I; n3 x - Cater_Name = '玄幻'. j( ]" d& L! U, v! R+ f
- if self.CaterId == 'wuxia':
6 v2 T- x. c+ z. s1 T - Cater_Name = '武侠'
+ E }; f3 w8 W( g* ]8 c0 f9 m& ]: } - if self.CaterId == 'lishi':, _, T( c* a% A5 K! z
- Cater_Name = '历史' + N: C- r: z' t; Y
- if self.CaterId == 'yanqing':
8 C: T; r7 E( n - Cater_Name = '都市' 4 e+ J/ d' t7 z* K$ P, Z& k
- if self.CaterId == 'nvsheng':
7 L% Z6 k0 e. _; M: i+ R- I5 ] - Cater_Name = '都市' 1 w' n w8 x( Y# @
- if self.CaterId == 'kehuan':* v4 U0 w2 E$ C0 Z6 Y- [9 S
- Cater_Name = '科幻'
( p6 t0 @ j a* J3 U! G - if self.CaterId == 'kongbu':' R( _$ C1 G! q; q6 W X2 V9 Q
- Cater_Name = '游戏' . b7 T/ O) X w* Y( H7 l+ d- B
- print self.CaterId
# v$ I- n' a6 M' l" A6 j - url = self.base_url1 + str(self.CaterId) + self.base_url2 + str(self.page_num) + "/" + O" L8 k8 F P Y. `- L% g' Y
- self.crawl(url, callback=self.list_Caterg,save=Cater_Name)1 q9 S. f0 f$ k) q
- self.page_num += 1 ' A) W6 a3 Q2 v8 I% Z% D
- ' V( A) o0 F3 p' K( p7 a5 m8 B
- def list_Caterg(self, response):' L. k( |7 x4 t: L( x9 B# s! ?
- Cater_Name = response.save, w9 F }. K, T! s
- for each in response.doc('.pic-list a[href^="http"]').items():
* d9 ^/ n; B. D/ Q) ]9 D+ n& L, w3 Z - self.crawl(each.attr.href, callback=self.list_Caterg_detail,save=Cater_Name)+ E- P% r- w* u" `3 u
- % \& q0 m# p- ^0 O+ Z9 \3 S
- def list_Caterg_detail(self, response):$ {' H" I# p- R$ [' W+ C
- Cater_Name = response.save
: b: S+ ^: U* K0 ]1 Z0 _ - # print Cater_Name; H; i/ n& L l& u# \
- Bookname = response.doc('h1').text()6 D" d* u% @: s9 u8 Q1 P2 E
- print Bookname4 ~5 I0 N4 {, w7 \7 q
- Book_author = response.doc('.authorname > a').text()1 e @9 v+ O: Y
- # print Book_author
+ m3 P7 B- S4 o/ C3 U - Book_Introduction = response.doc('.book-intro > div').text()
4 w: i: K; J+ z) e- } - # print Book_Introduction
4 H% F6 u% x* d/ B4 \ - Book_Synopsis = response.doc('b').eq(1).text()- g( @8 @- O$ [* J) n2 S, ^
- # print Book_Synopsis/ O a. k! M, @
- Book_Palabras = response.doc('.booktitle p').text().split(' ')[1].split('|')[0]
# H# Z; r z9 A8 Y5 M: r - # print Book_Palabras
8 f! f2 P# j; C - BookIDs = response.url.split("xiaoshuo/")[-1].split("/")[0] #小说ID
Y1 K. x; y+ ?9 _ - # print BookIDs
s6 ^+ _ c9 o2 [: a - Book_Dates = str(datetime.datetime.now()) * J$ f* D w; J" q8 g
- for imgs in response.doc('.bigpic > img[src^="http"]').items():( I# F% Z! b- z( L r7 }- c
- img = imgs.attr.src
+ ^+ h# d* g" \# O2 z - print img; u. ^3 v- e5 i2 l( u' q) g0 r
- #小说封面下载
+ q- u+ I' M }" X; M6 l* w; m! d - extension = self.getExtension(img): }+ v4 Z# L* f. t& {5 s; }
- name = self.getname(img)
; Z" X7 `4 F# A! ?2 h* v1 @7 g0 l - file_name = name + "." + extension
- l3 s; F- ^' u' G1 Z y - imgDir = P_dir + name8 [* r8 K3 O: e6 r
- Locaimg = imgDir + "/" + file_name5 I, a M8 x/ I: J1 E
- print Locaimg _! s% ^2 [& K9 k1 X5 p2 l
- if(self.download(P_dir, imgDir, file_name, img)): #这2行可注译,图片下载到本地( n7 l+ V9 R5 V0 M; m
- print('attachment url is ' + img) #
; S9 y& I3 O. ^7 @7 `, u/ i( C7 O - Datos = {1 o# B8 ?4 I6 m- ~6 ?0 U- ^, h
- "Cater_Name":Cater_Name,- U7 j; m" T7 I: ?2 k: ]2 Y
- "Book_author":Book_author,
& x) ~# F; k7 C& v' U1 b+ e& ] - "Book_Introduction":Book_Introduction,
+ }7 |" a7 y/ x% x, V+ x n7 X4 D: S - "Book_Synopsis":Book_Synopsis,
' j( ?! L D' S0 B6 x9 n - "Book_Palabras":Book_Palabras,# f: `3 T8 }8 P
- "img":img,
0 m; X u% o4 i. d1 g( n - }
! z$ @; h) `$ j" r, t( V - self.add_BookFile(Bookname, BookIDs, img, Locaimg, Book_Dates) #这行可注译,数据库发布接口,方便其他系统的发布
* J! }$ A' O# c O$ P' w& s& T - for each in response.doc('div[class="bookbtn-txt"] a[class="catalogbtn"]').items():
4 |# t6 \3 `: l/ L. S2 Y - self.crawl(each.attr.href, callback=self.index_page,save=Datos)5 I' y- t" s2 c0 m) G
- 0 s1 h/ ~, A( p7 Z( I6 ?" S' \
- @config(age=8 * 60 * 60)
; P! d6 n2 ?: C! c - def index_page(self, response): 8 ]+ G. D) E# W
- Datos = {
; Y& `. f7 d' x. e' E - "Cater_Name":response.save['Cater_Name'],+ a% G: {9 q5 V% A
- "Book_author":response.save['Book_author'],
2 F" I6 N0 n" V) @* t% m( B - "Book_Introduction":response.save['Book_Introduction'],. {- C. D# a% R- G4 [+ c' @
- "Book_Synopsis":response.save['Book_Synopsis'],
* N; \: _) v2 C* k# ] - "Book_Palabras":response.save['Book_Palabras'],
! F3 @2 R: k% \0 g4 [ - "img":response.save['img'],
. j) U/ k3 U/ j! l+ r+ |/ h K - }
+ \0 m1 K9 p9 }" y - for each in response.doc('.chapter-list li:first-child a[href^="http"]').items():
" C* m, D# X9 ]3 |' r! V - # for each in response.doc('.chapter-list a[href^="http"]').items(): $ u5 m8 p) K0 N! x0 O
- self.crawl(each.attr.href, callback=self.detail_page,save=Datos)7 i$ Z! A3 t U9 L" v( c% L# n
- @config(priority=2)3 z, Z$ T! L+ z8 E, f1 E
- @catch_status_code_error
, ], \9 p$ R% s/ C5 `1 K - def detail_page(self, response):
+ _1 z1 Y5 b8 n( K - NewRe1 = u'哈书'
( p; O8 \5 P4 \7 d2 L g. d( V - NewRe2 = u'huhjsd.CC'
_. a& r+ q7 `8 |: n, Q - NewRe3 = r'^\\n\\n'
/ Q9 u. W5 q3 W8 J [) O f6 X# ~ - NewRe5 = u'小说网', T9 I% h7 ] Q9 t9 \
- NewRe6 = u'fgdfgf'
& G! }: R, A# C3 K/ Z, L - NewRe7 = u'fgfgf'6 c! |1 e+ T3 h4 o5 ^
- NewRe8 = u'ffhgf'
8 |. W/ T- c: M$ k, T - NewRe4 = r'[\f\t\v+\.\{\(\)\}\!\/_,$%^*(+"\')]+|[+——()?【】“”!,。?、~@#¥%……&*()]+'
6 J0 @; H, |" I2 r1 W - ReC1 = u'静思'" o0 ]5 O; w; ]8 v% O+ N5 T' ^% @6 d U
- ReC2 = u'aghgf.com'
$ i. \. T0 I! g+ o. H* p6 Z - ReC3 = u'aghgfh.com'; G' g9 T7 B a2 n
- ReC4 = u''( k \2 W3 w9 B
- ReC5 = u'文学网'
0 ? g# q) H* h+ j" N( | - ReC6 = r'<BR>'
0 r, M* S! W& B* S) D+ G! Y - Bookname = response.doc('.readlocation a').eq(2).text() #小说名称
/ \8 _: u, R/ H" { - print Bookname
! E) y! }- d e; o3 v& \7 w8 T& m4 b) ? - Cater_Name = response.save['Cater_Name'] # 小说分类 C% b# [3 u9 r# v; G/ L$ `
- Book_author = response.save['Book_author'] #小说作者
1 T( U! F8 u. R2 t4 Y - Book_Introduction1 = response.save['Book_Introduction'] #小说简介
9 T0 K; m' p4 Y5 { - Book_Synopsis = response.save['Book_Synopsis'] #最近更新6 d3 o$ n% Y/ o" V# h9 T
- Book_Palabras = response.save['Book_Palabras'] #小说字数8 A$ f' Y" H l4 m6 f* u
- Bookurl = response.url #小说网址/ g1 F+ C4 F- _' a
- Booktitle = response.doc('.article-title').text() #章节名称$ X4 X; E4 w" u( N. _8 r' v3 d
- BookID = response.doc('.readset-r span').text() #小说ID y2 d2 a( V+ [& K
- BookConte1 = response.doc('.article-con').text() #小说章节内容" @" p! e1 I' w1 ]
- abover = response.doc('.article-title').text() + response.save['Book_Synopsis'] + response.save['Book_Palabras'] + response.save['Book_Introduction'] #小说状态(连载还是完成)6 j% e! C/ w' E
- Book_Date = str(datetime.datetime.now()) # 采集时间; m7 u# e: e8 z ~& i& t1 p, V
- BookConte2 = BookConte1.replace(NewRe1 , ReC1)
0 q$ R; Q/ F; N& {+ Z - BookConte3 = BookConte2.replace(NewRe2 , ReC2)% T( ?, H( @& s( b
- BookConte5 = BookConte3.replace(NewRe5 , ReC5)% N& d6 c$ @0 C& Q0 P+ U
- BookConte6 = BookConte5.replace(NewRe6 , ReC2)
5 e9 S# s2 b9 P$ ^2 ` - BookConte7 = BookConte6.replace(NewRe7 , ReC2)$ `& d+ B9 J9 o4 L8 o
- BookConte8 = BookConte7.replace(NewRe3 , ReC6)
$ z3 I9 U6 g; t - BookConte4 = re.sub(NewRe4 , ReC4 , BookConte8)- l; J8 A9 c4 ~
- BookConte = BookConte4.replace("\n\n","<br>")
N. G; P2 N6 f: p. J - print BookConte" [% Q9 f6 n) p" P5 p1 u+ N5 Z
- Book_Introduction2 = Book_Introduction1.replace(NewRe1 , ReC1)$ {3 G+ Y( z7 z7 e' R
- Book_Introduction3 = Book_Introduction2.replace(NewRe2 , ReC2)
% b9 |' ^& _. ?$ Z4 O - Book_Introduction4 = Book_Introduction3.replace(NewRe3 , ReC3)
7 N! e! `( X/ v5 D: G7 a) k$ L - Book_Introduction = re.sub(NewRe4 , ReC4 , Book_Introduction4)
6 s' ]! c& Q5 C - Titleid = response.url.split(BookID + "/")[-1].split("/")[0] : r6 h! m7 f) n% w) f) F E* {
- Book_img = response.save['img'], #小说图片6 \8 ~8 y- `' J
- - [! s0 D- v3 M9 N
- #insert into MySQL 小说入库
8 r! S) p; w( S - self.add_question(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Synopsis,Book_Palabras,Bookurl,Booktitle,BookID,BookConte,Titleid,abover,Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布# o" a" d1 F0 S/ S7 v7 X0 J
- self.add_comment(Bookname, Booktitle, BookID, Titleid, Book_Date) #这行可注译,数据库发布接口,方便其他系统的发布
; y) Q) E [' n! ] - #post提交发布: p- a( Q6 d8 ^. M. F8 n% j& d
- self.add_locoy(Bookname,Cater_Name,Book_author,Book_Introduction,Book_Palabras,Book_img,Booktitle,BookConte,abover) #这行可注译,火车头发布接口,不需要可取消
6 R! c7 B7 T7 r/ y( ?/ r, E( X - Datos = {
; {( i4 J7 s' X - "Cater_Name":response.save['Cater_Name'],5 ?4 g% m2 }! J# R( j6 i
- "Book_author":response.save['Book_author'],
1 c5 ?0 R4 O9 y) d6 R4 A, U - "Book_Introduction":response.save['Book_Introduction'], f: @, j; R' S2 Z! m3 c; B
- "Book_Synopsis":response.save['Book_Synopsis'],
s- a7 L2 z6 p - "Book_Palabras":response.save['Book_Palabras'],
; @) K6 ~* I. r& @1 g/ r - "img":response.save['img'],3 a, F3 Z y. z( ]+ X$ d5 ~% E+ w
- }: y' Z* C" \/ Q# o6 G I/ @. f
- for each in response.doc('.articlebtn > a:nth-child(4)[href*="/xiaoshuo"]').items():
+ n1 I' W! |0 r) X, \' e0 t$ ? - self.crawl(each.attr.href, callback=self.detail_page,save=Datos) ) _" N% @4 ~( W. ]7 Z
- return {
% L( ^! T9 b4 T8 V) U* Q6 Q - "Cater_Name":Cater_Name,% u0 Q6 j0 ^% U- b4 ]8 w3 [ c6 A* l+ P& k
- "Bookname":Bookname,) D& u$ @9 q, G* }. e
- "Book_author":Book_author,1 b- f1 B3 N, y1 [
- "Book_Introduction":Book_Introduction,
( j, Z w. Q: r) a. `3 G8 w3 q - "Book_Synopsis":Book_Synopsis,
2 d! L& F! s5 A4 Z. Y6 @. V: K - "Book_Palabras":Book_Palabras,4 u5 h4 T$ e: e% h5 x6 x
- "Book_img":Book_img,. T: Z* ]6 H4 D
- "Bookurl": response.url,2 r* D+ I0 | _* B) i! X
- "Booktitle": Booktitle,% A0 T. e3 H; {9 [$ p2 W
- "BookID": BookID,
3 x" P; X" K& I$ e" H - "BookConte": BookConte,
; `2 g; c V' W' t# ]+ O - "Titleid": Titleid,. I+ j$ S2 i8 _: N) D! o
- "abover":abover,; X" r% P+ @& ?( m% \
- # "Book_Date" = str(datetime.datetime.now()), u5 I; g# R( B$ V
- }6 P a( d+ p) E- ?
- def download(self, P_dir, imgDir, file_name, Book_img):. {: D8 v. x. z+ a$ b: d
- if not os.path.exists(imgDir):
, \6 Y- B3 ^% |& U6 A' D - os.makedirs(imgDir)
, p2 d, y" u+ o; o# U7 K* S - file = imgDir + "/" + file_name
' a, R8 s( X5 h8 X0 {6 ` - # print file
$ N( |$ z. P0 E9 u - f = open(file, 'wb+')
8 T2 f8 T3 ^8 Q, L - imag = requests.get(Book_img)
6 v% D$ }2 P4 X" t. j* @2 ~2 }% d - f.write(imag.content)
, D: P* J- T) a" h - f.close()+ V2 J1 B, ?* ]' T {" {8 s3 g" M
- #保存图片前& t2 \ @" z6 w6 G. V
- def save_imgs(self,response):
7 d+ x! h6 }9 }& g2 j - content = response.content1 g, k- y7 t, k8 Q# E
- file_name = response.save["file_name"]3 h3 t* I# O+ U6 T8 O
- imgDir = response.save["imgDir"]" R( a5 r9 L6 v* f1 p" w" b
- file_path = imgDir + file_name
7 a1 a9 T. F' H' Z2 w - self.save_img(content,imgDir,file_path)0 y5 _; }0 g* l6 i' v
- #保存图片
, P! L g' j. k - def save_img(self,content,imgDir,path):2 e* t' Z3 b& Y; j
- if not os.path.exists(imgDir):
. U8 C0 x' E, h# s5 l8 R - os.makedirs(imgDir)
9 x( C! b% H5 ~, E+ ^; R. T! M - f = open(path,"wb" ): x" t% B" D8 p
- f.write(content) D, m; i) O) G# D
- f.close()
+ o% I" {5 N& g& U! I! B - #获取url后缀名
. d. W0 ~$ I1 G. ^2 b - def getExtension(self,url): 6 B, k) [0 \5 V
- extension = url.split(".")[-1]
1 x# g) I$ e0 ? - return extension
- X3 R% w$ F( v9 @0 W -
- t4 e6 n* B) C- l - #获取图片名
% X9 ~1 R3 _0 c1 ~5 e6 e& G - def getname(self,url):
' @2 P9 t" B6 y3 n+ _ - name=url.split("/")[-1].split(".")[0]
; [: P- {; Q+ w( _& L5 y - return name
复制代码
- ?9 t& A3 f B' u7 p, Q( V
7 e$ R# |+ C3 b9 c- ~ |