spider/selenium_web.py at master · bsbforever/spider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
__author__ = 'bsbfo'
#coding=utf8
import requests
import json
import selenium.webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import time
import os
import random

os.environ["LANG"] = "en_US.UTF-8"


def scroll(n,i):
    return "window.scrollTo(0,(document.body.scrollHeight/{0})*{1}*80);".\
        format(n,i)

def comic(url):
    comics={}
    comic_url=[]
    #driver = selenium.webdriver.Chrome()
    driver = selenium.webdriver.PhantomJS()
    #driver.implicitly_wait(30)
    driver.set_page_load_timeout(30)
    #url='http://ac.qq.com/ComicView/index/id/543606/cid/1'
    #url='https://manhua.163.com/reader/4639712296520118385/4643138479170091194#scale=7@imgIndex=8'
    driver.get(url)
    time.sleep(4)
    n = 20
    for i in range(0,n+1):
        s = scroll(n,i)
        print(s)
        driver.execute_script(s)
        time.sleep(random.randint(1, 10))


    content=driver.page_source

    #print (content)
    soup = BeautifulSoup(content,"lxml")
    if '163' in url:
        comic_list=soup.find_all('img',attrs = {'draggable' : 'false'})
        for i in comic_list:
            print(i)
            try:
                image_src=i['src']
            except Exception as e:
                print (e)
            print (image_src)

    else:

        comic_list=soup.find_all('li',attrs = {'style' : True})
        comic_title=soup.find('span',attrs = {'class' : 'title-comicHeading'}).text.strip()
        comic_title=comic_title.replace('/',' ')
        for i in comic_list:
            #image_num=0
            try:
                #image_num=i.find('em').text.strip()
                image_url=i.find('img')['src']
                #print(image_url)
                comic_url.append(image_url)
            except Exception as e:
                print (e)

        comics['title']=comic_title
        comics['url']=comic_url
            #print (image_num)
            #print (image_url)

            #break
    return comics
    #print (content)
    #soup = BeautifulSoup(content,"lxml")


def download(comics):
    #directory='d:\\manhua\\'+directory
    title=comics['title']
    urls=comics['url']
    directory='d:\\manhua\\幽游白书\\'+title

    if os.path.exists(directory):
        for download_link in urls:
            time.sleep(random.randint(1, 10))
            fname=directory+'\\'+str(urls.index(download_link)+1)+'.png'
            if os.path.exists(fname):
                #pass
                print ('File '+fname+' is already exists,SKIP......')
            else:
           #     print ('Folder is already exists,Downloding '+directory+'.....Please Waiting')
                r=requests.get(download_link)
                with open (fname,"wb") as code:
                    code.write(r.content)
                print(fname+' Download Compelte')
    else:
        os.mkdir(directory)
        #     print ('Downloding '+directory+'.....Please Waiting')
        for download_link in urls:
            time.sleep(random.randint(1, 10))
            fname=directory+'\\'+str(urls.index(download_link)+1)+'.png'
            if os.path.exists(fname):
                #pass
                print ('File '+fname+' is already exists,SKIP......')
            else:
           #     print ('Folder is already exists,Downloding '+directory+'.....Please Waiting')
                r=requests.get(download_link)
                with open (fname,"wb") as code:
                    code.write(r.content)
                print(fname+' Download Compelte')

if __name__=="__main__":

    url='http://ac.tc.qq.com/store_file_download?buid=15017&uin=1422363399&dir_path=/&name=27_20_56_3faad5b60275c2829819a8c41cb0a919_953.ori'

    for i in range(71,72):
        url='http://ac.qq.com/ComicView/index/id/543606/cid/'+str(i)

        print ('Now Analyzing  Chapter '+ str(i)+',waiting for 60 seconds')
        time.sleep(random.randint(50, 60))
        result=comic(url)
        download(result)


    #print (result)