基于selenium+phantomJS的动态网页抓取

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

# set phantomJS's agent to Firefox
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = \
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0 "

# system path you need to config by yourself
phantomjsPath = "/Users/Shawn/Downloads/phantomjs-2.1.1-macosx/bin/phantomjs"

driver = webdriver.PhantomJS(executable_path=phantomjsPath, desired_capabilities=dcap)

1
2

login_url = "http://mis.sse.ustc.edu.cn/"
driver.get(login_url)

1
2
3

idXpath = ".//*[@id='winLogin_sfLogin_txtUserLoginID']"
pwXpath = ".//*[@id='winLogin_sfLogin_txtPassword']"
loginXpath = ".//*[@id='ext-gen5']"

1
2
3
4

wait = WebDriverWait(driver, 10)
wait.until(EC.presence_of_element_located((By.XPATH, idXpath)))
wait.until(EC.presence_of_element_located((By.XPATH, pwXpath)))
wait.until(EC.element_to_be_clickable((By.XPATH, loginXpath)))

1
2
3
4

driver.find_element_by_xpath(idXpath).send_keys(username)
driver.find_element_by_xpath(pwXpath).send_keys(password)
driver.find_element_by_xpath(loginXpath).click()
print("Login Success!")

1
2

redirectXpath = ".//*[@id='RegionPanel1_UpRegion_ContentPanel1_content']/table/tbody/tr/td[2]"
wait.until(EC.presence_of_element_located((By.XPATH, redirectXpath)))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

def save_cookies(driver, file_path):
    # The format could vary
    LINE = "document.cookie = '{name}={value}; path={path}; domain={domain}';\n"
    with open(file_path, 'w') as fd:
        for cookie in driver.get_cookies():
            fd.write(LINE.format(**cookie))


def load_cookies(driver, file_path):
    with open(file_path, 'r') as fd:
        driver.execute_script(fd.read())

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

def login(login_url, username, password):
    driver = webdriver.PhantomJS(executable_path=phantomjsPath,
                                 desired_capabilities=dcap)
    try:
        driver.get(login_url)
        wait = WebDriverWait(driver, 10)
        wait.until(EC.presence_of_element_located((By.XPATH, idXpath)))
        wait.until(EC.presence_of_element_located((By.XPATH, pwXpath)))
        wait.until(EC.element_to_be_clickable((By.XPATH, loginXpath)))

        driver.find_element_by_xpath(idXpath).send_keys(username)
        driver.find_element_by_xpath(pwXpath).send_keys(password)
        driver.find_element_by_xpath(loginXpath).click()
        print("Login Success!")

        wait.until(EC.presence_of_element_located((By.XPATH, redirectXpath)))
        print("Current url is: " + driver.current_url)
        print("Current cookies are: " + str(driver.get_cookies()))
        save_cookies(driver, r'cookies.js')
    except Exception as ex:
        print("Login failed!")
        print(ex)
    finally:
        driver.close()

1
2
3
4
5
6

scrape_url = "http://mis.sse.ustc.edu.cn/homepage/StuDefault.aspx"

driver.get(login_url)
driver.delete_all_cookies()
load_cookies(driver, r'cookies.js')
driver.get(scrape_url)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

def scraper(login_url, scrape_url):
    driver = webdriver.PhantomJS(executable_path=phantomjsPath,
                                 desired_capabilities=dcap)
    try:
        driver.get(login_url)
        driver.delete_all_cookies()
        load_cookies(driver, r'cookies.js')
        driver.get(scrape_url)

        if driver.current_url == login_url:
            raise NameError('Redirect failed!')
        print("Scraper login through cookie success!")
        print("scrapping...")
        # Do anything here! Scrape!
    except Exception as ex:
        print("Scraper login through cookie failed!")
        print(ex)
    finally:
        logout(driver, login_url)

1
2
3
4
5
6
7
8

import re
infoXpath = ".//*[@id='ext-gen51']"

elem = driver.find_element_by_xpath(infoXpath)
table_source = elem.get_attribute('innerHTML')

reg = re.compile("/Base/NoticeInfo/ViewNotice\.aspx\?ID=[-a-z0-9]+")
link_list = re.findall(reg, table_source)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# 设置保存地址
savePath = "/Users/Shawn/Documents/webscraper/data/"

for link in link_list:
    page_link = login_url + link
    # 取得链接内容的源代码
    driver.get(page_link)
    pagesource = driver.page_source
    # 用beautifulsoup解析页面内容并提取标题
    from bs4 import BeautifulSoup
    bsObj = BeautifulSoup(pagesource, "html.parser")
    title = bsObj.find("span", {"id": "lblNoticeName"}).get_text().encode('utf-8')
    # 保存到本地, 文件名就是标题名
    with open(savePath + str(title)+".html", 'w') as fd:
        fd.write(bsObj.get_text().encode('utf-8') + '\n')