示例

一些例子以便于你更好地理解crawlist

StaticPager

StaticRedirectPager

import crawlist as cl

if __name__ == '__main__':
    pager = cl.StaticRedirectPager(uri="https://www.douban.com/doulist/893264/?start=0&sort=seq&playable=0&sub_type=",
                                   uri_split="https://www.douban.com/doulist/893264/?start=%v&sort=seq&playable=0&sub_type=",
                                   start=0,
                                   offset=25)
    selector = cl.CssSelector(pattern=".doulist-item")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))

StaticListRedirectPager

import crawlist as cl

if __name__ == '__main__':
    uris = ["https://www.douban.com/doulist/893264/?start=0&sort=seq&playable=0&sub_type=",
            "https://www.douban.com/doulist/893264/?start=25&sort=seq&playable=0&sub_type=",
            "https://www.douban.com/doulist/893264/?start=50&sort=seq&playable=0&sub_type=",
            "https://www.douban.com/doulist/893264/?start=75&sort=seq&playable=0&sub_type="]
    pager = cl.StaticListRedirectPager(uris=uris)
    selector = cl.CssSelector(pattern=".doulist-item")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))

DynamicPager

DynamicRedirectPager

import crawlist as cl

if __name__ == '__main__':
    pager = cl.DynamicRedirectPager(uri="https://www.boc.cn/sourcedb/whpj/index.html",
                                    uri_split="https://www.boc.cn/sourcedb/whpj/index_%v.html",
                                    start=0)
    selector = cl.CssSelector(pattern="body > div > div.BOC_main > div.publish > div:nth-child(3) > table > tbody > tr")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

DynamicListRedirectPager

import crawlist as cl

if __name__ == '__main__':
    uris = ["https://www.boc.cn/sourcedb/whpj/index.html",
            "https://www.boc.cn/sourcedb/whpj/index_1.html",
            "https://www.boc.cn/sourcedb/whpj/index_2.html",
            "https://www.boc.cn/sourcedb/whpj/index_3.html",
            "https://www.boc.cn/sourcedb/whpj/index_4.html"]
    pager = cl.DynamicListRedirectPager(uris=uris)
    selector = cl.CssSelector(pattern="body > div > div.BOC_main > div.publish > div:nth-child(3) > table > tbody > tr")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

DynamicScrollPager

import crawlist as cl

if __name__ == '__main__':
    pager = cl.DynamicScrollPager(uri="https://ec.ltn.com.tw/list/international")
    selector = cl.CssSelector(pattern="#ec > div.content > section > div.whitecon.boxTitle.boxText > ul > li")
    analyzer = cl.AnalyzerLinks(pager=pager, selector=selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

DynamicLineButtonPager

import crawlist as cl

if __name__ == '__main__':
    button_selector = cl.CssWebElementSelector(pattern='#newslist > button')
    pager = cl.DynamicLineButtonPager(uri="https://www.yicai.com/news/kechuang/",
                                      button_selector=button_selector)
    selector = cl.XpathSelector(pattern='//*[@id="newslist"]/a')
    analyzer = cl.AnalyzerPrettify(pager=pager, selector=selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

DynamicNumButtonPager

import crawlist as cl

if __name__ == '__main__':
    button_selector = cl.CssWebElementSelector(pattern='#kkpager > div:nth-child(1) > span.pageBtnWrap > a')
    pager = cl.DynamicNumButtonPager(uri="https://pccz.court.gov.cn/pcajxxw/gkaj/gkaj",
                                     button_selector=button_selector)
    selector = cl.CssSelector(pattern='#gkajlb ul')
    analyzer = cl.AnalyzerPrettify(pager=pager, selector=selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

DynamicNextButtonPager

import crawlist as cl

if __name__ == '__main__':
    button_selector = cl.CssWebElementSelector(pattern='#downpage')
    pager = cl.DynamicNextButtonPager(uri="https://news.lnd.com.cn/xwyw/index.shtml",
                                      button_selector=button_selector, start=2, offset=2)
    selector = cl.CssSelector(
        pattern='body > div:nth-child(5) > div:nth-child(2) > div:nth-child(1) > div.main_more > span > ul > li')
    analyzer = cl.AnalyzerPrettify(pager=pager, selector=selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

Pre-loading

from selenium.webdriver.remote.webdriver import WebDriver
import crawlist as cl

if __name__ == '__main__':
    class MyPager(cl.DynamicLineButtonPager):
        def pre_load(self, webdriver: WebDriver = None) -> None:
            webdriver.get("https://kuaixun.eastmoney.com/")
            cl.Action.click(webdriver,
                                '/html/body/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[3]/label/span[1]')
        
    pager = MyPager(uri="https://kuaixun.eastmoney.com/",
                                      button_selector=cl.CssWebElementSelector('#news_list > div.load_more'))
    selector = cl.CssSelector(pattern="#news_item_collection > div")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))

from selenium.webdriver.remote.webdriver import WebDriver
from selenium.webdriver.common.by import By

import crawlist as cl

if __name__ == '__main__':
    button_selector = cl.CssWebElementSelector(pattern='#frs_list_pager > a')
    uri = "https://tieba.baidu.com/f?kw=%E8%B4%B4%E5%90%A7&ie=utf-8&pn=0"


    class MyPager(cl.DynamicNumButtonPager):
        def pre_load(self, webdriver: WebDriver = None) -> None:
            webdriver.get(uri)
            #  You also could use the selenium methods.
            element = webdriver.find_element(By.XPATH, '//*[@id="tiebaCustomPassLogin"]/div[2]/span')
            element.click()

    pager = MyPager(uri=uri,
                    button_selector=button_selector)
    selector = cl.CssSelector(pattern='#thread_list > li')
    analyzer = cl.AnalyzerPrettify(pager=pager, selector=selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

Pre-loading with scripts

from selenium.webdriver.remote.webdriver import WebDriver
import crawlist as cl

if __name__ == '__main__':
    class MyPager(cl.DynamicNumButtonPager):
        def pre_load(self, webdriver: WebDriver) -> None:
            script = {
                "method": "redirect",
                "url": "https://www.baidu.com/",
                "next": {
                    "method": "inputKeyword",
                    "xpath": "//*[@id=\"kw\"]",
                    "keyword": "和泉雾纱",
                    "next": {
                        "method": "click",
                        "xpath": "//*[@id=\"su\"]"
                    }
                }
            }  # You could save the script as Json on your physical storage
            cl.Script(script)(webdriver)

    pager = MyPager(uri=baidu_uri,
                    button_selector=cl.XpathWebElementSelector('//*[@id="page"]/div/a/span'),
                    webdriver=cl.DefaultDriver(isDebug=True), interval=5)
    selector = cl.XpathSelector(pattern='/html/body/div[3]/div[3]/div[1]/div[3]/div')
    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(TestCase.limit):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()

Implement your own Request

import requests
from requests import HTTPError

import crawlist as cl

if __name__ == '__main__':
    class MyRequest(cl.Request):
        def request(self, uri: str) -> str:
            try:
                headers = {
                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",
                    'Cache-Control': 'max-age=0',
                }
                r = requests.get(uri, headers=headers)
                r.raise_for_status()
                r.encoding = r.apparent_encoding
                return r.text
            except HTTPError:
                return ""


    pager = cl.StaticRedirectPager(uri="https://www.douban.com/doulist/893264/?start=0&sort=seq&playable=0&sub_type=",
                                   uri_split="https://www.douban.com/doulist/893264/?start=%v&sort=seq&playable=0&sub_type=",
                                   start=0,
                                   offset=25, request=MyRequest())
    selector = cl.CssSelector(pattern=".doulist-item")

    analyzer = cl.AnalyzerPrettify(pager, selector)
    res = []
    for tr in analyzer(100):
        print(tr)
        res.append(tr)
    print(len(res))

Use your own webdriver

import crawlist as cl
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver as wd
from selenium.webdriver.chrome.service import Service

if __name__ == '__main__':
    class MyDriver(cl.Driver):
        def __init__(self):
            pass

        def get_driver(self) -> WebDriver:
            option = wd.ChromeOptions()
            option.add_argument("start-maximized")
            option.add_argument("--headless")
            option.add_argument("window-size=1920x3000")
            agent = 'user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"'
            option.add_argument(agent)
            my_webdriver = wd.Chrome(service=Service(ChromeDriverManager().install()), options=option)
            return my_webdriver

    pager = cl.DynamicScrollPager(uri="https://ec.ltn.com.tw/list/international", webdriver=MyDriver())
    # pager = cl.DynamicScrollPager(uri="https://ec.ltn.com.tw/list/international", webdriver=MyDriver().get_driver())
    selector = cl.CssSelector(pattern="#ec > div.content > section > div.whitecon.boxTitle.boxText > ul > li")
    analyzer = cl.AnalyzerPrettify(pager=pager, selector=selector)
    res = []
    for tr in analyzer(TestCase.limit):
        print(tr)
        res.append(tr)
    print(len(res))
    pager.webdriver.quit()