​🎬 攻城狮7号：个人主页

pip install selenium pyquery

# 可选：pip install pymongo

​

​

<ul class="goods-list">

  <li>

    <a href="商品详情链接">

      <img src="图片链接" ...>

      <div class="title">商品名称</div>

      <div class="price">价格</div>

      ...

    </a>

  </li>

  ...

</ul>

​

<div class="mi-pagenav">

  <span class="numbers current">1</span>

  <a class="numbers">2</a>

  <a class="numbers">3</a>

  ...

</div>

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.chrome.service import Service

from selenium.webdriver.chrome.options import Options

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from pyquery import PyQuery as pq

import time


KEYWORD = '空调'

MAX_PAGE = 3


chrome_options = Options()

# chrome_options.add_argument('--headless')  # 如需无头模式可取消注释

chrome_options.add_argument('--disable-gpu')

chrome_options.add_argument('--no-sandbox')

chrome_options.add_argument('--disable-dev-shm-usage')

chrome_options.add_experimental_option("detach", True)


service = Service()

browser = webdriver.Chrome(service=service, options=chrome_options)

wait = WebDriverWait(browser, 20)


def get_products(page):

    html = browser.page_source

    doc = pq(html)

    products = []

    for item in doc('.goods-list li, .product-list li, .list li, .goods-item, .product-item').items():

        title = item.find('.title, .name, .pro-title, .text').text()

        price = item.find('.price, .pro-price, .num').text()

        img = item.find('img').attr('src') or item.find('img').attr('data-src')

        link = item.find('a').attr('href')

        if title and ('空调' in title or '空调' in item.text()):

            products.append({

                'title': title,

                'price': price,

                'img': img,

                'link': link

            })

    if not products:

        for a in doc('a').items():

            text = a.text()

            if '空调' in text:

                img = a.find('img').attr('src') or a.find('img').attr('data-src')

                price = ''

                parent = a.parent()

                for p in parent.parents():

                    price = p.find('.price, .pro-price, .num').text()

                    if price:

                        break

                products.append({

                    'title': text,

                    'price': price,

                    'img': img,

                    'link': a.attr('href')

                })

    print(f'第{page}页空调商品:')

    for p in products:

        print(p)

    return products


def main():

    browser.get(f'https://www.mi.com/shop/search?keyword={KEYWORD}')

    for page in range(1, MAX_PAGE + 1):

        wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, 'body')))

        time.sleep(2)

        get_products(page)

        if page < MAX_PAGE:

            try:

                page_btn = browser.find_element(By.XPATH, f'//div[contains(@class,"mi-pagenav")]//a[@class="numbers" and text()="{page+1}"]')

                browser.execute_script("arguments[0].click();", page_btn)

                time.sleep(2)

            except Exception as e:

                print(f'点击第{page+1}页失败:', e)

                break

    browser.quit()


if __name__ == '__main__':

    main()

​

import pymongo

MONGO_URL = 'localhost'

MONGO_DB = 'xiaomi'

MONGO_COLLECTION = 'ac_products'

client = pymongo.MongoClient(MONGO_URL)

db = client[MONGO_DB]

def save_to_mongo(result):

    try:

        if db[MONGO_COLLECTION].insert_one(result):

            print('存储到 MongoDB 成功')

    except Exception:

        print('存储到 MongoDB 失败')

chrome_options.add_argument('--headless')