from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import webdriverwait
from selenium.webdriver.support import expected_conditions as ec
from lxml import etree
import csv
import requests,re,time
#搜索的商品名称
shopname="Python设计模式"
#声明浏览器对象
browser=webdriver.Chrome()
browser.get("https://www.jd.com")
#查找节点
inputtext = browser.find_element_by_class_name('text')
#输入数据
inputtext.send_keys(shopname)
#提交
btn = browser.find_element_by_class_name('button')
btn.click()
#搜索后的页面
#显式等待
wait = webdriverwait(browser, 10)
wait.until(ec.title_contains(shopname))
with open(shopname+".csv",'a') as f:
wr= csv.DictWriter(f,['name','price','shop'])
wr.writeheader()
while True:
#判断是否为反爬虫机制窗体 是否正常
if len(browser.window_handles)>1:
handles=browser.window_handles[1]
browser.switch_to_window(handles)
browser.close()
# 滚动条
browser.execute_script("window.scrollTo(0, document.body.scrollHeight)")
wait.until(ec.presence_of_element_located((By.CLASS_NAME, 'pn-next')))
# 爬取内容
html = etree.HTML(browser.page_source)
# 读取每个商品
shops = html.xpath('//div[contains(@class,"gl-i-wrap")]')
# 下一页
npage =html.xpath('//a[@class="pn-next disabled"]/em//text()')
for shop in shops:
name = shop.xpath('.//div[contains(@class,"p-name")]//em//text()')
name = "".join(name)
price = shop.xpath('.//div[contains(@class,"p-price")]//i//text()')
price = "".join(price)
sname = shop.xpath('.//div[contains(@class,"p-shop")]//a//@title')
sname = "".join(sname)
if sname.strip() == '':
sname = "京东自营"
wr.writerow({'name':name,'price':price,'shop':sname})
if len(npage)>0:
break
try:
pbtn = browser.find_element_by_class_name("pn-next")
pbtn.click()
except:
pass
browser.close()
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。