应用requests、Xpath和selenium编写爬虫脚本

requests.get(url, params=None, **kwargs)
requests.post(url, data=None, **kwargs)

# url：想要获取的网页链接
# params：显示在url中的参数，字典形式
# data：不显示在url中，通过提交表单的方式提交参数，也是字典形式
# **kwargs：控制访问的参数，字典形式

import requests

url = 'https://movie.douban.com/j/chart/top_list'
# 传入的url参数设置
param = {
    'type': '30',	# 影片类型代号
    'interval_id': '100:90',
    'action':'',
    'start': '0',	# 从第一个影片开始
    'limit': '10',	# 需要抓取的影片数
}
# 伪装浏览器
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.46'
}
# 抓取上面定义范围内所有影片信息
response = requests.get(url = url, params = param, headers = headers)
# json字符串反序列化为list类型
ls = response.json()
# 解析电影图片地址，并抓取和保存图片
for p in ls:
    with open('./' + str(p['title']) + '.jpg','wb') as img:
        response1 = requests.get(url = p["cover_url"], headers = headers)
        img.write(response1.content)
        
print('over!!!')

import requests
from lxml import etree
import os
# 举个例子，这里抓取的是第二页数据
for i in range(2,3):
    url = 'http://pic.netbian.com/4kdongman/index_'+str(i)+'.html'
    headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.3'
    }
    response = requests.get(url = url, headers = headers)
    # 获取网页内容字符串
    page_text = response.text

	# 实例化etree对象，获取所有图片li标签信息（列表格式）
    tree = etree.HTML(page_text)
    li_list = tree.xpath('//div[@class="slist"]/ul/li')
    
	# 创建文件夹
    if not os.path.exists('./pic'):
            os.mkdir('./pic')
            
    # 解析li标签下孙子标签img的src属性和alt属性
    for li in li_list:
        img_src = 'http://pic.netbian.com' + li.xpath('./a/img/@src')[0]
        img_name = li.xpath('./a/img/@alt')[0] + '.jpg'
        # 这里需要处理中文乱码问题，重新编码和解码，gbk是中文最常用的
        img_name = img_name.encode('iso-8859-1').decode('gbk')
        img_data = requests.get(url = img_src, headers = headers).content
        img_path = 'pic/' + img_name
        with open(img_path,'wb') as fp:
                fp.write(img_data)
                print(img_name,'下载成功！！！')

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from time import sleep
# 启动前先将驱动程序放在当前页面
bro = Service(executable_path = './chromedriver.exe')
# 启动谷歌浏览器
driver = webdriver.Chrome(service = bro)
# 进入淘宝网页页面
driver.get('https://www.taobao.com/')
# 标签定位，输入栏id属性值为'q'，id属性是整个html中唯一的，不会重复
search_input = driver.find_element(By.ID, 'q')
# 节点交互，向输入栏中输入数据'Iphone'
search_input.send_keys('Iphone')
# 标签定位，找到搜索按钮（可以用css选择器、id值或者Xpath等定位，这里用css选择器）
btn = driver.find_element(By.CSS_SELECTOR, '.btn-search')
# 节点交互，点击搜索按钮
btn.click()
driver.get('https://www.baidu.com')
sleep(2)
# 回退
driver.back()
sleep(2)
# 前进
driver.forward()
sleep(2)
# 退出浏览器
driver.quit()

from selenium.webdriver.common.by import By

find_element(By.ID, 'value')	
find_element(By.NAME, 'value')
find_element(By.CLASS_NAME, 'value')
find_element(By.TAG_NAME, 'value')
find_element(By.LINK_TEXT, 'value')
find_element(By.PARTIAL_LINK_TEXT, 'value')
find_element(By,XPATH, 'value')
find_element(By.CSS_SELECTOR, 'value')

driver.get("https://www.baidu.com")		# 打开网站
driver.back()		# 浏览器后退
driver.forward()		# 浏览器前进
driver.refresh()		# 浏览器刷新
driver.add_cookie({"name": "key", "value": "value"})		# 当前浏览器添加cookie
driver.find_element(By.LINK_TEXT, "new window").click()		# 新窗口中打开链接
driver.find_element(By.ID, 'value').send_keys('value')		# 定位并发送内容
driver.switch_to.new_window('tab')		# 打开新标签页并切换到新标签页
driver.switch_to.new_window('window')	# 打开新窗口并切换到新窗口
driver.switch_to.window(original_window)	# 切回之前的标签页或窗口
driver.close()		# 关闭标签页或窗口
driver.quit()		# 关闭浏览器

import time
from selenium.webdriver.common.by import By
from selenium import webdriver
from time import sleep
from selenium.webdriver import ActionChains

bro = webdriver.Chrome(executable_path = './chromedriver')
bro.get('https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable')

# 如果定位的标签是存在于iframe标签之中的则必须通过如下操作在进行标签定位
bro.switch_to.frame('iframeResult')		# 切换浏览器标签定位的作用域
div = bro.find_element(By.ID, 'draggable')

# 动作链
action = ActionChains(bro)
# 点击长按指定的标签
action.click_and_hold(div)

for i in range(10):
    action.move_by_offset(5,0)		# move_by_offset(x,y):x水平方向 y竖直方向
    action.perform()	# 执行动作链操作
time.sleep(3)
#释放动作链
action.release()
bro.quit()

import random
import re
from pandas import DataFrame
import os
import requests
import time
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from time import sleep
from selenium.webdriver import ChromeOptions
from lxml import etree
# moder可以为author或者article，前者为按公众号搜索，后者为按文章关键字搜索
modern='author'
# 搜索的关键字，如果modern = author，输入公众号名字，否则输入文章关键字
keyword = '冷兔'
# 爬取多少页，建议先手动搜索最大页码数，page大于最大页码将会报错
page = 2
headers = {
        'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'
    }
# 实现规避检测
option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
bro = webdriver.Chrome(executable_path = './chromedriver', options=option)
bro.get("https://www.sogou.com/index.php")
wait = WebDriverWait(bro, 2)
search_input = bro.find_element(By.ID, 'query')
search_input.send_keys(keyword)
# 点击搜索按钮
btn = bro.find_element(By.ID, 'stb')
btn.click()
# 点击微信登陆
time.sleep(2)
btn = bro.find_element(By.XPATH, '//*[@id="loginBtn"]')
btn.click()
# 用微信扫码，只有十秒
time.sleep(2)
btn = bro.find_element(By.XPATH, '/html/body/div[3]/div[3]/div[2]/div[4]/div/a[2]')
btn.click()
time.sleep(10)
button = bro.find_element(By.ID, 'sogou_weixin')
button.click()
article_button = bro.find_element(By.XPATH, '//*[@id="scroll-header"]/form/div/input[1]')
article_button.click()
# 最后需要爬取的文章url都在这里
url_list=[]
for i in range(page):
    page_text = bro.page_source

	# 解析
    tree = etree.HTML(page_text)
    author=tree.xpath('/html/body/div[2]/div[1]/div[3]/ul/li/div[2]/div/a/text()')
    print(author)
    url_page_list=tree.xpath('/html/body/div[2]/div[1]/div[3]/ul/li/div[2]/h3/a/@href')
    # 下面这个循环判断按author爬取还是按照aiticle爬取
    for j in range(len(author)):
        if author[j]==keyword:
            url_list.append(url_page_list[j])
        elif modern=='article':
            url_list.append(url_page_list[j])
        else:
            continue
    if i != page - 1:
        next_button = bro.find_element(By.ID, 'sogou_next')
        next_button.click()
    time.sleep(2)
# 拼接地址
url_list=['https://weixin.sogou.com/'+i for i in url_list]
title_list=[]
content_list=[]
time_list=[]
author_list=[]
for url in url_list:
    response = bro.get(url=url)
    time.sleep(random.randint(1, 3))	    # 爬一个，休息1-3秒，怕被抓
    page_text = bro.page_source
    tree_content = etree.HTML(page_text)	# 获取爬到的动态页面源码
    title = tree_content.xpath('/html/body/div[1]/div[2]/div[1]/div/div[1]/h1/text()')
    title = re.sub(r'\s', '', ''.join(title))	# 获取到的文章题目
    content = tree_content.xpath('/html/body/div[1]/div[2]/div[1]/div/div[1]/div[3]//text()')
    content = re.sub(r'\s*', '', ''.join(content))	# 获取到的文章内容
    time1=tree_content.xpath('//*[@id="publish_time"]/text()')[0]
    author=tree_content.xpath('//*[@id="js_name"]/text()')[0]
    author=re.sub(r'\s', '', ''.join(author))
    title_list.append(title)
    content_list.append(content)
    time_list.append(time1)
    author_list.append(author)

# 写入xlsx文件中
data = {'title': title_list, 'time':time_list,'author':author_list,'content': content_list}
df = DataFrame(data)
df.to_excel('./微信公众号_'+keyword+'.xlsx')