爬虫：xpath模块及昵图网实例

xpath模块

from lxml import etree  str1 = """ <div>     <ul>          <li class="item-0"><a href="link1.html">first item</a></li>          <li class="item-1"><a href="link2.html">second item</a></li>          <li class="item-inactive"><a href="link3.html"><span class="bold">third item</span></a></li>          <li class="item-1"><a href="link4.html">fourth item</a></li>          <li class="item-0"><a class="test" href="link5.html">fifth item</a>111</li>          <li class="item-0" vmid="10023898942">bsajkfhsdbfhjsdgbhfds</li>      </ul>  </div> """ # 将字符串转化为html对象 strTree1 = etree.HTML(str1) print(strTree1.xpath('//li/text()')) #获取所有li里面的内容，这里只能获取直属li里面的内容 print(strTree1.xpath('//li/@class')) # xpath中的下标，是从1开始的 print(strTree1.xpath('//li[3]/a/@href')) # xpath中没有负数下标，若从右向左取，利用last获取最后一个 通过减一减二的方式获取倒数第二个倒数第三个 print(strTree1.xpath('//li[last()-1]/text()')) # 获取指定属性值的标签 print(strTree1.xpath('//li[@class="item-1"]//text()'))

昵图网实例

from requests_html import HTMLSession from lxml import etree  session = HTMLSession() url = 'https://soso.nipic.com/?q=%E7%BE%8E%E5%A5%B3'  response = session.get(url) html = etree.HTML(response.text)  imgNames = [] imgUrls = [] for i in range(1, 6):     img_name = html.xpath(f'//ul[@id="img-list-outer"]/li[{i}]/a/img/@alt')     img_url = html.xpath(f'//ul[@id="img-list-outer"]/li[{i}]/a/img/@data-original')     imgNames.append(img_name[0])     imgUrls.append('https:' + img_url[0])  for i in range(len(imgUrls)):     with open('美女\\' + imgNames[i] + '.jpg', 'wb') as f:         f.write(session.get(imgUrls[i]).content)  # content就是将响应转化为二进制内容