requests和lxml实现爬虫的方法


            如下所示：

# requests模块来请求页面

# lxml模块的html构建selector选择器(格式化响应response)

# from lxml import html

# import requests

# response = requests.get(url).content

# selector = html.formatstring(response)

# hrefs = selector.xpath('/html/body//div[@class='feed-item _j_feed_item']/a/@href')

# 以url = 'http://www.mafengwo.cn/gonglve/ziyouxing/2033.html'为例子





# python 2.7

import requests

from lxml import html

import os







# 获取首页中子页的url链接

def get_page_urls(url):

  response = requests.get(url).content

  # 通过lxml的html来构建选择器

  selector = html.fromstring(response)

  urls = []

  for i in selector.xpath("/html/body//div[@class='feed-item _j_feed_item']/a/@href"):

    urls.append(i)

  return urls







# get title from a child's html(div[@class='title'])

def get_page_a_title(url):

  '''url is ziyouxing's a@href'''

  response = requests.get(url).content

  selector = html.fromstring(response)

  # get xpath by chrome's tool --> /html/body//div[@class='title']/text()

  a_title = selector.xpath("/html/body//div[@class='title']/text()")

  return a_title







# 获取页面选择器(通过lxml的html构建)

def get_selector(url):

  response = requests.get(url).content

  selector = html.fromstring(response)

  return selector







# 通过chrome的开发者工具分析html页面结构后发现，我们需要获取的文本内容主要显示在div[@class='l-topic']和div[@class='p-section']中







# 获取所需的文本内容

 def get_page_content(selector):

   # /html/body/div[2]/div[2]/div[1]/div[@class='l-topic']/p/text()

   page_title = selector.xpath("//div[@class='l-topic']/p/text()")

   # /html/body/div[2]/div[2]/div[1]/div[2]/div[15]/div[@class='p-section']/text()

   page_content = selector.xpath("//div[@class='p-section']/text()")

   return page_title,page_content







# 获取页面中的图片url地址

def get_image_urls(selector):

  imagesrcs = selector.xpath("//img[@class='_j_lazyload']/@src")

  return imagesrcs







# 获取图片的标题



def get_image_title(selector, num)

  # num 是从2开始的

  url = "/html/body/div[2]/div[2]/div[1]/div[2]/div["+num+"]/span[@class='img-an']/text()"

  if selector.xpath(url) is not None:

    image_title = selector.xpath(url)

  else:

    image_title = "map"+str(num) # 没有就起一个

  return image_title







# 下载图片



def downloadimages(selector,number):

  '''number是用来计数的'''

  urls = get_image_urls()

  num = 2

  amount = len(urls)

  for url in urls:

    image_title = get_image_title(selector, num)

    filename = "/home/WorkSpace/tour/words/result"+number+"/+"image_title+".jpg"

    if not os.path.exists(filename):

      os.makedirs(filename)

    print('downloading %s image %s' %(number, image_title))

    with open(filename, 'wb') as f:

      f.write(requests.get(url).content)

    num += 1

  print "已经下载了%s张图" %num







# 入口，启动并把获取的数据存入文件中

if __name__ =='__main__':

  url = 'http://www.mafengwo.cn/gonglve/ziyouxing/2033.html'

  urls = get_page_urls(url)

  # turn to get response from html

  number = 1

  for i in urls:

    selector = get_selector(i)

    # download images

    downloadimages(selector,number)

    # get text and write into a file

    page_title, page_content = get_page_content(selector)

    result = page_title+'\n'+page_content+'\n\n'

    path = "/home/WorkSpace/tour/words/result"+num+"/"

    if not os.path.exists(filename):

      os.makedirs(filename)

    filename = path + "num"+".txt"

    with open(filename,'wb') as f:

      f.write(result)

    print result



到此就结束了该爬虫，爬取页面前一定要认真分析html结构，有些页面是由js生成，该页面比较简单，没涉及到js的处理，日后的随笔中会有相关分享

以上这篇requests和lxml实现爬虫的方法就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持中文源码网。