Python通过解析网页实现看报程序的方法


            本文所述实例可以实现基于Python的查看图片报纸《参考消息》并将当天的图片报纸自动下载到本地供查看的功能，具体实现代码如下：





# coding=gbk

import urllib2

import socket

import re

import time

import os



# timeout in seconds

#timeout = 10

#socket.setdefaulttimeout(timeout)

timeout = 10

urllib2.socket.setdefaulttimeout(timeout)



home_url = "http://www.hqck.net"

home_page = ""

try:

  home_page_context = urllib2.urlopen(home_url)

  home_page = home_page_context.read()



  print "Read home page finishd."

  print "-------------------------------------------------"

except urllib2.URLError,e:

  print e.code

  exit()

except:

  print e.code

  exit()



reg_str = r'<a class="item-baozhi" href="/arc/jwbt/ckxx/\d{4}/\d{4}/\w+\.html" rel="external nofollow" ><span class.+>.+</span></a>'



news_url_reg = re.compile(reg_str)



today_cankao_news = news_url_reg.findall(home_page)



if len(today_cankao_news) == 0:

  print "Cannot find today's news!"

  exit()



my_news = today_cankao_news[0]

print "Latest news link = " + my_news

print



url_s = my_news.find("/arc/")

url_e = my_news.find(".html")

url_e = url_e + 5



print "Link index = [" + str(url_s) + "," + str(url_e) + "]"

my_news = my_news[url_s:url_e]

print "part url = " + my_news



full_news_url = home_url + my_news

print "full url = " + full_news_url

print



image_folder = "E:\\new_folder\\"



if (os.path.exists(image_folder) == False):

  os.makedirs(image_folder)

today_num = time.strftime('%Y-%m-%d',time.localtime(time.time()))

image_folder = image_folder + today_num + "\\"

if (os.path.exists(image_folder) == False):

  os.makedirs(image_folder)

print "News image folder = " + image_folder

print



context_uri = full_news_url[0:-5]



first_page_url = context_uri + ".html"

try:

  first_page_context = urllib2.urlopen(first_page_url)

  first_page = first_page_context.read()

except urllib2.HTTPError, e:

  print e.code

  exit()



tot_page_index = first_page.find("共")

tot_page_index = tot_page_index



tmp_str = first_page[tot_page_index:tot_page_index+10]

end_s = tmp_str.find("页")



page_num = tmp_str[2:end_s]

print page_num



page_count = int(page_num)

print "Total " + page_num + " pages:"

print



page_index = 1

download_suc = True

while page_index <= page_count:

  page_url = context_uri

  if page_index > 1:

    page_url = page_url + "_" + str(page_index)

  page_url = page_url + ".html"

  print "News page link = " + page_url



  try:

    news_img_page_context = urllib2.urlopen(page_url)

  except urllib2.URLError,e:

    print e.reason

    download_suc = False

    break

  

  news_img_page = news_img_page_context.read()



  #f = open("e:\\page.html", "w")

  #f.write(news_img_page)

  #f.close()



  reg_str = r'http://image\S+jpg'

  image_reg = re.compile(reg_str)

  image_results = image_reg.findall(news_img_page)

  if len(image_results) == 0:

    print "Cannot find news page" + str(page_index) + "!"

    download_suc = False

    break

  

  image_url = image_results[0]



  print "News image url = " + image_url

  news_image_context = urllib2.urlopen(image_url)



  image_name = image_folder + "page_" + str(page_index) + ".jpg"

  imgf = open(image_name, 'wb')

  print "Getting image..."

  try:

    while True:

      date = news_image_context.read(1024*10)

      if not date:

        break

      imgf.write(date)

    imgf.close()

  except:

    download_suc = False

    print "Save image " + str(page_index) + " failed!"

    print "Unexpected error: " + sys.exc_info()[0] + sys.exc_info()[1]

  else:

    print "Save image " + str(page_index) + " succeed!"

    print

  page_index = page_index + 1



if download_suc == True:

  print "News download succeed! Path = \"" + str(image_folder) + "\""

  print "Enjoy it! ^^"

else:

  print "news download failed!"