【下载文档:  Python爬虫单线程爬取网页.txt 】

---

Python爬虫单线程爬取网页
源代码 # coding=utf-8 import re import requests import sys reload(sys) sys.setdefaultencoding("utf8") class spider(): def __init__(self): print "开始爬取内容。。。" def changePage(self, url, total_page): nowpage = int(re.search('/0-(\d+)/', url, re.S).group(1)) pagegroup = [] for i in range(nowpage, total_page + 1): link = re.sub('/0-(\d+)/', '/0-%s/' % i, url, re.S) pagegroup.append(link) return pagegroup def getsource(self, url): html = requests.get(url) return html.text def getclasses(self, source): classes = re.search('

(.*?)

', source, re.S).group(1) return classes def geteach(self, classes): eachclasses = re.findall('

(.*?)

', classes, re.S) return eachclasses def getinfo(self, eachclass): info = {} info['title'] = re.search('(.*?)

', eachclass, re.S).group(1) return info def saveinfo(self, classinfo): f = open('info.txt', 'a') for each in classinfo: f.writelines('title : ' + each['title'] + '\n') f.writelines('people : ' + each['people'] + '\n\n') f.close() if __name__ == '__main__': classinfo = [] url = 'http://www.maiziedu.com/course/list/all-all/0-1/' maizispider = spider() all_links = maizispider.changePage(url, 30) for each in all_links: htmlsources = maizispider.getsource(each) classes = maizispider.getclasses(htmlsources) eachclasses = maizispider.geteach(classes) for each in eachclasses: info = maizispider.getinfo(each) classinfo.append(info) maizispider.saveinfo(classinfo)以上代码并不难懂，基本就是正则表达式的使用，然后直接运行就可以看到开头我们的截图内容了，由于这是单线程爬虫，所以运行速度感觉有点慢，接下来还会继续更新多线程爬虫。应小伙伴们的要求，下面附上requests爬虫库的安装和简单示例首先安装pip包管理工具,下载get-pip.py. 我的机器上安装的既有python2也有python3。 安装pip到python2：python get-pip.py安装到python3：python3 get-pip.pypip安装完成以后，安装requests库开启python爬虫学习。安装requestspip3 install requests我使用的python3，python2可以直接用pip install requests.入门例子import requestshtml=requests.get("http://gupowang.baijia.baidu.com/article/283878")html.encoding='utf-8'print(html.text)第一行引入requests库，第二行使用requests的get方法获取网页源代码，第三行设置编码格式，第四行文本输出。 把获取到的网页源代码保存到文本文件中：import requestsimport oshtml=requests.get("http://gupowang.baijia.baidu.com/article/283878")html_file=open("news.txt","w")html.encoding='utf-8'print(html.text,file=html_file)