建设论坛网站步骤,用php写的网站,六安公司做网站,兰州做公司网站一条爬虫抓取一个小网站所有数据
今天闲来无事#xff0c;写一个爬虫来玩玩。在网上冲浪的时候发现了一个搞笑的段子网#xff0c;发现里面的内容还是比较有意思的#xff0c;于是心血来潮#xff0c;就想着能不能写一个Python程序#xff0c;抓取几条数据下来看看写一个爬虫来玩玩。在网上冲浪的时候发现了一个搞笑的段子网发现里面的内容还是比较有意思的于是心血来潮就想着能不能写一个Python程序抓取几条数据下来看看一不小心就把这个网站的所有数据都拿到了。 这个网站主要的数据都是详情在HTML里面的可以采用lxml模块的xpath对HTML标签的内容解析获取到自己想要的数据然后再保存在本地文件中整个过程是一气呵成的。能够抓取到一页的数据之后加一个循环就可以抓取到所有页的数据下面的就是数据展示。 废话少说直接上Python代码
import requests
import csv
from lxml import etree
import timeclass Page:def __init__(self):self.pre_url https://www.biedoul.comself.start_page 1self.end_page 15233def askHTML(self, current_page, opportunity):print( current page str(current_page) )try:pre_url self.pre_url /index/ str(current_page)page requests.get(urlpre_url)html etree.HTML(page.content)articles html.xpath(/html/body/div/div/div/dl)return articlesexcept Exception as e:if opportunity 0:time.sleep(500)print( retry str(opportunity) )return self.askHTML(current_page, opportunity - 1)else:return Nonedef analyze(self, articles):lines []for article in articles:data {}data[link] article.xpath(./span/dd/a/href)[0]data[title] article.xpath(./span/dd/a/strong/text())[0]data[content] self.analyze_content(article)picture_links article.xpath(./dd/img/src)if (picture_links is not None and len(picture_links) 0):# print(picture_links)data[picture_links] picture_linkselse:data[picture_links] []# data[good_zan] article.xpath(./div/div/a[classpinattn good]/p/text())[0]# data[bad_bs] article.xpath(./div/div/a[classpinattn bad]/p/text())[0]data[good_zan] self.analyze_zan(article, good)# article.xpath(./div/div/a[classpinattn good]/p/text())[0]data[bad_bs] self.analyze_zan(article, bad)# article.xpath(./div/div/a[classpinattn bad]/p/text())[0]lines.append(data)return lines# 解析文章内容def analyze_content(self, article):# 1. 判断dd标签下是否为文本内容content article.xpath(./dd/text())if content is not None and len(content) 0 and not self.is_empty_list(content):return contentcontent []p_list article.xpath(./dd)for p in p_list:# 2. 判断dd/.../font标签下是否为文本内容if len(content) 0 or content is None:fonts p.xpath(.//font)for font_html in fonts:font_content font_html.xpath(./text())if font_content is not None and len(font_content) 0:content.append(font_content)# 3. 判断dd/.../p标签下是否为文本内容if len(content) 0 or content is None:fonts p.xpath(.//p)for font_html in fonts:font_content font_html.xpath(./text())if font_content is not None and len(font_content) 0:content.append(font_content)return contentdef analyze_zan(self, article, type):num article.xpath(./div/div/a[classpinattn type ]/p/text())if num is not None and len(num) 0:return num[0]return 0def do_word(self):fieldnames [index, link, title, content, picture_links, good_zan, bad_bs]with open(article.csv, a, encodingUTF8, newline) as f:writer csv.DictWriter(f, fieldnamesfieldnames)# writer.writeheader()for i in range(self.start_page, self.end_page):articles self.askHTML(i, 3)if articles is None:continuearticle_list self.analyze(articles)self.save(writer, article_list)# 保存到文件中def save(self, writer, lines):print(##### 保存中到文件中...)# python2可以用file替代openprint(lines)writer.writerows(lines)print(##### 保存成功...)def is_empty_list(self, list):for l in list:if not self.empty(l):return Falsereturn Truedef empty(self, content):result content.replace(\r, ).replace(\n, )if result :return Truereturn False# 递归解析文章内容def analyze_font_content(self, font_html, depth):content []print(depth)font_content_list font_html.xpath(./font/text())if font_content_list is not None and len(font_content_list) 0 and not self.is_empty_list(font_content_list):for font_content in font_content_list:content.append(font_content)else:if depth 0:return []return self.analyze_font_content(font_html.xpath(./font), depth - 1)return contentif __name__ __main__:page Page()page.do_word()在运行下面的代码之前需要先按照好requests、lxml两个模块安装命令为
pip installl requests
pip install lxml大家对这个爬虫有什么疑问欢迎给我留言。如果大家对于我这个爬虫创意还不错的话记得关注微信公众号【智享学习】哟后续我会分享更多有意思的编程项目。 本文由博客一文多发平台 OpenWrite 发布
