爬虫实例

### requests 与 beautifulsoup结合使用
```
>>> import requests
>>> from bs4 import BeautifulSoup
>>> def get_html(url):
... html=urllib.request.urlopen(url).read()
... soup=BeautifulSoup(html,'lxml')
... for i in soup.find(id='content').find_all('ul'):
... for j in i.find_all('li'):
... for k in j.find_all('a'):
... yield 'http://www.runoob.com'+k['href']
... def get_html_link(link):
... for i in link:
... link_list=urllib.request.urlopen(i).read()
... soup=BeautifulSoup(link_list,'lxml')
... content=soup.find(id='content')
... if content:
... title=content.find('h1').string
... conten_list=content.find_all('p',limit=3)
... subject=''
... for j in conten_list:
... subject+=j.get_text()
... yield (title,subject)
... def save_suject(title_content):
... with open('G:/2.txt','w+',encoding='utf+8') as f:
... for tile,content in title_content:
... f.write(tile+'\n')
... f.write(content+'\n')
... f.write('#'*80+'\n')
... def fun_call(url):
... link=get_html(url)
... title_content=get_html_link(link)
... save_suject(title_content)
... def main():
... url='http://www.runoob.com/python/python-100-examples.html'
... fun_call(url)
... if __name__=='__main__':
... main()

```

代码来自CSDN
```
import urllib.request
from bs4 import BeautifulSoup
# 1、获取所有页面链接
def get_html(url):
# 获取页面HTML源码
html=urllib.request.urlopen(url).read()
# 格式化html
soup=BeautifulSoup(html,'lxml')
# 首先找到第一个id='content'的标签，并找到子标签ul(2个)
# 其次遍历子标签ul,并获取到所有的ul的子标签li
# 然后遍历li标签，并获取到li标签下的所有a标签
# 使用yield返回超链接
for i in soup.find(id='content').find_all('ul'):
for j in i.find_all('li'):
for k in j.find_all('a'):
yield 'http://www.runoob.com'+k['href']
# 2、获取详细的页面数据
def get_html_link(link):
# 遍历所有的超链接
for i in link:
# 请求超链接页面HTML
link_list=urllib.request.urlopen(i).read()
# 格式化HTML
soup=BeautifulSoup(link_list,'lxml')
# 获取id='content'的标签
content=soup.find(id='content')
if content:
# 获取h1标签的内容
title=content.find('h1').string
# 获取前3个p标签的内容
conten_list=content.find_all('p',limit=3)
subject=''
for j in conten_list:
subject+=j.get_text()
yield (title,subject)
# 3、数据保存
def save_suject(title_content):
with open('G:/2.txt','w+',encoding='utf+8') as f:
for tile,content in title_content:
f.write(tile+'\n')
f.write(content+'\n')
f.write('#'*80+'\n')
# 4、函数回调
def fun_call(url):
link=get_html(url)
title_content=get_html_link(link)
save_suject(title_content)
# 5、主函数
def main():
url='http://www.runoob.com/python/python-100-examples.html'
fun_call(url)
if __name__=='__main__':
main()

```

No repository file, test, entry point, or concrete requested change is identified. First clarify whether this code is intended as a feature and what behavior or output is required; completion cannot be verified from the issue as written.

Written by the indexing model from the issue text.