快手 creator模式 爬一下就会connect timeout

今天爬快手的数据

Traceback (most recent call last):
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/streams/tls.py", line 140, in _call_sslobject_method
result = func(*args)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/ssl.py", line 944, in do_handshake
self._sslobj.do_handshake()
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:1129)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 115, in fail_after
yield cancel_scope
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 69, in start_tls
ssl_stream = await anyio.streams.tls.TLSStream.wrap(
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/streams/tls.py", line 132, in wrap
await wrapper._call_sslobject_method(ssl_object.do_handshake)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/streams/tls.py", line 147, in _call_sslobject_method
data = await self.transport_stream.receive()
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 1227, in receive
await self._protocol.read_event.wait()
File "/opt/anaconda3/envs/Cabian/lib/python3.9/asyncio/locks.py", line 226, in wait
await fut
asyncio.exceptions.CancelledError: Cancelled by cancel scope 1266bcf40

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_exceptions.py", line 10, in map_exceptions
yield
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 78, in start_tls
raise exc
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 69, in start_tls
ssl_stream = await anyio.streams.tls.TLSStream.wrap(
File "/opt/anaconda3/envs/Cabian/lib/python3.9/contextlib.py", line 135, in exit
self.gen.throw(type, value, traceback)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 118, in fail_after
raise TimeoutError
TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
yield
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 262, in handle_async_request
raise exc
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 245, in handle_async_request
response = await connection.handle_async_request(request)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_async/http_proxy.py", line 299, in handle_async_request
stream = await stream.start_tls(**kwargs)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_backends/anyio.py", line 78, in start_tls
raise exc
File "/opt/anaconda3/envs/Cabian/lib/python3.9/contextlib.py", line 135, in exit
self.gen.throw(type, value, traceback)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
raise to_exc(exc) from exc
httpcore.ConnectTimeout

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/main.py", line 66, in
asyncio.get_event_loop().run_until_complete(main())
File "/opt/anaconda3/envs/Cabian/lib/python3.9/asyncio/base_events.py", line 642, in run_until_complete
return future.result()
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/main.py", line 56, in main
await crawler.start()
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/core.py", line 86, in start
await self.get_creators_and_videos()
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/core.py", line 275, in get_creators_and_videos
all_video_list = await self.ks_client.get_all_videos_by_creator(
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/client.py", line 317, in get_all_videos_by_creator
await callback(videos)
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/core.py", line 293, in fetch_creator_video_detail
video_details = await asyncio.gather(*task_list)
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/core.py", line 146, in get_video_info_task
result = await self.ks_client.get_video_info(video_id)
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/client.py", line 128, in get_video_info
return await self.post("", post_data)
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/client.py", line 68, in post
return await self.request(method="POST", url=f"{self._host}{uri}",
File "/Users/jasonyang/workspace/python_workspace/Cabian/MediaCrawler/media_platform/kuaishou/client.py", line 49, in request
response = await client.request(
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_client.py", line 1530, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_client.py", line 1617, in send
response = await self._send_handling_auth(
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_client.py", line 1645, in _send_handling_auth
response = await self._send_handling_redirects(
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_client.py", line 1682, in _send_handling_redirects
response = await self._send_single_request(request)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_client.py", line 1719, in _send_single_request
response = await transport.handle_async_request(request)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/contextlib.py", line 135, in exit
self.gen.throw(type, value, traceback)
File "/opt/anaconda3/envs/Cabian/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
raise mapped_exc(message) from exc
httpx.ConnectTimeout

只能爬出来20条之后程序就崩了

Start with media_platform/kuaishou/core.py, especially get_creators_and_videos, fetch_creator_video_detail, and get_video_info_task, then trace the request through media_platform/kuaishou/client.py. Reproduce creator-mode crawling and inspect the httpx.ConnectTimeout after about 20 videos. Done means the crawl no longer crashes at that point and its timeout behavior is handled.