起因
如题，最近在编写一个使用Fastapi调用Playwright进行网页渲染截图的工具
环境如下：
| 环境 | 版本 |
|---|---|
| Python | 3.11.2 |
| Playwright | latest |
| Windows | 10 |
| Vscode | latest |
试图在FastApi路由中运行子进程，但执行结果是 NotImplementedError
原始代码如下
from playwright.async_api import async_playwright,TimeoutError
import asyncio
from typing import Literal
from io import BytesIO
import uvicorn
from fastapi import FastAPI,Form
from fastapi.responses import JSONResponse,StreamingResponse
from asyncio.windows_events import ProactorEventLoop
from uvicorn import Config, Server
app = FastAPI(
title="Playwright",
description="Playwright服务",
version="0.0.2",
docs_url="/apidoc",
redoc_url=None
)
async def AsyncPWscreenshot(
content:str = "",
url:str = "https://www.baidu.com",
referer:str = "",
wait_until:Literal["commit", "domcontentloaded", "load", "networkidle"] = "networkidle",
timeout:int = 30000,
viewport_height:int = 1080,
viewport_width: int = 1920,
proxy_address:str = "",
proxy_username:str = "",
proxy_password:str = "",
screenshot_quality:int = 100,
scale: Literal["css", "device"] = "device",
format: Literal["png","jpeg"] = "device",
ifRAW:bool = False
):
async with async_playwright() as playwright:
try:
browser = await playwright.firefox.launch(
headless=True,
timeout=timeout,
)
context = await browser.new_context(
viewport={'width': viewport_width, 'height': viewport_height}
)
if proxy_address:
proxy={
"server":proxy_address,
"username": proxy_username,
"password": proxy_password
}
context = await browser.new_context(
viewport={'width': viewport_width, 'height': viewport_height},
proxy=proxy
)
page = await context.new_page()
if ifRAW and content:
return JSONResponse(status_code=200,content={"message":"","result":content})
if content:
await page.set_content(content,timeout=timeout,wait_until=wait_until)
else:
await page.goto(
url,
referer=referer,
wait_until=wait_until
)
if ifRAW:
return JSONResponse(status_code=200,content={"message":"","result":await page.content()})
if format == "png":
screenshot_quality = None
data = await page.screenshot(
full_page=True,
scale=scale,
type=format,
quality=screenshot_quality
)
except TimeoutError:
return JSONResponse(status_code=502,content={"message":"Time Out","result":None})
bytes_io = BytesIO(data)
bytes_io.seek(0)
return StreamingResponse(content=bytes_io,media_type=f"image/{format}")
@app.post("/playwright",tags=["pw代理"])
async def root(
content:str = Form("",description="直接使用传入字符串生成"),
url:str = Form("https://www.baidu.com",description="使用网址生成"),
referer:str = Form("",description="来源"),
wait_until:Literal["commit", "domcontentloaded", "load", "networkidle"] = Form("networkidle",description="等待模式"),
timeout:int = Form(30000,description="超时时间"),
viewport_height:int = Form(1080,description="浏览器高度"),
viewport_width: int = Form(1920,description="浏览器宽度"),
proxy_address:str = Form("",description="代理地址"),
proxy_username:str = Form("",description="代理账号"),
proxy_password:str = Form("",description="代理密码"),
screenshot_quality:int = Form(100,description="图片质量，仅JPEG生效"),
scale: Literal["css", "device"] = Form("device",description="scale"),
format: Literal["png","jpeg"] = Form("png",description="图片格式"),
ifRAW:bool = Form(False,description="是否仅返回页面源码"),
):
return await AsyncPWscreenshot(
content=content,
url=url,
referer=referer,
wait_until=wait_until,
timeout=timeout,
viewport_height=viewport_height,
viewport_width=viewport_width,
proxy_address=proxy_address,
proxy_password=proxy_password,
proxy_username=proxy_username,
screenshot_quality=screenshot_quality,
scale=scale,
format=format,
ifRAW=ifRAW
)
import multiprocessing
if __name__ == '__main__':
vcpus = multiprocessing.cpu_count()
workers = int((vcpus*2)+1)
uvicorn.run(app='pw:app', host="0.0.0.0", port=8003,workers=2,reload=False)启动后，Fastapi工作正常，但在请求目标路由的时候，开始报错
INFO: Started server process [9192]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Started server process [21696]
INFO: Waiting for application startup.
INFO: Application startup complete.
Task exception was never retrieved
future: <Task finished name='Task-5' coro=<Connection.run() done, defined at D:\Python311\Lib\site-packages\playwright\_impl\_connection.py:240> exception=NotImplementedError()>
Traceback (most recent call last):
File "D:\Python311\Lib\site-packages\playwright\_impl\_connection.py", line 247, in run
await self._transport.connect()
File "D:\Python311\Lib\site-packages\playwright\_impl\_transport.py", line 127, in connect
raise exc
File "D:\Python311\Lib\site-packages\playwright\_impl\_transport.py", line 116, in connect
self._proc = await asyncio.create_subprocess_exec(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\subprocess.py", line 218, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\base_events.py", line 1688, in subprocess_exec
transport = await self._make_subprocess_transport(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\base_events.py", line 502, in _make_subprocess_transport
raise NotImplementedError
NotImplementedError
INFO: 127.0.0.1:58295 - "POST /playwright HTTP/1.1" 500 Internal Server Error
ERROR: Exception in ASGI application
Traceback (most recent call last):
File "D:\Python311\Lib\site-packages\uvicorn\protocols\http\h11_impl.py", line 407, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\site-packages\fastapi\applications.py", line 271, in __call__
await super().__call__(scope, receive, send)
File "D:\Python311\Lib\site-packages\starlette\applications.py", line 118, in __call__
await self.middleware_stack(scope, receive, send)
File "D:\Python311\Lib\site-packages\starlette\middleware\errors.py", line 184, in __call__
raise exc
File "D:\Python311\Lib\site-packages\starlette\middleware\errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "D:\Python311\Lib\site-packages\starlette\middleware\exceptions.py", line 79, in __call__
raise exc
File "D:\Python311\Lib\site-packages\starlette\middleware\exceptions.py", line 68, in __call__
await self.app(scope, receive, sender)
File "D:\Python311\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 21, in __call__
raise e
File "D:\Python311\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "D:\Python311\Lib\site-packages\starlette\routing.py", line 706, in __call__
await route.handle(scope, receive, send)
File "D:\Python311\Lib\site-packages\starlette\routing.py", line 276, in handle
await self.app(scope, receive, send)
File "D:\Python311\Lib\site-packages\starlette\routing.py", line 66, in app
response = await func(request)
^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\site-packages\fastapi\routing.py", line 237, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\site-packages\fastapi\routing.py", line 163, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "e:\Desktop\pw.py", line 112, in root
return await AsyncPWscreenshot(
^^^^^^^^^^^^^^^^^^^^^^^^
File "e:\Desktop\pw.py", line 38, in AsyncPWscreenshot
async with async_playwright() as playwright:
File "D:\Python311\Lib\site-packages\playwright\async_api\_context_manager.py", line 46, in __aenter__
playwright = AsyncPlaywright(next(iter(done)).result())
^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\site-packages\playwright\_impl\_transport.py", line 116, in connect
self._proc = await asyncio.create_subprocess_exec(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\subprocess.py", line 218, in create_subprocess_exec
transport, protocol = await loop.subprocess_exec(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\base_events.py", line 1688, in subprocess_exec
transport = await self._make_subprocess_transport(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Python311\Lib\asyncio\base_events.py", line 502, in _make_subprocess_transport
raise NotImplementedError
NotImplementedError当我试图使用这样的方法来调用异步的PlaywrightAPI的时候，它提示 create_subprocess 失败
解决
通过询问大佬和查阅相关文档，我注意到这是因为uvicorn默认使用的Loop循环在Windows下会产生很迷惑的问题
因此需要手动对Fastapi指定Loop循环，可以将循环修改为 ProactorEventLoop 并且关闭uvicorn的reload功能，因为这个功能可能会在下一次重载的时候，再次把Loop恢复为默认。
因此需要强制修改循环，部分修改代码如下
import asyncio
from asyncio.windows_events import ProactorEventLoop
from fastapi import FastAPI
from uvicorn import Config, Server
app = FastAPI()
class ProactorServer(Server):
def run(self, sockets=None):
loop = ProactorEventLoop()
asyncio.set_event_loop(loop) # since this is the default in Python 3.10, explicit selection can also be omitted
asyncio.run(self.serve(sockets=sockets))
config = Config(app=app, host="0.0.0.0", port=8000, reload=True)
server = ProactorServer(config=config)
server.run()经过这种魔法后的Fastapi就能正常启动Playwright的子进程了。
次魔方仅适用Windows，Linux下经过测试可能不需要此魔法
评论 (1)