Simon
ead5f83bee
dependencies.py 拆分为 dependencies/ 包; 新增 vision/ragflow_ingestion/neo4j 客户端与 h5_ai_task; alembic 045 图置信度迁移; 响应契约统一收尾。
293 lines
10 KiB
Python
293 lines
10 KiB
Python
# =============================================================================
|
||
# 企微IT智能服务台 — 视觉理解服务（D5 / P1-3）
|
||
# =============================================================================
|
||
# 说明：封装 Qwen-VL 截图理解能力，通过 Dify vision workflow 调用本地
|
||
# Qwen3-VL-8B-Instruct 模型，将员工截屏转换为结构化描述文本，
|
||
# 注入会话上下文参与后续 AI 推理。
|
||
#
|
||
# 核心能力：
|
||
# 1. analyze_screenshot: 接收图片字节流 → Dify vision workflow → 结构化描述
|
||
# 2. _preprocess_image: 图片预处理（resize/compress）
|
||
# 3. inject_to_conversation_context: 将视觉描述注入会话消息上下文
|
||
#
|
||
# 设计决策：
|
||
# - 视觉理解经 Dify 后端 → Qwen-VL 本地推理（D5 硬约束）
|
||
# - 图片预处理：Pillow resize max 1024px + JPEG quality=85
|
||
# - 视觉模型可配置（settings.qwen_vl_model，默认 Qwen3-VL-8B-Instruct）
|
||
# =============================================================================
|
||
|
||
import base64
|
||
import io
|
||
import logging
|
||
from typing import Any, Dict, Optional
|
||
|
||
import httpx
|
||
from PIL import Image
|
||
|
||
from app.config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class VisionService:
|
||
"""视觉理解服务 — 截图→结构化描述。
|
||
|
||
调用本地 Qwen-VL（经 Dify vision workflow）分析员工截图，
|
||
生成结构化文本描述并注入会话上下文。
|
||
|
||
使用方式：
|
||
service = VisionService()
|
||
result = await service.analyze_screenshot(image_bytes, conversation_id)
|
||
await service.inject_to_conversation_context(result["description"], conversation_id)
|
||
|
||
Attributes:
|
||
dify_vision_api_url: Dify Vision Workflow API 端点
|
||
dify_vision_api_key: Dify Vision Workflow API Key
|
||
model: 视觉模型名称（默认 Qwen3-VL-8B-Instruct）
|
||
"""
|
||
|
||
# 图片预处理参数
|
||
_MAX_DIMENSION: int = 1024 # 最大边长（像素）
|
||
_JPEG_QUALITY: int = 85 # JPEG 压缩质量
|
||
|
||
def __init__(
|
||
self,
|
||
dify_vision_api_url: Optional[str] = None,
|
||
dify_vision_api_key: Optional[str] = None,
|
||
model: Optional[str] = None,
|
||
):
|
||
"""初始化视觉理解服务。
|
||
|
||
Args:
|
||
dify_vision_api_url: Dify Vision Workflow API 端点
|
||
dify_vision_api_key: Dify Vision Workflow API Key
|
||
model: 视觉模型名称
|
||
"""
|
||
self.dify_vision_api_url: str = (
|
||
dify_vision_api_url or settings.dify_vision_api_url
|
||
)
|
||
self.dify_vision_api_key: str = (
|
||
dify_vision_api_key or settings.dify_vision_api_key
|
||
)
|
||
self.model: str = model or settings.qwen_vl_model
|
||
self._client: Optional[httpx.AsyncClient] = None
|
||
|
||
async def _get_client(self) -> httpx.AsyncClient:
|
||
"""获取或创建 httpx 异步客户端。"""
|
||
if self._client is None or self._client.is_closed:
|
||
self._client = httpx.AsyncClient(
|
||
timeout=httpx.Timeout(60.0), # 视觉推理可能需要更长时间
|
||
headers={
|
||
"Authorization": f"Bearer {self.dify_vision_api_key}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
)
|
||
return self._client
|
||
|
||
async def close(self):
|
||
"""关闭 httpx 客户端。"""
|
||
if self._client and not self._client.is_closed:
|
||
await self._client.aclose()
|
||
self._client = None
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 核心方法
|
||
# --------------------------------------------------------------------------
|
||
|
||
async def analyze_screenshot(
|
||
self, image_bytes: bytes, conversation_id: str
|
||
) -> Dict[str, Any]:
|
||
"""分析截图，返回结构化视觉描述。
|
||
|
||
Args:
|
||
image_bytes: 图片字节流
|
||
conversation_id: 会话ID（用于上下文关联）
|
||
|
||
Returns:
|
||
Dict: {
|
||
"description": str, # 结构化的视觉描述文本
|
||
"confidence": float, # 视觉理解置信度
|
||
"metadata": dict, # 元数据（detected_ui_elements, error_codes等）
|
||
}
|
||
"""
|
||
# 默认降级响应
|
||
default_response: Dict[str, Any] = {
|
||
"description": "",
|
||
"confidence": 0.0,
|
||
"metadata": {},
|
||
}
|
||
|
||
if not self.dify_vision_api_url:
|
||
logger.warning("Dify Vision API 未配置，跳过视觉分析")
|
||
return default_response
|
||
|
||
try:
|
||
# 1. 预处理图片
|
||
processed_image = await self._preprocess_image(image_bytes)
|
||
|
||
# 2. 调用 Dify vision workflow
|
||
result = await self._call_vision_workflow(processed_image, conversation_id)
|
||
|
||
if result is None:
|
||
return default_response
|
||
|
||
return {
|
||
"description": result.get("description", ""),
|
||
"confidence": float(result.get("confidence", 0.0)),
|
||
"metadata": result.get("metadata", {}),
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"截图视觉分析失败: {e}")
|
||
return default_response
|
||
|
||
async def inject_to_conversation_context(
|
||
self, description: str, conversation_id: str
|
||
) -> bool:
|
||
"""将视觉描述注入会话消息上下文。
|
||
|
||
以 system 消息形式将视觉理解结果写入会话消息表，
|
||
后续 AI 推理时可读取此描述作为上下文。
|
||
|
||
Args:
|
||
description: 视觉理解描述文本
|
||
conversation_id: 会话ID
|
||
|
||
Returns:
|
||
bool: 注入成功返回 True
|
||
"""
|
||
if not description:
|
||
logger.debug("视觉描述为空，跳过上下文注入")
|
||
return False
|
||
|
||
try:
|
||
from app.database import _get_session_factory
|
||
from app.models.message import Message
|
||
|
||
session_factory = _get_session_factory()
|
||
async with session_factory() as db:
|
||
msg = Message(
|
||
conversation_id=conversation_id,
|
||
sender_type="system",
|
||
content=f"[视觉理解] {description}",
|
||
)
|
||
db.add(msg)
|
||
await db.commit()
|
||
logger.info(
|
||
f"视觉描述已注入会话 {conversation_id}: "
|
||
f"description_length={len(description)}"
|
||
)
|
||
return True
|
||
|
||
except ImportError:
|
||
logger.warning("Message 模型不可用，无法注入视觉描述")
|
||
return False
|
||
except Exception as e:
|
||
logger.error(f"注入视觉描述失败: {e}")
|
||
return False
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 内部方法
|
||
# --------------------------------------------------------------------------
|
||
|
||
async def _preprocess_image(self, image_bytes: bytes) -> bytes:
|
||
"""预处理图片：resize + compress。
|
||
|
||
使用 Pillow 将图片缩小到最大 1024px，压缩为 JPEG quality=85，
|
||
减少传输大小和视觉模型推理开销。
|
||
|
||
Args:
|
||
image_bytes: 原始图片字节流
|
||
|
||
Returns:
|
||
bytes: 预处理后的图片字节流
|
||
"""
|
||
try:
|
||
img = Image.open(io.BytesIO(image_bytes))
|
||
|
||
# 转换为 RGB（处理 RGBA/PNG 等格式）
|
||
if img.mode in ("RGBA", "P", "LA"):
|
||
img = img.convert("RGB")
|
||
|
||
# 按最大边长等比缩放
|
||
w, h = img.size
|
||
max_dim = max(w, h)
|
||
if max_dim > self._MAX_DIMENSION:
|
||
ratio = self._MAX_DIMENSION / max_dim
|
||
new_w, new_h = int(w * ratio), int(h * ratio)
|
||
img = img.resize((new_w, new_h), Image.LANCZOS)
|
||
logger.debug(f"图片缩放: {w}x{h} → {new_w}x{new_h}")
|
||
|
||
# 输出为 JPEG
|
||
buffer = io.BytesIO()
|
||
img.save(buffer, format="JPEG", quality=self._JPEG_QUALITY)
|
||
result = buffer.getvalue()
|
||
|
||
logger.debug(
|
||
f"图片预处理完成: input_size={len(image_bytes)}, "
|
||
f"output_size={len(result)}"
|
||
)
|
||
return result
|
||
|
||
except Exception as e:
|
||
logger.warning(f"图片预处理失败，使用原始图片: {e}")
|
||
return image_bytes
|
||
|
||
async def _call_vision_workflow(
|
||
self, processed_image: bytes, conversation_id: str
|
||
) -> Optional[Dict[str, Any]]:
|
||
"""调用 Dify Vision Workflow 进行视觉理解。
|
||
|
||
将预处理后的图片以 base64 格式发送到 Dify vision workflow。
|
||
|
||
Args:
|
||
processed_image: 预处理后的图片字节流
|
||
conversation_id: 会话ID
|
||
|
||
Returns:
|
||
Optional[Dict]: 视觉理解结果，失败返回 None
|
||
"""
|
||
try:
|
||
# Base64 编码图片
|
||
image_base64 = base64.b64encode(processed_image).decode("utf-8")
|
||
|
||
payload: Dict[str, Any] = {
|
||
"inputs": {
|
||
"image_base64": image_base64,
|
||
"conversation_id": conversation_id,
|
||
},
|
||
"response_mode": "blocking",
|
||
"user": f"vision-{conversation_id[:8]}",
|
||
}
|
||
|
||
client = await self._get_client()
|
||
logger.info(
|
||
f"调用 Dify Vision Workflow: conversation_id={conversation_id}, "
|
||
f"model={self.model}"
|
||
)
|
||
response = await client.post(self.dify_vision_api_url, json=payload)
|
||
response.raise_for_status()
|
||
data = response.json()
|
||
|
||
# 解析 Dify workflow 返回
|
||
outputs = data.get("data", {}).get("outputs", {})
|
||
if not outputs:
|
||
logger.warning("Dify Vision Workflow 返回空 outputs")
|
||
return None
|
||
|
||
return {
|
||
"description": outputs.get("description", ""),
|
||
"confidence": float(outputs.get("confidence", 0.0)),
|
||
"metadata": outputs.get("metadata", {}),
|
||
}
|
||
|
||
except httpx.TimeoutException:
|
||
logger.error("Dify Vision Workflow 超时")
|
||
return None
|
||
except httpx.HTTPStatusError as e:
|
||
logger.error(f"Dify Vision Workflow HTTP 错误: status={e.response.status_code}")
|
||
return None
|
||
except Exception as e:
|
||
logger.error(f"Dify Vision Workflow 调用失败: {e}")
|
||
return None
|