164 lines
5.8 KiB
Python
164 lines
5.8 KiB
Python
|
# =============================================================================
|
|||
|
|
# 企微IT智能服务台 — 视觉理解 API（Tier1 新增 / D5 / P1-3）
|
|||
|
|
# =============================================================================
|
|||
|
|
# 说明：截图视觉理解接口，调用本地 Qwen-VL（经 Dify vision workflow）
|
|||
|
|
# 分析员工截图，返回结构化描述文本。
|
|||
|
|
#
|
|||
|
|
# 1. POST /api/vision/analyze — 分析截图（multipart: image + conversation_id）
|
|||
|
|
# 2. GET /api/vision/models — 可用的视觉模型列表
|
|||
|
|
#
|
|||
|
|
# D5 硬约束：
|
|||
|
|
# - 视觉理解经 Dify 后端调用本地 Qwen-VL（Qwen3-VL-8B-Instruct）
|
|||
|
|
# - 预留 vision_model 参数以便后续升级
|
|||
|
|
# - 截图隐私仅保留接口（D6），不阻断消息
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
import logging
|
|||
|
|
from typing import List
|
|||
|
|
|
|||
|
|
from fastapi import APIRouter, Depends, File, Form, UploadFile
|
|||
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|||
|
|
|
|||
|
|
from app.config import settings
|
|||
|
|
from app.database import get_db
|
|||
|
|
from app.dependencies import get_current_user, require_any_user, UserInfo
|
|||
|
|
from app.services.vision_service import VisionService
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
router = APIRouter()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# -----------------------------------------------------------------------------
|
|||
|
|
# 分析截图（Tier1 新增）
|
|||
|
|
# -----------------------------------------------------------------------------
|
|||
|
|
# POST /api/vision/analyze
|
|||
|
|
@router.post("/analyze")
|
|||
|
|
@require_any_user
|
|||
|
|
async def analyze_screenshot(
|
|||
|
|
image: UploadFile = File(..., description="截图文件（支持 PNG/JPG/GIF）"),
|
|||
|
|
conversation_id: str = Form(..., description="会话ID（用于上下文关联）"),
|
|||
|
|
vision_model: str = Form(
|
|||
|
|
default="",
|
|||
|
|
description="视觉模型名称（可选，默认使用配置中的模型）",
|
|||
|
|
),
|
|||
|
|
current_user: UserInfo = Depends(get_current_user),
|
|||
|
|
db: AsyncSession = Depends(get_db),
|
|||
|
|
):
|
|||
|
|
"""分析截图，返回 AI 视觉理解的结构化描述。
|
|||
|
|
|
|||
|
|
员工发送截图后，前端调用此接口将图片交给 Qwen-VL 视觉模型分析。
|
|||
|
|
分析结果将自动注入到对应会话的上下文中，参与后续 AI 推理。
|
|||
|
|
|
|||
|
|
**请求格式**: multipart/form-data
|
|||
|
|
|
|||
|
|
**字段说明**:
|
|||
|
|
- **image**: 截图文件（必填）
|
|||
|
|
- **conversation_id**: 会话ID（必填）
|
|||
|
|
- **vision_model**: 视觉模型名称（可选，默认使用 Qwen3-VL-8B-Instruct）
|
|||
|
|
|
|||
|
|
**支持的文件格式**: PNG、JPG、GIF、WebP
|
|||
|
|
**文件大小限制**: 最大 10MB
|
|||
|
|
|
|||
|
|
**D5 隐私说明**: 截图分析结果仅供 AI 理解上下文使用，
|
|||
|
|
隐私检测接口已预留（D6），当前不阻断消息。
|
|||
|
|
"""
|
|||
|
|
# 校验文件类型
|
|||
|
|
allowed_types = {"image/png", "image/jpeg", "image/gif", "image/webp"}
|
|||
|
|
if image.content_type and image.content_type not in allowed_types:
|
|||
|
|
return {
|
|||
|
|
"code": 400,
|
|||
|
|
"message": f"不支持的图片格式: {image.content_type}，仅支持 PNG/JPG/GIF/WebP",
|
|||
|
|
"data": None,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 读取图片字节流
|
|||
|
|
image_bytes = await image.read()
|
|||
|
|
|
|||
|
|
# 校验文件大小（最大 10MB）
|
|||
|
|
max_size = 10 * 1024 * 1024
|
|||
|
|
if len(image_bytes) > max_size:
|
|||
|
|
return {
|
|||
|
|
"code": 400,
|
|||
|
|
"message": f"图片过大（{len(image_bytes) / 1024 / 1024:.1f}MB），最大支持 10MB",
|
|||
|
|
"data": None,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 调用视觉理解服务
|
|||
|
|
service = VisionService(
|
|||
|
|
model=vision_model if vision_model else None,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
result = await service.analyze_screenshot(image_bytes, conversation_id)
|
|||
|
|
|
|||
|
|
# 将视觉描述注入会话上下文
|
|||
|
|
if result.get("description"):
|
|||
|
|
injected = await service.inject_to_conversation_context(
|
|||
|
|
result["description"], conversation_id
|
|||
|
|
)
|
|||
|
|
if injected:
|
|||
|
|
logger.info(
|
|||
|
|
f"视觉描述已注入会话 {conversation_id}: "
|
|||
|
|
f"confidence={result.get('confidence', 0):.2f}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
await service.close()
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"code": 0,
|
|||
|
|
"message": "视觉分析完成",
|
|||
|
|
"data": {
|
|||
|
|
"description": result.get("description", ""),
|
|||
|
|
"confidence": result.get("confidence", 0.0),
|
|||
|
|
"metadata": result.get("metadata", {}),
|
|||
|
|
"injected": result.get("description", "") != "",
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
await service.close()
|
|||
|
|
logger.error(f"视觉分析异常: {e}")
|
|||
|
|
return {
|
|||
|
|
"code": 500,
|
|||
|
|
"message": f"视觉分析失败: {str(e)}",
|
|||
|
|
"data": None,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# -----------------------------------------------------------------------------
|
|||
|
|
# 可用的视觉模型列表（Tier1 新增）
|
|||
|
|
# -----------------------------------------------------------------------------
|
|||
|
|
# GET /api/vision/models
|
|||
|
|
@router.get("/models")
|
|||
|
|
async def list_vision_models():
|
|||
|
|
"""获取当前可用的视觉模型列表。
|
|||
|
|
|
|||
|
|
返回系统配置的视觉模型信息，包括当前默认模型和可升级选项。
|
|||
|
|
|
|||
|
|
**无需鉴权（公开查询）。**
|
|||
|
|
"""
|
|||
|
|
models: List[dict] = [
|
|||
|
|
{
|
|||
|
|
"id": "Qwen3-VL-8B-Instruct",
|
|||
|
|
"name": "Qwen3-VL-8B-Instruct（默认）",
|
|||
|
|
"provider": "Qwen",
|
|||
|
|
"description": "本地部署的千问视觉模型，8B 参数，适用于一般截图理解",
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"id": "Qwen3-VL-32B-Instruct",
|
|||
|
|
"name": "Qwen3-VL-32B-Instruct",
|
|||
|
|
"provider": "Qwen",
|
|||
|
|
"description": "千问视觉模型 32B 版本，精度更高但需要更多显存（≥48GB）",
|
|||
|
|
},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"code": 0,
|
|||
|
|
"message": "success",
|
|||
|
|
"data": {
|
|||
|
|
"models": models,
|
|||
|
|
"default_model": settings.qwen_vl_model,
|
|||
|
|
},
|
|||
|
|
}
|