Files
wecom_it_smart_desk/backend/app/tasks/h5_ai_task.py
T

904 lines
35 KiB
Python
Raw Normal View History

# =============================================================================
# 企微IT智能服务台 — H5 员工端 AI 回复后台任务
# =============================================================================
# 背景:原 h5_send_message 在同步 HTTP 请求内 await AI 推理(Dify 3~15s),
# 整条请求被阻塞,前端表现为"发送中"长时间卡顿。
# 本模块将 AI 推理移出请求,改为 asyncio 后台任务,结果经 WebSocket
# 流式推回(ai_reply_chunk / ai_reply),发送瞬时完成。
#
# 关键约束(详见 docs/02-需求分析/技术架构演进/员工端消息发送延时改造方案.md):
# 1. 必须单 worker 运行(docker-compose --workers 1):
# ws_manager 是进程内单例,多 worker 时后台任务与员工 WS 连接可能不在
# 同进程,broadcast 会静默丢失(约 50%)。
# 2. 使用独立 DB session_get_session_factory),不可复用请求的 db
# (请求返回后该 session 会被关闭)。
# =============================================================================
import asyncio
import logging
import os
from datetime import datetime, timedelta
from pathlib import Path
from sqlalchemy import select
from app.api.byod import _byod_keyword_prefilter
from app.database import _get_session_factory
from app.dependencies import get_shared_ai_handler
from app.models.conversation import Conversation
from app.models.message import Message
from app.services.routing_service import (
routing_keyword_prefilter,
detect_routing_intent,
get_contact_by_category,
send_contact_card,
record_routing_event,
_keyword_fallback_category,
)
from app.services.vision_service import VisionService
from app.services.ws_manager import manager as ws_manager
logger = logging.getLogger(__name__)
# =============================================================================
# Phase 4A: VisionService 接入 — 图片消息视觉理解
# =============================================================================
# 图片文件本地存储根目录(与 upload.py 中 UPLOAD_DIR 一致)
_UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "./uploads"))
# 视觉理解置信度阈值:低于此值不注入描述(避免错误描述误导 AI)
_VISION_CONFIDENCE_THRESHOLD = 0.6
def _media_url_to_local_path(media_url: str) -> Path:
"""将媒体 URL 路径转换为本地文件系统路径。
做什么:把 "/api/media/2026/07/13/abc.png" 转换为
"./uploads/2026/07/13/abc.png"
为什么:VisionService 需要读取原始图片字节流,
而媒体 URL 是 HTTP 访问路径,不是文件系统路径。
Args:
media_url: 媒体文件 URL(如 /api/media/2026/07/13/abc.png
Returns:
Path: 本地文件路径对象
"""
# 去掉 URL 前缀 /api/media/,拼接到 UPLOAD_DIR
# 例: "/api/media/2026/07/13/abc.png" → "2026/07/13/abc.png"
relative = media_url.replace("/api/media/", "", 1)
return _UPLOAD_DIR / relative
async def _fetch_recent_employee_text(
db, conversation_id: str, employee_id: str, within_seconds: int = 5
) -> str:
"""获取最近 N 秒内员工的文字消息(Phase 4B 消息融合)。
做什么:查询同一会话中,当前图片消息之前 within_seconds 秒内,
员工发送的文本消息内容。
为什么:用户经常先打字描述问题再发截图,或先发截图再补充文字。
将文字与图片视觉描述融合后一次性传给 Dify,
避免 AI 分别处理两条消息导致上下文割裂。
Args:
db: 异步 DB session
conversation_id: 会话 ID
employee_id: 员工企微 UserID
within_seconds: 时间窗口(秒),默认 5 秒
Returns:
str: 最近的员工文字消息内容(多条用换行拼接),无则返回空字符串
"""
cutoff = datetime.now() - timedelta(seconds=within_seconds)
stmt = (
select(Message)
.where(
Message.conversation_id == conversation_id,
Message.sender_type == "employee",
Message.sender_id == employee_id,
Message.msg_type == "text",
Message.created_at >= cutoff,
)
.order_by(Message.created_at.desc())
.limit(3) # 最多取 3 条,避免内容过长
)
result = await db.execute(stmt)
messages = result.scalars().all()
if not messages:
return ""
# 按时间正序拼接(先发的在前)
texts = [m.content for m in reversed(messages) if m.content]
return "\n".join(texts)
async def _enrich_image_content(
db,
media_url: str,
original_content: str,
conversation_id: str,
employee_id: str,
) -> str:
"""用 VisionService 分析图片,生成增强后的消息内容。
做什么:
1. 从本地文件系统读取图片
2. 调用 VisionService.analyze_screenshot() 获取视觉描述
3. 查询最近 5 秒内的员工文字消息(消息融合)
4. 拼接视觉描述 + 用户文字 → 传给 Dify
为什么:Dify 文本模型无法直接""图片,需要先将图片转为
文字描述,再与用户输入融合后传给 Dify 推理。
降级策略:
- 图片文件不存在 → 返回原始 content
- VisionService 调用失败 → 返回 "我收到了您的截图,但暂时无法识别内容"
- 置信度 < 0.6 → 不注入视觉描述,仅使用用户文字
Args:
db: 异步 DB session
media_url: 图片 URL(如 /api/media/2026/07/13/abc.png
original_content: 原始消息内容(如 "[图片] 截图"
conversation_id: 会话 ID
employee_id: 员工企微 UserID
Returns:
str: 增强后的消息内容(视觉描述 + 用户文字)
"""
# 1. 读取本地图片文件
local_path = _media_url_to_local_path(media_url)
if not local_path.exists():
logger.warning(f"图片文件不存在: {local_path} (media_url={media_url})")
return original_content
try:
image_bytes = local_path.read_bytes()
except Exception as e:
logger.error(f"读取图片文件失败: {local_path} - {e}")
return original_content
# 2. 调用 VisionService 分析截图
vision_service = VisionService()
try:
result = await vision_service.analyze_screenshot(
image_bytes, conversation_id
)
description = result.get("description", "")
confidence = result.get("confidence", 0.0)
logger.info(
f"VisionService 分析完成: conversation={conversation_id}, "
f"confidence={confidence:.2f}, desc_len={len(description)}"
)
# 3. 注入视觉描述到会话上下文(供后续多轮对话使用)
if description and confidence >= _VISION_CONFIDENCE_THRESHOLD:
await vision_service.inject_to_conversation_context(
description, conversation_id
)
except Exception as e:
logger.error(f"VisionService 调用异常: {e}")
description = ""
confidence = 0.0
finally:
await vision_service.close()
# 4. 消息融合:查询最近 5 秒内员工的文字消息
recent_text = await _fetch_recent_employee_text(
db, conversation_id, employee_id, within_seconds=5
)
# 5. 拼接增强内容
# 格式:[视觉描述] + [用户最近文字] + [原始消息内容]
parts = []
if description and confidence >= _VISION_CONFIDENCE_THRESHOLD:
parts.append(f"[用户发送了截图,视觉理解结果] {description}")
if recent_text:
parts.append(f"[用户最近的文字描述] {recent_text}")
# 原始内容如果不是纯占位符(如"[图片] 截图"),也加入
if original_content and not original_content.startswith("[图片]"):
parts.append(original_content)
if not parts:
# 降级:视觉分析失败且无文字补充
return "我收到了您的截图,但暂时无法识别内容,请描述一下您遇到的问题。"
return "\n".join(parts)
async def _persist_and_push(
db,
conversation: Conversation,
employee_id: str,
content: str,
is_guidance: bool,
should_count: bool,
should_transfer: bool,
dify_conversation_id,
):
"""持久化 AI 回复并推送给员工端 + 广播坐席端。
做什么:
1. 存 AI 消息到 DB
2. 更新会话状态(dify 上下文 / 计数 / 转人工)
3. 经 WS 向员工推 ai_reply 终态(前端据此替换打字机气泡)
4. 经 WS 向坐席端广播 new_message + conversation_updated
为什么:把"落库 + 推送"封装为单点,供同步路径与流式路径复用。
"""
# 1. 存 AI 消息
ai_message = Message(
conversation_id=conversation.id,
sender_type="ai",
sender_id="ai_bot",
sender_name="Duckula(达寇拉)",
content=content,
msg_type="text",
is_read=True,
)
db.add(ai_message)
await db.flush()
# 2. 更新会话状态
if dify_conversation_id:
conversation.dify_conversation_id = dify_conversation_id
if should_count:
conversation.ai_substantive_reply_count += 1
if should_transfer:
conversation.status = "queued"
conversation.updated_at = datetime.now()
db.add(conversation)
await db.flush()
await db.commit()
# 3. 推 ai_reply 终态给员工(前端替换打字机气泡)
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_reply",
"data": {
"message_id": str(ai_message.id),
"conversation_id": str(conversation.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": content,
"msg_type": "text",
"is_guidance": is_guidance,
"ai_reply_count": conversation.ai_substantive_reply_count,
"can_call_agent": conversation.ai_substantive_reply_count >= 3,
"conversation_status": conversation.status,
},
})
# 4. 广播坐席端(new_message + conversation_updated
try:
await ws_manager.broadcast({
"type": "new_message",
"data": {
"conversation_id": str(conversation.id),
"message_id": str(ai_message.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": content,
"msg_type": "text",
},
})
await ws_manager.broadcast({
"type": "conversation_updated",
"data": {
"conversation_id": str(conversation.id),
"status": conversation.status,
"assigned_agent_id": str(conversation.assigned_agent_id) if conversation.assigned_agent_id else None,
},
})
except Exception as ws_err:
# WS 广播失败不阻塞消息存储,只记录 warning
logger.warning(f"WS 广播 AI 回复给坐席失败(消息已存储): {ws_err}")
async def _persist_and_push_structured(
db,
conversation: Conversation,
employee_id: str,
result: dict,
):
"""持久化结构化 AI 回复并推送给员工端 + 广播坐席端(v2.0 双 WS 通道)。
改造后的核心变化(2026-07-13):
- Dify 返回 JSON {text, action, options},后端解析后同时发两条 WS:
① ai_reply → 聊天气泡(text + options
② dynamic_recommend → 侧边栏推荐(action 卡片)
- 两条消息同一时刻发出,零时间差到达
- 文字明确引用侧边栏内容(如"右侧已为您准备好入口"),语义强关联
命中判断规则:
- 结构化回复且有 action 或 options → 视为命中(AI 在主动引导)
- 纯文本回复 → 走原有 _check_knowledge_hit 判断
Args:
db: 异步 DB session
conversation: 当前会话对象
employee_id: 员工企微 UserID
result: get_structured_reply() 返回的结构化结果
"""
text = result.get("text", "")
action = result.get("action")
options = result.get("options")
hit = result.get("hit", False)
is_structured = result.get("is_structured", False)
dify_conv_id = result.get("conversation_id")
# Phase 6A: 提取诊断阶段
diagnosis_stage = result.get("diagnosis_stage")
# 结构化回复且有 action 或 options → 视为命中(AI 在主动引导/推荐)
if is_structured and (action or options):
hit = True
# Phase 6A: 基于 diagnosis_stage 调整会话状态
# escalating → AI 建议转人工
# resolved → AI 认为问题已解决
if diagnosis_stage == "escalating":
hit = False # 不计为有效回复,触发转人工
elif diagnosis_stage == "resolved":
hit = True # 计为有效回复
should_count = hit
should_transfer = not hit
# 确定消息类型
if is_structured and (options or action):
msg_type = "ai_structured"
else:
msg_type = "text"
# 构建 extra_data(存储 options 和 action 供前端渲染)
extra_data = {}
if options:
extra_data["options"] = options
if action:
extra_data["action"] = action
# 1. 存 AI 消息
ai_message = Message(
conversation_id=conversation.id,
sender_type="ai",
sender_id="ai_bot",
sender_name="Duckula(达寇拉)",
content=text,
msg_type=msg_type,
extra_data=extra_data if extra_data else None,
is_read=True,
)
db.add(ai_message)
await db.flush()
# 2. 更新会话状态
if dify_conv_id:
conversation.dify_conversation_id = dify_conv_id
if should_count:
conversation.ai_substantive_reply_count += 1
if should_transfer:
conversation.status = "queued"
# Phase 6A: 将 diagnosis_stage 存入 tags(无需迁移,利用现有 JSON 字段)
if diagnosis_stage:
tags = conversation.tags or {}
tags["diagnosis_stage"] = diagnosis_stage
tags["diagnosis_updated_at"] = datetime.now().isoformat()
conversation.tags = tags
conversation.updated_at = datetime.now()
db.add(conversation)
await db.flush()
await db.commit()
# 3. 推 ai_reply 给员工端(聊天气泡:text + options
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_reply",
"data": {
"message_id": str(ai_message.id),
"conversation_id": str(conversation.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": text,
"msg_type": msg_type,
"extra_data": extra_data if extra_data else None,
"is_guidance": False,
"ai_reply_count": conversation.ai_substantive_reply_count,
"can_call_agent": conversation.ai_substantive_reply_count >= 3,
"conversation_status": conversation.status,
# Phase 6A: 诊断阶段(前端可据此调整 UI/提示)
"diagnosis_stage": diagnosis_stage,
},
})
# 4. 推 dynamic_recommend 给员工端侧边栏(仅当 action 非空时)
# 与 ai_reply 同一时刻发出 → 零时间差到达
if action:
recommend_data = {
"recommend_id": f"rec_{ai_message.id}",
"card_type": action.get("type", "approval_card"),
"title": action.get("title", ""),
"description": action.get("description", ""),
"approval_type": action.get("approval_type"),
"confidence": action.get("confidence", 0.85),
"message_id": str(ai_message.id),
"conversation_id": str(conversation.id),
}
await ws_manager.broadcast_to_employees([employee_id], {
"type": "dynamic_recommend",
"data": recommend_data,
})
logger.info(
f"动态推荐已推送: employee={employee_id}, "
f"card_type={recommend_data['card_type']}, "
f"title={recommend_data['title']}"
)
# 5. 广播坐席端(new_message + conversation_updated
try:
await ws_manager.broadcast({
"type": "new_message",
"data": {
"conversation_id": str(conversation.id),
"message_id": str(ai_message.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": text,
"msg_type": msg_type,
"extra_data": extra_data if extra_data else None,
},
})
await ws_manager.broadcast({
"type": "conversation_updated",
"data": {
"conversation_id": str(conversation.id),
"status": conversation.status,
"assigned_agent_id": (
str(conversation.assigned_agent_id)
if conversation.assigned_agent_id else None
),
},
})
except Exception as ws_err:
logger.warning(f"WS 广播结构化 AI 回复给坐席失败(消息已存储): {ws_err}")
async def _handle_byod_query(db, conversation, employee_id, content):
"""处理 BYOD 自备电脑补贴查询。
在 H5 聊天消息流中拦截 BYOD 关键词后执行资格检查,并以 byod_card
卡片消息形式推送给员工端(前端 MessageBubble 据 msg_type 渲染
ByodSubsidyCard)。
流程:
1. 通过 WecomService 获取员工岗位(position
2. 与 BYOD 资格清单匹配(_match_position
3. 创建 byod_card 类型 AI 消息并落库
4. 经 WS 推送 ai_reply 给员工端(携带 extra_data.byod_result
5. 广播 new_message + conversation_updated 给坐席端(与 _persist_and_push 一致)
Args:
db: 异步 DB sessionprocess_h5_ai_reply 的 factory session
conversation: 当前会话对象(Conversation
employee_id: 员工企微 UserID
content: 用户消息原文(用于日志)
"""
# 延迟导入避免循环依赖(byod 模块注册路由时可能引用 app.main)
from app.api.byod import _match_position, BYOD_APPLICATION_URL, BYOD_NOTES, BYOD_REGISTER_NOTES
from app.services.wecom_service import WecomService
# 1. 获取员工岗位(企微通讯录 API)
position = ""
try:
wecom_service = WecomService()
try:
user_info = await wecom_service.get_user_info(employee_id)
position = user_info.get("position", "")
finally:
await wecom_service.close()
except Exception as e:
logger.error(f"BYOD: 获取员工岗位失败: {e}")
# 2. 岗位匹配(返回: 是否匹配, 匹配岗位, 匹配类别)
eligible, matched_pos, matched_category = _match_position(position)
# 3. 构建 BYOD 结果数据
# 字段与前端 ByodSubsidyCard.vue props 完全一致:
# eligible / position / matched_category / application_url / notes / reason
byod_result = {
"eligible": eligible,
"has_subsidy": eligible,
"position": position,
"matched_category": matched_category,
"application_url": BYOD_APPLICATION_URL, # 所有岗位都提供链接
"notes": BYOD_NOTES if eligible else BYOD_REGISTER_NOTES,
"reason": (
"" if eligible
else f"您的岗位「{position}」不在自备电脑补贴资格清单中,可进行自备电脑登记(无补贴)"
),
}
# 4. 展示文本(AI 气泡的 content,卡片下方不直接展示,但会话列表/坐席端可见)
if eligible:
display_text = f"您岗位为「{position}」,符合自备电脑补贴申请资格"
else:
display_text = f"您岗位为「{position}」,可进行自备电脑登记(无补贴)"
# 5. 创建 AI 消息(byod_card 类型,携带 byod_result
ai_message = Message(
conversation_id=conversation.id,
sender_type="ai",
sender_id="ai_bot",
sender_name="Duckula(达寇拉)",
content=display_text,
msg_type="byod_card",
extra_data={"byod_result": byod_result},
is_read=True,
)
db.add(ai_message)
await db.flush()
# 6. 更新会话状态(计数 + 时间,BYOD 视为一次实质性 AI 回复)
conversation.ai_substantive_reply_count += 1
conversation.updated_at = datetime.now()
db.add(conversation)
await db.flush()
await db.commit()
# 7. 推送 ai_reply 给员工端(前端据 msg_type="byod_card" 渲染卡片)
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_reply",
"data": {
"message_id": str(ai_message.id),
"conversation_id": str(conversation.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": display_text,
"msg_type": "byod_card",
"extra_data": {"byod_result": byod_result},
"is_guidance": False,
"ai_reply_count": conversation.ai_substantive_reply_count,
"can_call_agent": conversation.ai_substantive_reply_count >= 3,
"conversation_status": conversation.status,
},
})
# 8. 广播坐席端(new_message + conversation_updated,与 _persist_and_push 一致)
try:
await ws_manager.broadcast({
"type": "new_message",
"data": {
"conversation_id": str(conversation.id),
"message_id": str(ai_message.id),
"sender_type": "ai",
"sender_id": "ai_bot",
"sender_name": "Duckula(达寇拉)",
"content": display_text,
"msg_type": "byod_card",
"extra_data": {"byod_result": byod_result},
},
})
await ws_manager.broadcast({
"type": "conversation_updated",
"data": {
"conversation_id": str(conversation.id),
"status": conversation.status,
"assigned_agent_id": (
str(conversation.assigned_agent_id)
if conversation.assigned_agent_id else None
),
},
})
except Exception as ws_err:
logger.warning(f"BYOD: WS 广播给坐席失败: {ws_err}")
logger.info(
f"BYOD 查询完成: employee_id={employee_id}, position={position}, "
f"eligible={eligible}, matched_category={matched_category}"
)
async def _handle_routing(
db,
conversation: Conversation,
employee_id: str,
content: str,
) -> bool:
"""处理非IT业务路由推荐。
在 H5 聊天消息流中拦截路由关键词后调用 Dify 统一意图识别,
判定为 non_it_routing 且 routing_confidence ≥ 阈值时发送名片三段式消息。
流程:
1. 调用 Dify 统一意图识别(detect_routing_intent
2. 检查 intent_type == "non_it_routing" && routing_confidence ≥ 阈值
→ YES: 查询联系人 → 发送名片三段式消息 → 记录路由事件 → 返回 True
→ NO: 返回 False(继续走正常 AI 流程)
3. Dify 调用失败 → 关键词降级兜底(按 ROUTING_KEYWORD_TO_CATEGORY 映射)
Args:
db: 异步 DB sessionprocess_h5_ai_reply 的 factory session
conversation: 当前会话对象(Conversation
employee_id: 员工企微 UserID
content: 用户消息原文
Returns:
bool: True 表示已发送路由名片(应 return 中断后续流程),
False 表示未触发路由(继续走正常 AI 流程)
"""
from app.config import settings
# 1. 调用 Dify 统一意图识别
try:
result = await detect_routing_intent(content, employee_id)
intent_type = result.get("intent_type", "chitchat")
business_category = result.get("business_category")
routing_confidence = result.get("routing_confidence", 0.0)
# 如果是审批意图,不拦截(让审批流程处理)
if intent_type == "approval":
return False
logger.info(
f"路由意图检测(Dify): intent_type={intent_type}, "
f"business_category={business_category}, "
f"routing_confidence={routing_confidence}"
)
# 2. 检查是否触发路由推荐
threshold = settings.routing_confidence_threshold
if intent_type != "non_it_routing" or routing_confidence < threshold:
# 置信度不足或非路由意图,走正常 AI 流程
return False
if not business_category:
logger.warning("路由意图为 non_it_routing 但 business_category 为空,跳过")
return False
except Exception as e:
logger.warning(f"Dify 路由意图识别失败,降级为关键词匹配: {e}")
# 3. 降级为关键词匹配
business_category = _keyword_fallback_category(content)
if not business_category:
# 关键词也未命中,走正常 AI 流程
return False
routing_confidence = 0.75 # 降级兜底给一个略高于阈值的置信度
logger.info(f"路由意图检测(兜底): business_category={business_category}")
# 4. 查询联系人
contact = await get_contact_by_category(db, business_category)
if not contact:
logger.warning(f"未找到 {business_category} 类别的联系人,跳过路由推荐")
return False
# 5. 构建路由说明文本
category_display = business_category.replace("行政-物业", "物业")
reason = (
f"您的问题属于{category_display}业务范畴,不在IT服务台服务范围内 😊\n\n"
f"为您推荐{category_display}服务相关联系人,您可以直接点击名片联系TA:"
)
# 6. 发送名片三段式消息
await send_contact_card(
db=db,
conversation=conversation,
employee_id=employee_id,
contact=contact,
reason=reason,
business_category=business_category,
routing_confidence=routing_confidence,
)
# 7. 记录路由事件(P1
await record_routing_event(
db=db,
conversation_id=str(conversation.id),
employee_id=employee_id,
message_content=content,
business_category=business_category,
routing_confidence=routing_confidence,
contact=contact,
)
return True
async def process_h5_ai_reply(
conversation_id: str,
employee_id: str,
content: str,
dify_conversation_id=None,
msg_type: str = "text",
media_url: str = None,
):
"""H5 发送消息后的 AI 回复处理(asyncio.create_task 入口)。
v2.0 改造(2026-07-13):
- AI 回复从流式 SSE 改为 blocking + JSON 结构化输出
- Dify 返回 {text, action, options} JSON → 后端解析 → 双 WS 推送
- 聊天气泡收到 ai_replytext + options),侧边栏收到 dynamic_recommendaction
- 新增 ai_thinking 指示器,用户发送后立即看到"正在思考..."
v2.1 改造(2026-07-13 Phase 4):
- 新增图片消息处理分支(msg_type=image
- 图片 → VisionService.analyze_screenshot() → 视觉描述 → 融合到用户文字
- 消息融合:查询最近 5 秒内员工的文字消息,与图片描述合并后传给 Dify
- 降级:VisionService 失败/低置信度 → 使用原始文字或提示用户描述问题
流程:
1. BYOD 关键词拦截 → byod_card 卡片
2. 路由关键词拦截 → 名片推荐
3. 本地快判断(打招呼/呼叫人工)→ 同步引导
4. ★ 图片消息处理(Phase 4A)→ VisionService 分析 → 内容增强
5. ★ 结构化 AI 回复(blocking + JSON 解析 + 双 WS 推送)
6. 任意异常 → 推 ai_reply_failed
Args:
conversation_id: 会话 ID
employee_id: 员工企微 UserID
content: 消息文本内容
dify_conversation_id: Dify 会话 ID(用于多轮上下文)
msg_type: 消息类型(text/image/file),默认 text
media_url: 媒体文件 URL(图片消息时使用)
"""
ai_handler = get_shared_ai_handler()
factory = _get_session_factory()
async with factory() as db:
try:
conversation = await db.get(Conversation, conversation_id)
if not conversation:
logger.warning(f"后台 AI 任务:会话不存在 {conversation_id}")
return
# === BYOD 关键词拦截(仅文本消息)===
# 图片消息的 content 是占位符(如 "[图片] 截图"),跳过关键词拦截
if msg_type == "text" and _byod_keyword_prefilter(content):
await _handle_byod_query(db, conversation, employee_id, content)
return
# === 业务路由检测(仅文本消息)===
if msg_type == "text" and routing_keyword_prefilter(content):
routed = await _handle_routing(db, conversation, employee_id, content)
if routed:
return
# === 本地快判断:打招呼 / 呼叫人工(仅文本消息)===
if msg_type == "text" and (ai_handler.is_greeting(content) or ai_handler.is_call_human(content)):
result = await ai_handler.handle_message(
content=content,
dify_conversation_id=dify_conversation_id,
user_id=employee_id,
)
await _persist_and_push(
db, conversation, employee_id, result.content,
result.is_guidance, result.should_count,
result.should_transfer, result.dify_conversation_id,
)
return
# === ★ v2.1 图片消息处理(Phase 4A/4B===
# 做什么:检测到图片消息 → 调用 VisionService 分析截图 →
# 将视觉描述与用户文字融合 → 传给 Dify 推理
# 为什么:Dify 文本模型无法"看"图片,需要先将图片转为文字描述
# 降级:VisionService 失败 → 使用原始 content 继续流程
enriched_content = content # 默认使用原始内容
if msg_type == "image" and media_url:
logger.info(
f"图片消息检测: conversation={conversation_id}, "
f"media_url={media_url}"
)
try:
enriched_content = await _enrich_image_content(
db=db,
media_url=media_url,
original_content=content,
conversation_id=conversation_id,
employee_id=employee_id,
)
logger.info(
f"图片内容增强完成: original_len={len(content)}, "
f"enriched_len={len(enriched_content)}"
)
except Exception as vision_err:
logger.error(
f"VisionService 处理失败,降级为纯文本: {vision_err}"
)
# 降级:使用原始 contentAI 会收到 "[图片] 截图" 这样的占位符
# Dify 会回复"我收到了您的截图,请描述一下问题"
# === ★ v2.0 结构化 AI 回复(替代流式)===
# 1. 立即推送 "正在思考..." 指示器
# 同时推给员工(气泡动画)和坐席(状态指示)
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_thinking",
"data": {
"conversation_id": conversation_id,
},
})
# 坐席端也通知:AI 正在处理此会话的消息
try:
await ws_manager.broadcast({
"type": "ai_thinking",
"data": {
"conversation_id": conversation_id,
"employee_id": employee_id,
},
})
except Exception:
pass # 坐席端通知失败不影响主流程
# 2. 启动延迟 "仍在思考" 后台任务(15 秒后触发)
# 如果 Dify 在 15 秒内返回,此任务会被取消
async def _push_still_thinking():
"""15 秒后推送 "仍在思考" 提示,缓解用户等待焦虑。"""
await asyncio.sleep(15)
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_thinking",
"data": {
"conversation_id": conversation_id,
"status": "still_thinking",
},
})
thinking_task = asyncio.create_task(_push_still_thinking())
# 3. 调用 Difyblocking 模式 + JSON 解析 + 30 秒硬超时)
# get_structured_reply 内部处理 HTTP 错误和 JSON 解析失败
# asyncio.wait_for 处理 30 秒硬超时 → 建议转人工
# 注意:图片消息使用 enriched_content(视觉描述+用户文字融合)
try:
result = await asyncio.wait_for(
ai_handler.ai_service.get_structured_reply(
message=enriched_content,
conversation_id=dify_conversation_id,
user_id=employee_id,
),
timeout=30,
)
except asyncio.TimeoutError:
# 30 秒硬超时 → 建议转人工
thinking_task.cancel()
logger.warning(f"Dify 30 秒超时: conversation={conversation_id}")
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_reply_failed",
"data": {
"conversation_id": conversation_id,
"message": "AI 响应时间较长,建议转人工坐席处理。",
},
})
return
# 4. 取消 "仍在思考" 任务(Dify 已返回)
thinking_task.cancel()
try:
await thinking_task # 等待 task 真正取消,避免 warning
except asyncio.CancelledError:
pass
# 5. 持久化 + 双 WS 推送(ai_reply + dynamic_recommend
await _persist_and_push_structured(
db, conversation, employee_id, result,
)
except Exception as e:
logger.error(f"后台 AI 任务异常: {e}", exc_info=True)
try:
await ws_manager.broadcast_to_employees([employee_id], {
"type": "ai_reply_failed",
"data": {
"conversation_id": conversation_id,
"message": "AI 服务异常,请转人工坐席或稍后重试。",
},
})
except Exception:
# 推送失败也无所谓,员工端 3 秒轮询兜底
pass