# ============================================================================= # 企微IT智能服务台 — H5 员工端 AI 回复后台任务 # ============================================================================= # 背景:原 h5_send_message 在同步 HTTP 请求内 await AI 推理(Dify 3~15s), # 整条请求被阻塞,前端表现为"发送中"长时间卡顿。 # 本模块将 AI 推理移出请求,改为 asyncio 后台任务,结果经 WebSocket # 流式推回(ai_reply_chunk / ai_reply),发送瞬时完成。 # # 关键约束(详见 docs/02-需求分析/技术架构演进/员工端消息发送延时改造方案.md): # 1. 必须单 worker 运行(docker-compose --workers 1): # ws_manager 是进程内单例,多 worker 时后台任务与员工 WS 连接可能不在 # 同进程,broadcast 会静默丢失(约 50%)。 # 2. 使用独立 DB session(_get_session_factory),不可复用请求的 db # (请求返回后该 session 会被关闭)。 # ============================================================================= import asyncio import logging import os from datetime import datetime, timedelta from pathlib import Path from sqlalchemy import select from app.api.byod import _byod_keyword_prefilter from app.database import _get_session_factory from app.dependencies import get_shared_ai_handler from app.models.conversation import Conversation from app.models.message import Message from app.services.routing_service import ( routing_keyword_prefilter, detect_routing_intent, get_contact_by_category, send_contact_card, record_routing_event, _keyword_fallback_category, ) from app.services.vision_service import VisionService from app.services.ws_manager import manager as ws_manager logger = logging.getLogger(__name__) # ============================================================================= # Phase 4A: VisionService 接入 — 图片消息视觉理解 # ============================================================================= # 图片文件本地存储根目录(与 upload.py 中 UPLOAD_DIR 一致) _UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "./uploads")) # 视觉理解置信度阈值:低于此值不注入描述(避免错误描述误导 AI) _VISION_CONFIDENCE_THRESHOLD = 0.6 def _media_url_to_local_path(media_url: str) -> Path: """将媒体 URL 路径转换为本地文件系统路径。 做什么:把 "/api/media/2026/07/13/abc.png" 转换为 "./uploads/2026/07/13/abc.png" 为什么:VisionService 需要读取原始图片字节流, 而媒体 URL 是 HTTP 访问路径,不是文件系统路径。 Args: media_url: 媒体文件 URL(如 /api/media/2026/07/13/abc.png) Returns: Path: 本地文件路径对象 """ # 去掉 URL 前缀 /api/media/,拼接到 UPLOAD_DIR # 例: "/api/media/2026/07/13/abc.png" → "2026/07/13/abc.png" relative = media_url.replace("/api/media/", "", 1) return _UPLOAD_DIR / relative async def _fetch_recent_employee_text( db, conversation_id: str, employee_id: str, within_seconds: int = 5 ) -> str: """获取最近 N 秒内员工的文字消息(Phase 4B 消息融合)。 做什么:查询同一会话中,当前图片消息之前 within_seconds 秒内, 员工发送的文本消息内容。 为什么:用户经常先打字描述问题再发截图,或先发截图再补充文字。 将文字与图片视觉描述融合后一次性传给 Dify, 避免 AI 分别处理两条消息导致上下文割裂。 Args: db: 异步 DB session conversation_id: 会话 ID employee_id: 员工企微 UserID within_seconds: 时间窗口(秒),默认 5 秒 Returns: str: 最近的员工文字消息内容(多条用换行拼接),无则返回空字符串 """ cutoff = datetime.now() - timedelta(seconds=within_seconds) stmt = ( select(Message) .where( Message.conversation_id == conversation_id, Message.sender_type == "employee", Message.sender_id == employee_id, Message.msg_type == "text", Message.created_at >= cutoff, ) .order_by(Message.created_at.desc()) .limit(3) # 最多取 3 条,避免内容过长 ) result = await db.execute(stmt) messages = result.scalars().all() if not messages: return "" # 按时间正序拼接(先发的在前) texts = [m.content for m in reversed(messages) if m.content] return "\n".join(texts) async def _enrich_image_content( db, media_url: str, original_content: str, conversation_id: str, employee_id: str, ) -> str: """用 VisionService 分析图片,生成增强后的消息内容。 做什么: 1. 从本地文件系统读取图片 2. 调用 VisionService.analyze_screenshot() 获取视觉描述 3. 查询最近 5 秒内的员工文字消息(消息融合) 4. 拼接视觉描述 + 用户文字 → 传给 Dify 为什么:Dify 文本模型无法直接"看"图片,需要先将图片转为 文字描述,再与用户输入融合后传给 Dify 推理。 降级策略: - 图片文件不存在 → 返回原始 content - VisionService 调用失败 → 返回 "我收到了您的截图,但暂时无法识别内容" - 置信度 < 0.6 → 不注入视觉描述,仅使用用户文字 Args: db: 异步 DB session media_url: 图片 URL(如 /api/media/2026/07/13/abc.png) original_content: 原始消息内容(如 "[图片] 截图") conversation_id: 会话 ID employee_id: 员工企微 UserID Returns: str: 增强后的消息内容(视觉描述 + 用户文字) """ # 1. 读取本地图片文件 local_path = _media_url_to_local_path(media_url) if not local_path.exists(): logger.warning(f"图片文件不存在: {local_path} (media_url={media_url})") return original_content try: image_bytes = local_path.read_bytes() except Exception as e: logger.error(f"读取图片文件失败: {local_path} - {e}") return original_content # 2. 调用 VisionService 分析截图 vision_service = VisionService() try: result = await vision_service.analyze_screenshot( image_bytes, conversation_id ) description = result.get("description", "") confidence = result.get("confidence", 0.0) logger.info( f"VisionService 分析完成: conversation={conversation_id}, " f"confidence={confidence:.2f}, desc_len={len(description)}" ) # 3. 注入视觉描述到会话上下文(供后续多轮对话使用) if description and confidence >= _VISION_CONFIDENCE_THRESHOLD: await vision_service.inject_to_conversation_context( description, conversation_id ) except Exception as e: logger.error(f"VisionService 调用异常: {e}") description = "" confidence = 0.0 finally: await vision_service.close() # 4. 消息融合:查询最近 5 秒内员工的文字消息 recent_text = await _fetch_recent_employee_text( db, conversation_id, employee_id, within_seconds=5 ) # 5. 拼接增强内容 # 格式:[视觉描述] + [用户最近文字] + [原始消息内容] parts = [] if description and confidence >= _VISION_CONFIDENCE_THRESHOLD: parts.append(f"[用户发送了截图,视觉理解结果] {description}") if recent_text: parts.append(f"[用户最近的文字描述] {recent_text}") # 原始内容如果不是纯占位符(如"[图片] 截图"),也加入 if original_content and not original_content.startswith("[图片]"): parts.append(original_content) if not parts: # 降级:视觉分析失败且无文字补充 return "我收到了您的截图,但暂时无法识别内容,请描述一下您遇到的问题。" return "\n".join(parts) async def _persist_and_push( db, conversation: Conversation, employee_id: str, content: str, is_guidance: bool, should_count: bool, should_transfer: bool, dify_conversation_id, ): """持久化 AI 回复并推送给员工端 + 广播坐席端。 做什么: 1. 存 AI 消息到 DB 2. 更新会话状态(dify 上下文 / 计数 / 转人工) 3. 经 WS 向员工推 ai_reply 终态(前端据此替换打字机气泡) 4. 经 WS 向坐席端广播 new_message + conversation_updated 为什么:把"落库 + 推送"封装为单点,供同步路径与流式路径复用。 """ # 1. 存 AI 消息 ai_message = Message( conversation_id=conversation.id, sender_type="ai", sender_id="ai_bot", sender_name="Duckula(达寇拉)", content=content, msg_type="text", is_read=True, ) db.add(ai_message) await db.flush() # 2. 更新会话状态 if dify_conversation_id: conversation.dify_conversation_id = dify_conversation_id if should_count: conversation.ai_substantive_reply_count += 1 if should_transfer: conversation.status = "queued" conversation.updated_at = datetime.now() db.add(conversation) await db.flush() await db.commit() # 3. 推 ai_reply 终态给员工(前端替换打字机气泡) await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_reply", "data": { "message_id": str(ai_message.id), "conversation_id": str(conversation.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": content, "msg_type": "text", "is_guidance": is_guidance, "ai_reply_count": conversation.ai_substantive_reply_count, "can_call_agent": conversation.ai_substantive_reply_count >= 3, "conversation_status": conversation.status, }, }) # 4. 广播坐席端(new_message + conversation_updated) try: await ws_manager.broadcast({ "type": "new_message", "data": { "conversation_id": str(conversation.id), "message_id": str(ai_message.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": content, "msg_type": "text", }, }) await ws_manager.broadcast({ "type": "conversation_updated", "data": { "conversation_id": str(conversation.id), "status": conversation.status, "assigned_agent_id": str(conversation.assigned_agent_id) if conversation.assigned_agent_id else None, }, }) except Exception as ws_err: # WS 广播失败不阻塞消息存储,只记录 warning logger.warning(f"WS 广播 AI 回复给坐席失败(消息已存储): {ws_err}") async def _persist_and_push_structured( db, conversation: Conversation, employee_id: str, result: dict, ): """持久化结构化 AI 回复并推送给员工端 + 广播坐席端(v2.0 双 WS 通道)。 改造后的核心变化(2026-07-13): - Dify 返回 JSON {text, action, options},后端解析后同时发两条 WS: ① ai_reply → 聊天气泡(text + options) ② dynamic_recommend → 侧边栏推荐(action 卡片) - 两条消息同一时刻发出,零时间差到达 - 文字明确引用侧边栏内容(如"右侧已为您准备好入口"),语义强关联 命中判断规则: - 结构化回复且有 action 或 options → 视为命中(AI 在主动引导) - 纯文本回复 → 走原有 _check_knowledge_hit 判断 Args: db: 异步 DB session conversation: 当前会话对象 employee_id: 员工企微 UserID result: get_structured_reply() 返回的结构化结果 """ text = result.get("text", "") action = result.get("action") options = result.get("options") hit = result.get("hit", False) is_structured = result.get("is_structured", False) dify_conv_id = result.get("conversation_id") # Phase 6A: 提取诊断阶段 diagnosis_stage = result.get("diagnosis_stage") # 结构化回复且有 action 或 options → 视为命中(AI 在主动引导/推荐) if is_structured and (action or options): hit = True # Phase 6A: 基于 diagnosis_stage 调整会话状态 # escalating → AI 建议转人工 # resolved → AI 认为问题已解决 if diagnosis_stage == "escalating": hit = False # 不计为有效回复,触发转人工 elif diagnosis_stage == "resolved": hit = True # 计为有效回复 should_count = hit should_transfer = not hit # 确定消息类型 if is_structured and (options or action): msg_type = "ai_structured" else: msg_type = "text" # 构建 extra_data(存储 options 和 action 供前端渲染) extra_data = {} if options: extra_data["options"] = options if action: extra_data["action"] = action # 1. 存 AI 消息 ai_message = Message( conversation_id=conversation.id, sender_type="ai", sender_id="ai_bot", sender_name="Duckula(达寇拉)", content=text, msg_type=msg_type, extra_data=extra_data if extra_data else None, is_read=True, ) db.add(ai_message) await db.flush() # 2. 更新会话状态 if dify_conv_id: conversation.dify_conversation_id = dify_conv_id if should_count: conversation.ai_substantive_reply_count += 1 if should_transfer: conversation.status = "queued" # Phase 6A: 将 diagnosis_stage 存入 tags(无需迁移,利用现有 JSON 字段) if diagnosis_stage: tags = conversation.tags or {} tags["diagnosis_stage"] = diagnosis_stage tags["diagnosis_updated_at"] = datetime.now().isoformat() conversation.tags = tags conversation.updated_at = datetime.now() db.add(conversation) await db.flush() await db.commit() # 3. 推 ai_reply 给员工端(聊天气泡:text + options) await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_reply", "data": { "message_id": str(ai_message.id), "conversation_id": str(conversation.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": text, "msg_type": msg_type, "extra_data": extra_data if extra_data else None, "is_guidance": False, "ai_reply_count": conversation.ai_substantive_reply_count, "can_call_agent": conversation.ai_substantive_reply_count >= 3, "conversation_status": conversation.status, # Phase 6A: 诊断阶段(前端可据此调整 UI/提示) "diagnosis_stage": diagnosis_stage, }, }) # 4. 推 dynamic_recommend 给员工端侧边栏(仅当 action 非空时) # 与 ai_reply 同一时刻发出 → 零时间差到达 if action: recommend_data = { "recommend_id": f"rec_{ai_message.id}", "card_type": action.get("type", "approval_card"), "title": action.get("title", ""), "description": action.get("description", ""), "approval_type": action.get("approval_type"), "confidence": action.get("confidence", 0.85), "message_id": str(ai_message.id), "conversation_id": str(conversation.id), } await ws_manager.broadcast_to_employees([employee_id], { "type": "dynamic_recommend", "data": recommend_data, }) logger.info( f"动态推荐已推送: employee={employee_id}, " f"card_type={recommend_data['card_type']}, " f"title={recommend_data['title']}" ) # 5. 广播坐席端(new_message + conversation_updated) try: await ws_manager.broadcast({ "type": "new_message", "data": { "conversation_id": str(conversation.id), "message_id": str(ai_message.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": text, "msg_type": msg_type, "extra_data": extra_data if extra_data else None, }, }) await ws_manager.broadcast({ "type": "conversation_updated", "data": { "conversation_id": str(conversation.id), "status": conversation.status, "assigned_agent_id": ( str(conversation.assigned_agent_id) if conversation.assigned_agent_id else None ), }, }) except Exception as ws_err: logger.warning(f"WS 广播结构化 AI 回复给坐席失败(消息已存储): {ws_err}") async def _handle_byod_query(db, conversation, employee_id, content): """处理 BYOD 自备电脑补贴查询。 在 H5 聊天消息流中拦截 BYOD 关键词后执行资格检查,并以 byod_card 卡片消息形式推送给员工端(前端 MessageBubble 据 msg_type 渲染 ByodSubsidyCard)。 流程: 1. 通过 WecomService 获取员工岗位(position) 2. 与 BYOD 资格清单匹配(_match_position) 3. 创建 byod_card 类型 AI 消息并落库 4. 经 WS 推送 ai_reply 给员工端(携带 extra_data.byod_result) 5. 广播 new_message + conversation_updated 给坐席端(与 _persist_and_push 一致) Args: db: 异步 DB session(process_h5_ai_reply 的 factory session) conversation: 当前会话对象(Conversation) employee_id: 员工企微 UserID content: 用户消息原文(用于日志) """ # 延迟导入避免循环依赖(byod 模块注册路由时可能引用 app.main) from app.api.byod import _match_position, BYOD_APPLICATION_URL, BYOD_NOTES, BYOD_REGISTER_NOTES from app.services.wecom_service import WecomService # 1. 获取员工岗位(企微通讯录 API) position = "" try: wecom_service = WecomService() try: user_info = await wecom_service.get_user_info(employee_id) position = user_info.get("position", "") finally: await wecom_service.close() except Exception as e: logger.error(f"BYOD: 获取员工岗位失败: {e}") # 2. 岗位匹配(返回: 是否匹配, 匹配岗位, 匹配类别) eligible, matched_pos, matched_category = _match_position(position) # 3. 构建 BYOD 结果数据 # 字段与前端 ByodSubsidyCard.vue props 完全一致: # eligible / position / matched_category / application_url / notes / reason byod_result = { "eligible": eligible, "has_subsidy": eligible, "position": position, "matched_category": matched_category, "application_url": BYOD_APPLICATION_URL, # 所有岗位都提供链接 "notes": BYOD_NOTES if eligible else BYOD_REGISTER_NOTES, "reason": ( "" if eligible else f"您的岗位「{position}」不在自备电脑补贴资格清单中,可进行自备电脑登记(无补贴)" ), } # 4. 展示文本(AI 气泡的 content,卡片下方不直接展示,但会话列表/坐席端可见) if eligible: display_text = f"您岗位为「{position}」,符合自备电脑补贴申请资格" else: display_text = f"您岗位为「{position}」,可进行自备电脑登记(无补贴)" # 5. 创建 AI 消息(byod_card 类型,携带 byod_result) ai_message = Message( conversation_id=conversation.id, sender_type="ai", sender_id="ai_bot", sender_name="Duckula(达寇拉)", content=display_text, msg_type="byod_card", extra_data={"byod_result": byod_result}, is_read=True, ) db.add(ai_message) await db.flush() # 6. 更新会话状态(计数 + 时间,BYOD 视为一次实质性 AI 回复) conversation.ai_substantive_reply_count += 1 conversation.updated_at = datetime.now() db.add(conversation) await db.flush() await db.commit() # 7. 推送 ai_reply 给员工端(前端据 msg_type="byod_card" 渲染卡片) await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_reply", "data": { "message_id": str(ai_message.id), "conversation_id": str(conversation.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": display_text, "msg_type": "byod_card", "extra_data": {"byod_result": byod_result}, "is_guidance": False, "ai_reply_count": conversation.ai_substantive_reply_count, "can_call_agent": conversation.ai_substantive_reply_count >= 3, "conversation_status": conversation.status, }, }) # 8. 广播坐席端(new_message + conversation_updated,与 _persist_and_push 一致) try: await ws_manager.broadcast({ "type": "new_message", "data": { "conversation_id": str(conversation.id), "message_id": str(ai_message.id), "sender_type": "ai", "sender_id": "ai_bot", "sender_name": "Duckula(达寇拉)", "content": display_text, "msg_type": "byod_card", "extra_data": {"byod_result": byod_result}, }, }) await ws_manager.broadcast({ "type": "conversation_updated", "data": { "conversation_id": str(conversation.id), "status": conversation.status, "assigned_agent_id": ( str(conversation.assigned_agent_id) if conversation.assigned_agent_id else None ), }, }) except Exception as ws_err: logger.warning(f"BYOD: WS 广播给坐席失败: {ws_err}") logger.info( f"BYOD 查询完成: employee_id={employee_id}, position={position}, " f"eligible={eligible}, matched_category={matched_category}" ) async def _handle_routing( db, conversation: Conversation, employee_id: str, content: str, ) -> bool: """处理非IT业务路由推荐。 在 H5 聊天消息流中拦截路由关键词后调用 Dify 统一意图识别, 判定为 non_it_routing 且 routing_confidence ≥ 阈值时发送名片三段式消息。 流程: 1. 调用 Dify 统一意图识别(detect_routing_intent) 2. 检查 intent_type == "non_it_routing" && routing_confidence ≥ 阈值 → YES: 查询联系人 → 发送名片三段式消息 → 记录路由事件 → 返回 True → NO: 返回 False(继续走正常 AI 流程) 3. Dify 调用失败 → 关键词降级兜底(按 ROUTING_KEYWORD_TO_CATEGORY 映射) Args: db: 异步 DB session(process_h5_ai_reply 的 factory session) conversation: 当前会话对象(Conversation) employee_id: 员工企微 UserID content: 用户消息原文 Returns: bool: True 表示已发送路由名片(应 return 中断后续流程), False 表示未触发路由(继续走正常 AI 流程) """ from app.config import settings # 1. 调用 Dify 统一意图识别 try: result = await detect_routing_intent(content, employee_id) intent_type = result.get("intent_type", "chitchat") business_category = result.get("business_category") routing_confidence = result.get("routing_confidence", 0.0) # 如果是审批意图,不拦截(让审批流程处理) if intent_type == "approval": return False logger.info( f"路由意图检测(Dify): intent_type={intent_type}, " f"business_category={business_category}, " f"routing_confidence={routing_confidence}" ) # 2. 检查是否触发路由推荐 threshold = settings.routing_confidence_threshold if intent_type != "non_it_routing" or routing_confidence < threshold: # 置信度不足或非路由意图,走正常 AI 流程 return False if not business_category: logger.warning("路由意图为 non_it_routing 但 business_category 为空,跳过") return False except Exception as e: logger.warning(f"Dify 路由意图识别失败,降级为关键词匹配: {e}") # 3. 降级为关键词匹配 business_category = _keyword_fallback_category(content) if not business_category: # 关键词也未命中,走正常 AI 流程 return False routing_confidence = 0.75 # 降级兜底给一个略高于阈值的置信度 logger.info(f"路由意图检测(兜底): business_category={business_category}") # 4. 查询联系人 contact = await get_contact_by_category(db, business_category) if not contact: logger.warning(f"未找到 {business_category} 类别的联系人,跳过路由推荐") return False # 5. 构建路由说明文本 category_display = business_category.replace("行政-物业", "物业") reason = ( f"您的问题属于{category_display}业务范畴,不在IT服务台服务范围内 😊\n\n" f"为您推荐{category_display}服务相关联系人,您可以直接点击名片联系TA:" ) # 6. 发送名片三段式消息 await send_contact_card( db=db, conversation=conversation, employee_id=employee_id, contact=contact, reason=reason, business_category=business_category, routing_confidence=routing_confidence, ) # 7. 记录路由事件(P1) await record_routing_event( db=db, conversation_id=str(conversation.id), employee_id=employee_id, message_content=content, business_category=business_category, routing_confidence=routing_confidence, contact=contact, ) return True async def process_h5_ai_reply( conversation_id: str, employee_id: str, content: str, dify_conversation_id=None, msg_type: str = "text", media_url: str = None, ): """H5 发送消息后的 AI 回复处理(asyncio.create_task 入口)。 v2.0 改造(2026-07-13): - AI 回复从流式 SSE 改为 blocking + JSON 结构化输出 - Dify 返回 {text, action, options} JSON → 后端解析 → 双 WS 推送 - 聊天气泡收到 ai_reply(text + options),侧边栏收到 dynamic_recommend(action) - 新增 ai_thinking 指示器,用户发送后立即看到"正在思考..." v2.1 改造(2026-07-13 Phase 4): - 新增图片消息处理分支(msg_type=image) - 图片 → VisionService.analyze_screenshot() → 视觉描述 → 融合到用户文字 - 消息融合:查询最近 5 秒内员工的文字消息,与图片描述合并后传给 Dify - 降级:VisionService 失败/低置信度 → 使用原始文字或提示用户描述问题 流程: 1. BYOD 关键词拦截 → byod_card 卡片 2. 路由关键词拦截 → 名片推荐 3. 本地快判断(打招呼/呼叫人工)→ 同步引导 4. ★ 图片消息处理(Phase 4A)→ VisionService 分析 → 内容增强 5. ★ 结构化 AI 回复(blocking + JSON 解析 + 双 WS 推送) 6. 任意异常 → 推 ai_reply_failed Args: conversation_id: 会话 ID employee_id: 员工企微 UserID content: 消息文本内容 dify_conversation_id: Dify 会话 ID(用于多轮上下文) msg_type: 消息类型(text/image/file),默认 text media_url: 媒体文件 URL(图片消息时使用) """ ai_handler = get_shared_ai_handler() factory = _get_session_factory() async with factory() as db: try: conversation = await db.get(Conversation, conversation_id) if not conversation: logger.warning(f"后台 AI 任务:会话不存在 {conversation_id}") return # === BYOD 关键词拦截(仅文本消息)=== # 图片消息的 content 是占位符(如 "[图片] 截图"),跳过关键词拦截 if msg_type == "text" and _byod_keyword_prefilter(content): await _handle_byod_query(db, conversation, employee_id, content) return # === 业务路由检测(仅文本消息)=== if msg_type == "text" and routing_keyword_prefilter(content): routed = await _handle_routing(db, conversation, employee_id, content) if routed: return # === 本地快判断:打招呼 / 呼叫人工(仅文本消息)=== if msg_type == "text" and (ai_handler.is_greeting(content) or ai_handler.is_call_human(content)): result = await ai_handler.handle_message( content=content, dify_conversation_id=dify_conversation_id, user_id=employee_id, ) await _persist_and_push( db, conversation, employee_id, result.content, result.is_guidance, result.should_count, result.should_transfer, result.dify_conversation_id, ) return # === ★ v2.1 图片消息处理(Phase 4A/4B)=== # 做什么:检测到图片消息 → 调用 VisionService 分析截图 → # 将视觉描述与用户文字融合 → 传给 Dify 推理 # 为什么:Dify 文本模型无法"看"图片,需要先将图片转为文字描述 # 降级:VisionService 失败 → 使用原始 content 继续流程 enriched_content = content # 默认使用原始内容 if msg_type == "image" and media_url: logger.info( f"图片消息检测: conversation={conversation_id}, " f"media_url={media_url}" ) try: enriched_content = await _enrich_image_content( db=db, media_url=media_url, original_content=content, conversation_id=conversation_id, employee_id=employee_id, ) logger.info( f"图片内容增强完成: original_len={len(content)}, " f"enriched_len={len(enriched_content)}" ) except Exception as vision_err: logger.error( f"VisionService 处理失败,降级为纯文本: {vision_err}" ) # 降级:使用原始 content,AI 会收到 "[图片] 截图" 这样的占位符 # Dify 会回复"我收到了您的截图,请描述一下问题" # === ★ v2.0 结构化 AI 回复(替代流式)=== # 1. 立即推送 "正在思考..." 指示器 # 同时推给员工(气泡动画)和坐席(状态指示) await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_thinking", "data": { "conversation_id": conversation_id, }, }) # 坐席端也通知:AI 正在处理此会话的消息 try: await ws_manager.broadcast({ "type": "ai_thinking", "data": { "conversation_id": conversation_id, "employee_id": employee_id, }, }) except Exception: pass # 坐席端通知失败不影响主流程 # 2. 启动延迟 "仍在思考" 后台任务(15 秒后触发) # 如果 Dify 在 15 秒内返回,此任务会被取消 async def _push_still_thinking(): """15 秒后推送 "仍在思考" 提示,缓解用户等待焦虑。""" await asyncio.sleep(15) await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_thinking", "data": { "conversation_id": conversation_id, "status": "still_thinking", }, }) thinking_task = asyncio.create_task(_push_still_thinking()) # 3. 调用 Dify(blocking 模式 + JSON 解析 + 30 秒硬超时) # get_structured_reply 内部处理 HTTP 错误和 JSON 解析失败 # asyncio.wait_for 处理 30 秒硬超时 → 建议转人工 # 注意:图片消息使用 enriched_content(视觉描述+用户文字融合) try: result = await asyncio.wait_for( ai_handler.ai_service.get_structured_reply( message=enriched_content, conversation_id=dify_conversation_id, user_id=employee_id, ), timeout=30, ) except asyncio.TimeoutError: # 30 秒硬超时 → 建议转人工 thinking_task.cancel() logger.warning(f"Dify 30 秒超时: conversation={conversation_id}") await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_reply_failed", "data": { "conversation_id": conversation_id, "message": "AI 响应时间较长,建议转人工坐席处理。", }, }) return # 4. 取消 "仍在思考" 任务(Dify 已返回) thinking_task.cancel() try: await thinking_task # 等待 task 真正取消,避免 warning except asyncio.CancelledError: pass # 5. 持久化 + 双 WS 推送(ai_reply + dynamic_recommend) await _persist_and_push_structured( db, conversation, employee_id, result, ) except Exception as e: logger.error(f"后台 AI 任务异常: {e}", exc_info=True) try: await ws_manager.broadcast_to_employees([employee_id], { "type": "ai_reply_failed", "data": { "conversation_id": conversation_id, "message": "AI 服务异常,请转人工坐席或稍后重试。", }, }) except Exception: # 推送失败也无所谓,员工端 3 秒轮询兜底 pass