5.3 能力扩展概览

图像 → [视觉编码器 (ViT)] → 视觉 token 序列
                                    ↓
                            [投影模块 (MLP/Cross-Attention)]
                                    ↓
文本 → [Tokenizer] → 文本 token ──→ [LLM 骨干] → 回复

# 使用 Qwen3-VL 进行视觉问答的推理示例
from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor

model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen3-VL-2B-Instruct",
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen3-VL-2B-Instruct")

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": "https://example.com/photo.jpg"},
            {"type": "text", "text": "请详细描述这张图片的内容。"},
        ],
    }
]

text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = processor(text=text, images=[image], return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=256)
response = processor.decode(outputs[0], skip_special_tokens=True)

# 工具调用训练数据格式示例
training_sample = {
    "messages": [
        {
            "role": "system",
            "content": "你是一个有用的助手。你可以使用以下工具：\n"
                       "[get_weather, search_web, calculate]"
        },
        {
            "role": "user",
            "content": "明天北京会下雨吗？"
        },
        {
            "role": "assistant",
            "content": None,
            "tool_calls": [{
                "type": "function",
                "function": {
                    "name": "get_weather",
                    "arguments": '{"city": "北京", "forecast": true}'
                }
            }]
        },
        {
            "role": "tool",
            "name": "get_weather",
            "content": '{"forecast": "明天多云转晴，降水概率 10%"}'
        },
        {
            "role": "assistant",
            "content": "根据天气预报，明天北京多云转晴，降水概率仅 10%，不太可能下雨。"
        }
    ]
}

# Qwen3 函数调用示例
from transformers import AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-1.7B", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-1.7B")

tools = [
    {
        "type": "function",
        "function": {
            "name": "search_knowledge_base",
            "description": "搜索内部知识库获取相关信息",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {"type": "string", "description": "搜索关键词"},
                    "top_k": {"type": "integer", "description": "返回结果数量"}
                },
                "required": ["query"]
            }
        }
    }
]

messages = [
    {"role": "system", "content": "You are a helpful assistant with access to tools."},
    {"role": "user", "content": "帮我查一下 LoRA 微调的最佳学习率是多少？"}
]

# 使用 apply_chat_template 自动格式化工具定义
text = tokenizer.apply_chat_template(
    messages, tools=tools, tokenize=False, add_generation_prompt=True
)
inputs = tokenizer(text, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=256)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(response)

# 使用 Outlines 进行约束解码
import outlines

model = outlines.models.transformers("Qwen/Qwen3-1.7B")

# 定义 JSON Schema
schema = {
    "type": "object",
    "properties": {
        "name": {"type": "string"},
        "arguments": {"type": "object"}
    },
    "required": ["name", "arguments"]
}

generator = outlines.generate.json(model, schema)
result = generator("Call the weather tool for Beijing")
# 保证输出一定是合法的 JSON

用户请求 → 模型思考 → 调用工具 A → 获得结果 A
                → 模型思考 → 调用工具 B(结果 A) → 获得结果 B
                → 模型思考 → 生成最终回复

# RAG 基本流程示例
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter

# 1. 构建知识库
documents = load_documents("knowledge_base/")
text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
chunks = text_splitter.split_documents(documents)

embeddings = HuggingFaceEmbeddings(model_name="BAAI/bge-large-zh-v1.5")
vectorstore = FAISS.from_documents(chunks, embeddings)

# 2. 检索相关内容
query = "Qwen3 的后训练流程是什么？"
relevant_docs = vectorstore.similarity_search(query, k=3)

# 3. 增强生成
context = "\n".join([doc.page_content for doc in relevant_docs])
prompt = f"""基于以下参考信息回答问题：

参考信息：
{context}

问题：{query}
回答："""

# 送入 LLM 生成回复
response = llm.generate(prompt)