LangChain3 多模态能力指南 🎨🎵🎤

from langchain_core.messages import HumanMessage

# 方法1：使用图片URL
image_url = "https://static.xiaohoucode.com/xh-python/a7f83564c2025e0096fdb8b22ff83d66/342111/53f28b7b-e009-47cd-b826-7c42ddb81fc4.jpg"

messages = HumanMessage(
    content=[
        {"type": "text", "text": "用中文详细描述一下这张图片的内容"},
        {"type": "image_url", "image_url": {"url": image_url}}
    ]
)

response = llm.invoke([messages])
print(response.content)

import base64
import httpx
from langchain_core.messages import HumanMessage

# 方法2：使用Base64编码（推荐：兼容性更好）
image_data = base64.b64encode(httpx.get(image_url).content).decode("utf-8")

messages = HumanMessage(
    content=[
        {"type": "text", "text": "分析这张图片中的主要元素"},
        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_data}"}}
    ]
)

response = llm.invoke([messages])
print(response.content)

from langchain_core.messages import HumanMessage
import base64

def process_audio_file(audio_path: str, prompt: str = "请描述这段音频的内容"):
    """处理音频文件的通用函数"""
    
    with open(audio_path, "rb") as audio_file:
        audio_data = base64.b64encode(audio_file.read()).decode("utf-8")
    
    messages = HumanMessage(
        content=[
            {"type": "text", "text": prompt},
            {"type": "audio_url", "audio_url": {"url": f"data:audio/wav;base64,{audio_data}"}}
        ]
    )
    
    return llm.invoke([messages])

def process_audio_url(audio_url: str, prompt: str = "分析这段音频"):
    """处理音频URL的函数"""
    
    messages = HumanMessage(
        content=[
            {"type": "text", "text": prompt},
            {"type": "audio_url", "audio_url": {"url": audio_url}}
        ]
    )
    
    return llm.invoke([messages])

import openai

openai.api_key = "xxx"

with open("assets/test_audio.wav", "rb") as audio_file:
    response = openai.audio.transcriptions.create(
        model="whisper-1",
        file=audio_file,
        response_format="text",
    )

messages = HumanMessage(
    content=[
        # 文本内容
        {"type": "text", "text": "请分析以下内容"},
        
        # 图片内容
        {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
    ]
)