1
2
3
4
5
6
7

# 只能处理文字
text_ai = GPT3()
response = text_ai.chat("今天天气怎么样？")
# ✅ 能回答

response = text_ai.chat("[图片: 窗外风景]")
# ❌ 看不懂图片

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# 能处理文字、图片、音频、视频
multimodal_ai = GPT4V()

# 文字 ✅
response = multimodal_ai.chat("今天天气怎么样？")

# 图片 ✅
response = multimodal_ai.chat(
    text="这是什么？",
    image="photo.jpg"
)

# 音频 ✅
response = multimodal_ai.chat(
    text="这段音乐是什么风格？",
    audio="music.mp3"
)

# 视频 ✅
response = multimodal_ai.chat(
    text="视频里的人在做什么？",
    video="video.mp4"
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

class Modality:
    """AI能理解的信息类型"""
    
    types = {
        "文本": "Text",           # 文字、代码
        "图像": "Image",          # 照片、图表、截图
        "音频": "Audio",          # 语音、音乐、声音
        "视频": "Video",          # 动态画面
        "3D": "3D Model",         # 三维模型
        "传感器": "Sensor Data"   # 温度、压力等
    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

from openai import OpenAI

client = OpenAI()

# 上传图片，AI生成描述
response = client.chat.completions.create(
    model="gpt-4-vision-preview",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "详细描述这张图片"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://example.com/photo.jpg"
                    }
                }
            ]
        }
    ]
)

print(response.choices[0].message.content)
# 输出: "这是一张在海边拍摄的日落照片。天空呈现出橙红色的渐变，
#        海面波光粼粼，远处有一艘帆船..."

1
2
3
4
5

# DALL-E 3 / Midjourney / Stable Diffusion
prompt = "一只穿着宇航服的猫在月球上弹吉他，赛博朋克风格，8K高清"

image = generate_image(prompt)
# 生成符合描述的图片

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

class CrossModalGeneration:
    """跨模态生成能力"""
    
    capabilities = {
        "文 → 图": "DALL-E, Midjourney, Stable Diffusion",
        "文 → 音": "MusicGen, AudioLDM",
        "文 → 视频": "Sora, Runway Gen-2",
        "图 → 文": "GPT-4V, Claude 3.5",
        "音 → 文": "Whisper, Qwen-Audio",
        "视频 → 文": "Gemini 2.0, GPT-4V"
    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 上传一张手写数学题的照片
image = "math_problem.jpg"  # 图片内容: "解方程 2x + 5 = 13"

response = gpt4v.chat(
    text="解这道题，并给出详细步骤",
    image=image
)

print(response)
# 输出:
# "这是一个一元一次方程：
#  步骤1: 2x + 5 = 13
#  步骤2: 2x = 13 - 5
#  步骤3: 2x = 8
#  步骤4: x = 4
#  答案: x = 4"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 场景：医疗诊断
inputs = {
    "X光片": "chest_xray.jpg",
    "病历": "患者男性，65岁，咳嗽两周...",
    "血液检测": "blood_test.pdf"
}

diagnosis = multimodal_ai.analyze(inputs)
# 输出: "根据X光片显示的肺部阴影、病史和血液指标，
#        建议进一步做CT检查排除肺部感染..."

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# 实战：分析商品评论的配图
from openai import OpenAI

client = OpenAI()

def analyze_product_review(image_url, review_text):
    """分析带图片的商品评论"""
    
    response = client.chat.completions.create(
        model="gpt-4-vision-preview",
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": f"用户评论：{review_text}\n请结合图片分析这个评论是否真实可信"
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url}
                    }
                ]
            }
        ],
        max_tokens=500
    )
    
    return response.choices[0].message.content

# 使用示例
review = "这个键盘手感超好，RGB灯效炫酷！"
image = "https://example.com/keyboard.jpg"

analysis = analyze_product_review(image, review)
print(analysis)
# 输出: "图片显示的确实是一款机械键盘，RGB背光清晰可见，
#        与评论描述一致。从键帽磨损程度看，应该是新品。
#        评论可信度：高"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

import google.generativeai as genai

genai.configure(api_key="YOUR_API_KEY")

# Gemini的杀手锏：理解长视频
model = genai.GenerativeModel('gemini-2.0-flash')

# 上传一个1小时的会议录像
video_file = genai.upload_file(path="meeting.mp4")

# 让AI总结会议内容
response = model.generate_content([
    "请总结这次会议的关键决策和行动项",
    video_file
])

print(response.text)
# 输出: "会议主要讨论了Q4产品路线图：
#        1. 决定推迟Feature A的发布至明年Q1
#        2. 增加移动端开发资源
#        3. 行动项：@张三 本周完成技术方案
#        ..."

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

import anthropic

client = anthropic.Anthropic()

# Claude擅长复杂的视觉推理
message = client.messages.create(
    model="claude-3-5-sonnet-20241022",
    max_tokens=1024,
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": "image/jpeg",
                        "data": base64_image,
                    },
                },
                {
                    "type": "text",
                    "text": "这个电路图有什么问题？"
                }
            ],
        }
    ],
)

print(message.content[0].text)
# 输出: "电路图中存在以下问题：
#        1. R2电阻的阻值标注错误（应该是10kΩ而不是1kΩ）
#        2. C1电容的极性接反了
#        3. 缺少保护二极管
#        建议修改..."

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

from transformers import AutoModelForCausalLM, AutoTokenizer

# 加载Qwen-VL模型
model = AutoModelForCausalLM.from_pretrained(
    "Qwen/Qwen-VL-Chat",
    device_map="auto",
    trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(
    "Qwen/Qwen-VL-Chat",
    trust_remote_code=True
)

# 中文图片问答
query = tokenizer.from_list_format([
    {'image': 'https://example.com/image.jpg'},
    {'text': '图片里的人在做什么？'},
])

response, history = model.chat(tokenizer, query=query, history=None)
print(response)
# 输出: "图片中有两个人在打羽毛球，背景是室内体育馆"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

class SmartShoppingAssistant:
    """拍照即可搜索商品"""
    
    def __init__(self):
        self.vision_model = GPT4V()
        self.search_engine = TaobaoAPI()
    
    def find_product(self, image):
        """通过图片找商品"""
        
        # Step 1: AI识别商品
        description = self.vision_model.describe(image)
        # "这是一双白色的Nike Air Force 1运动鞋，鞋码约为42"
        
        # Step 2: 提取关键信息
        keywords = self.vision_model.extract_keywords(description)
        # ["Nike", "Air Force 1", "白色", "42码"]
        
        # Step 3: 搜索商品
        products = self.search_engine.search(keywords)
        
        # Step 4: 匹配相似度
        best_match = self.vision_model.find_most_similar(
            image,
            [p.image for p in products]
        )
        
        return best_match

# 使用
assistant = SmartShoppingAssistant()
result = assistant.find_product("shoe_photo.jpg")
print(f"找到商品：{result.name}，价格：¥{result.price}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

class MedicalAIAssistant:
    """辅助医生诊断"""
    
    def analyze_xray(self, xray_image, patient_info):
        """分析X光片"""
        
        # 多模态输入
        inputs = {
            "image": xray_image,
            "text": f"""
                患者信息：
                - 年龄：{patient_info['age']}
                - 性别：{patient_info['gender']}
                - 症状：{patient_info['symptoms']}
                - 病史：{patient_info['history']}
            """
        }
        
        # AI分析
        analysis = multimodal_ai.analyze(inputs)
        
        return {
            "findings": analysis.findings,      # 发现的异常
            "diagnosis": analysis.diagnosis,    # 初步诊断
            "confidence": analysis.confidence,  # 置信度
            "recommendations": analysis.recommendations  # 建议
        }

# 使用示例
patient = {
    "age": 45,
    "gender": "男",
    "symptoms": "胸痛、咳嗽",
    "history": "吸烟20年"
}

result = assistant.analyze_xray("chest_xray.jpg", patient)

print(f"发现：{result['findings']}")
print(f"建议：{result['recommendations']}")
# 输出:
# 发现：左肺下叶可见片状阴影
# 建议：建议进行CT检查以进一步确认，排除肺部感染或肿瘤

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

class SmartSecuritySystem:
    """智能安防系统"""
    
    def __init__(self):
        self.video_model = Gemini2()
        self.alert_system = AlertSystem()
    
    async def monitor_camera(self, camera_stream):
        """实时监控摄像头"""
        
        while True:
            # 获取视频帧
            frame = await camera_stream.get_frame()
            
            # AI分析
            analysis = self.video_model.analyze(
                frame,
                prompt="检测是否有异常行为：打架、摔倒、闯入等"
            )
            
            # 发现异常
            if analysis.has_anomaly:
                # 生成详细报告
                report = self.video_model.generate_report(
                    frame,
                    prompt=f"详细描述发生了什么：{analysis.anomaly_type}"
                )
                
                # 发送警报
                await self.alert_system.send_alert(
                    type=analysis.anomaly_type,
                    description=report,
                    image=frame,
                    timestamp=datetime.now()
                )
            
            await asyncio.sleep(1)  # 每秒分析一次

# 部署
system = SmartSecuritySystem()
await system.monitor_camera(camera)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

class AITutor:
    """AI家教"""
    
    def help_with_homework(self, homework_image):
        """帮助解答作业"""
        
        # Step 1: OCR识别题目
        problem = vision_model.extract_text(homework_image)
        
        # Step 2: 理解题目类型
        problem_type = vision_model.classify(
            homework_image,
            categories=["数学", "物理", "化学", "语文", "英语"]
        )
        
        # Step 3: 生成解答
        if problem_type == "数学":
            # 识别手写公式
            equation = vision_model.parse_math(homework_image)
            
            # 逐步求解
            solution = math_solver.solve_step_by_step(equation)
            
            return {
                "problem": equation,
                "steps": solution.steps,
                "answer": solution.answer,
                "explanation": solution.explanation
            }
        
        elif problem_type == "英语":
            # 识别作文
            essay = vision_model.extract_text(homework_image)
            
            # 批改作文
            feedback = english_tutor.grade_essay(essay)
            
            return {
                "score": feedback.score,
                "grammar_errors": feedback.grammar_errors,
                "suggestions": feedback.suggestions,
                "corrected_version": feedback.corrected_essay
            }

# 使用
tutor = AITutor()
result = tutor.help_with_homework("homework.jpg")
print(result)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

class MultimodalAI:
    """多模态AI的基本架构"""
    
    def __init__(self):
        # 各模态的编码器
        self.text_encoder = TextEncoder()      # BERT, GPT
        self.image_encoder = ImageEncoder()    # ViT, CLIP
        self.audio_encoder = AudioEncoder()    # Whisper
        self.video_encoder = VideoEncoder()    # VideoMAE
        
        # 融合层
        self.fusion_layer = MultimodalFusion()
        
        # 解码器
        self.decoder = UnifiedDecoder()
    
    def process(self, inputs):
        """处理多模态输入"""
        
        # Step 1: 各模态编码
        embeddings = []
        
        if "text" in inputs:
            text_emb = self.text_encoder(inputs["text"])
            embeddings.append(text_emb)
        
        if "image" in inputs:
            image_emb = self.image_encoder(inputs["image"])
            embeddings.append(image_emb)
        
        if "audio" in inputs:
            audio_emb = self.audio_encoder(inputs["audio"])
            embeddings.append(audio_emb)
        
        # Step 2: 融合
        fused_embedding = self.fusion_layer(embeddings)
        
        # Step 3: 解码生成输出
        output = self.decoder(fused_embedding)
        
        return output

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

# CLIP的训练方式
class CLIP:
    def __init__(self):
        self.image_encoder = ViT()  # Vision Transformer
        self.text_encoder = Transformer()
    
    def train(self, image_text_pairs):
        """对比学习"""
        
        for image, text in image_text_pairs:
            # 编码
            image_emb = self.image_encoder(image)
            text_emb = self.text_encoder(text)
            
            # 目标：匹配的图文对相似度高，不匹配的相似度低
            similarity = cosine_similarity(image_emb, text_emb)
            
            # 损失函数
            loss = contrastive_loss(similarity, is_match=True)
            
            # 反向传播
            loss.backward()

# 使用CLIP
clip = CLIP()

# 图片搜索
image = load_image("cat.jpg")
texts = ["一只猫", "一只狗", "一辆车"]

# 计算相似度
similarities = [
    clip.similarity(image, text)
    for text in texts
]

best_match = texts[np.argmax(similarities)]
print(best_match)  # 输出: "一只猫"

1
2
3
4
5
6
7
8

# 多模态AI要学习的映射关系
mappings = {
    "图片中的猫" ↔ "文字'猫'",
    "笑脸表情" ↔ "开心的情绪",
    "红色" ↔ "热情、危险、停止",
    "钢琴声" ↔ "优雅、古典",
    # ... 数十亿种映射关系
}

1
2
3
4
5
6
7

# 真实案例
image = "empty_room.jpg"  # 一个空房间的照片

response = ai.describe(image)
print(response)
# 错误输出: "房间里有一张桌子和两把椅子"
# （实际上房间是空的！）

1
2
3
4
5
6
7
8

# 使用置信度阈值
response = ai.describe(image, min_confidence=0.8)

# 或者要求AI标注不确定的部分
response = ai.describe(
    image,
    instruction="如果不确定，请说'不确定'而不是猜测"
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 成本对比
costs = {
    "纯文本": {
        "GPT-4": "$0.03 / 1K tokens",
        "Claude": "$0.015 / 1K tokens"
    },
    "多模态": {
        "GPT-4V": "$0.01 / image + $0.03 / 1K tokens",
        "Gemini Pro Vision": "$0.0025 / image"
    }
}

# 处理1000张图片 + 对话
text_only_cost = 0.03 * 10  # $0.30
multimodal_cost = 0.01 * 1000 + 0.03 * 10  # $10.30

print(f"多模态成本是纯文本的 {multimodal_cost / text_only_cost:.0f} 倍")
# 输出: 多模态成本是纯文本的 34 倍

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 风险场景
class PrivacyRisks:
    risks = [
        "人脸识别 → 隐私泄露",
        "医疗图像 → 敏感信息",
        "监控视频 → 滥用风险",
        "深度伪造 → 虚假信息"
    ]
    
    # 防护措施
    protections = [
        "数据脱敏",
        "本地部署（不上传云端）",
        "访问控制",
        "水印技术"
    ]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

future_capabilities = {
    "2026": [
        "实时多模态对话（像人类一样边看边聊）",
        "3D场景理解（理解空间关系）",
        "情感识别（从表情、语气判断情绪）",
        "跨模态生成（说一句话，生成视频）"
    ],
    
    "2027": [
        "具身智能（机器人 + 多模态AI）",
        "全感官AI（视觉+听觉+触觉+嗅觉）",
        "实时翻译（包括手语、表情）",
        "AI导演（自动拍摄剪辑视频）"
    ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 人类的智能 = 多模态
human_intelligence = {
    "视觉": "看",
    "听觉": "听",
    "触觉": "摸",
    "嗅觉": "闻",
    "味觉": "尝",
    "综合": "理解世界"
}

# AI要达到人类水平，必须也是多模态的
agi = MultimodalAI(
    vision=True,
    audio=True,
    touch=True,  # 未来
    smell=True,  # 未来
    taste=True   # 未来
)

1
2
3
4
5
6
7
8

# 你可以做的事情
opportunities = [
    "开发多模态应用（医疗、教育、安防）",
    "训练垂直领域的多模态模型",
    "创建多模态数据集",
    "研究新的融合算法",
    "探索新的应用场景"
]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 1. 试用GPT-4V
from openai import OpenAI
client = OpenAI()
# 上传图片，开始对话

# 2. 试用Gemini
import google.generativeai as genai
genai.configure(api_key="YOUR_KEY")
# 上传视频，让AI总结

# 3. 本地部署Qwen-VL
# git clone https://github.com/QwenLM/Qwen-VL
# 完全免费，可商用