多模态大模型技术

人类理解世界并非只依赖文字。我们通过视觉、听觉、触觉等多种感官协同工作，构建对世界的完整认知。多模态大模型让机器首次能够像人类一样跨模态理解和生成内容。

一、CLIP：对比学习的突破#

核心思想#

CLIP (Contrastive Language-Image Pre-training) 通过对比学习，让图像和文本在同一语义空间中对齐。

对比损失（InfoNCE）：

\mathcal{L} = -\frac{1}{N}\sum_{i=1}^N \log \frac{\exp(\text{sim}(I_i, T_i) / \tau)}{\sum_{j=1}^N \exp(\text{sim}(I_i, T_j) / \tau)}

PyTorch实现#

1
import torch
2
import torch.nn as nn
3
import torch.nn.functional as F
4

5
class CLIP(nn.Module):
6
    def __init__(self, image_encoder, text_encoder, embed_dim=512):
7
        super().__init__()
8
        self.image_encoder = image_encoder
9
        self.text_encoder = text_encoder
10
        self.temperature = nn.Parameter(torch.tensor(0.07))
11

12
        self.image_projection = nn.Linear(image_encoder.output_dim, embed_dim)
13
        self.text_projection = nn.Linear(text_encoder.output_dim, embed_dim)
14

15
    def forward(self, images, texts):
16
        # 编码
17
        image_features = self.image_encoder(images)
18
        text_features = self.text_encoder(texts)
19

20
        # 投影到共享空间
21
        image_embeds = F.normalize(self.image_projection(image_features), dim=-1)
22
        text_embeds = F.normalize(self.text_projection(text_features), dim=-1)
23

24
        # 计算相似度矩阵
25
        logits = image_embeds @ text_embeds.T / self.temperature
26

27
        return logits, logits.T
28

29
def clip_loss(logits):
30
    labels = torch.arange(logits.size(0), device=logits.device)
31
    loss_i = F.cross_entropy(logits, labels)
32
    loss_t = F.cross_entropy(logits.T, labels)
33
    return (loss_i + loss_t) / 2

Zero-Shot分类#

1
def zero_shot_classify(model, image, class_names):
2
    texts = [f"a photo of a {c}" for c in class_names]
3

4
    image_embed = model.encode_image(image)
5
    text_embeds = model.encode_text(texts)
6

7
    image_embed = F.normalize(image_embed, dim=-1)
8
    text_embeds = F.normalize(text_embeds, dim=-1)
9

10
    logits = (image_embed @ text_embeds.T) / model.temperature
11
    probs = F.softmax(logits, dim=-1)
12

13
    return probs[0]

二、BLIP：生成式视觉语言模型#

三种架构模式#

Image-Text Contrastive (ITC)：类似CLIP
Image-grounded Text Generation (ITG)：图像描述生成
Image-Text Matching (ITM)：二分类判断

图像描述生成#

1
class BLIP_ITG(nn.Module):
2
    def __init__(self, image_encoder, text_decoder):
3
        super().__init__()
4
        self.image_encoder = image_encoder
5
        self.text_decoder = text_decoder
6
        self.cross_attention = nn.MultiheadAttention(embed_dim=768, num_heads=12)
7

8
    @torch.no_grad()
9
    def generate_caption(self, image, max_length=50):
10
        image_embeds = self.image_encoder(image.unsqueeze(0))
11

12
        generated = [self.text_decoder.bos_token_id]
13
        for _ in range(max_length):
14
            text_embeds = self.text_decoder.embed(torch.tensor([generated]))
15

16
            # 跨模态注意力
17
            text_embeds, _ = self.cross_attention(text_embeds, image_embeds, image_embeds)
18

19
            logits = self.text_decoder.lm_head(text_embeds)
20
            next_token = logits[0, -1].argmax().item()
21

22
            if next_token == self.text_decoder.eos_token_id:
23
                break
24
            generated.append(next_token)
25

26
        return self.text_decoder.tokenizer.decode(generated)

三、GPT-4V：统一的多模态LLM#

视觉Token化#

方法1：Linear Projection (LLaVA)

1
class VisualTokenizer(nn.Module):
2
    def __init__(self, vision_encoder, d_model):
3
        super().__init__()
4
        self.encoder = vision_encoder
5
        self.projection = nn.Linear(vision_encoder.output_dim, d_model)
6

7
    def forward(self, images):
8
        features = self.encoder(images)  # [batch, 196, 1024]
9
        tokens = self.projection(features)  # [batch, 196, 4096]
10
        return tokens

方法2：Q-Former (BLIP-2)

1
class QFormer(nn.Module):
2
    def __init__(self, num_queries=32, d_model=768):
3
        super().__init__()
4
        self.queries = nn.Parameter(torch.randn(num_queries, d_model))
5
        self.cross_attention = nn.TransformerDecoder(...)
6

7
    def forward(self, image_features):
8
        batch_size = image_features.size(0)
9
        queries = self.queries.unsqueeze(0).repeat(batch_size, 1, 1)
10
        output = self.cross_attention(queries, image_features)
11
        return output

多模态Prompt#

1
def build_multimodal_prompt(image, text_prompt):
2
    visual_tokens = vision_tokenizer(image)
3
    text_tokens = text_tokenizer(text_prompt)
4

5
    # 前缀方式
6
    input_embeds = torch.cat([visual_tokens, text_tokens], dim=0)
7

8
    return input_embeds
9

10
# 使用
11
prompt = "Describe this image in detail:"
12
response = multimodal_llm.generate(image=img, prompt=prompt)

四、训练技巧#

硬负样本挖掘#

1
def hard_negative_mining(image_embeds, text_embeds, num_hard=5):
2
    sim_matrix = image_embeds @ text_embeds.T
3

4
    batch_size = sim_matrix.size(0)
5
    hard_negatives = []
6

7
    for i in range(batch_size):
8
        mask = torch.ones_like(sim_matrix[i]).bool()
9
        mask[i] = False
10

11
        neg_sims = sim_matrix[i][mask]
12
        hard_neg_idx = neg_sims.topk(num_hard).indices
13
        hard_negatives.append(hard_neg_idx)
14

15
    return hard_negatives

温度缩放#

1
class LearnableTemperature(nn.Module):
2
    def __init__(self, init_value=0.07):
3
        super().__init__()
4
        self.logit_scale = nn.Parameter(torch.log(torch.tensor(1 / init_value)))
5

6
    def forward(self, similarity):
7
        temperature = torch.clamp(self.logit_scale.exp(), 0.01, 100)
8
        return similarity * temperature

五、评估与基准#

主要数据集#

数据集	任务	规模
MS-COCO	图像描述/VQA	120K图
Flickr30K	图文检索	30K图
VQAv2	视觉问答	1.1M问题
LAION-5B	预训练	50亿图文对

评估指标#

1
def calculate_recall(image_embeds, text_embeds, k=[1, 5, 10]):
2
    sim_matrix = image_embeds @ text_embeds.T
3

4
    recalls = {}
5
    for K in k:
6
        topk_indices = sim_matrix.topk(K, dim=1).indices
7
        correct = (topk_indices == torch.arange(len(sim_matrix)).unsqueeze(1)).any(dim=1)
8
        recalls[f'I2T_R@{K}'] = correct.float().mean().item()
9

10
    return recalls

六、实战：构建简易CLIP#

1
class SimpleCLIP(nn.Module):
2
    def __init__(self):
3
        super().__init__()
4

5
        from torchvision.models import resnet50
6
        from transformers import AutoModel
7

8
        resnet = resnet50(pretrained=True)
9
        self.vision_encoder = nn.Sequential(*list(resnet.children())[:-1])
10
        self.vision_projection = nn.Linear(2048, 512)
11

12
        self.text_encoder = AutoModel.from_pretrained('distilbert-base-uncased')
13
        self.text_projection = nn.Linear(768, 512)
14

15
        self.temperature = nn.Parameter(torch.tensor(0.07))
16

17
    def encode_image(self, images):
18
        features = self.vision_encoder(images).squeeze()
19
        return F.normalize(self.vision_projection(features), dim=-1)
20

21
    def encode_text(self, input_ids, attention_mask):
22
        outputs = self.text_encoder(input_ids, attention_mask)
23
        pooled = outputs.last_hidden_state[:, 0]
24
        return F.normalize(self.text_projection(pooled), dim=-1)
25

26
# 训练
27
model = SimpleCLIP()
28
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)
29

30
for images, texts in dataloader:
31
    image_embeds = model.encode_image(images)
32
    text_embeds = model.encode_text(texts['input_ids'], texts['attention_mask'])
33

34
    logits = (image_embeds @ text_embeds.T) / model.temperature
35

36
    labels = torch.arange(len(images), device=logits.device)
37
    loss = (F.cross_entropy(logits, labels) +
38
            F.cross_entropy(logits.T, labels)) / 2
39

40
    optimizer.zero_grad()
41
    loss.backward()
42
    optimizer.step()

七、总结#

多模态AI的核心：

对齐是关键：CLIP用对比学习对齐图文
架构选择：编码器 vs 编解码器 vs 统一LLM
数据规模：大规模预训练是基础

前沿方向：

ImageBind：六模态统一空间
Any-to-Any生成：任意模态输入输出
世界模型：物理推理与3D理解

多模态大模型正在成为AI的下一个制高点！🚀

音乐

音乐

一、CLIP：对比学习的突破#

核心思想#

PyTorch实现#

Zero-Shot分类#

二、BLIP：生成式视觉语言模型#

三种架构模式#

图像描述生成#

三、GPT-4V：统一的多模态LLM#

视觉Token化#

多模态Prompt#

四、训练技巧#

硬负样本挖掘#

温度缩放#

五、评估与基准#

主要数据集#

评估指标#

六、实战：构建简易CLIP#

七、总结#

🧠 文章思维导图

支持与分享

引用本文

评论区

音乐

文章目录