Transformer架构详解
931 字
5 分钟
Transformer架构详解
Transformer架构的诞生标志着深度学习进入新纪元。2017年，Google团队在论文《Attention is All You Need》中提出Transformer，彻底颠覆了序列建模的传统范式。它不仅催生了BERT、GPT等里程碑模型，更成为现代大语言模型的基石。
一、核心架构
Encoder块结构：
Input Embedding + Positional Encoding ↓Multi-Head Self-Attention ↓Add & Norm ↓Feed-Forward Network ↓Add & Norm完整实现：
import torch.nn as nn
class TransformerEncoderLayer(nn.Module): def __init__(self, d_model, num_heads, d_ff, dropout=0.1): super().__init__()
# 多头自注意力 self.self_attn = MultiHeadAttention(d_model, num_heads, dropout)
# 前馈网络 self.ffn = PositionWiseFeedForward(d_model, d_ff, dropout)
# Layer Normalization self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model)
# Dropout self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout)
def forward(self, x, mask=None): # 自注意力 + 残差连接 + LayerNorm attn_output, _ = self.self_attn(x, x, x, mask) x = self.norm1(x + self.dropout1(attn_output))
# FFN + 残差连接 + LayerNorm ffn_output = self.ffn(x) x = self.norm2(x + self.dropout2(ffn_output))
return xDecoder块增加交叉注意力：
class TransformerDecoderLayer(nn.Module): def __init__(self, d_model, num_heads, d_ff, dropout=0.1): super().__init__()
# 三个注意力层 self.self_attn = MultiHeadAttention(d_model, num_heads, dropout) self.cross_attn = MultiHeadAttention(d_model, num_heads, dropout) self.ffn = PositionWiseFeedForward(d_model, d_ff, dropout)
self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) self.norm3 = nn.LayerNorm(d_model)
self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.dropout3 = nn.Dropout(dropout)
def forward(self, x, encoder_output, src_mask=None, tgt_mask=None): # 1. Masked Self-Attention（防止看到未来） attn_output, _ = self.self_attn(x, x, x, tgt_mask) x = self.norm1(x + self.dropout1(attn_output))
# 2. Cross-Attention（attend to encoder） attn_output, _ = self.cross_attn(x, encoder_output, encoder_output, src_mask) x = self.norm2(x + self.dropout2(attn_output))
# 3. FFN ffn_output = self.ffn(x) x = self.norm3(x + self.dropout3(ffn_output))
return x二、完整Transformer模型
class Transformer(nn.Module): def __init__( self, src_vocab_size, tgt_vocab_size, d_model=512, num_heads=8, num_encoder_layers=6, num_decoder_layers=6, d_ff=2048, dropout=0.1, max_len=5000 ): super().__init__()
# Embedding层 self.src_embedding = nn.Embedding(src_vocab_size, d_model) self.tgt_embedding = nn.Embedding(tgt_vocab_size, d_model)
# 位置编码 self.pos_encoding = PositionalEncoding(d_model, max_len, dropout)
# Encoder self.encoder_layers = nn.ModuleList([ TransformerEncoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_encoder_layers) ])
# Decoder self.decoder_layers = nn.ModuleList([ TransformerDecoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_decoder_layers) ])
# 输出层 self.fc_out = nn.Linear(d_model, tgt_vocab_size)
self.dropout = nn.Dropout(dropout) self.d_model = d_model
def generate_mask(self, src, tgt): # Encoder mask（padding） src_mask = (src != 0).unsqueeze(1).unsqueeze(2) # [batch, 1, 1, src_len]
# Decoder mask（padding + causal） tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3) # [batch, 1, tgt_len, 1]
seq_len = tgt.size(1) nopeak_mask = torch.tril(torch.ones(seq_len, seq_len)).bool() nopeak_mask = nopeak_mask.to(tgt.device)
tgt_mask = tgt_mask & nopeak_mask
return src_mask, tgt_mask
def encode(self, src, src_mask): # Embedding + 位置编码 x = self.src_embedding(src) * math.sqrt(self.d_model) x = self.pos_encoding(x)
# 通过Encoder层 for layer in self.encoder_layers: x = layer(x, src_mask)
return x
def decode(self, tgt, encoder_output, src_mask, tgt_mask): # Embedding + 位置编码 x = self.tgt_embedding(tgt) * math.sqrt(self.d_model) x = self.pos_encoding(x)
# 通过Decoder层 for layer in self.decoder_layers: x = layer(x, encoder_output, src_mask, tgt_mask)
return x
def forward(self, src, tgt): """ Args: src: [batch, src_len] tgt: [batch, tgt_len] Returns: output: [batch, tgt_len, tgt_vocab_size] """ src_mask, tgt_mask = self.generate_mask(src, tgt)
encoder_output = self.encode(src, src_mask) decoder_output = self.decode(tgt, encoder_output, src_mask, tgt_mask)
output = self.fc_out(decoder_output) return output三、训练技巧
Label Smoothing
class LabelSmoothingLoss(nn.Module): def __init__(self, smoothing=0.1): super().__init__() self.smoothing = smoothing
def forward(self, pred, target): # pred: [batch * seq_len, vocab_size] # target: [batch * seq_len]
vocab_size = pred.size(-1) confidence = 1.0 - self.smoothing
true_dist = torch.zeros_like(pred) true_dist.fill_(self.smoothing / (vocab_size - 1)) true_dist.scatter_(1, target.unsqueeze(1), confidence)
return F.kl_div(F.log_softmax(pred, dim=-1), true_dist, reduction='batchmean')Warmup Learning Rate
class NoamOpt: def __init__(self, d_model, warmup_steps, optimizer): self.d_model = d_model self.warmup_steps = warmup_steps self.optimizer = optimizer self._step = 0
def step(self): self._step += 1 lr = self.d_model ** (-0.5) * min( self._step ** (-0.5), self._step * self.warmup_steps ** (-1.5) )
for param_group in self.optimizer.param_groups: param_group['lr'] = lr
self.optimizer.step()四、应用场景
机器翻译：
# 训练model = Transformer(src_vocab_size, tgt_vocab_size)criterion = LabelSmoothingLoss()
for src, tgt in dataloader: output = model(src, tgt[:, :-1]) # 输入去掉最后一个token loss = criterion( output.reshape(-1, output.size(-1)), tgt[:, 1:].reshape(-1) # 目标去掉第一个token ) loss.backward() optimizer.step()文本生成（Greedy Decoding）：
@torch.no_grad()def generate(model, src, max_len=50): model.eval()
src_mask, _ = model.generate_mask(src, src[:, :1]) encoder_output = model.encode(src, src_mask)
ys = torch.ones(1, 1).fill_(start_token).long()
for i in range(max_len): _, tgt_mask = model.generate_mask(src, ys) decoder_output = model.decode(ys, encoder_output, src_mask, tgt_mask)
prob = model.fc_out(decoder_output[:, -1]) next_word = prob.argmax(dim=-1).unsqueeze(0)
ys = torch.cat([ys, next_word], dim=1)
if next_word.item() == end_token: break
return ys五、总结
Transformer的关键创新：
- 自注意力机制：O(n²)复杂度换取全局视野
- 位置编码：巧妙注入顺序信息
- 残差连接 + LayerNorm：稳定深层网络训练
- 并行化：GPU友好，训练效率高
局限性：
- 序列长度的平方复杂度
- 对位置编码的依赖
- 缺乏归纳偏置（如CNN的平移不变性）
后续改进方向：
- Sparse Attention（Longformer）
- Linear Attention（Performer）
- 相对位置编码（T5, DeBERTa）
Transformer彻底改变了NLP，并逐渐渗透到CV、多模态、强化学习等领域，堪称深度学习的”万能架构”！🚀
🧠 文章思维导图
mindmap 一、核心架构# 二、完整Transformer模型# 三、训练技巧# Label Smoothing# Warmup Learning Rate# 四、应用场景# 五、总结#
支持与分享
如果这篇文章对你有帮助，欢迎分享给更多人或打赏支持！
Transformer架构详解
https://meiluosi.github.io/posts/2024-09-01-transformer架构详解/引用本文
@article{2024-09-01-transformer架构详解,
author = {枫语},
title = {Transformer架构详解},
journal = {枫语},
year = {2024},
month = {sep},
url = {https://meiluosi.github.io/posts/2024-09-01-transformer%E6%9E%B6%E6%9E%84%E8%AF%A6%E8%A7%A3/}
}相关文章智能推荐
1
多模态大模型技术
Deep Learning从CLIP到GPT-4V的视觉语言理解
2
LLM训练技术详解
Deep Learning预训练、SFT与RLHF全流程
3
1B模型全链路实验计划：MacBook Air M4 本地实战
Deep Learning从SFT到DPO再到部署，用1B参数大模型在Apple Silicon上跑通LLM训练全流程
4
LLM推理优化与部署
Deep Learning量化、加速与生产环境部署
5
高效微调技术：LoRA与PEFT
Deep Learning参数高效微调方法实战
随机文章随机推荐