在计算机视觉任务中处理高分辨率图像时常常面临计算复杂度爆炸的挑战。传统注意力机制虽然能够捕捉全局依赖关系但其二次方的计算复杂度限制了在长序列任务中的应用。近期CCF-A类期刊TIFS 2026年发表的重构滑动窗口注意力RSWAtt通过创新的窗口重构机制实现了像素级依赖建模为CV任务提供了即插即用的高效解决方案。本文将完整解析RSWAtt的核心原理、实现细节和实战应用包含从基础概念到代码实现的完整流程。无论你是刚入门Transformer架构的新手还是希望优化现有视觉模型的开发者都能从中获得可直接复用的技术方案。1. 注意力机制基础与演进1.1 传统自注意力机制的局限性自注意力机制是Transformer架构的核心组件通过计算序列中每个元素与其他所有元素的相关性权重实现全局上下文信息的捕捉。其标准计算公式为import torch import torch.nn as nn import torch.nn.functional as F class StandardSelfAttention(nn.Module): def __init__(self, d_model, n_heads): super().__init__() self.d_model d_model self.n_heads n_heads self.head_dim d_model // n_heads self.wq nn.Linear(d_model, d_model) self.wk nn.Linear(d_model, d_model) self.wv nn.Linear(d_model, d_model) self.wo nn.Linear(d_model, d_model) def forward(self, x): # x: (batch_size, seq_len, d_model) batch_size, seq_len, _ x.shape Q self.wq(x) # (batch_size, seq_len, d_model) K self.wk(x) # (batch_size, seq_len, d_model) V self.wv(x) # (batch_size, seq_len, d_model) # 计算注意力分数 scores torch.matmul(Q, K.transpose(-2, -1)) / (self.head_dim ** 0.5) # scores: (batch_size, n_heads, seq_len, seq_len) attn_weights F.softmax(scores, dim-1) output torch.matmul(attn_weights, V) return self.wo(output)这种全局注意力机制的计算复杂度为O(n²)当处理高分辨率图像时如1024×1024的图像会产生超过100万个像素点内存消耗和计算时间会呈指数级增长严重限制了实际应用。1.2 滑动窗口注意力的演进滑动窗口注意力通过限制每个token只关注局部邻域内的其他token将计算复杂度从O(n²)降低到O(n×k)其中k为窗口大小。这种局部注意力机制特别适合具有空间局部相关性的视觉任务。class SlidingWindowAttention(nn.Module): def __init__(self, d_model, n_heads, window_size): super().__init__() self.window_size window_size self.d_model d_model self.n_heads n_heads self.head_dim d_model // n_heads self.wq nn.Linear(d_model, d_model) self.wk nn.Linear(d_model, d_model) self.wv nn.Linear(d_model, d_model) self.wo nn.Linear(d_model, d_model) def create_window_mask(self, seq_len, window_size): 创建滑动窗口掩码 mask torch.zeros(seq_len, seq_len) for i in range(seq_len): start max(0, i - window_size // 2) end min(seq_len, i window_size // 2 1) mask[i, start:end] 1 return mask def forward(self, x): batch_size, seq_len, _ x.shape Q self.wq(x) K self.wk(x) V self.wv(x) # 计算注意力分数 scores torch.matmul(Q, K.transpose(-2, -1)) / (self.head_dim ** 0.5) # 应用窗口掩码 window_mask self.create_window_mask(seq_len, self.window_size) scores scores.masked_fill(window_mask 0, float(-inf)) attn_weights F.softmax(scores, dim-1) output torch.matmul(attn_weights, V) return self.wo(output)1.3 RSWAtt的创新之处RSWAtt在传统滑动窗口注意力的基础上引入了重构机制通过动态调整窗口形状和大小更好地适应图像中的不规则结构和长距离依赖关系。其主要创新点包括自适应窗口重构根据输入特征动态调整窗口形状多尺度特征融合在不同尺度上捕捉依赖关系像素级精度实现真正的像素级依赖建模即插即用设计可无缝集成到现有视觉架构中2. RSWAtt核心原理详解2.1 窗口重构机制RSWAtt的核心创新在于其窗口重构机制。传统滑动窗口使用固定大小的矩形窗口而RSWAtt通过可学习的参数动态调整窗口形状class WindowReconstructor(nn.Module): def __init__(self, d_model, max_window_size16): super().__init__() self.max_window_size max_window_size self.window_predictor nn.Sequential( nn.Linear(d_model, d_model // 2), nn.ReLU(), nn.Linear(d_model // 2, 4) # 预测窗口的四个边界偏移 ) def forward(self, x, reference_points): x: 输入特征 (batch_size, seq_len, d_model) reference_points: 参考点坐标 (batch_size, seq_len, 2) batch_size, seq_len, _ x.shape # 预测每个位置的窗口边界偏移 offsets self.window_predictor(x) # (batch_size, seq_len, 4) # 将偏移量转换为实际的窗口边界 window_bounds self.offsets_to_bounds(offsets, reference_points) return window_bounds def offsets_to_bounds(self, offsets, reference_points): # 将归一化的偏移量转换为实际的坐标边界 left reference_points[..., 0] offsets[..., 0] * self.max_window_size right reference_points[..., 0] offsets[..., 1] * self.max_window_size top reference_points[..., 1] offsets[..., 2] * self.max_window_size bottom reference_points[..., 1] offsets[..., 3] * self.max_window_size return torch.stack([left, right, top, bottom], dim-1)2.2 多尺度注意力融合RSWAtt通过多尺度注意力机制捕捉不同范围的依赖关系class MultiScaleRSWAtt(nn.Module): def __init__(self, d_model, n_heads, window_sizes[4, 8, 16]): super().__init__() self.window_sizes window_sizes self.attentions nn.ModuleList([ RSWAttLayer(d_model, n_heads, window_size) for window_size in window_sizes ]) self.fusion_weights nn.Parameter(torch.ones(len(window_sizes))) self.fusion_layer nn.Linear(d_model * len(window_sizes), d_model) def forward(self, x, reference_points): outputs [] for i, attn in enumerate(self.attentions): output attn(x, reference_points) outputs.append(output) # 加权融合多尺度注意力结果 weighted_outputs [] for i, output in enumerate(outputs): weight torch.sigmoid(self.fusion_weights[i]) weighted_outputs.append(output * weight) fused_output torch.cat(weighted_outputs, dim-1) return self.fusion_layer(fused_output)2.3 像素级依赖建模RSWAtt通过精细化的位置编码和注意力计算实现真正的像素级依赖建模class PixelLevelRSWAtt(nn.Module): def __init__(self, d_model, n_heads, image_size): super().__init__() self.d_model d_model self.n_heads n_heads self.image_size image_size # 像素级位置编码 self.pos_encoding self.create_pixel_pos_encoding(image_size, d_model) self.rswatt RSWAttLayer(d_model, n_heads, adaptiveTrue) def create_pixel_pos_encoding(self, image_size, d_model): 创建像素级位置编码 height, width image_size pos_encoding torch.zeros(1, height * width, d_model) position torch.arange(0, height * width).unsqueeze(1) div_term torch.exp(torch.arange(0, d_model, 2) * -(math.log(10000.0) / d_model)) pos_encoding[0, :, 0::2] torch.sin(position * div_term) pos_encoding[0, :, 1::2] torch.cos(position * div_term) return nn.Parameter(pos_encoding) def forward(self, x): # x: (batch_size, channels, height, width) batch_size, channels, height, width x.shape # 展平为序列格式 x_flat x.flatten(2).transpose(1, 2) # (batch_size, height*width, channels) # 添加位置编码 x_flat x_flat self.pos_encoding[:, :height*width, :] # 生成参考点坐标 reference_points self.generate_reference_points(height, width, batch_size) # 应用RSWAtt output self.rswatt(x_flat, reference_points) # 恢复空间维度 output output.transpose(1, 2).view(batch_size, channels, height, width) return output def generate_reference_points(self, height, width, batch_size): 生成每个像素的参考点坐标 y_coords torch.arange(0, height).float() / height x_coords torch.arange(0, width).float() / width yy, xx torch.meshgrid(y_coords, x_coords, indexingij) reference_points torch.stack([xx.flatten(), yy.flatten()], dim-1) reference_points reference_points.unsqueeze(0).repeat(batch_size, 1, 1) return reference_points3. 环境准备与依赖配置3.1 基础环境要求实现RSWAtt需要以下环境配置# 创建conda环境 conda create -n rswatt python3.9 conda activate rswatt # 安装核心依赖 pip install torch2.0.1 torchvision0.15.2 pip install numpy1.24.3 matplotlib3.7.1 pip install opencv-python4.8.0.74 Pillow9.5.03.2 项目结构规划rswatt-implementation/ ├── src/ │ ├── __init__.py │ ├── attention/ │ │ ├── __init__.py │ │ ├── rswatt.py # RSWAtt核心实现 │ │ └── multi_scale.py # 多尺度注意力 │ ├── models/ │ │ ├── __init__.py │ │ └── vision_transformer.py # ViT集成RSWAtt │ └── utils/ │ ├── __init__.py │ └── visualization.py # 注意力可视化工具 ├── configs/ │ └── default.yaml # 配置文件 ├── tests/ │ └── test_attention.py # 单元测试 └── requirements.txt3.3 核心依赖版本说明# requirements.txt torch2.0.0 torchvision0.15.0 numpy1.24.0 opencv-python4.7.0 Pillow9.0.0 tqdm4.65.0 tensorboard2.13.0 # 检查环境兼容性 import torch print(fPyTorch版本: {torch.__version__}) print(fCUDA可用: {torch.cuda.is_available()}) print(fCUDA版本: {torch.version.cuda}) import sys print(fPython版本: {sys.version})4. RSWAtt完整实现4.1 基础RSWAtt层实现import math import torch import torch.nn as nn import torch.nn.functional as F class RSWAttLayer(nn.Module): def __init__(self, d_model, n_heads, base_window_size8, adaptiveTrue): super().__init__() self.d_model d_model self.n_heads n_heads self.head_dim d_model // n_heads self.base_window_size base_window_size self.adaptive adaptive # 线性变换层 self.wq nn.Linear(d_model, d_model) self.wk nn.Linear(d_model, d_model) self.wv nn.Linear(d_model, d_model) self.wo nn.Linear(d_model, d_model) # 窗口重构器 if adaptive: self.window_reconstructor WindowReconstructor(d_model) # 层归一化 self.norm1 nn.LayerNorm(d_model) self.norm2 nn.LayerNorm(d_model) # 前馈网络 self.ffn nn.Sequential( nn.Linear(d_model, d_model * 4), nn.GELU(), nn.Linear(d_model * 4, d_model) ) def forward(self, x, reference_pointsNone): # 残差连接 residual x # 层归一化 x self.norm1(x) # 生成注意力掩码 if self.adaptive and reference_points is not None: attention_mask self.generate_adaptive_mask(x, reference_points) else: attention_mask self.generate_fixed_mask(x.shape[1]) # 多头注意力计算 attn_output self.multi_head_attention(x, attention_mask) # 残差连接和层归一化 x residual attn_output residual x x self.norm2(x) # 前馈网络 ff_output self.ffn(x) x residual ff_output return x def generate_adaptive_mask(self, x, reference_points): 生成自适应窗口掩码 batch_size, seq_len, _ x.shape if self.adaptive: window_bounds self.window_reconstructor(x, reference_points) mask self.bounds_to_mask(window_bounds, seq_len) else: mask self.generate_fixed_mask(seq_len) return mask def bounds_to_mask(self, window_bounds, seq_len): 将窗口边界转换为注意力掩码 batch_size window_bounds.shape[0] mask torch.zeros(batch_size, seq_len, seq_len, devicewindow_bounds.device) for b in range(batch_size): for i in range(seq_len): left, right, top, bottom window_bounds[b, i] # 简化实现实际应根据2D坐标计算 start max(0, int(left * seq_len)) end min(seq_len, int(right * seq_len)) mask[b, i, start:end] 1 return mask def generate_fixed_mask(self, seq_len): 生成固定窗口掩码 mask torch.zeros(seq_len, seq_len) for i in range(seq_len): start max(0, i - self.base_window_size // 2) end min(seq_len, i self.base_window_size // 2 1) mask[i, start:end] 1 return mask.unsqueeze(0) # 添加batch维度 def multi_head_attention(self, x, attention_mask): 多头注意力计算 batch_size, seq_len, _ x.shape Q self.wq(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) K self.wk(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) V self.wv(x).view(batch_size, seq_len, self.n_heads, self.head_dim).transpose(1, 2) # 计算注意力分数 scores torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # 应用注意力掩码 scores scores.masked_fill(attention_mask.unsqueeze(1) 0, float(-inf)) # Softmax归一化 attn_weights F.softmax(scores, dim-1) # 注意力加权求和 attn_output torch.matmul(attn_weights, V) attn_output attn_output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model) return self.wo(attn_output)4.2 增强型窗口重构器class EnhancedWindowReconstructor(nn.Module): def __init__(self, d_model, max_window_size32, num_anchors8): super().__init__() self.max_window_size max_window_size self.num_anchors num_anchors # 锚点预测网络 self.anchor_predictor nn.Sequential( nn.Linear(d_model, d_model // 2), nn.GELU(), nn.Linear(d_model // 2, num_anchors * 4) # 每个锚点预测4个参数 ) # 注意力权重预测 self.attention_predictor nn.Sequential( nn.Linear(d_model, d_model // 2), nn.GELU(), nn.Linear(d_model // 2, num_anchors) ) def forward(self, x, reference_points): batch_size, seq_len, _ x.shape # 预测锚点参数 anchor_params self.anchor_predictor(x) # (batch_size, seq_len, num_anchors*4) anchor_params anchor_params.view(batch_size, seq_len, self.num_anchors, 4) # 预测注意力权重 attention_weights self.attention_predictor(x) # (batch_size, seq_len, num_anchors) attention_weights F.softmax(attention_weights, dim-1) # 生成最终窗口边界 window_bounds self.fuse_anchors(anchor_params, attention_weights, reference_points) return window_bounds def fuse_anchors(self, anchor_params, attention_weights, reference_points): 融合多个锚点生成最终窗口 batch_size, seq_len, num_anchors, _ anchor_params.shape # 将参数转换为边界偏移 bounds_offsets anchor_params * self.max_window_size # 加权融合 weighted_bounds torch.einsum(bsna,bsa-bsn, bounds_offsets, attention_weights) weighted_bounds weighted_bounds.view(batch_size, seq_len, 4) # 应用到参考点 window_bounds torch.zeros_like(weighted_bounds) window_bounds[..., 0] reference_points[..., 0] weighted_bounds[..., 0] # left window_bounds[..., 1] reference_points[..., 0] weighted_bounds[..., 1] # right window_bounds[..., 2] reference_points[..., 1] weighted_bounds[..., 2] # top window_bounds[..., 3] reference_points[..., 1] weighted_bounds[..., 3] # bottom return window_bounds4.3 完整RSWAtt模块集成class CompleteRSWAtt(nn.Module): def __init__(self, d_model, n_heads, num_layers6, adaptiveTrue): super().__init__() self.layers nn.ModuleList([ RSWAttLayer(d_model, n_heads, adaptiveadaptive) for _ in range(num_layers) ]) self.norm nn.LayerNorm(d_model) def forward(self, x, reference_pointsNone): for layer in self.layers: x layer(x, reference_points) return self.norm(x) class RSWAttViT(nn.Module): 集成RSWAtt的Vision Transformer def __init__(self, image_size224, patch_size16, num_classes1000, d_model768, n_heads12, num_layers12, adaptiveTrue): super().__init__() self.image_size image_size self.patch_size patch_size self.num_patches (image_size // patch_size) ** 2 # Patch嵌入层 self.patch_embed nn.Conv2d(3, d_model, kernel_sizepatch_size, stridepatch_size) # 位置编码 self.pos_embed nn.Parameter(torch.randn(1, self.num_patches 1, d_model) * 0.02) # 类别token self.cls_token nn.Parameter(torch.zeros(1, 1, d_model)) # RSWAtt编码器 self.encoder CompleteRSWAtt(d_model, n_heads, num_layers, adaptive) # 分类头 self.head nn.Linear(d_model, num_classes) self.initialize_weights() def initialize_weights(self): 权重初始化 # 省略初始化细节... pass def forward(self, x): batch_size x.shape[0] # 提取patch嵌入 x self.patch_embed(x) # (batch_size, d_model, H, W) x x.flatten(2).transpose(1, 2) # (batch_size, num_patches, d_model) # 添加类别token cls_tokens self.cls_token.expand(batch_size, -1, -1) x torch.cat((cls_tokens, x), dim1) # 添加位置编码 x x self.pos_embed # 生成参考点简化实现 reference_points self.generate_reference_points(batch_size) # 通过RSWAtt编码器 x self.encoder(x, reference_points) # 使用类别token进行分类 x x[:, 0] # 取类别token x self.head(x) return x def generate_reference_points(self, batch_size): 生成patch级别的参考点 grid_size self.image_size // self.patch_size y_coords torch.arange(0, grid_size).float() / grid_size x_coords torch.arange(0, grid_size).float() / grid_size yy, xx torch.meshgrid(y_coords, x_coords, indexingij) reference_points torch.stack([xx.flatten(), yy.flatten()], dim-1) # 添加类别token的参考点居中位置 cls_point torch.tensor([[0.5, 0.5]]) # 图像中心 reference_points torch.cat([cls_point, reference_points], dim0) reference_points reference_points.unsqueeze(0).repeat(batch_size, 1, 1) return reference_points5. 实战应用与性能测试5.1 图像分类任务集成def train_rswatt_vit(): 训练RSWAtt-ViT模型 import torch.optim as optim from torch.utils.data import DataLoader from torchvision import datasets, transforms # 数据预处理 transform transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean[0.485, 0.456, 0.406], std[0.229, 0.224, 0.225]) ]) # 加载数据集示例使用CIFAR-10 train_dataset datasets.CIFAR10(root./data, trainTrue, downloadTrue, transformtransform) train_loader DataLoader(train_dataset, batch_size32, shuffleTrue) # 初始化模型 model RSWAttViT(image_size224, patch_size16, num_classes10, d_model768, n_heads12, num_layers6) # 优化器 optimizer optim.AdamW(model.parameters(), lr1e-4, weight_decay0.01) criterion nn.CrossEntropyLoss() # 训练循环 model.train() for epoch in range(10): total_loss 0 for batch_idx, (data, target) in enumerate(train_loader): optimizer.zero_grad() output model(data) loss criterion(output, target) loss.backward() optimizer.step() total_loss loss.item() if batch_idx % 100 0: print(fEpoch: {epoch}, Batch: {batch_idx}, Loss: {loss.item():.4f}) print(fEpoch {epoch} completed. Average Loss: {total_loss/len(train_loader):.4f}) def benchmark_performance(): 性能基准测试 import time from thop import profile model RSWAttViT() input_tensor torch.randn(1, 3, 224, 224) # 计算FLOPs和参数数量 flops, params profile(model, inputs(input_tensor,)) print(f模型参数: {params/1e6:.2f}M) print(f计算量: {flops/1e9:.2f}G FLOPs) # 推理速度测试 model.eval() start_time time.time() with torch.no_grad(): for _ in range(100): _ model(input_tensor) end_time time.time() avg_inference_time (end_time - start_time) / 100 * 1000 # 毫秒 print(f平均推理时间: {avg_inference_time:.2f}ms) # 运行性能测试 if __name__ __main__: benchmark_performance()5.2 目标检测任务适配class RSWAttDetectionHead(nn.Module): 基于RSWAtt的目标检测头 def __init__(self, in_channels, num_classes, num_anchors9): super().__init__() self.num_classes num_classes self.num_anchors num_anchors # RSWAtt特征增强 self.rswatt RSWAttLayer(in_channels, n_heads8) # 检测头 self.cls_head nn.Conv2d(in_channels, num_anchors * num_classes, 3, padding1) self.reg_head nn.Conv2d(in_channels, num_anchors * 4, 3, padding1) def forward(self, x): # x: (batch_size, channels, height, width) batch_size, channels, height, width x.shape # 转换为序列格式 x_seq x.flatten(2).transpose(1, 2) # (batch_size, height*width, channels) # 生成参考点 reference_points self.generate_reference_points(height, width, batch_size) # 应用RSWAtt enhanced_seq self.rswatt(x_seq, reference_points) # 恢复空间维度 enhanced_x enhanced_seq.transpose(1, 2).view(batch_size, channels, height, width) # 分类和回归预测 cls_output self.cls_head(enhanced_x) reg_output self.reg_head(enhanced_x) return cls_output, reg_output def generate_reference_points(self, height, width, batch_size): 生成特征图每个位置的参考点 y_coords torch.arange(0, height).float() / height x_coords torch.arange(0, width).float() / width yy, xx torch.meshgrid(y_coords, x_coords, indexingij) reference_points torch.stack([xx.flatten(), yy.flatten()], dim-1) reference_points reference_points.unsqueeze(0).repeat(batch_size, 1, 1) return reference_points5.3 语义分割应用class RSWAttSegmentation(nn.Module): 基于RSWAtt的语义分割模型 def __init__(self, backbone, num_classes, d_model256): super().__init__() self.backbone backbone self.num_classes num_classes # RSWAtt解码器 self.rswatt_decoder CompleteRSWAtt(d_model, n_heads8, num_layers4) # 分割头 self.segmentation_head nn.Sequential( nn.Conv2d(d_model, d_model // 2, 3, padding1), nn.BatchNorm2d(d_model // 2), nn.ReLU(), nn.Conv2d(d_model // 2, num_classes, 1) ) def forward(self, x): # 骨干网络提取特征 features self.backbone(x) # 选择高分辨率特征图 high_res_feat features[-1] # 假设是最高分辨率的特征 batch_size, channels, height, width high_res_feat.shape # 转换为序列格式 feat_seq high_res_feat.flatten(2).transpose(1, 2) # 生成参考点 reference_points self.generate_reference_points(height, width, batch_size) # RSWAtt增强 enhanced_seq self.rswatt_decoder(feat_seq, reference_points) # 恢复空间维度 enhanced_feat enhanced_seq.transpose(1, 2).view(batch_size, channels, height, width) # 分割预测 seg_map self.segmentation_head(enhanced_feat) seg_map F.interpolate(seg_map, sizex.shape[2:], modebilinear, align_cornersFalse) return seg_map def generate_reference_points(self, height, width, batch_size): 生成参考点坐标 y_coords torch.arange(0, height).float() / height x_coords torch.arange(0, width).float() / width yy, xx torch.meshgrid(y_coords, x_coords, indexingij) reference_points torch.stack([xx.flatten(), yy.flatten()], dim-1) reference_points reference_points.unsqueeze(0).repeat(batch_size, 1, 1) return reference_points6. 性能优化与调参技巧6.1 计算效率优化class OptimizedRSWAtt(nn.Module): 优化版的RSWAtt实现 def __init__(self, d_model, n_heads, window_size8, use_flash_attentionFalse): super().__init__() self.d_model d_model self.n_heads n_heads self.window_size window_size self.use_flash_attention use_flash_attention # 线性变换 self.wqkv nn.Linear(d_model, 3 * d_model) # 合并QKV计算 self.wo nn.Linear(d_model, d_model) # 使用Flash Attention如果可用 if use_flash_attention and hasattr(F, scaled_dot_product_attention): self.attention_fn self.flash_attention else: self.attention_fn self.standard_attention def flash_attention(self, Q, K, V, attention_mask): 使用PyTorch的Flash Attention # 简化实现实际需要处理掩码 return F.scaled_dot_product_attention(Q, K, V, attn_maskattention_mask) def standard_attention(self, Q, K, V, attention_mask): 标准注意力实现 scores torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(Q.size(-1)) scores scores.masked_fill(attention_mask 0, float(-inf)) attn_weights F.softmax(scores, dim-1) return torch.matmul(attn_weights, V) def forward(self, x): batch_size, seq_len, _ x.shape # 合并QKV计算 qkv self.wqkv(x).reshape(batch_size, seq_len, 3, self.n_heads, self.d_model // self.n_heads) qkv qkv.permute(2, 0, 3, 1, 4) # (3, batch_size, n_heads, seq_len, head_dim) Q, K, V qkv[0], qkv[1], qkv[2] # 生成注意力掩码 attention_mask self.generate_window_mask(seq_len, batch_size) # 注意力计算 attn_output self.attention_fn(Q, K, V, attention_mask) # 输出变换 attn_output attn_output.transpose(1, 2).reshape(batch_size, seq_len, self.d_model) return self.wo(attn_output) def generate_window_mask(self, seq_len, batch_size): 生成优化的窗口掩码 mask torch.ones(batch_size, self.n_heads, seq_len, seq_len) # 应用滑动窗口限制 for i in range(seq_len): start max(0, i - self.window_size // 2) end min(seq_len, i self.window_size // 2 1) mask[:, :, i, :start]