أقنعة الانتباه وتخطيط الدفعة

import torch
from torch import nn
from torch.nn import functional as F
attention = nn.MultiheadAttention(2,1,dropout=0.0,bias=False,batch_first=True,dtype=torch.float64,device="cpu")
with torch.no_grad():
    attention.in_proj_weight.copy_(torch.cat([torch.eye(2,dtype=torch.float64,device="cpu")]*3))
    attention.out_proj.weight.copy_(torch.eye(2,dtype=torch.float64,device="cpu"))
x = torch.tensor([[[1.0,0.0],[0.0,1.0],[1.0,1.0]]],dtype=torch.float64,device="cpu")
blocked = torch.ones(3,3,dtype=torch.bool,device="cpu").triu(diagonal=1)
output, weights = attention(x,x,x,attn_mask=blocked,need_weights=True,average_attn_weights=False)
q = x.unsqueeze(1)
direct = F.scaled_dot_product_attention(q,q,q,attn_mask=~blocked,dropout_p=0.0)
print("weight shape", tuple(weights.shape))
print("first output", output[0,0].tolist())
print("future weights", weights[0,0,0,1:].tolist())
print("same after mask conversion", torch.allclose(output,direct.squeeze(1),atol=1e-12,rtol=1e-12))

weight shape (1, 1, 3, 3)
first output [1.0, 0.0]
future weights [0.0, 0.0]
same after mask conversion True

تجعل إسقاطات الهوية الوحدة والانتباه المباشر قابلين للمقارنة رياضياً؛ وللقناعين المنطقيين معنيان متعاكسان. لم تنفذ نواة انتباه أو إسقاط أو مقارنة allclose في torch.

from math import exp, sqrt
rows = [[1.0,0.0],[0.0,1.0],[1.0,1.0]]
blocked = [[j>i for j in range(3)] for i in range(3)]
allowed = [[not value for value in row] for row in blocked]
keys = [[j for j,ok in enumerate(row) if ok] for row in allowed]
scores = [sum(a*b for a,b in zip(rows[0],key))/sqrt(2) for key in rows]
maximum = max(score for score,ok in zip(scores,allowed[0]) if ok)
raw = [exp(score-maximum) if ok else 0.0 for score,ok in zip(scores,allowed[0])]
weights = [value/sum(raw) for value in raw]
context = [sum(weight*row[j] for weight,row in zip(weights,rows)) for j in range(2)]
print("allowed key indices", keys)
print("first context", context)

allowed key indices [[0], [0, 1], [0, 1, 2]]
first context [1.0, 0.0]

يفحص المرافق المنفذ مجموعة السماح السببية وسياقاً موزوناً واحداً؛ لا بث أقنعة PyTorch أو تكافؤ التنفيذ.