交易中的神经网络：优化时间序列预测变换器（LSEAttention）·进阶篇
（2/3）· 当自注意力矩阵在不同行情里收敛成同一张脸，长期预测还能信几分？
「GPU 核里归约出注意力权重」
这段 OpenCL 内核在做多头相对注意力里的 softmax 前归约：先并行求输入张量的最大值，再把每个元素减去最大值并截断到 -120，避免 exp 溢出成 inf。 临时数组 temp 用 local memory 做树形求和，count 从 ls 开始每次 (count+1)/2 折半，CLK_LOCAL_MEM_FENCE 保证工作组内写读可见；最终 sum 是归一化分母，输出写成 inp_exp/(sum+1.2e-7f)，那个 1.2e-7f 就是防零除的 epsilon。 内核签名里 q/k/v 是查询键值的全局矩阵，bk/bv 是位置偏置，gc/gp 分别是全局内容偏置与位置偏置向量，score 为输出分数矩阵；在 MT5 里用 CLExecute 跑这段，重点看 local size(ls) 设多大，ls 直接决定归约轮数和显存冲突概率。 外汇与贵金属行情跳变频繁，这类 GPU 指标在高波动段可能给出滞后权重，上实盘前务必用历史 tick 回测验证。
temp[l] = (l < count && (l + count) < total ? fmax(temp[l + count],temp[l]) : temp[l]); barrier(CLK_LOCAL_MEM_FENCE); } class="kw">while(count > class="num">1); class="type">float max_value = temp[class="num">0]; barrier(CLK_LOCAL_MEM_FENCE); inp = fmax(inp - max_value, -class="num">120); class="type">float inp_exp = exp(inp); if(isinf(inp_exp) || isnan(inp_exp)) inp_exp = class="num">0; for(class="type">int i = class="num">0; i < total; i += ls) { if(l >= i && l < (i + ls)) temp[l] = (i > class="num">0 ? temp[l] : class="num">0) + inp_exp; barrier(CLK_LOCAL_MEM_FENCE); } class=class="str">"cmt">//--- count = min(ls, (class="type">uint)total); do { count = (count + class="num">1) / class="num">2; if(l < ls) temp[l] += (l < count && (l + count) < total ? temp[l + count] : class="num">0); if(l + count < ls) temp[l + count] = class="num">0; barrier(CLK_LOCAL_MEM_FENCE); } class="kw">while(count > class="num">1); class=class="str">"cmt">//--- class="type">float sum = temp[class="num">0]; outputs[shift_head+l] = inp_exp / (sum + class="num">1.2e-7f); } __kernel class="type">void MHRelativeAttentionOut(__global class="kw">const class="type">float *q, class=class="str">"cmt">///<-[in] Matrix of Querys __global class="kw">const class="type">float *k, class=class="str">"cmt">///<-[in] Matrix of Keys __global class="kw">const class="type">float *v, class=class="str">"cmt">///<-[in] Matrix of Values __global class="kw">const class="type">float *bk, class=class="str">"cmt">///<-[in] Matrix of Positional Bias Keys __global class="kw">const class="type">float *bv, class=class="str">"cmt">///<-[in] Matrix of Positional Bias Values __global class="kw">const class="type">float *gc, class=class="str">"cmt">///<-[in] Global content bias vector __global class="kw">const class="type">float *gp, class=class="str">"cmt">///<-[in] Global positional bias vector __global class="type">float *score) class=class="str">"cmt">///<-[out] Matrix of Scores
多头注意力里的分数归约与溢出防护
在 MT5 的 OpenCL 内核里做多头注意力，最容易被忽视的是跨 key 维度归约时的数值稳定。上面这段内核先用 get_global_id / get_local_id 把 query、key、bias 的偏移算清楚，dimension 决定单头向量长度，heads 是头数，shift_q 和 shift_kv 分别把 Q、K 铺到全局/局部内存的正确位置。 sc 的累加不是单纯 Q·K，而是把 bias key（bk）、gate（gc、gp）都揉进一个式子：val_q*val_k + val_q*val_bk + val_k*val_bk + gc*val_k + gp*val_bk，最后除以 sqrt(dimension)。若 dimension=64，koef 就是 8.0，这一步缩放直接决定 softmax 前数值会不会爆。 找最大值的归约用 __local float temp[LOCAL_ARRAY_SIZE] 做分段 fmax，ls 取 min(local_size, LOCAL_ARRAY_SIZE)，避免 AMD 卡上本地数组越界。归约完 sc = exp(fmax(sc - temp[0], -120))，那个 -120 是防 exp 上溢的硬底线，exp(>88) 在 float 就 inf 了，所以先减最大值再卡 -120。 isnan / isinf 兜底把异常 sc 置 0，否则一个 NaN 会顺着 barrier 污染整个 warp。开 MT5 把 LOCAL_ARRAY_SIZE 调到 256 跑 EURUSD 的 M15 注意力特征，可能比 128 快 15%~20%，但得先确认显卡 local memory 够用。
__global class="type">float *out, class=class="str">"cmt">///<[out] Matrix of attention class="kw">const class="type">int dimension class=class="str">"cmt">///< Dimension of Key ) { class=class="str">"cmt">//--- init class="kw">const class="type">int q_id = get_global_id(class="num">0); class="kw">const class="type">int k_id = get_local_id(class="num">1); class="kw">const class="type">int h = get_global_id(class="num">2); class="kw">const class="type">int qunits = get_global_size(class="num">0); class="kw">const class="type">int kunits = get_local_size(class="num">1); class="kw">const class="type">int heads = get_global_size(class="num">2); class="kw">const class="type">int shift_q = dimension * (q_id * heads + h); class="kw">const class="type">int shift_kv = dimension * (heads * k_id + h); class="kw">const class="type">int shift_gc = dimension * h; class="kw">const class="type">int shift_s = kunits * (q_id * heads + h) + k_id; class="kw">const class="type">int shift_pb = q_id * kunits + k_id; class="kw">const class="type">uint ls = min((class="type">uint)get_local_size(class="num">1), (class="type">uint)LOCAL_ARRAY_SIZE); class="type">float koef = sqrt((class="type">float)dimension); __local class="type">float temp[LOCAL_ARRAY_SIZE]; class=class="str">"cmt">//--- score class="type">float sc = class="num">0; for(class="type">int d = class="num">0; d < dimension; d++) { class="type">float val_q = q[shift_q + d]; class="type">float val_k = k[shift_kv + d]; class="type">float val_bk = bk[shift_kv + d]; sc += val_q * val_k + val_q * val_bk + val_k * val_bk + gc[shift_q + d] * val_k + gp[shift_q + d] * val_bk; } sc = sc / koef; class=class="str">"cmt">//--- max value for(class="type">int cur_k = class="num">0; cur_k < kunits; cur_k += ls) { if(k_id >= cur_k && k_id < (cur_k + ls)) { class="type">int shift_local = k_id % ls; temp[shift_local] = (cur_k == class="num">0 ? sc : fmax(temp[shift_local], sc)); } barrier(CLK_LOCAL_MEM_FENCE); } class="type">uint count = min(ls, (class="type">uint)kunits); class=class="str">"cmt">//--- do { count = (count + class="num">1) / class="num">2; if(k_id < ls) temp[k_id] = (k_id < count && (k_id + count) < kunits ? fmax(temp[k_id + count], temp[k_id]) : temp[k_id]); barrier(CLK_LOCAL_MEM_FENCE); } class="kw">while(count > class="num">1); sc = exp(fmax(sc - temp[class="num">0], -class="num">120)); if(isnan(sc) || isinf(sc)) sc = class="num">0; barrier(CLK_LOCAL_MEM_FENCE); class=class="str">"cmt">//--- sum of exp for(class="type">int cur_k = class="num">0; cur_k < kunits; cur_k += ls) { if(k_id >= cur_k && k_id < (cur_k + ls))
◍ 核函数里的归约与激活落地
这段 OpenCL 核代码干了两件硬事：先对局部内存 temp 做并行归约算出 score 权重，再把加权后的价值向量求和写回 out 缓冲区。归约用的经典折半法，count 从 ls 与 kunits 的较小值起，每轮 (count+1)/2 向下折，直到 count 不大于 1，本地线程同步靠 barrier(CLK_LOCAL_MEM_FENCE) 卡住。 权重归一化有个防呆判断：sum 若 NaN、Inf 或小于 1.2e-7f 直接置 1，随后 sc /= sum。这意味着当累计得分趋近于零时，单点得分不会被放大成异常值，外汇与贵金属行情里跳空造成的脏数据能被压住，但高频噪声下仍可能误导，属高风险场景。 输出阶段对每个 dimension 跑一遍：val = sc * (val_v + val_bv)，同样做归约求和，仅 k_id==0 的线程负责写 out[shift_q + d]。若 temp[0] 是 NaN/Inf 则写 0，避免显存脏写。 Activation 函数开头先拦 NaN/Inf 返回 0，再按 function 分支；case 0 用 tanh(clamp(value, -20.0f, 20.0f)) 把输入截断到 ±20 再压到 (-1,1)。在 MT5 里把这段贴进自定义指标核，调 clamp 上限能明显改变激活饱和度，建议用 EURUSD 的 M1 数据跑一遍看输出分布。
{
class="type">int shift_local = k_id % ls;
temp[shift_local] = (cur_k == class="num">0 ? class="num">0 : temp[shift_local]) + sc;
}
barrier(CLK_LOCAL_MEM_FENCE);
}
class=class="str">"cmt">//---
count = min(ls, (class="type">uint)kunits);
do
{
count = (count + class="num">1) / class="num">2;
if(k_id < ls)
temp[k_id] += (k_id < count && (k_id + count) < kunits ? temp[k_id + count] : class="num">0);
if(k_id + count < ls)
temp[k_id + count] = class="num">0;
barrier(CLK_LOCAL_MEM_FENCE);
}
class="kw">while(count > class="num">1);
class=class="str">"cmt">//--- score
class="type">float sum = temp[class="num">0];
if(isnan(sum) || isinf(sum) || sum <= class="num">1.2e-7f)
sum = class="num">1;
sc /= sum;
score[shift_s] = sc;
barrier(CLK_LOCAL_MEM_FENCE);
class=class="str">"cmt">//--- out
for(class="type">int d = class="num">0; d < dimension; d++)
{
class="type">float val_v = v[shift_kv + d];
class="type">float val_bv = bv[shift_kv + d];
class="type">float val = sc * (val_v + val_bv);
if(isnan(val) || isinf(val))
val = class="num">0;
class=class="str">"cmt">//--- sum of value
for(class="type">int cur_v = class="num">0; cur_v < kunits; cur_v += ls)
{
if(k_id >= cur_v && k_id < (cur_v + ls))
{
class="type">int shift_local = k_id % ls;
temp[shift_local] = (cur_v == class="num">0 ? class="num">0 : temp[shift_local]) + val;
}
barrier(CLK_LOCAL_MEM_FENCE);
}
class=class="str">"cmt">//---
count = min(ls, (class="type">uint)kunits);
do
{
count = (count + class="num">1) / class="num">2;
if(k_id < count && (k_id + count) < kunits)
temp[k_id] += temp[k_id + count];
if(k_id + count < ls)
temp[k_id + count] = class="num">0;
barrier(CLK_LOCAL_MEM_FENCE);
}
class="kw">while(count > class="num">1);
class=class="str">"cmt">//---
if(k_id == class="num">0)
out[shift_q + d] = (isnan(temp[class="num">0]) || isinf(temp[class="num">0]) ? class="num">0 : temp[class="num">0]);
barrier(CLK_LOCAL_MEM_FENCE);
}
}
class="type">float Activation(class="kw">const class="type">float value, class="kw">const class="type">int function)
{
if(isnan(value) || isinf(value))
class="kw">return class="num">0;
class=class="str">"cmt">//---
class="type">float result = value;
class="kw">switch(function)
{
case class="num">0:
result = tanh(clamp(value, -class="num">20.0f, class="num">20.0f));「反向传播里的激活与反激活配对」
把梯度回传写进 EA 之前，得先确认前向激活和反向反激活用的是同一套函数编号，否则权重更新方向可能偏掉。下面这段 Deactivation 接收梯度 grad 和前向输入 inp_value，按 function 分支把梯度映射到当前激活的导数尺度。 先判 isnan / isinf，任一为脏值直接把 result 置 0，避免 NaN 在神经网络里传染整张权表。TANH 分支里用 clamp(grad+inp_value,-1,1)-inp_value 做数值保护，再乘 (1 - inp_value^2)，和双曲正切导数公式一致。 Sigmoid 分支乘的是 inp_value*(1-inp_value)，LReLU 在 inp_value<0 时梯度乘 0.01f，和前向的负值缩放完全对称。SoftPlus 与 GELU 则借用了前向 Activation 来算导数，GELU 仅在 inp_value<0.9f 时套用 5*inp_value 的 Sigmoid 近似，超出后梯度倾向保持原值。 外汇与贵金属行情跳空频繁，这类数值保护不是可选项；在 MT5 里把 function 改成 4 跑一遍 EURUSD 的 M15 样本，观察 grad 是否出现溢出，比直接上实盘更稳妥。
break; case class="num">1: class=class="str">"cmt">//Sigmoid result = class="num">1 / (class="num">1 + exp(clamp(-value, -class="num">20.0f, class="num">20.0f))); break; case class="num">2: class=class="str">"cmt">//LReLU if(value < class="num">0) result *= class="num">0.01f; break; case class="num">3: class=class="str">"cmt">//SoftPlus result = (value >= class="num">20.0f ? class="num">1.0f : (value <= -class="num">20.0f ? class="num">0.0f : log(class="num">1 + exp(value)))); break; case class="num">4: class=class="str">"cmt">//GELU result = value / (class="num">1 + exp(clamp(-class="num">1.702f * value, -class="num">20.0f, class="num">20.0f))); break; class="kw">default: break; } class=class="str">"cmt">//--- class="kw">return result; } class="type">float Deactivation(class="kw">const class="type">float grad, class="kw">const class="type">float inp_value, class="kw">const class="type">int function) { class="type">float result = grad; class=class="str">"cmt">//--- if(isnan(inp_value) || isinf(inp_value) || isnan(grad) || isinf(grad)) result = class="num">0; else class="kw">switch(function) { case class="num">0: class=class="str">"cmt">//TANH result = clamp(grad + inp_value, -class="num">1.0f, class="num">1.0f) - inp_value; result *= class="num">1.0f - pow(inp_value, class="num">2.0f); break; case class="num">1: class=class="str">"cmt">//Sigmoid result = clamp(grad + inp_value, class="num">0.0f, class="num">1.0f) - inp_value; result *= inp_value * (class="num">1.0f - inp_value); break; case class="num">2: class=class="str">"cmt">//LReLU if(inp_value < class="num">0) result *= class="num">0.01f; break; case class="num">3: class=class="str">"cmt">//SoftPlus result *= Activation(inp_value, class="num">1); break; case class="num">4: class=class="str">"cmt">//GELU if(inp_value < class="num">0.9f) result *= Activation(class="num">5 * inp_value, class="num">1); break; class="kw">default: