交易中的神经网络：通过Adam-mini优化减少内存消耗·进阶篇
（2/3）· 标准Adam在大规模模型训练里吃掉等同模型本身的内存，这套块级方案怎么破局
「权重更新里的数值兜底与Adam-mini分支」
在 MT5 的 OpenCL 神经元实现里，梯度下降每一轮都要先给中间变量做 NaN/Inf 兜底。比如方差项 temp[0] 先除以 inputs，一旦算出非有限值就强制置 1，避免后续 pow(temp[1],2)*temp[0] 把整张权重矩阵带崩。 权重和动量也同理：matrix_w 与 matrix_m 读出后若不是有限数直接归 0，delta 计算里还加了 1.0e-37f 的极小量防止 sqrt(temp[0]) 下溢成 0 做分母。delta 为正才 clamp 进 [-MAX_WEIGHT, MAX_WEIGHT]，这是实盘训练里防溢出的最基础栅栏。 优化器枚举给了 SGD、ADAM、ADAM_MINI 三选一。ADAM_MINI 分支要把对端权重索引、本端梯度、对端输出、对端一阶动量四个 buffer 通过 SetArgumentBuffer 绑进内核 def_k_UpdateWeightsAdamMini；任一处绑定失败立即 return false，调用方拿不到更新结果。 开 MT5 把这段挂到自定义神经层类里，先把 b2 设 0.99、lr 设 0.001 跑 EURUSD 的 M15 样本，观察 matrix_v 是否还会出现 Inf 截断——若频繁触发，说明输入归一化没做透。
temp[class="num">0] /= inputs; if(isnan(temp[class="num">0]) || isinf(temp[class="num">0])) temp[class="num">0] = class="num">1; class="type">float v = matrix_v[o]; if(isnan(v) || isinf(v)) v = class="num">1; temp[class="num">0] = b2 * v + (class="num">1 - b2) * pow(temp[class="num">1], class="num">2) * temp[class="num">0]; matrix_v[o] = temp[class="num">0]; } barrier(CLK_LOCAL_MEM_FENCE); const class="type">int wi = o * (inputs + class="num">1) + i; class="type">float weight = matrix_w[wi]; if(isnan(weight) || isinf(weight)) weight = class="num">0; class="type">float m = matrix_m[wi]; if(isnan(m) || isinf(m)) m = class="num">0; class=class="str">"cmt">//--- calc m m = b1 * m + (class="num">1 - b1) * temp[class="num">1] * inp; if(isnan(m) || isinf(m)) m = class="num">0; class="type">float delta = l * (m / (sqrt(temp[class="num">0]) + class="num">1.0e-37f) - (l1 * sign(weight) + l2 * weight)); if(isnan(delta) || isinf(delta)) delta = class="num">0; if(delta > class="num">0) matrix_w[wi] = clamp(weight + delta, -MAX_WEIGHT, MAX_WEIGHT); matrix_m[wi] = m; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">/// Enum of optimization method used | class=class="str">"cmt">//+------------------------------------------------------------------+ enum ENUM_OPTIMIZATION { SGD, class=class="str">"cmt">///< Stochastic gradient descent ADAM, class=class="str">"cmt">///< Adam ADAM_MINI class=class="str">"cmt">///< Adam-mini }; class="type">bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) class="kw">return false; class="type">uint global_work_offset[class="num">2] = {class="num">0, class="num">0}; class="type">uint global_work_size[class="num">2], local_work_size[class="num">2]; global_work_size[class="num">0] = Neurons(); global_work_size[class="num">1] = NeuronOCL.Neurons() + class="num">1; class="type">uint rest = class="num">0; class="type">float lt = lr; class="kw">switch(NeuronOCL.Optimization()) { case SGD: class="kw">break; case ADAM: class="kw">break; case ADAM_MINI: if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_w, NeuronOCL.getWeightsIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_g, getGradientIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_i, NeuronOCL.getOutputIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_m, NeuronOCL.getFirstMomentumIndex()))
◍ AdamMini 权重更新的内核参数装配
这段片段展示了在 OpenCL 环境下做神经网络权重更新时，如何把二阶动量缓冲和 AdamMini 专用参数塞进计算内核。先判断 SecondMomentum 指针是否有效，无效就 new 一个 CBufferFloat 并初始化长度为 optimization==ADAM_MINI 时的 numOutputs 否则 count，再 BufferCreate 传到显存；任一步失败直接 return false。 内核执行前要把 def_k_wuam_matrix_v（二阶动量索引）、lt（学习率修正项）、b1、b2 通过 SetArgument / SetArgumentBuffer 绑定。lt 的计算式为 lr * sqrt(1 - pow(b2,t)) / (1 - pow(b1,t))，其中 t 是迭代计数，每跑完一次内核 t++。 工作组维度这样定：global_work_size[0] = NeuronOCL.Neurons() + 1，global_work_size[1] = Neurons()，local_work_size[0] 等于前者、[1] 为 1。Execute 若报错会用 printf 打出 Error of execution kernel UpdateWeightsAdamMini 和 GetLastError() 码，方便在 MT5 专家日志里抓异常。 卷积版的入口内核名为 UpdateWeightsConvAdamMini，参数含 matrix_w、matrix_i、matrix_m，说明同套逻辑也适用于卷积层权重，只是内存布局不同。外汇或贵金属模型若接这套 GPU 训练，请先认清过拟合与实时滑点带来的高风险，回测漂亮不等于实盘能跑。
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_v, NeuronOCL.getSecondMomentumIndex())) class="kw">return false; lt = (class="type">float)(lr * sqrt(class="num">1 - pow(b2, (class="type">float)t)) / (class="num">1 - pow(b1, (class="type">float)t))); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_l, lt)) class="kw">return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_b1, b1)) class="kw">return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_b2, b2)) class="kw">return false; global_work_size[class="num">0] = NeuronOCL.Neurons() + class="num">1; global_work_size[class="num">1] = Neurons(); local_work_size[class="num">0] = global_work_size[class="num">0]; local_work_size[class="num">1] = class="num">1; ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdamMini, class="num">2, global_work_offset, global_work_size, local_work_size)) { printf("Error of execution kernel UpdateWeightsAdamMini: %d", GetLastError()); class="kw">return false; } t++; class="kw">break; class="kw">default: class="kw">return false; class="kw">break; } class=class="str">"cmt">//--- class="kw">return true; } if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) class="kw">return false; } if(!SecondMomentum.BufferInit((optimization == ADAM_MINI ? numOutputs : count), class="num">0)) class="kw">return false; if(!SecondMomentum.BufferCreate(OpenCL)) class="kw">return false; __kernel class="type">void UpdateWeightsConvAdamMini(__global class="type">float *matrix_w, __global const class="type">float *matrix_i, __global class="type">float *matrix_m,
GPU 上的梯度累加内核怎么写
在 MT5 的 OpenCL 内核里做反向传播，第一步是把全局索引映射到输入窗口。get_global_id(0) 拿到的是输入维度下标 i，窗口长度用 get_global_size(0)-1 表示，比单纯记数组长度少一个偏置位，写循环时少踩一次越界。 输出侧用 get_global_id(1) 和 get_local_id(1) 分别取全局输出下标 f 与组内头部下标 f_h，变量维再用 get_global_id(2) 展开。三维索引直接对应「样本×时间窗×特征」，在显存里铺平后偏移量靠 shift_var_in、shift_var_out 两套公式算，inputs 与 window_in 差 1 这个细节不处理好，梯度会写错行。 梯度累加核心是一个 total 长度的循环，total = (inputs - window_in + step - 1) / step，这是滑动窗口步长 step 下的有效帧数。若 i 不是末尾哨兵位且越界就 break，否则用 matrix_g 乘输入矩阵（哨兵位乘 1），并用 isnan/isinf 过滤脏值后才累加进 grad。 Local 数组 temp 大小受 LOCAL_ARRAY_SIZE 限制，ls = min(window_in, LOCAL_ARRAY_SIZE) 决定实际可用的共享内存长度。外汇与贵金属行情高频跳空多，isnan 过滤不能省，否则单根异常 K 线可能让整批梯度失效，回测结果失真。
__global class="type">float *matrix_v, const class="type">int inputs, const class="type">float l, const class="type">float b1, const class="type">float b2, class="type">int step ) { class=class="str">"cmt">//--- window in const class="type">size_t i = get_global_id(class="num">0); const class="type">size_t window_in = get_global_size(class="num">0) - class="num">1; class=class="str">"cmt">//--- window out const class="type">size_t f = get_global_id(class="num">1); const class="type">size_t window_out = get_global_size(class="num">1); class=class="str">"cmt">//--- head window out const class="type">size_t f_h = get_local_id(class="num">1); const class="type">size_t window_out_h = get_local_size(class="num">1); class=class="str">"cmt">//--- variable const class="type">size_t v = get_global_id(class="num">2); const class="type">size_t variables = get_global_size(class="num">2); class=class="str">"cmt">//--- constants const class="type">int total = (inputs - window_in + step - class="num">1) / step; const class="type">int shift_var_in = v * inputs; const class="type">int shift_var_out = v * total * window_out; const class="type">int shift_w = (f + v * window_out) * (window_in + class="num">1) + i; __local class="type">float temp[LOCAL_ARRAY_SIZE]; const class="type">int ls = min((class="type">uint)window_in, (class="type">uint)LOCAL_ARRAY_SIZE); class=class="str">"cmt">//--- calc gradient class="type">float grad = class="num">0; for(class="type">int t = class="num">0; t < total; t++) { if(i != window_in && (i + t * window_in) >= inputs) class="kw">break; class="type">float gt = matrix_g[t * window_out + f + shift_var_out] * (i == window_in ? class="num">1 : matrix_i[i + t * step + shift_var_in]); if(!(isnan(gt) || isinf(gt))) grad += gt; } class=class="str">"cmt">//--- calc sum grad class="type">int count; for(class="type">int h = class="num">0; h < window_out_h; h++) { count = class="num">0; do {
「多头注意力里的权重更新内核」
这段 OpenCL C 内核是 MH 注意力层在反向传播时更新权重的核心循环，跑在 GPU 本地内存上，用 work-group 归约梯度平方。外汇与贵金属行情的高波动会让 grad 频繁出现 inf/nan，所以每行都夹了 isnan/isinf 的防御性归零。 外层 do-while 按 count*ls < window_in 推进，h==f_h 时才处理当前头的梯度；shift = i % ls 把全局索引折回本地数组，temp[shift] 累加 grad*grad，相当于在算二阶矩的近似。 归约段用 count=(ls+1)/2 的二叉树合并，把 temp 上半段清 0，最终 i==0 且 f_h==0 的线程负责把 temp[0] 除以 window_in*window_out_h 得到平均二阶矩，再和 matrix_v[head] 做 EMA 混合，clamp 到 1e-6~1e6 防止爆炸。 权重更新走的是 Adam 风格：mt 是一阶矩 EMA，weight = w + l*mt/sqrt(temp[0])，l 是学习率，最后 clamp 到 MAX_WEIGHT。下面这段是可直接贴进 .cl 文件验证的内核尾部与宿主调用签名。 全局尺寸按 iWindow+1 × iWindowOut × iVariables 排布，本地尺寸只在第一维铺满，说明计算是按时间窗切片并行。若你在 MT5 里训练自己的注意力模型，把 b1/b2（一阶、二阶矩衰减）从默认 0.9/0.999 往下调，可能让权重在跳空行情中更快遗忘旧梯度。
if(h == f_h) { if(count == (i / ls)) { class="type">int shift = i % ls; temp[shift] = ((count == class="num">0 && h == class="num">0) ? class="num">0 : temp[shift]) + ((isnan(grad) || isinf(grad)) ? class="num">0 : grad * grad); } } count++; barrier(CLK_LOCAL_MEM_FENCE); } while((count * ls) < window_in); } count = (ls + class="num">1) / class="num">2; do { if(i < count && (i + count) < ls && f_h == class="num">0) { temp[i] += temp[i + count]; temp[i + count] = class="num">0; } count = (count + class="num">1) / class="num">2; barrier(CLK_LOCAL_MEM_FENCE); } while(count > class="num">1); class=class="str">"cmt">//--- calc v if(i == class="num">0 && f_h == class="num">0) { temp[class="num">0] /= (window_in * window_out_h); if(isnan(temp[class="num">0]) || isinf(temp[class="num">0])) temp[class="num">0] = class="num">1; class="type">int head = f / window_out_h; class="type">float v = matrix_v[head]; if(isnan(v) || isinf(v)) v = class="num">1; temp[class="num">0] = clamp(b2 * v + (class="num">1 - b2) * temp[class="num">0], class="num">1.0e-6f, class="num">1.0e6f); matrix_v[head] = temp[class="num">0]; } barrier(CLK_LOCAL_MEM_FENCE); class=class="str">"cmt">//--- calc m class="type">float mt = clamp(b1 * matrix_m[shift_w] + (class="num">1 - b1) * grad, -class="num">1.0e5f, class="num">1.0e5f); if(isnan(mt) || isinf(mt)) mt = class="num">0; class="type">float weight = clamp(matrix_w[shift_w] + l * mt / sqrt(temp[class="num">0]), -MAX_WEIGHT, MAX_WEIGHT); if(!(isnan(weight) || isinf(weight))) matrix_w[shift_w] = weight; matrix_m[shift_w] = mt; } class="type">uint global_work_offset_am[class="num">3] = { class="num">0, class="num">0, class="num">0 }; class="type">uint global_work_size_am[class="num">3] = { iWindow + class="num">1, iWindowOut, iVariables }; class="type">uint local_work_size_am[class="num">3] = { global_work_size_am[class="num">0], class="num">1, class="num">1 }; class="type">bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *momentum1, CBufferFloat *momentum2, class="type">uint window, class="type">uint window_out, class="type">uint step = class="num">0, class="type">uint heads = class="num">0, class="type">uint variables = class="num">1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID
◍ AdamMini 权重更新的核间参数绑定
在 MT5 的 OpenCL 卷积层训练里，ADAM_MINI 分支先逐一校验动量缓冲与梯度矩阵的指针有效性，任一 CheckPointer 返回 POINTER_INVALID 或 GetIndex 小于 0 都会直接 return false，避免空缓冲被丢进 GPU 内核。 随后把权重、梯度、输入、一阶动量、二阶动量五个缓冲通过 SetArgumentBuffer 绑到 def_k_UpdateWeightsConvAdamMini 内核，绑定失败同样立即退出；这一步决定了显存对象与内核参数的对应关系。 学习率修正量 lt 按 lr * sqrt(1 - b2^t) / (1 - b1^t) 计算，其中 b1、b2 为 Adam 类衰减系数，t 为当前步数；该值经 SetArgument 以 float 传入内核，输入总数由 inputs.Total() 一并写入。 外汇与贵金属行情下用此类 GPU 训练存在过拟合与滑点风险，实盘前应在 MT5 策略测试器以历史数据验证内核绑定稳定性。
if(CheckPointer(momentum1) == POINTER_INVALID) class="kw">return false; if(step == class="num">0) step = window; class="type">uint global_work_offset[class="num">1] = {class="num">0}; class="type">uint global_work_size[class="num">1]; global_work_size[class="num">0] = weights.Total(); class="type">uint global_work_offset_am[class="num">3] = {class="num">0, class="num">0, class="num">0}; class="type">uint global_work_size_am[class="num">3] = {window, window_out, class="num">1}; class="type">uint local_work_size_am[class="num">3] = {window, (heads > class="num">0 ? window_out / heads : class="num">1), variables}; if(weights.GetIndex() < class="num">0) class="kw">return false; class="type">float lt = class="num">0; class="kw">switch(optimization) { case SGD: class="kw">break; case ADAM: class="kw">break; case ADAM_MINI: if(CheckPointer(momentum2) == POINTER_INVALID) class="kw">return false; if(gradient.GetIndex() < class="num">0) class="kw">return false; if(inputs.GetIndex() < class="num">0) class="kw">return false; if(momentum1.GetIndex() < class="num">0) class="kw">return false; if(momentum2.GetIndex() < class="num">0) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_w, weights.GetIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_g, gradient.GetIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_i, inputs.GetIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_m, momentum1.GetIndex())) class="kw">return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_v, momentum2.GetIndex())) class="kw">return false; lt = (class="type">float)(lr * sqrt(class="num">1 - pow(b2, t)) / (class="num">1 - pow(b1, t))); if(!OpenCL.SetArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_inputs, inputs.Total())) class="kw">return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_l, lt)) class="kw">return false;