神经网络变得轻松（第七部分）：自适应优化方法·进阶篇
（2/3）·固定学习率在稀有样本前为何失灵，AdaGrad 到 Adam 的数学切换给你答案
◍ 神经元基类的权重与动量缓冲布局
在 MT5 用 OpenCL 跑神经网络时，基类 CNeuronBaseOCL 把权重和梯度都塞进 CBufferDouble 指针里，默认构造函数一次性 new 出 Output、PrevOutput、Weights、DeltaWeights、Gradient、FirstMomentum、SecondMomentum 共 7 个双精度缓冲。 优化器切换靠 ENUM_OPTIMIZATION 枚举，只给了 SGD 与 ADAM 两个选项；缓冲区角色则由 ENUM_BUFFERS 定义，WEIGHTS 到 SECOND_MOMENTUM 共 6 类，和类里那一串指针一一对应。 ADAM 相关的超参在构造时写死：b1=0.9、b2=0.999、t 从 1 起始，如果你在外汇 5 分钟线上做权重更新，直接改这两个 decay 率可能让收敛节奏明显变化（高风险，回测验证后再上实盘）。 下面这段宏定义把 UpdateWeightsAdam 的内核参数槽位用数字固定下来，def_k_uwa_matrix_w 到 def_k_uwa_l 分别是 0~6，b1、b2 占 7、8——调 OpenCL 内核传参时按这个序号填，错一位会直接算飞。
class="macro">#define def_k_UpdateWeightsAdam class="num">4 class="macro">#define def_k_uwa_matrix_w class="num">0 class="macro">#define def_k_uwa_matrix_g class="num">1 class="macro">#define def_k_uwa_matrix_i class="num">2 class="macro">#define def_k_uwa_matrix_m class="num">3 class="macro">#define def_k_uwa_matrix_v class="num">4 class="macro">#define def_k_uwa_inputs class="num">5 class="macro">#define def_k_uwa_l class="num">6 class="macro">#define def_k_uwa_b1 class="num">7 class="macro">#define def_k_uwa_b2 class="num">8 enum ENUM_OPTIMIZATION { SGD, ADAM }; class=class="str">"cmt">//--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; class CNeuronBaseOCL : class="kw">public CObject { class="kw">protected: ......... ......... ......... CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; class=class="str">"cmt">//--- ......... ......... class="kw">const class="type">class="kw">double b1; class="kw">const class="type">class="kw">double b2; class="type">int t; class=class="str">"cmt">//--- ......... ......... ENUM_OPTIMIZATION optimization; CNeuronBaseOCL::CNeuronBaseOCL(class="type">void) : alpha(momentum), activation(TANH), optimization(SGD), b1(class="num">0.9), b2(class="num">0.999), t(class="num">1) { OpenCL=NULL; Output=new CBufferDouble(); PrevOutput=new CBufferDouble(); Weights=new CBufferDouble(); DeltaWeights=new CBufferDouble(); Gradient=new CBufferDouble(); FirstMomentum=new CBufferDouble(); SecondMomentum=new CBufferDouble(); } CNeuronBaseOCL::~CNeuronBaseOCL(class="type">void) { if(CheckPointer(Output)!=POINTER_INVALID) class="kw">delete Output; if(CheckPointer(PrevOutput)!=POINTER_INVALID) class="kw">delete PrevOutput; if(CheckPointer(Weights)!=POINTER_INVALID) class="kw">delete Weights; if(CheckPointer(DeltaWeights)!=POINTER_INVALID) class="kw">delete DeltaWeights;
神经网络对象的指针回收与权重初始化
在 MT5 的 OpenCL 神经网络封装里，析构前必须逐个校验指针再 delete，否则在 EA 重载时可能抛出无效指针访问。下面这段清理逻辑先确认 Gradient、FirstMomentum、SecondMomentum 不是 POINTER_INVALID，才执行释放，最后把 OpenCL 置空。 if(CheckPointer(Gradient)!=POINTER_INVALID) delete Gradient; if(CheckPointer(FirstMomentum)!=POINTER_INVALID) delete FirstMomentum; if(CheckPointer(SecondMomentum)!=POINTER_INVALID) delete SecondMomentum; OpenCL=NULL; Init 函数接管权重缓冲的创建：当 numOutputs>0 时，按 (numNeurons+1)*numOutputs 算出连接数，例如 numNeurons=10、numOutputs=1 时 count=11。权重用 (MathRand()+1)/32768.0-0.5 落到约 [-0.5,0.5) 区间，若随机到 0 则强制设为 0.001 避免死权重。 若优化类型选 SGD，还会额外建 DeltaWeights 缓冲并清零，同时误把 FirstMomentum/SecondMomentum 的 POINTER_INVALID 判断后接了 delete——这里原逻辑有笔误倾向，真实写法应是无效才 new、有效才 delete，复制时务必核对。外汇与贵金属行情下跑这类 GPU 计算负荷高，显存泄漏可能让策略在实盘中断，建议开 MT5 专家日志盯 CheckPointer 返回值。
if(CheckPointer(Gradient)!=POINTER_INVALID) class="kw">delete Gradient; if(CheckPointer(FirstMomentum)!=POINTER_INVALID) class="kw">delete FirstMomentum; if(CheckPointer(SecondMomentum)!=POINTER_INVALID) class="kw">delete SecondMomentum; OpenCL=NULL; } class="type">bool CNeuronBaseOCL::Init(class="type">uint numOutputs,class="type">uint myIndex,COpenCLMy *open_cl,class="type">uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(open_cl)==POINTER_INVALID || numNeurons<=class="num">0) class="kw">return class="kw">false; OpenCL=open_cl; optimization=optimization_type; class=class="str">"cmt">//--- .................... .................... .................... .................... class=class="str">"cmt">//--- if(numOutputs>class="num">0) { if(CheckPointer(Weights)==POINTER_INVALID) { Weights=new CBufferDouble(); if(CheckPointer(Weights)==POINTER_INVALID) class="kw">return class="kw">false; } class="type">int count=(class="type">int)((numNeurons+class="num">1)*numOutputs); if(!Weights.Reserve(count)) class="kw">return class="kw">false; for(class="type">int i=class="num">0;i<count;i++) { class="type">class="kw">double weigh=(MathRand()+class="num">1)/class="num">32768.0-class="num">0.5; if(weigh==class="num">0) weigh=class="num">0.001; if(!Weights.Add(weigh)) class="kw">return class="kw">false; } if(!Weights.BufferCreate(OpenCL)) class="kw">return class="kw">false; class=class="str">"cmt">//--- if(optimization==SGD) { if(CheckPointer(DeltaWeights)==POINTER_INVALID) { DeltaWeights=new CBufferDouble(); if(CheckPointer(DeltaWeights)==POINTER_INVALID) class="kw">return class="kw">false; } if(!DeltaWeights.BufferInit(count,class="num">0)) class="kw">return class="kw">false; if(!DeltaWeights.BufferCreate(OpenCL)) class="kw">return class="kw">false; if(CheckPointer(FirstMomentum)==POINTER_INVALID) class="kw">delete FirstMomentum; if(CheckPointer(SecondMomentum)==POINTER_INVALID)
「动量缓冲的指针回收与权重更新内核装配」
在神经网络基类的初始化分支里，先对 SecondMomentum 做空指针判断，若 CheckPointer 返回 POINTER_INVALID 就直接 delete，避免悬空句柄占用显存。 若走 else 分支，则依次校验 DeltaWeights 与 FirstMomentum：任一缓冲无效就 new 一个 CBufferDouble，再次 CheckPointer 失败便 return false 中断。FirstMomentum 与 SecondMomentum 都需 BufferInit(count,0) 清零，再 BufferCreate(OpenCL) 挂到 OpenCL 上下文；这两步任一返回 false 同样终止。 当最外层条件不满足时，对 Weights 与 DeltaWeights 做非空判断后 delete，确保切换优化器时不残留旧缓冲。 updateInputWeights 里先用 CheckPointer 卡住 OpenCL 与传入神经元指针，无效直接撤出。随后设 global_work_size[0]=Neurons()、[1]=NeuronOCL.Neurons()，以二维网格覆盖全连接权矩阵。SGD 优化下调用 OpenCL.SetArgumentBuffer 把权重、梯度、输出、DeltaWeights 的索引依次绑到 def_k_UpdateWeightsMomentum 内核，再设输入神经元数与学习率 eta，交由 GPU 并行更新。外汇与贵金属行情下用此类 GPU 加速网络做推断，仍属高风险，过拟合可能导致样本外表现显著衰减。
class="kw">delete SecondMomentum; } else { if(CheckPointer(DeltaWeights)==POINTER_INVALID) class="kw">delete DeltaWeights; class=class="str">"cmt">//--- if(CheckPointer(FirstMomentum)==POINTER_INVALID) { FirstMomentum=new CBufferDouble(); if(CheckPointer(FirstMomentum)==POINTER_INVALID) class="kw">return class="kw">false; } if(!FirstMomentum.BufferInit(count,class="num">0)) class="kw">return class="kw">false; if(!FirstMomentum.BufferCreate(OpenCL)) class="kw">return class="kw">false; class=class="str">"cmt">//--- if(CheckPointer(SecondMomentum)==POINTER_INVALID) { SecondMomentum=new CBufferDouble(); if(CheckPointer(SecondMomentum)==POINTER_INVALID) class="kw">return class="kw">false; } if(!SecondMomentum.BufferInit(count,class="num">0)) class="kw">return class="kw">false; if(!SecondMomentum.BufferCreate(OpenCL)) class="kw">return class="kw">false; } } else { if(CheckPointer(Weights)!=POINTER_INVALID) class="kw">delete Weights; if(CheckPointer(DeltaWeights)!=POINTER_INVALID) class="kw">delete DeltaWeights; } class=class="str">"cmt">//--- class="kw">return true; } class="type">bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL)==POINTER_INVALID || CheckPointer(NeuronOCL)==POINTER_INVALID) class="kw">return class="kw">false; class="type">uint global_work_offset[class="num">2]={class="num">0,class="num">0}; class="type">uint global_work_size[class="num">2]; global_work_size[class="num">0]=Neurons(); global_work_size[class="num">1]=NeuronOCL.Neurons(); if(optimization==SGD) { OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum,def_k_uwm_matrix_w,NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum,def_k_uwm_matrix_g,getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum,def_k_uwm_matrix_i,NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum,def_k_uwm_matrix_dw,NeuronOCL.getDeltaWeightsIndex()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum,def_k_uwm_inputs,NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum,def_k_uwm_learning_rates,eta);
◍ 动量法与Adam在GPU上的权重更新分叉
神经网络层在OpenCL下做权重更新时，按优化器类型走了两条kernel路径：动量法走 UpdateWeightsMomentum，Adam走 UpdateWeightsAdam，二者都先塞参数再Execute，任一步SetArgument/Execute失败就直接return false并打出GetLastError()。 动量分支只设了alpha一个标量参数，随后用2维global_work_size启动kernel；Adam分支则要绑定5个显存缓冲区——权重w、梯度g、输出i、一阶矩m、二阶矩v，分别对应NeuronOCL和本层的索引句柄。 Adam的学习率做了偏差修正：lt = eta * sqrt(1 - pow(b2,t)) / (1 - pow(b1,t))，t是内部步数计数，每次更新后t++。注意global_work_size[1]被4整除向上取整（rest=size%4，补1个group当rest>0），这是kernel里向量化宽度=4的硬约束。 Save方法另起炉灶，先校验file_handle!=INVALID_HANDLE，再写Type()和activation枚举（各占一个INT_VALUE），任一次FileWriteInteger返回值小于INT_VALUE即判失败。外汇/贵金属模型训练耗GPU且过拟合风险高，参数请先在离线样本验证。
OpenCL.SetArgument(def_k_UpdateWeightsMomentum,def_k_uwm_momentum,alpha); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsMomentum,class="num">2,global_work_offset,global_work_size)) { printf("Error of execution kernel UpdateWeightsMomentum: %d",GetLastError()); class="kw">return class="kw">false; } } else { if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam,def_k_uwa_matrix_w,NeuronOCL.getWeightsIndex())) class="kw">return class="kw">false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam,def_k_uwa_matrix_g,getGradientIndex())) class="kw">return class="kw">false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam,def_k_uwa_matrix_i,NeuronOCL.getOutputIndex())) class="kw">return class="kw">false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam,def_k_uwa_matrix_m,NeuronOCL.getFirstMomentumIndex())) class="kw">return class="kw">false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam,def_k_uwa_matrix_v,NeuronOCL.getSecondMomentumIndex())) class="kw">return class="kw">false; class="type">class="kw">double lt=eta*sqrt(class="num">1-pow(b2,t))/(class="num">1-pow(b1,t)); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam,def_k_uwa_inputs,NeuronOCL.Neurons())) class="kw">return class="kw">false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam,def_k_uwa_l,lt)) class="kw">return class="kw">false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam,def_k_uwa_b1,b1)) class="kw">return class="kw">false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam,def_k_uwa_b2,b2)) class="kw">return class="kw">false; class="type">uint rest=global_work_size[class="num">1]%class="num">4; global_work_size[class="num">1]=(global_work_size[class="num">1]-rest)/class="num">4 + (rest>class="num">0 ? class="num">1 : class="num">0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdam,class="num">2,global_work_offset,global_work_size)) { printf("Error of execution kernel UpdateWeightsAdam: %d",GetLastError()); class="kw">return class="kw">false; } t++; } class=class="str">"cmt">//--- class="kw">return NeuronOCL.Weights.BufferRead(); } class="type">bool CNeuronBaseOCL::Save(class="kw">const class="type">int file_handle) { if(file_handle==INVALID_HANDLE) class="kw">return class="kw">false; if(FileWriteInteger(file_handle,Type())<INT_VALUE) class="kw">return class="kw">false; class=class="str">"cmt">//--- if(FileWriteInteger(file_handle,(class="type">int)activation,INT_VALUE)<INT_VALUE) class="kw">return class="kw">false;
把网络状态落盘时的指针与优化器分支
这段保存逻辑的核心是先写标量再写对象缓冲：optimization 和 t 各用 FileWriteInteger 以 INT_VALUE 长度落盘，返回值小于 INT_VALUE 就直接 return false，说明任何整数写入截断都会让整个存档失效。 接着对 Output、PrevOutput、Gradient 三个对象做 CheckPointer 非空判断，并依次调用各自的 BufferRead 与 Save；只要其中一个指针无效或 Save 失败，存档立即中断。 Weights 的处理带了一个标志位：指针无效时写 0 并 return true，有效时写 1 再继续存权重缓冲。这里 0/1 的写入量就是 4 字节整数，用来标记该连接是否带权重。 优化器分支在末尾才分叉：SGD 只额外存 DeltaWeights，而 Adam 类优化器要存 FirstMomentum 和 SecondMomentum（对应 mt、vt）。CConnection 里 mt、vt 初值都是 0，Save 时分别用 FileWriteDouble 写出，任一个写入字节数 ≤0 就返回 false。 在 MT5 里跑这套，建议先打印 CheckPointer 各返回值，确认训练中途存档不会因某个动量缓冲未初始化而静默丢状态。外汇与贵金属模型训练本身高风险，存档完整性只解决复现问题，不改善过拟合。
if(FileWriteInteger(file_handle,(class="type">int)optimization,INT_VALUE)<INT_VALUE) class="kw">return class="kw">false; if(FileWriteInteger(file_handle,(class="type">int)t,INT_VALUE)<INT_VALUE) class="kw">return class="kw">false; if(CheckPointer(Output)==POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle)) class="kw">return class="kw">false; if(CheckPointer(PrevOutput)==POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle)) class="kw">return class="kw">false; if(CheckPointer(Gradient)==POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle)) class="kw">return class="kw">false; class=class="str">"cmt">//--- if(CheckPointer(Weights)==POINTER_INVALID) { FileWriteInteger(file_handle,class="num">0); class="kw">return true; } else FileWriteInteger(file_handle,class="num">1); class=class="str">"cmt">//--- if(CheckPointer(Weights)==POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle)) class="kw">return class="kw">false; if(optimization==SGD) { if(CheckPointer(DeltaWeights)==POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle)) class="kw">return class="kw">false; } else { if(CheckPointer(FirstMomentum)==POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle)) class="kw">return class="kw">false; if(CheckPointer(SecondMomentum)==POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle)) class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true; } class CConnection : class="kw">public CObject { class="kw">public: class="type">class="kw">double weight; class="type">class="kw">double deltaWeight; class="type">class="kw">double mt; class="type">class="kw">double vt; CConnection(class="type">class="kw">double w) { weight=w; deltaWeight=class="num">0; mt=class="num">0; vt=class="num">0; } class="type">bool CConnection::Save(class="type">int file_handle) { ........... ........... ........... if(FileWriteDouble(file_handle,mt)<=class="num">0) class="kw">return class="kw">false; if(FileWriteDouble(file_handle,vt)<=class="num">0) class="kw">return class="kw">false; class=class="str">"cmt">//--- class="kw">return true; }
「权重更新里的 Adam 与 SGD 分叉」
神经元的输入权重更新在 updateInputWeights 里走两条路：当 optimization 标记为 SGD 时用最朴素的梯度下降加动量，否则进入 Adam 分支。Adam 分支里 mt 是一阶矩、vt 是二阶矩，各自按 b1、b2 衰减，权重增量用 lt*mt/sqrt(vt) 表达，lt 由 eta 与两个衰减值按 sqrt(1-b2^t)/(1-b1^t) 修正。 代码里那行 lt 的计算值得在 MT5 里单步看：eta 是学习率，b1 与 b2 是类里 const double 成员，t 从 0 开始每次更新后自增。当 t 较小，分母 (1-b1^t) 偏小，lt 会被放大，初期步长比稳态大，这可能加速早期收敛。 非 SGD 分支给 vt 加了 0.00000001 的 epsilon 防除零，这是数值稳健的常见做法。外汇与贵金属行情噪声大，用 Adam 训练这类网络过拟合概率不低，开 MT5 把 b1=0.9、b2=0.999 先跑一遍再调。
class="type">bool CConnection::Load(class="type">int file_handle) { ............ ............ ............ mt=FileReadDouble(file_handle); vt=FileReadDouble(file_handle); class=class="str">"cmt">//--- class="kw">return true; } class CNeuronBase : class="kw">public CObject { class="kw">protected: ......... ......... ......... ENUM_OPTIMIZATION optimization; class="kw">const class="type">class="kw">double b1; class="kw">const class="type">class="kw">double b2; class="type">int t; class="type">bool CNeuronBase::Init(class="type">uint numOutputs,class="type">uint myIndex, ENUM_OPTIMIZATION optimization_type) { optimization=optimization_type; class="type">bool CNeuron::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer)==POINTER_INVALID) class="kw">return class="kw">false; class=class="str">"cmt">//--- class="type">class="kw">double lt=eta*sqrt(class="num">1-pow(b2,t))/(class="num">1-pow(b1,t)); class="type">int total=prevLayer.Total(); for(class="type">int n=class="num">0; n<total && !IsStopped(); n++) { CNeuron *neuron= prevLayer.At(n); CConnection *con=neuron.Connections.At(m_myIndex); if(CheckPointer(con)==POINTER_INVALID) class="kw">continue; if(optimization==SGD) con.weight+=con.deltaWeight=(gradient!=class="num">0 ? eta*neuron.getOutputVal()*gradient : class="num">0)+(con.deltaWeight!=class="num">0 ? alpha*con.deltaWeight : class="num">0); else { con.mt=b1*con.mt+(class="num">1-b1)*gradient; con.vt=b2*con.vt+(class="num">1-b2)*pow(gradient,class="num">2)+class="num">0.00000001; con.weight+=con.deltaWeight=lt*con.mt/sqrt(con.vt); t++; } } class=class="str">"cmt">//--- class="kw">return true; } class CLayerDescription : class="kw">public CObject { class="kw">public: CLayerDescription(class="type">void); ~CLayerDescription(class="type">void) {}; class=class="str">"cmt">//--- class="type">int type; class="type">int count; class="type">int window; class="type">int step; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; }; CNet::CNet(CArrayObj *Description) { if(CheckPointer(Description)==POINTER_INVALID) class="kw">return; class=class="str">"cmt">//--- class="type">int total=Description.Total(); if(total<=class="num">0)