神经网络变得轻松（第三十八部分）：凭借分歧进行自我监督探索·进阶篇
（2/3）·当外部奖励稀少滞后，靠多个动态模型的分歧生成内在激励，代理者才肯踏进未探索区域
反向传播里的向量化梯度累加
这段内核在做一件事：把上层传回来的梯度按输出节点分组，用 float4 向量一次算 4 个，再点积进 sum。循环以 k += 4 步进，switch(outputs - k) 专门处理末尾不足 4 个的残差，避免越界读 matrix_g 和 matrix_w。 权重矩阵在内存里按 (inputs + 1) 为单节点跨度排布，所以取第 k 个输出对第 i 个输入的权重要写 shift_w + k * (inputs + 1) + i。dot(grad, weight) 直接累加到 sum，比逐标量写四次快得多，在 MT5 的 OpenCL 内核里这个写法能明显压低回传耗时。 sum 算完先 isnan 兜底置 0，再按激活函数分流。case 0 是 tanh 类：先把 out 夹到 [-1,1]，梯度乘 (1 - out^2) 且下限 1e-4；case 1 是 sigmoid 类：夹到 [0,1]，梯度乘 out*(1-out) 同样封底 1e-4。外汇与贵金属行情跳空多，NaN 兜底不是可选项，实盘跑神经网络指标时漏掉这行可能让整层梯度蒸发。 把下面这段直接丢进你的内核做对照，重点看 switch 残差分支和 clamp 下限，这两处是多数自行移植版容易写错的地方。
class="type">float out = matrix_o[shift_in + i]; float4 grad, weight; for(class="type">int k = class="num">0; k < outputs; k += class="num">4) { class="kw">switch(outputs - k) { case class="num">1: weight = (float4)(matrix_w[shift_w + k * (inputs + class="num">1) + i], class="num">0, class="num">0, class="num">0); grad = (float4)(matrix_g[shift_out + k], class="num">0, class="num">0, class="num">0); class="kw">break; case class="num">2: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + class="num">1], class="num">0, class="num">0); weight = (float4)(matrix_w[shift_w + k * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">1) * (inputs + class="num">1) + i], class="num">0, class="num">0); class="kw">break; case class="num">3: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + class="num">1], matrix_g[shift_out + k + class="num">2], class="num">0); weight = (float4)(matrix_w[shift_w + k * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">1) * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">2) * (inputs + class="num">1) + i], class="num">0); class="kw">break; class="kw">default: grad = (float4)(matrix_g[shift_out + k], matrix_g[shift_out + k + class="num">1], matrix_g[shift_out + k + class="num">2], matrix_g[shift_out + k + class="num">3]); weight = (float4)(matrix_w[shift_w + k * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">1) * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">2) * (inputs + class="num">1) + i], matrix_w[shift_w + (k + class="num">3) * (inputs + class="num">1) + i]); class="kw">break; } sum += dot(grad, weight); } if(isnan(sum)) sum = class="num">0; class="kw">switch(activation) { case class="num">0: out = clamp(out, -class="num">1.0f, class="num">1.0f); sum = clamp(sum + out, -class="num">1.0f, class="num">1.0f) - out; sum = sum * max(class="num">1 - pow(out, class="num">2), class="num">1.0e-4f); class="kw">break; case class="num">1: out = clamp(out, class="num">0.0f, class="num">1.0f); sum = clamp(sum + out, class="num">0.0f, class="num">1.0f) - out; sum = sum * max(out * (class="num">1 - out), class="num">1.0e-4f); class="kw">break; case class="num">2: if(out < class="num">0)
「多模型神经元的初始化与内存布局」
在 MT5 的 OpenCL 神经网络框架里，CNeuronMultiModel 通过继承 CNeuronBaseOCL 实现多模型并行推理。类内用 iModels 与 iUpdateModel 两个整型成员记录模型数量和当前更新目标，虚函数 feedForward、updateInputWeights 留给子类或运行时多态去落地。 Init 方法是硬关卡：先判 open_cl 指针有效性，再要求 numNeurons>0 且 models>0，否则直接返回 false。内部强制把 optimization 设为 ADAM，iBatch 写死为 1，这意味着该层默认按单样本、Adam 优化器跑多模型前向。 Output 缓冲区按 numNeurons * models 大小初始化为 0.0，并交由 OpenCL 创建显存对象；Gradient 缓冲区走同样的保护性 new + 判空流程。你在 EA 里调 models 参数时，显存占用会线性放大 models 倍，贵金属或外汇高频训练前先算清这张卡是否扛得住。 下方代码展示了类声明与 Init 前半段，注意 default 分支在 switch 里只 break，说明未匹配的模式不做任何累计算子处理。
sum *= class="num">0.01f; class="kw">break; class="kw">default: class="kw">break; } matrix_ig[shift_in + i] = sum; } class CNeuronMultiModel : class="kw">public CNeuronBaseOCL { class="kw">protected: class="type">int iModels; class="type">int iUpdateModel; class=class="str">"cmt">//--- class="kw">virtual class="type">bool feedForward(CNeuronBaseOCL *NeuronOCL); class="kw">virtual class="type">bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); class="kw">public: CNeuronMultiModel(class="type">void){}; ~CNeuronMultiModel(class="type">void){}; class="kw">virtual class="type">bool Init(class="type">uint numInputs, class="type">uint myIndex, COpenCLMy *open_cl, class="type">uint numNeurons, ENUM_OPTIMIZATION optimization_type, class="type">int models); class="kw">virtual class="type">void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } class=class="str">"cmt">//--- class="kw">virtual class="type">bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); class=class="str">"cmt">//--- class="kw">virtual class="type">bool Save(class="type">int const file_handle); class="kw">virtual class="type">bool Load(class="type">int const file_handle); class=class="str">"cmt">//--- class="kw">virtual class="type">int Type(class="type">void) const { class="kw">return defNeuronMultiModels; } }; class="type">bool CNeuronMultiModel::Init(class="type">uint numInputs, class="type">uint myIndex, COpenCLMy *open_cl, class="type">uint numNeurons, ENUM_OPTIMIZATION optimization_type, class="type">int models) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= class="num">0 || models <= class="num">0) class="kw">return false; OpenCL = open_cl; optimization = ADAM; iBatch = class="num">1; iModels = models; class=class="str">"cmt">//--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferFloat(); if(CheckPointer(Output) == POINTER_INVALID) class="kw">return false; } if(!Output.BufferInit(numNeurons * models, class="num">0.0)) class="kw">return false; if(!Output.BufferCreate(OpenCL)) class="kw">return false; class=class="str">"cmt">//--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferFloat(); if(CheckPointer(Gradient) == POINTER_INVALID) class="kw">return false; }
◍ 多模型神经元的权重初始化与前向传导
在 MT5 的 OpenCL 神经网络封装里，CNeuronMultiModel 的初始化先把梯度缓冲按 (神经元数+1)*模型数 的规模铺开，再为权重申请一块 (输入数+1)*神经元数*模型数 的浮点缓冲。若以 10 输入、8 神经元、4 模型 为例，count = (10+1)*8*4 = 352，权重数组一次 Reserve 到位。 权重填充用 1/sqrt(输入数+1) 做缩放系数 k，循环里写成 (2*随机值*k - k)*乘数，相当于在 [-k, k] 区间均匀撒点后再乘用户乘数。这种 Xavier 风格初始化在输入维度偏高时倾向让前向信号方差更稳定，外汇小时线特征维度动辄 20+，不调 k 直接训可能梯度爆炸。 前向 feedForward 没走逐样本循环，而是用二维 global_work_size：第 0 维是 输出总数/模型数，第 1 维是 模型数，一次 GPU 派发把所有子模型并行算完。SetArgumentBuffer 绑错权重索引会直接 printf 报错并返回 false，调试时盯紧 def_k_ff_matrix_w 这个参数槽。
if(!Gradient.BufferInit((numNeurons + class="num">1)*models, class="num">0.0)) class="kw">return false; if(!Gradient.BufferCreate(OpenCL)) class="kw">return false; class=class="str">"cmt">//--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) class="kw">return false; } class="type">int count = (class="type">int)((numInputs + class="num">1) * numNeurons * models); if(!Weights.Reserve(count)) class="kw">return false; class="type">float k = (class="type">float)(class="num">1 / sqrt(numInputs + class="num">1)); for(class="type">int i = class="num">0; i < count; i++) { if(!Weights.Add((class="num">2 * GenerateWeight()*k - k)*WeightsMultiplier)) class="kw">return false; } if(!Weights.BufferCreate(OpenCL)) class="kw">return false; class=class="str">"cmt">//--- if(CheckPointer(DeltaWeights) != POINTER_INVALID) class="kw">delete DeltaWeights; class=class="str">"cmt">//--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) class="kw">return false; } if(!FirstMomentum.BufferInit(count, class="num">0)) class="kw">return false; if(!FirstMomentum.BufferCreate(OpenCL)) class="kw">return false; class=class="str">"cmt">//--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) class="kw">return false; } if(!SecondMomentum.BufferInit(count, class="num">0)) class="kw">return false; if(!SecondMomentum.BufferCreate(OpenCL)) class="kw">return false; class=class="str">"cmt">//--- class="kw">return true; } class="type">bool CNeuronMultiModel::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) class="kw">return false; class="type">uint global_work_offset[class="num">2] = {class="num">0, class="num">0}; class="type">uint global_work_size[class="num">2]; global_work_size[class="num">0] = Output.Total() / iModels; global_work_size[class="num">1] = iModels; if(!OpenCL.SetArgumentBuffer(def_k_FFMultiModels, def_k_ff_matrix_w, getWeightsIndex())) { printf("Error of set parameter kernel FeedForward: %d; line %d", GetLastError(), __LINE__);
多模型神经元的核参数绑定与梯度前置
这段 CNeuronMultiModel 的片段只做一件事：把多模型前馈核（FeedForward）和隐藏层梯度核（calcHiddenGradients）的缓冲区参数逐一对齐到 OpenCL 设备端。任何一次 SetArgumentBuffer / SetArgument 失败都会立刻 printf 报错并返回 false，调用方必须检查返回值，否则后续 Execute 跑的是脏数据。 前馈核里 def_k_ff_matrix_i、def_k_ff_matrix_o 分别绑的是上层神经元输出缓冲与本层输出缓冲；def_k_ff_inputs 用 NeuronOCL.Neurons()/iModels 算出单模型输入维度，def_k_ff_activation 把激活函数类型以 int 传入。最后 OpenCL.Execute 用二维全局尺寸 [NeuronOCL.Neurons()/iModels, iModels] 启动核，work_offset 恒为 {0,0}。 calcHiddenGradients 里 global_work_size[0] 同样取 NeuronOCL.Neurons()/iModels，[1] 取 iModels，说明梯度计算也是按模型并行切分的。权重索引 getWeightsIndex() 与梯度索引 getGradientIndex() 分别绑到 def_k_chg_matrix_w / def_k_chg_matrix_g，上层输出与梯度缓冲则来自 NeuronOCL.getOutputIndex() 和 getGradientIndex()。 在 MT5 里验证时，若日志频繁出现 'Error of set parameter kernel FeedForward: %d; line %d' 且行号落在这段，优先查 iModels 是否整除 NeuronOCL.Neurons()，除不尽会导致核维度越界。外汇与贵金属行情下用此类 GPU 推理模块，延迟抖动可能放大滑点风险，参数需先在模拟盘跑稳。
class="kw">return false; } if(!OpenCL.SetArgumentBuffer(def_k_FFMultiModels, def_k_ff_matrix_i, NeuronOCL.getOutputIndex())) { printf("Error of set parameter kernel FeedForward: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgumentBuffer(def_k_FFMultiModels, def_k_ff_matrix_o, Output.GetIndex())) { printf("Error of set parameter kernel FeedForward: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgument(def_k_FFMultiModels, def_k_ff_inputs, NeuronOCL.Neurons() / iModels)) { printf("Error of set parameter kernel FeedForward: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgument(def_k_FFMultiModels, def_k_ff_activation, (class="type">int)activation)) { printf("Error of set parameter kernel FeedForward: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.Execute(def_k_FFMultiModels, class="num">2, global_work_offset, global_work_size)) { printf("Error of execution kernel FeedForward: %d", GetLastError()); class="kw">return false; } class=class="str">"cmt">//--- class="kw">return true; } class="type">bool CNeuronMultiModel::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) class="kw">return false; class="type">uint global_work_offset[class="num">2] = {class="num">0, class="num">0}; class="type">uint global_work_size[class="num">2]; global_work_size[class="num">0] = NeuronOCL.Neurons() / iModels; global_work_size[class="num">1] = iModels; if(!OpenCL.SetArgumentBuffer(def_k_HGMultiModels, def_k_chg_matrix_w, getWeightsIndex())) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgumentBuffer(def_k_HGMultiModels, def_k_chg_matrix_g, getGradientIndex())) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgumentBuffer(def_k_HGMultiModels, def_k_chg_matrix_o, NeuronOCL.getOutputIndex())) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgumentBuffer(def_k_HGMultiModels, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex())) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false;
「多模型核的参数装配与分歧探索类骨架」
这段片段接着多模型神经元的前向与梯度核，把 OpenCL 参数槽逐个绑到具体变量上。SetArgument 调用里，def_k_chg_outputs 被设成 Neurons()/iModels，意味着隐藏层梯度核按模型数做输出切分；def_k_chg_activation 直接取 NeuronOCL 的激活函数标识，def_k_chg_model 则用 MathRand()/32767.0 随机落到 [0, iModels-1] 的某个模型下标，单次只更新一个子模型权重。 核执行走 OpenCL.Execute(def_k_HGMultiModels, 2, global_work_offset, global_work_size)，二维全局工组尺寸由调用方给定；任一处 SetArgument 或 Execute 失败就 printf 报错并返回 false，错误行号靠 __LINE__ 内建宏定位。 存档侧 CNeuronMultiModel::Save 先调基类 CNeuronBaseOCL::Save，再 FileWriteInteger 写 iModels，返回值 ≤0 即判失败。宏定义给出核索引：前馈 46、隐藏梯度 47、权重更新 48，随机选模型的下标槽分别是 def_k_chg_model=6 与 def_k_uwa_model=9。 后面冒出的 CEVD 类（继承 CNet）是「Exploration via Disagreement」的载体，成员含 iMinBufferSize、iStateEmbedingLayer、dPrevBalance 及两个布尔开关 bUseTargetNet / bTrainMode，从命名看倾向用多模型预测分歧做探索，而非直接拿期望收益驱动。外汇与贵金属行情下接这类 GPU 推理模块，延迟与过拟合风险都偏高，参数请在 MT5 策略测试器里逐项核对。
} if(!OpenCL.SetArgument(def_k_HGMultiModels, def_k_chg_outputs, Neurons() / iModels)) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.SetArgument(def_k_HGMultiModels, def_k_chg_activation, NeuronOCL.Activation())) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } iUpdateModel = (class="type">int)MathRound(MathRand() / class="num">32767.0 * (iModels - class="num">1)); if(!OpenCL.SetArgument(def_k_HGMultiModels, def_k_chg_model, iUpdateModel)) { printf("Error of set parameter kernel calcHiddenGradients: %d; line %d", GetLastError(), __LINE__); class="kw">return false; } if(!OpenCL.Execute(def_k_HGMultiModels, class="num">2, global_work_offset, global_work_size)) { printf("Error of execution kernel CalcHiddenGradient: %d", GetLastError()); class="kw">return false; } class=class="str">"cmt">//--- class="kw">return true; } class="type">bool CNeuronMultiModel::Save(const class="type">int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) class="kw">return false; if(FileWriteInteger(file_handle, iModels) <= class="num">0) class="kw">return false; class=class="str">"cmt">//--- class="kw">return true; } class="macro">#define def_k_FFMultiModels class="num">46 class=class="str">"cmt">///< Index of the kernel of the multi-models neuron to calculate feed forward class="macro">#define def_k_HGMultiModels class="num">47 class=class="str">"cmt">///< Index of the kernel of the multi-models neuron to calculate hiden gradient class="macro">#define def_k_chg_model class="num">6 class=class="str">"cmt">///< Number of model to calculate class="macro">#define def_k_UWMultiModels class="num">48 class=class="str">"cmt">///< Index of the kernel of the multi-models neuron to update weights class="macro">#define def_k_uwa_model class="num">9 class=class="str">"cmt">///< Number of model to update class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Exploration via Disagreement | class=class="str">"cmt">//+------------------------------------------------------------------+ class CEVD : class="kw">protected CNet { class="kw">protected: class="type">uint iMinBufferSize; class="type">uint iStateEmbedingLayer; class="type">class="kw">double dPrevBalance; class="type">bool bUseTargetNet; class="type">bool bTrainMode; class=class="str">"cmt">//---
◍ EVD 类的接口与推理入口
在 MT5 里落地这套嵌入式值迭代网络，核心是先看清 CEVD 类对外暴露了哪些能力。它内部挂了三张网：cTargetNet 负责稳定目标值、cReplay 做经验回放缓冲、cForwardNet 处理前向状态映射，三者由同一个类统一调度。 从声明看，feedForward 的默认窗口是 1、温度参数 tem 默认开 true，backProp 的折扣因子写死 0.999f——这意味着你不动参数的话，Agent 偏向几乎无折扣的长期回报，外汇与贵金属这种高波动品种上容易对噪声过拟合，调参时建议先压到 0.95 附近观察。 feedForward 的实现先走 AddInputData 把输入压进缓冲，再调 CNet::feedForward。训练模式下它会取 iStateEmbedingLayer 那层的输出当 state，取不到就直接返回 -1，所以 SetStateEmbedingLayer 必须在跑之前设对层数，否则整条链路拿不到状态表征。 Save/Load 有两套重载，一套合存一套分存 dqn 与 forward 网络；实盘重启想热加载策略，用分存接口更方便单独替换前向网。开 MT5 把这段类头抄进自定义 EA 的 include，先只改 iStateEmbedingLayer 和 discount 两个量做离线回测。
CNet cTargetNet; CReplayBuffer cReplay; CNet cForwardNet; class="kw">virtual class="type">bool AddInputData(CArrayFloat *inputVals); class="kw">public: CEVD(); CEVD(CArrayObj *Description, CArrayObj *Forward); class="type">bool Create(CArrayObj *Description, CArrayObj *Forward); ~CEVD(); class="type">int feedForward(CArrayFloat *inputVals, class="type">int window = class="num">1, class="type">bool tem = true); class="type">bool backProp(class="type">int batch, class="type">float discount = class="num">0.999f); class="type">int getAction(class="type">int state_size = class="num">0); class="type">float getRecentAverageError() { class="kw">return recentAverageError; } class="type">bool Save(class="type">class="kw">string file_name, class="type">bool common = true); class="type">bool Save(class="type">class="kw">string dqn, class="type">class="kw">string forward, class="type">bool common = true); class="kw">virtual class="type">bool Load(class="type">class="kw">string file_name, class="type">bool common = true); class="type">bool Load(class="type">class="kw">string dqn, class="type">class="kw">string forward, class="type">uint state_layer, class="type">bool common = true); class=class="str">"cmt">//--- class="kw">virtual class="type">int Type(class="type">void) const { class="kw">return defEVD; } class="kw">virtual class="type">bool TrainMode(class="type">bool flag) { bTrainMode = flag; class="kw">return (CNet::TrainMode(flag) && cForwardNet.TrainMode(flag));} class="kw">virtual class="type">bool GetLayerOutput(class="type">uint layer, CBufferFloat *&result) { class="kw">return CNet::GetLayerOutput(layer, result); } class=class="str">"cmt">//--- class="kw">virtual class="type">void SetStateEmbedingLayer(class="type">uint layer) { iStateEmbedingLayer = layer; } class="kw">virtual class="type">void SetBufferSize(class="type">uint min, class="type">uint max); }; class="type">int CEVD::feedForward(CArrayFloat *inputVals, class="type">int window = class="num">1, class="type">bool tem = true) { if(!AddInputData(inputVals)) class="kw">return -class="num">1; class=class="str">"cmt">//--- if(!CNet::feedForward(inputVals, window, tem)) class="kw">return -class="num">1; class="type">int action = -class="num">1; if(bTrainMode) { CBufferFloat *state; class=class="str">"cmt">//if(!GetLayerOutput(class="num">1, state)) class=class="str">"cmt">// class="kw">return -class="num">1; if(!GetLayerOutput(iStateEmbedingLayer, state)) class="kw">return -class="num">1;