📘
神经网络变得简单（第 70 部分）：封闭式政策改进运算器（CFPI）·进阶篇
本章目录
Critic 网络的层堆叠与初始化
在强化学习智能体里，Critic 负责给状态-动作对打分，这段逻辑就是把它的网络结构一层层塞进 CNet 对象。先清掉旧描述，再建输入层：类型用 defNeuronBaseOCL，节点数等于 LatentCount，激活函数关成 None，优化器统一挂 ADAM。 紧接着是拼接层（defNeuronConcatenate），window 设为上一层节点数 prev_count，step 绑定 NActions，激活走 LReLU。后面三层都是 defNeuronBaseOCL，节点保持 LatentCount、LReLU 激活，直到第 4 层把输出压到 NRewards 且激活为 None，完成价值估计头。 每一层都用 new CLayerDescription() 拿描述符，Add 失败就 delete 并 return false，避免内存泄漏。外层输入参数 Iterations 默认 1e6，意味着训练循环上限百万步，外汇与贵金属行情下过拟合风险高，实盘前建议在 MT5 策略测试器里先跑小样本验证。 下方全局对象 StateEncoder 与 Critic1 已声明，配合 Buffer[] 存轨迹，开 MT5 把这段贴进 EA 的 Critic 构建函数即可观察网络是否按预期组装。
MQL5 / C++
descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- Critic critic.Clear(); class=class="str">"cmt">//--- Input layer if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; prev_count = descr.count = LatentCount; descr.activation = None; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronConcatenate; descr.count = LatentCount; descr.window = prev_count; descr.step = NActions; descr.optimization = ADAM; descr.activation = LReLU; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = NRewards; descr.optimization = ADAM; descr.activation = None; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- class="kw">return true; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Input parameters | class=class="str">"cmt">//+------------------------------------------------------------------+ input class="type">int Iterations = class="num">1e6; class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| | class=class="str">"cmt">//+------------------------------------------------------------------+ STrajectory Buffer[]; CNet StateEncoder; CNet Critic1;
「EA 初始化时神经网络模型的加载与维度校验」
MT5 的 OnInit 阶段先把历史学习数据通过 LoadTotalBase 读入，失败就直接返回 INIT_FAILED 并打出错误码，这一步卡住后续所有模型构建。 若三个网络文件（Enc.nnw、Crt1.nnw、Crt2.nnw）不存在或加载失败，代码会新建 actor / critic / encoder 三套描述对象，调用 CreateDescriptions 生成结构后分别 Create，任何一步失败均释放内存并终止初始化。 模型就位后，把 Critic1 的 OpenCL 上下文共享给 Critic2 与 StateEncoder，避免重复占用显存；随后用 getResults 与 GetLayerOutput 做四道维度断言：隐空间数须等于 LatentCount，编码器第 0 层输出须等于 HistoryBars * BarDescr，两个 Critic 输入须等于 LatentCount。 这些数字不吻合会在日志抛出 'scope does not match' 或 'Input size doesn't match' 并回 INIT_FAILED——实盘前务必确认 HistoryBars=120、BarDescr=5 这类宏与训练时一致，否则 EA 起不来。外汇与贵金属杠杆高，模型加载异常时勿手动强制跳过校验。
MQL5 / C++
CNet Critic2; class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert initialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- load models class="type">float temp; if(!StateEncoder.Load(FileName + "Enc.nnw", temp, temp, temp, dtStudied, true) || !Critic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !Critic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true)) { Print("Init new models"); CArrayObj *actor = new CArrayObj(); CArrayObj *critic = new CArrayObj(); CArrayObj *encoder = new CArrayObj(); if(!CreateDescriptions(actor, critic, encoder)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete encoder; class="kw">return INIT_FAILED; } if(!Critic1.Create(critic) || !Critic2.Create(critic) || !StateEncoder.Create(encoder)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete encoder; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete critic; class="kw">delete encoder; class=class="str">"cmt">//--- } class=class="str">"cmt">//--- OpenCL = Critic1.GetOpenCL(); Critic2.SetOpenCL(OpenCL); StateEncoder.SetOpenCL(OpenCL); class=class="str">"cmt">//--- StateEncoder.getResults(Result); if(Result.Total() != LatentCount) { PrintFormat("The scope of the State Encoder does not match the latent size count(%d <> %d)", LatentCount, Result.Total()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- StateEncoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of State Encoder doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Critic1.GetLayerOutput(class="num">0, Result); if(Result.Total() != LatentCount) { PrintFormat("Input size of Critic1 doesn&class="macro">#x27;t match State Encoder output(%d <> %d)", Result.Total(), LatentCount); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Critic2.GetLayerOutput(class="num">0, Result); if(Result.Total() != LatentCount) {
◍ 初始化失败与网络落盘的细节
这段 MT5 智能交易系统的片段，暴露了两个工程上容易踩的点：初始化阶段若 Critic2 输入尺寸和状态编码器输出对不上，会直接 PrintFormat 报出具体维度差异并 return INIT_FAILED，这种显式维度校验能避免后续训练时张量形状崩溃。 OnDeinit 里只在 reason 不是 REASON_INITFAILED 和 REASON_RECOMPILE 时才把 StateEncoder、Critic1、Critic2 存成 .nnw 文件，意味着重编译或初始化失败不会覆盖上一轮权重；三个网络分别用 getRecentAverageError() 记录近期平均误差，方便你对比训练漂移。 Train 函数开头用 GetProbTrajectories(Buffer, 0.9) 拿到概率轨迹，循环里用 MathRand() 的平方除以 32767 的平方做非线性采样，把索引 i 限制在 Buffer[tr].Total - 3 以内，若 i<0 就回退一次迭代。外汇与贵金属市场高杠杆、高波动，这类强化学习模型在历史数据上表现不代表实盘概率优势，开 MT5 把这段接进自己的 EA 框架，先跑 Init 事件看维度报错有没有触发。
MQL5 / C++
PrintFormat("Input size of Critic2 doesn&class="macro">#x27;t match State Encoder output(%d <> %d)", Result.Total(), LatentCount); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Gradient.BufferInit(AccountDescr, class="num">0); class=class="str">"cmt">//--- if(!EventChartCustom(ChartID(), class="num">1, class="num">0, class="num">0, "Init")) { PrintFormat("Error of create study event: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- class="kw">return(INIT_SUCCEEDED); } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert deinitialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void OnDeinit(const class="type">int reason) { class=class="str">"cmt">//--- if(!(reason == REASON_INITFAILED || reason == REASON_RECOMPILE)) { StateEncoder.Save(FileName + "Enc.nnw", class="num">0, class="num">0, class="num">0, TimeCurrent(), true); Critic1.Save(FileName + "Crt1.nnw", Critic1.getRecentAverageError(), class="num">0, class="num">0, TimeCurrent(), true); Critic2.Save(FileName + "Crt2.nnw", Critic2.getRecentAverageError(), class="num">0, class="num">0, TimeCurrent(), true); } class="kw">delete Result; class="kw">delete OpenCL; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Train function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void Train(class="type">void) { class=class="str">"cmt">//--- vector<class="type">float> probability = GetProbTrajectories(Buffer, class="num">0.9); vector<class="type">float> rewards, rewards1, rewards2, target_reward; class="type">uint ticks = GetTickCount(); class=class="str">"cmt">//--- for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++) { class="type">int tr = SampleTrajectory(probability); class="type">int i = (class="type">int)((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">3)); if(i < class="num">0) { iter--; class="kw">continue; } class=class="str">"cmt">//--- Q-function study State.AssignArray(Buffer[tr].States[i].state); class="type">float PrevBalance = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">0]; class="type">float PrevEquity = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">1]; Account.Clear();
账户特征与时间周期的编码注入
这段逻辑把每笔状态的账户维度逐个塞进 Account 容器，前 8 项分别是余额变动率、当前余额占比、权益变动率、自由保证金、预付款、手续费占比等，全部除以 PrevBalance 做归一化，避免不同资金规模下量纲撕裂。 时间特征用了 4 个三角函数：以 2023.01.01 到 2024.01.01 的秒差（约 31536000 秒）为年周期，再叠加月线、周线、日线的 PeriodSeconds 周期，分别算 MathSin / MathCos。这样把绝对时间戳转成 [−1,1] 的循环向量，模型能感知季节节律而非单纯递增数字。 喂完特征后立刻走 StateEncoder.feedForward，失败就 PrintFormat 打函数名和行号并 break；随后 Actions 和双 Critic 网络同步前向，rewards 用 Buffer 里 i+1 与 i+2 的奖励差乘折扣因子 DiscFactor 做时序修正。外汇与贵金属杠杆高，这类特征工程只解决输入尺度，不预示任何收益。
MQL5 / C++
Account.Add((Buffer[tr].States[i].account[class="num">0] - PrevBalance) / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">1] / PrevBalance); Account.Add((Buffer[tr].States[i].account[class="num">1] - PrevEquity) / PrevEquity); Account.Add(Buffer[tr].States[i].account[class="num">2]); Account.Add(Buffer[tr].States[i].account[class="num">3]); Account.Add(Buffer[tr].States[i].account[class="num">4] / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">5] / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">6] / PrevBalance); class="type">class="kw">double time = (class="type">class="kw">double)Buffer[tr].States[i].account[class="num">7]; class="type">class="kw">double x = time / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = time / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); Account.Add((class="type">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = time / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = time / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); if(Account.GetIndex() >= class="num">0) Account.BufferWrite(); class=class="str">"cmt">//--- if(!StateEncoder.feedForward(GetPointer(State), class="num">1, false, GetPointer(Account))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } class=class="str">"cmt">//--- Actions.AssignArray(Buffer[tr].States[i].action); if(Actions.GetIndex() >= class="num">0) Actions.BufferWrite(); class=class="str">"cmt">//--- if(!Critic1.feedForward(GetPointer(StateEncoder), -class="num">1, GetPointer(Actions)) || !Critic2.feedForward(GetPointer(StateEncoder), -class="num">1, GetPointer(Actions))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } class=class="str">"cmt">//--- Critic1.getResults(rewards1); Critic2.getResults(rewards2); rewards.Assign(Buffer[tr].States[i + class="num">1].rewards); target_reward.Assign(Buffer[tr].States[i + class="num">2].rewards); rewards = rewards - target_reward * DiscFactor; Result.AssignArray(CAGrad(rewards - rewards1) + rewards1); if(!Critic1.backProp(Result, GetPointer(Actions), GetPointer(Gradient)) ||
「双评论家网络的回传与训练监控」
这段逻辑跑在强化学习训练循环里，Critic1 与 Critic2 各自独立做反向传播，任何一次 backProp 或状态编码器的梯度回传失败，就立刻 PrintFormat 打出函数名和行号然后 break，避免脏梯度污染权重。 训练进度不是每一步都刷屏，而是用 GetTickCount 做了节流：距上次打印超过 500 毫秒才更新 Comment，显示两个 Critic 的近期平均误差，格式精确到 15.8f。默认 Iterations=10000、BatchSize=256，意味着全量跑完可能要上万次 tick 累加，MT5 策略测试器里开非可视化模式更稳。 循环结束后清空 Comment，并把 Critic1 / Critic2 的最终平均误差用 PrintFormat 落到日志（%10.7f 精度），随后调 ExpertRemove 自卸载——说明这是一次性离线训练 EA，不是常驻实盘策略。外汇与贵金属杠杆高，这类自训练模型仅用于回测研究，实盘接入前须重做样本外验证。
MQL5 / C++
if(!StateEncoder.backPropGradient(GetPointer(Account), GetPointer(Gradient))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } Result.AssignArray(CAGrad(rewards - rewards2) + rewards2); if(!Critic2.backProp(Result, GetPointer(Actions), GetPointer(Gradient)) || !StateEncoder.backPropGradient(GetPointer(Account), GetPointer(Gradient))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } class=class="str">"cmt">//--- if(GetTickCount() - ticks > class="num">500) { class="type">class="kw">string str = StringFormat("%-15s %class="num">5.2f%% -> Error %class="num">15.8f\n", "Critic1", iter * class="num">100.0 / (class="type">class="kw">double)(Iterations), Critic1.getRecentAverageError()); str += StringFormat("%-15s %class="num">5.2f%% -> Error %class="num">15.8f\n", "Critic2", iter * class="num">100.0 / (class="type">class="kw">double)(Iterations), Critic2.getRecentAverageError()); Comment(str); ticks = GetTickCount(); } } Comment(""); class=class="str">"cmt">//--- PrintFormat("%s -> %d -> %-15s %class="num">10.7f", __FUNCTION__, __LINE__, "Critic1", Critic1.getRecentAverageError()); PrintFormat("%s -> %d -> %-15s %class="num">10.7f", __FUNCTION__, __LINE__, "Critic2", Critic2.getRecentAverageError()); ExpertRemove(); class=class="str">"cmt">//--- } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Input parameters | class=class="str">"cmt">//+------------------------------------------------------------------+ input class="type">int Iterations = class="num">10000; input class="type">int BatchSize = class="num">256; CNet Actor; CNet Critic1; CNet Critic2; CNet StateEncoder; class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert initialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED;
◍ 加载与维度校验这条链路
EA 初始化收尾阶段先把 StateEncoder、Critic1、Critic2 三个网络从 .nnw 文件读入，文件名统一拼成 FileName+"Enc.nnw" 这类形式。只要任意一个 Load 返回失败，终端会打印 Can't load Critic models 并以 INIT_FAILED 退出，说明模型文件缺失或路径不对。
Actor 若 Load 失败则走新建分支：用 CreateDescriptions 构造 actor/critic 描述数组，再 Actor.Create 建网；任一环节出错同样 INIT_FAILED，且务必 delete 掉临时数组避免内存泄漏。建完网后把 OpenCL 上下文从 Actor 取出并共享给 Critic1、Critic2、StateEncoder，推理阶段统一关掉 TrainMode(false)。
随后做硬维度对齐：Actor 输出节点数须等于 NActions，否则报 scope of the actor does not match the actions count；StateEncoder 第 0 层输出须等于 HistoryBars*BarDescr，这是输入状态的总特征数。取 StateEncoder 实际输出数记为 latent_state，Critic1、Critic2、Actor 的第 0 层输入都必须等于该值，偏差会分别打印 Input size of Critic1/2/Actor doesn't match output State Encoder。最后 Gradient.BufferInit(AccountDescr, 0) 收尾，外汇与贵金属行情高波动，模型维度对不齐时宁可 INIT_FAILED 也不要带错网络上线。
MQL5 / C++
} class=class="str">"cmt">//--- load models class="type">float temp; if(!StateEncoder.Load(FileName + "Enc.nnw", temp, temp, temp, dtStudied, true) || !Critic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !Critic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true)) { Print("Can&class="macro">#x27;t load Critic models"); class="kw">return INIT_FAILED; } if(!Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true)) { Print("Init new models"); CArrayObj *actor = new CArrayObj(); CArrayObj *critic = new CArrayObj(); if(!CreateDescriptions(actor, critic, critic)) { class="kw">delete actor; class="kw">delete critic; class="kw">return INIT_FAILED; } if(!Actor.Create(actor)) { class="kw">delete actor; class="kw">delete critic; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete critic; } OpenCL = Actor.GetOpenCL(); Critic1.SetOpenCL(OpenCL); Critic2.SetOpenCL(OpenCL); StateEncoder.SetOpenCL(OpenCL); class=class="str">"cmt">//--- StateEncoder.TrainMode(false); Critic1.TrainMode(false); Critic2.TrainMode(false); class=class="str">"cmt">//--- Actor.getResults(Result); if(Result.Total() != NActions) { PrintFormat("The scope of the actor does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } StateEncoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of State Encoder doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } StateEncoder.getResults(Result); class="type">int latent_state = Result.Total(); Critic1.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Critic1 doesn&class="macro">#x27;t match output State Encoder(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Critic2.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Critic2 doesn&class="macro">#x27;t match output State Encoder(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Actor.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Actor doesn&class="macro">#x27;t match output State Encoder(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Gradient.BufferInit(AccountDescr, class="num">0); class=class="str">"cmt">//---
常见问题
先核对模型文件维度与 EA 里声明的输入层是否一致，再看初始化日志里落盘的网络文件是否完整，缺维度校验就会直接中断加载。
一般取账户杠杆、余额比例、当前周期代号做归一化后拼到状态向量尾部，避免用未缩放的绝对数值，否则 critic 回传容易发散。
可以，小布能定时拉取训练监控指标并标出回传异常，你打开对应品种页就能看到双 critic 的loss曲线和维度校验提示。
维度不一致会导致矩阵乘法越界或静默出错，校验能在 EA 初始化阶段就拦截错误并触发网络重新落盘。
初始化问题通常首次加载就报错或 loss 为 NaN，学习率问题多是前几百步正常后缓慢发散，分开看监控日志即可定位。