📘
神经网络变得轻松（第四十五部分）：训练状态探索技能·进阶篇
调度器与策略网络的后段层定义
这段代码片段承接前序网络搭建，继续往 scheduler 里塞第 9 到 12 层，并初始化 Actor 网络的前几层。外汇与贵金属品种的强化学习模型对层宽敏感，改一个 count 就可能让回测曲线漂移，MT5 上跑前先确认 NSkills 和 AccountDescr 的实际取值。 scheduler 的第 9 层用 defNeuronVAEOCL、节点数等于 NSkills，无显式激活；第 10、11 层分别是 128 和 256 个节点的 defNeuronBaseOCL，激活函数选 LReLU，优化器统一 ADAM。第 12 层节点数绑定 AccountDescr，激活为 None，用来对接账户状态描述。 Actor 网络先 Clear 再建层：输入层节点数 = NSkills、window=0、无激活；随后 layer 1 是 256 节点 TANH 激活，layer 2 是 256 节点 LReLU 激活，layer 3 开始继续往下接。任何一步 new 失败或 Add 失败都直接 delete 并返回 false，避免悬空指针拖垮 EA 初始化。
MQL5 / C++
if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">2 * NSkills; descr.activation = None; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">9 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronVAEOCL; descr.count = NSkills; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">10 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">128; descr.optimization = ADAM; descr.activation = LReLU; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">11 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">256; descr.optimization = ADAM; descr.activation = LReLU; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">12 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = AccountDescr; descr.optimization = ADAM; descr.activation = None; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Actor actor.Clear(); class=class="str">"cmt">//--- Input layer if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = NSkills; descr.window = class="num">0; descr.activation = None; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">256; descr.optimization = ADAM; descr.activation = TANH; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">256; descr.activation = LReLU; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription()))
◍ Actor 网络层定义与 OnTick 推理链路
在 MT5 里搭强化学习交易框架，网络结构靠 CLayerDescription 逐层堆。上面这段把 Actor 的第 3、4 层塞进去了：第 3 层用 defNeuronBaseOCL 类型、256 个节点、LReLU 激活加 ADAM 优化；第 4 层换成 defNeuronFQF、节点数绑定 NActions、window_out 设 32、同样走 ADAM。任何一层 Add 失败就 delete 描述符并 return false，避免野指针。 行情侧靠 OnTick 驱动，但先用 IsNewBar() 拦截，非新 K 线直接 return，省掉重复计算。新棒出现后把 State1 喂给 Scheduler.feedForward，取 LatentLayer 的输出 Result，再丢进 Actor.feedForward，最后 Actor.getSample() 拿动作编号——这条链路是策略实时决策的最小闭环。 训练函数 Train 里用 MathRand 做回放采样：tr 在 [0, total_tr-1] 均匀抽，i 用两次随机相乘压到 Buffer[tr].Total-2 以内，偏向近期样本。状态向量除了市场特征，还手工拼了三组账户比值——(余额差/前余)、(权益/前余)、(权益差/前权益)，把资金曲线波动也当成观测量送进网络。外汇和贵金属杠杆高，这类自反馈特征可能放大回撤，上实盘前先在历史数据跑通再谈。
MQL5 / C++
class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">256; descr.activation = LReLU; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronFQF; descr.count = NActions; descr.window_out = class="num">32; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert tick function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void OnTick() { class=class="str">"cmt">//--- if(!IsNewBar()) class="kw">return; class=class="str">"cmt">//--- ........ ........ class=class="str">"cmt">//--- if(!Scheduler.feedForward(GetPointer(State1), class="num">1, class="kw">false)) class="kw">return; if(!Scheduler.GetLayerOutput(LatentLayer, Result)) class="kw">return; class=class="str">"cmt">//--- if(!Actor.feedForward(Result, class="num">1, class="kw">false)) class="kw">return; class="type">int act = Actor.getSample(); class=class="str">"cmt">//--- ........ ........ class=class="str">"cmt">//--- } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Train function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void Train(class="type">void) { class="type">int total_tr = ArraySize(Buffer); class="type">uint ticks = GetTickCount(); vector<class="type">class="kw">float> account, reward; class="type">int bar, action; class=class="str">"cmt">//--- for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++) { class="type">int tr = (class="type">int)(((class="type">class="kw">double)MathRand() / class="num">32767.0) * (total_tr - class="num">1)); class="type">int i = (class="type">int)((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">2)); State.AssignArray(Buffer[tr].States[i].state); class="type">class="kw">float PrevBalance = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">0]; class="type">class="kw">float PrevEquity = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">1]; State.Add((Buffer[tr].States[i].account[class="num">0] - PrevBalance) / PrevBalance); State.Add(Buffer[tr].States[i].account[class="num">1] / PrevBalance); State.Add((Buffer[tr].States[i].account[class="num">1] - PrevEquity) / PrevEquity);
「账户状态归一化与单根收益的动作判定」
这段逻辑在每根历史棒上先把账户维度的特征灌进神经网络输入向量 State，其中下标 2、4、5、6、7、8 的账户字段分别除以前序余额 PrevBalance 做归一化，只有 5 和 6（持仓类计数）直接原值入参，避免量纲把反向传播带偏。 计算单根盈亏 prof_1l 时，取下一根状态里的 close-open 差值 cl_op，乘上 SYMBOL_TRADE_TICK_VALUE_PROFIT 再除 SYMBOL_POINT，得到以账户货币计价的点数价值；外汇与贵金属杠杆品种点值随合约变动，跑之前务必在 MT5 符号规格里核对这两个宏返回的是实时值。 动作分支以 prof_1l 的绝对值 5 与 10 为阈值：大于 5 时若低于 10 或账户下标 6 为正则给动作 2（倾向减仓），否则 0；小于 -5 时对称处理给 1 或 2；夹在 ±5 内给 3（观望）。这套硬阈值只是特征工程示例，实盘高波动时段可能频繁触发 3。 每次迭代末尾把新账户状态算出的 9 个比值塞进 Result 做 backProp，任一阶段 Scheduler 返回失败或 IsStopped() 为真就 PrintFormat 打行号并 ExpertRemove 退出，防止半截模型污染全局权重。
MQL5 / C++
State.Add(Buffer[tr].States[i].account[class="num">2] / PrevBalance); State.Add(Buffer[tr].States[i].account[class="num">4] / PrevBalance); State.Add(Buffer[tr].States[i].account[class="num">5]); State.Add(Buffer[tr].States[i].account[class="num">6]); State.Add(Buffer[tr].States[i].account[class="num">7] / PrevBalance); State.Add(Buffer[tr].States[i].account[class="num">8] / PrevBalance); class=class="str">"cmt">//--- bar = (HistoryBars - class="num">1) * BarDescr; class="type">class="kw">double cl_op = Buffer[tr].States[i + class="num">1].state[bar]; class="type">class="kw">double prof_1l = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_VALUE_PROFIT) * cl_op / SymbolInfoDouble(_Symbol, SYMBOL_POINT); PrevBalance = Buffer[tr].States[i].account[class="num">0]; PrevEquity = Buffer[tr].States[i].account[class="num">1]; if(IsStopped()) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">break; } class=class="str">"cmt">//--- if(!Scheduler.feedForward(GetPointer(State), class="num">1, class="kw">false)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">break; } if(prof_1l > class="num">5 ) action = (prof_1l < class="num">10 || Buffer[tr].States[i].account[class="num">6] > class="num">0 ? class="num">2 : class="num">0); else { if(prof_1l < -class="num">5) action = (prof_1l > -class="num">10 || Buffer[tr].States[i].account[class="num">5] > class="num">0 ? class="num">2 : class="num">1); else action = class="num">3; } account = GetNewState(Buffer[tr].States[i].account, action, prof_1l); Result.Clear(); Result.Add((account[class="num">0] - PrevBalance) / PrevBalance); Result.Add(account[class="num">1] / PrevBalance); Result.Add((account[class="num">1] - PrevEquity) / PrevEquity); Result.Add(account[class="num">2] / PrevBalance); Result.Add(account[class="num">4] / PrevBalance); Result.Add(account[class="num">5]); Result.Add(account[class="num">6]); Result.Add(account[class="num">7] / PrevBalance); Result.Add(account[class="num">8] / PrevBalance); if(!Scheduler.backProp(Result)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">break; }
初始化阶段把神经网络模型加载校验做扎实
EA 启动时不急着跑策略，先在 OnInit 里把学习数据和两套神经网络（Scheduler 与 Actor）的模型文件读进来。LoadTotalBase 失败就直接打印错误码并返回 INIT_FAILED，避免带着空底表进场。 模型加载分两层：Scheduler 从 FileName+"Sch.nnw" 读取，Actor 从 FileName+"Act.nnw" 读取；若 Actor 模型不存在，代码现场用 CreateDescriptions 建描述数组再 Actor.Create 造网络，任何一步失败都释放 CArrayObj 并返回 INIT_FAILED。 加载完必须核对输出维度——Actor 结果数要等于 NActions，Scheduler 结果数要等于 AccountDescr，不一致就报错退出。最后 Actor.SetUpdateTarget(MathMax(Iterations/100, 10000)) 把目标更新步数设为迭代次数的百分之一但不少于 1 万步，外汇与贵金属杠杆品种波动剧烈，模型维度对不齐就硬跑大概率产生无效信号。 主循环里每 500 毫秒（GetTickCount 差值 > 500）用 Comment 刷一次 Scheduler 的训练进度与近期平均误差，跑完调 ExpertRemove 卸载并 PrintFormat 留痕，方便你开 MT5 看日志核对误差数量级。
MQL5 / C++
if(GetTickCount() - ticks > class="num">500) { class="type">class="kw">string str = StringFormat("%-15s %class="num">5.2f%% -> Error %class="num">15.8f\n", "Scheduler", iter * class="num">100.0 / (class="type">class="kw">double)(Iterations), Scheduler.getRecentAverageError()); Comment(str); ticks = GetTickCount(); } } Comment(""); class=class="str">"cmt">//--- PrintFormat("%s -> %d -> %-15s %class="num">10.7f", __FUNCTION__, __LINE__, "Scheduler", Scheduler.getRecentAverageError()); ExpertRemove(); class=class="str">"cmt">//--- } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert initialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- load models class="type">class="kw">float temp; if(!Scheduler.Load(FileName + "Sch.nnw", temp, temp, temp, dtStudied, true)) { PrintFormat("Error of load scheduler model: %d", GetLastError()); class="kw">return INIT_FAILED; } if(!Actor.Load(FileName + "Act.nnw", dtStudied, true)) { CArrayObj *actor = new CArrayObj(); CArrayObj *scheduler = new CArrayObj(); if(!CreateDescriptions(actor, scheduler)) { class="kw">delete actor; class="kw">delete scheduler; class="kw">return INIT_FAILED; } if(!Actor.Create(actor)) { class="kw">delete actor; class="kw">delete scheduler; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete scheduler; class=class="str">"cmt">//--- } class=class="str">"cmt">//--- Actor.getResults(Result); if(Result.Total() != NActions) { PrintFormat("The scope of the actor does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } Actor.SetOpenCL(Scheduler.GetOpenCL()); Actor.SetUpdateTarget(MathMax(Iterations / class="num">100, class="num">10000)); class=class="str">"cmt">//--- Scheduler.getResults(Result); if(Result.Total() != AccountDescr) { PrintFormat("The scope of the scheduler does not match the account description(%d <> %d)", AccountDescr, Result.Total()); class="kw">return INIT_FAILED; }
◍ 初始化校验与训练回路的收口动作
EA 初始化收尾时，先取 Actor 第 0 层输出拿到输入维度 inputs，再尝试从 Scheduler 取 LatentLayer 的输出。若两层维度不一致（inputs != Result.Total()），直接打印 'Size of latent layer does not match input size of Actor (%d <> %d)' 并返回 INIT_FAILED，这一步能在加载阶段暴露网络结构错配，而不是等跑单才崩。 通过维度校验后，用 EventChartCustom(ChartID(), 1, 0, 0, "Init") 向图表抛自定义事件；返回 false 时打印 'Error of create study event: %d' 并 INIT_FAILED，成功才 return INIT_SUCCEEDED。外汇与贵金属杠杆高，这类初始化失败若被忽略，实盘可能以错误权重跑单。 训练函数 Train 里，每轮先 Scheduler.GetLayerOutput 取隐层，失败就 ExpertRemove 并 break；接着 Actor.feedForward(Result, 1, false) 做前向，同样失败即自退。之后对 NActions 个动作循环：用 GetNewState 拿 reward，并把 reward[0] 归一化为 reward[0]/PrevBalance - 1.0f、reward[2] 除以 PrevEquity 减 1，说明奖励信号是以余额/净值变化率为基准的。 最后 ActorResult 减去自身最小值做平移，State 赋值为下一状态并追加账户相对余额变化。整段没有显式收益结论，只是把『维度对齐 + 事件通知 + 奖励归一』三件事钉死，你开 MT5 把 LatentLayer 层数改错一个，编译能过但初始化必报那行维度不匹配。
MQL5 / C++
Actor.GetLayerOutput(class="num">0, Result); class="type">int inputs = Result.Total(); if(!Scheduler.GetLayerOutput(LatentLayer, Result)) { PrintFormat("Error of load latent layer %d", LatentLayer); class="kw">return INIT_FAILED; } if(inputs != Result.Total()) { PrintFormat("Size of latent layer does not match input size of Actor(%d <> %d)", Result.Total(), inputs); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- if(!EventChartCustom(ChartID(), class="num">1, class="num">0, class="num">0, "Init")) { PrintFormat("Error of create study event: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- class="kw">return(INIT_SUCCEEDED); } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Train function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void Train(class="type">void) { ........ ........ class=class="str">"cmt">//--- if(!Scheduler.GetLayerOutput(LatentLayer, Result)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">break; } class=class="str">"cmt">//--- if(!Actor.feedForward(Result, class="num">1, class="kw">false)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">break; } Scheduler.getResults(SchedulerResult); ActorResult = vector<class="type">class="kw">float>::Zeros(NActions); for(action = class="num">0; action < NActions; action++) { reward = GetNewState(Buffer[tr].States[i].account, action, prof_1l); reward[class="num">0] = reward[class="num">0] / PrevBalance - class="num">1.0f; reward[class="num">3] = reward[class="num">2] / PrevBalance; reward[class="num">2] = reward[class="num">1] / PrevEquity - class="num">1.0f; reward[class="num">1] /= PrevBalance; reward[class="num">4] /= PrevBalance; reward[class="num">7] /= PrevBalance; reward[class="num">8] /= PrevBalance; reward=MathPow(SchedulerResult - reward, class="num">2.0); ActorResult[action] = -reward.Sum(); } ActorResult = ActorResult - ActorResult.Min(); State.AssignArray(Buffer[tr].States[i+class="num">1].state); State.Add((Buffer[tr].States[i+class="num">1].account[class="num">0] - PrevBalance) / PrevBalance);
常见问题
把调度器单独负责动作节奏，后段层只做状态到动作值的映射，避免在前段混入决策逻辑，代码里用独立类封装两层最稳。
在 OnTick 入口和每层前向计算后打时间戳，重点看归一化与全连接层耗时，单根 K 线内推理应控制在毫秒级。
小布可接入账户与模型状态流，自动提示训练中断或收益异常，你把重复巡检交给它，只复核关键动作就行。
建议先用历史回测标出单根收益的正负 1.5 倍标准差带，穿越上沿才偏多头动作，外汇贵金属波动大需保守些。
加载后必须校验层维度、权重非空与输入归一参数一致，训练回路开头加一次完整性断言，缺一项就终止启动。