🧠
神经网络变得简单（第 69 部分）：基于密度的行为政策支持约束（SPOT）·进阶篇
（2/3）·离线训练里 Critic 对分布外动作的高估，可能让智能体反复踩坑，SPOT 用 VAE 密度把政策拉回训练集支持集
离线强化学习用静态数据集训模型，省了和环境反复试错的成本，但智能体一旦跑出训练集分布，Critic 给的估值就不可信。高估比低估更危险，会让策略在实盘里重复危险动作。SPOT 直接拿训练集密度做约束，把政策钉在支持集内。
本章目录
「解码器堆叠与模型冷启动的坑」
上面这段把 CVAE 的解码器后半段拼完了：第三层用 LReLU 激活、ADAM 优化，节点数等于 LatentCount；第四层切到 SIGMOID，节点数等于 NActions，同样挂 ADAM。任何一层 Add 失败就 delete 描述符并回 false，避免内存悬挂。 OnInit 里先 LoadTotalBase 读训练数据，失败直接 INIT_FAILED 并打印错误码。若 Encoder.Load / Decoder.Load 找不到现成 nnw 权重文件，就现场 CreateCVAEDescriptions 建结构再 Create，相当于从零初始化一个新 CVAE——这条分支在第一次挂 EA 时必然走到。 加载完后做尺寸校验：Encoder 第 0 层输出总数必须等于 HistoryBars * BarDescr，否则报「Input size doesn't match」并 INIT_FAILED。实测若 HistoryBars=128、BarDescr=5，期望输入维度为 640，改任一参数都要同步训练端，否则 EA 起不来。外汇与贵金属市场高波动，模型冷启动不代表实盘倾向盈利，先跑历史回测验证。 OpenCL 上下文从 Encoder 取出后喂给 Decoder 共用，保证前后向计算在同一设备队列。最后比对 Encoder 潜变量维度和 Decoder 第 0 层输入，不一致就拦在初始化阶段，省得后面推理时才崩。
MQL5 / C++
class="kw">return false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!decoder.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = NActions; descr.optimization = ADAM; descr.activation = SIGMOID; if(!decoder.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- class="kw">return true; } CNet Encoder; CNet Decoder; class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert initialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- load models class="type">float temp; if(!Encoder.Load(FileName + "Enc.nnw", temp, temp, temp, dtStudied, true) || !Decoder.Load(FileName + "Dec.nnw", temp, temp, temp, dtStudied, true)) { Print("Init new CVAE"); CArrayObj *encoder = new CArrayObj(); CArrayObj *decoder = new CArrayObj(); if(!CreateCVAEDescriptions(encoder,decoder)) { class="kw">delete encoder; class="kw">delete decoder; class="kw">return INIT_FAILED; } if(!Encoder.Create(encoder) || !Decoder.Create(decoder)) { class="kw">delete encoder; class="kw">delete decoder; class="kw">return INIT_FAILED; } class="kw">delete encoder; class="kw">delete decoder; } OpenCL = Encoder.GetOpenCL(); Decoder.SetOpenCL(OpenCL); Encoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of Encoder doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Encoder.getResults(Result); class="type">int latent_state = Result.Total(); Decoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) {
编码器解码器对齐与训练落盘
EA 初始化时若解码器输入维度与编码器输出不一致，会直接打印 'Input size of Decoder doesn't match result of Encoder (%d <> %d)' 并返回 INIT_FAILED，这里 %d 分别是 Result.Total() 与 latent_state 的实际值，开 MT5 跑前务必确认两边张量尺寸相等。 图表事件用 EventChartCustom(ChartID(), 1, 0, 0, "Init") 触发，失败则取 GetLastError() 报 'Error of create study event: %d' 并终止加载，这种错在自定义指标未就绪时容易出现。 反初始化里 Encoder.Save 与 Decoder.Save 会把权重写到 FileName+'Enc.nnw' / 'Dec.nnw'，其中 Decoder 存盘带 getRecentAverageError() 作为误差记录，训练中断也能续上。 Train 函数用 MathRand() 平方再除以 32767 平方做偏向小值的随机采样，挑样本 tr 与帧 i；若 Encoder.feedForward 或 Decoder.feedForward 任一返回 false，就打印函数名加行号并 break，这种断点在外汇与贵金属高频重训时可能频繁触发，属高波动品种常见风险。
MQL5 / C++
PrintFormat("Input size of Decoder doesn&class="macro">#x27;t match result of Encoder(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } if(!EventChartCustom(ChartID(), class="num">1, class="num">0, class="num">0, "Init")) { PrintFormat("Error of create study event: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- class="kw">return(INIT_SUCCEEDED); } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert deinitialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void OnDeinit(const class="type">int reason) { class=class="str">"cmt">//--- Encoder.Save(FileName + "Enc.nnw", class="num">0, class="num">0, class="num">0, TimeCurrent(), true); Decoder.Save(FileName + "Dec.nnw", Decoder.getRecentAverageError(), class="num">0, class="num">0, TimeCurrent(), true); class="kw">delete Result; class="kw">delete OpenCL; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Train function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">void Train(class="type">void) { class="type">int total_tr = ArraySize(Buffer); class="type">uint ticks = GetTickCount(); class="type">int bar = (HistoryBars - class="num">1) * BarDescr; for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++) { class="type">int tr = class="type">int((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (total_tr)); class="type">int i = class="type">int((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">2)); if(i < class="num">0) class="kw">continue; State.AssignArray(Buffer[tr].States[i].state); Actions.AssignArray(Buffer[tr].States[i].action); if(Actions.GetIndex() >= class="num">0) Actions.BufferWrite(); if(!Encoder.feedForward(GetPointer(State), class="num">1,false, GetPointer(Actions)) || !Decoder.feedForward(GetPointer(Encoder), -class="num">1, GetPointer(Encoder),class="num">1)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } if(!Decoder.backProp(GetPointer(Actions), GetPointer(Encoder), class="num">1) ||
◍ CVAE与Actor-Critic网络的初始化落点
这段片段展示了在 MT5 专家顾问里把训练好的 CVAE（编码器/解码器）和 SAC 风格策略网络一次性挂进内存的做法。Encoder、Decoder 分别读取 Enc.nnw、Dec.nnw，Actor 与双 Critic 网络（含 Target 副本）则从 Act.nnw、Crt1.nnw、Crt2.nnw 载入；任一带权文件缺失时 OnInit 直接返回 INIT_FAILED，EA 不会偷偷用随机权重跑实盘。 训练循环里有个硬超时保护：用 GetTickCount() 减去起始 ticks，一旦超过 500 毫秒就通过 Comment 把 Decoder 的近期平均误差和迭代进度（iter * 100.0 / Iterations）刷到图表左上角，避免主线程被反向传播卡死。回测时你能直接看到误差从初始量级往下降，若 500ms 内没动静说明这批样本计算量偏大，可考虑减隐层或降批大小。 若 Encoder.backPropGradient 返回失败，代码会打印函数名与行号并 break 跳出迭代，随后 ExpertRemove() 结束 EA——这意味着权重没更新成功就别指望它继续交易。外汇与贵金属行情下用这类自编码+强化学习结构，参数敏感度高，过拟合可能导致样本外信号反转，务必先在策略测试器用历史数据验证。
MQL5 / C++
if(!Encoder.backPropGradient(GetPointer(Actions), GetPointer(Actions))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } if(GetTickCount() - ticks > class="num">500) { class="type">class="kw">string str = StringFormat("%-15s %class="num">5.2f%% -> Error %class="num">15.8f\n", "Decoder", iter * class="num">100.0 / (class="type">class="kw">double)(Iterations), Decoder.getRecentAverageError()); Comment(str); ticks = GetTickCount(); } } Comment(""); class=class="str">"cmt">//--- PrintFormat("%s -> %d -> %-15s %class="num">10.7f", __FUNCTION__, __LINE__, "Decoder", Decoder.getRecentAverageError()); ExpertRemove(); class=class="str">"cmt">//--- } STrajectory Buffer[]; CNet Actor; CNet Critic1; CNet Critic2; CNet TargetCritic1; CNet TargetCritic2; CNet Convolution; CNet Encoder; CNet Decoder; class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Expert initialization function | class=class="str">"cmt">//+------------------------------------------------------------------+ class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- load models class="type">float temp; if(!Encoder.Load(FileName + "Enc.nnw", temp, temp, temp, dtStudied, true) || !Decoder.Load(FileName + "Dec.nnw", temp, temp, temp, dtStudied, true)) { Print("Cann&class="macro">#x27;t load CVAE"); class="kw">return INIT_FAILED; } if(!Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true) || !Critic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !Critic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true) || !TargetCritic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !TargetCritic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true)) { Print("Init new models"); CArrayObj *actor = new CArrayObj(); CArrayObj *critic = new CArrayObj(); CArrayObj *convolution = new CArrayObj();
「双评论员与目标网络权重的初始化陷阱」
这段初始化逻辑里，Twin Delayed DDPG 的 actor、双 critic 与卷积编码器是分开构件的。只要 CreateDescriptions 或任意网络的 Create 失败，就立刻 delete 三个容器指针并返回 INIT_FAILED，避免半残对象挂在内存里。 注意 Actor.Create 之后还要分别建 Critic1、Critic2 以及 TargetCritic1、TargetCritic2，任何一环返回 false 都会走同一套清理。最后用 WeightsUpdate(GetPointer(Critic1), 1.0f) 把在线 critic 权重整份拷给目标网络，系数 1.0 表示硬更新而非软更新。 卷积部分若 Load 失败（文件名 + "CNN.nnw" 读不到），会重新 new 三个 CArrayObj 并只建 Convolution，此时 actor/critic 容器建完即删，不进编码器训练。 所有网络都要 SetOpenCL 指向同一上下文，否则 Encoder/Decoder 与 actor 不在同设备会直接推理报错。最后 Actor.getResults(Result) 后必须校验 Result.Total() == NActions，不一致就 INIT_FAILED——这是 EA 加载时最容易漏掉的动作数对齐检查。外汇与贵金属杠杆高，这类初始化失败会让策略完全不触发，实盘前务必在 MT5 策略测试器看 Init 日志。
MQL5 / C++
if(!CreateDescriptions(actor, critic, convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } if(!Actor.Create(actor) || !Critic1.Create(critic) || !Critic2.Create(critic) || !Convolution.Create(convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } if(!TargetCritic1.Create(critic) || !TargetCritic2.Create(critic)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class=class="str">"cmt">//--- TargetCritic1.WeightsUpdate(GetPointer(Critic1), class="num">1.0f); TargetCritic2.WeightsUpdate(GetPointer(Critic2), class="num">1.0f); StartTargetIter = StartTargetIteration; } else StartTargetIter = class="num">0; if(!Convolution.Load(FileName + "CNN.nnw", temp, temp, temp, dtStudied, true)) { Print("Init new Encoder model"); CArrayObj *actor = new CArrayObj(); CArrayObj *critic = new CArrayObj(); CArrayObj *convolution = new CArrayObj(); if(!CreateDescriptions(actor, critic, convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } if(!Convolution.Create(convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; } OpenCL = Actor.GetOpenCL(); Critic1.SetOpenCL(OpenCL); Critic2.SetOpenCL(OpenCL); TargetCritic1.SetOpenCL(OpenCL); TargetCritic2.SetOpenCL(OpenCL); Convolution.SetOpenCL(OpenCL); Encoder.SetOpenCL(OpenCL); Decoder.SetOpenCL(OpenCL); Encoder.TrainMode(false); Decoder.TrainMode(false); Actor.getResults(Result); if(Result.Total() != NActions) { PrintFormat("The scope of the actor does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } Actor.GetLayerOutput(class="num">0, Result);
神经网络层维度对齐的初始化校验
在 MT5 里用强化学习框架搭交易智能体时，最容易被忽略却最致命的一步是各网络层输入输出的维度对齐。上面这段初始化代码在 OnInit 阶段连续做了多组断言：Actor 的输入维度必须等于 HistoryBars * BarDescr，Critic1 的首层输出必须匹配 Actor 的隐状态维度 latent_state，Decoder 的输出必须严格等于动作数 NActions，Encoder 的输入也要等于状态描述长度。任何一处对不上，直接 return INIT_FAILED，EA 连加载都过不了。 维度错位往往不是写错公式，而是 HistoryBars 或 BarDescr 在外部被改了没同步。比如你把回看 bar 数从 50 调到 100，但网络结构文件没重训，Result.Total() 就会报出 (100*BarDescr <> 50*BarDescr) 这种差异，PrintFormat 会把实际值和期望值都打在日志里，照着改参数即可。 校验全过之后，代码用 EventChartCustom 往图表抛了一个自定义事件 "Init"，失败同样返回 INIT_FAILED。最后 return INIT_SUCCEEDED 才意味着智能体框架在当前品种上完成了装配——外汇与贵金属杠杆高，实盘前务必在策略测试器用历史数据跑通这套初始化，避免实盘加载直接崩。
MQL5 / C++
if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of Actor doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } Actor.GetLayerOutput(LatentLayer, Result); class="type">int latent_state = Result.Total(); Critic1.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Critic doesn&class="macro">#x27;t match latent state Actor(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Decoder.getResults(Result); if(Result.Total() != NActions) { PrintFormat("The scope of the Decoder does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } Encoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of Encoder doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } Encoder.getResults(Result); latent_state = Result.Total(); Decoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Decoder doesn&class="macro">#x27;t match result of Encoder(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Gradient.BufferInit(AccountDescr, class="num">0); class=class="str">"cmt">//--- if(!EventChartCustom(ChartID(), class="num">1, class="num">0, class="num">0, "Init")) { PrintFormat("Error of create study event: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- class="kw">return(INIT_SUCCEEDED); } class="type">void Train(class="type">void) { class="type">int total_tr = ArraySize(Buffer); class="type">uint ticks = GetTickCount(); class=class="str">"cmt">//--- class="type">int total_states = Buffer[class="num">0].Total; for(class="type">int i = class="num">1; i < total_tr; i++) total_states += Buffer[i].Total; vector<class="type">float> temp, next; Convolution.getResults(temp); matrix<class="type">float> state_embedding = matrix<class="type">float>::Zeros(total_states, temp.Size()); matrix<class="type">float> rewards = matrix<class="type">float>::Zeros(total_states, NRewards); matrix<class="type">float> actions = matrix<class="type">float>::Zeros(total_states, NActions); class="type">int state = class="num">0; for(class="type">int tr = class="num">0; tr < total_tr; tr++) { for(class="type">int st = class="num">0; st < Buffer[tr].Total; st++) { State.AssignArray(Buffer[tr].States[st].state); class="type">float PrevBalance = Buffer[tr].States[MathMax(st - class="num">1, class="num">0)].account[class="num">0];
◍ 把账户状态喂给卷积网络前的特征拼装
这段逻辑干的事很直接：把每一帧的账户快照换算成相对变化率，再叠一圈周期相位编码，最后塞进卷积层做状态嵌入。外汇与贵金属杠杆高，这类特征若直接用于实盘信号，回测漂移可能很大，只建议在 MT5 策略测试器里先验证数值稳定性。 PrevEquity 取上一状态的权益（账户[1]），用 MathMax 兜底 st-1 不越界；随后连续 Add 了余额收益率、权益绝对值比、权益差值率，以及账户[2]~[6]除以 PrevBalance 的缩放值。注意账户[2]、[3]没除基线，是裸值进向量。 周期相位那几行才是重点：账户[7]疑似累计tick或时长，分别除以 2023 全年秒数、月线秒数、周线秒数、日线秒数，再乘 2π 做 sin/cos。年周期用 sin，月用 cos，周和日用 sin，等于手动铺了四路周期基底，让网络能嗅到季节节律。 拼完 State 后补一段零向量占位动作维度，卷积 feedForward 失败就打印行号并 ExpertRemove 自毁；通过后把结果写进 state_embedding，奖励行用 temp - next*DiscFactor 算折扣回报。每 500ms 用 GetTickCount 切一下，避免历史回放卡死主线程。
MQL5 / C++
class="type">float PrevEquity = Buffer[tr].States[MathMax(st - class="num">1, class="num">0)].account[class="num">1]; State.Add((Buffer[tr].States[st].account[class="num">0] - PrevBalance) / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">1] / PrevBalance); State.Add((Buffer[tr].States[st].account[class="num">1] - PrevEquity) / PrevEquity); State.Add(Buffer[tr].States[st].account[class="num">2]); State.Add(Buffer[tr].States[st].account[class="num">3]); State.Add(Buffer[tr].States[st].account[class="num">4] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">5] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); State.Add((class="type">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); State.AddArray(vector<class="type">float>::Zeros(NActions)); if(!Convolution.feedForward((CBufferFloat *)GetPointer(State), class="num">1, false, (CBufferFloat *)NULL)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">return; } Convolution.getResults(temp); if(!state_embedding.Row(temp, state)) class="kw">continue; if(!temp.Assign(Buffer[tr].States[st].action) || !actions.Row(temp, state)) class="kw">continue; if(!temp.Assign(Buffer[tr].States[st].rewards) || !next.Assign(Buffer[tr].States[st + class="num">1].rewards) || !rewards.Row(temp - next * DiscFactor, state)) class="kw">continue; state++; if(GetTickCount() - ticks > class="num">500)
让小布替你跑这套密度诊断
把 SPOT 的 CVAE 密度估算接进小布盯盘的工作流，打开对应品种页就能看到策略是否滑出训练集支持边界，你只管判断要不要切回在线微调。
常见问题
低估只会让模型放弃某些动作、学到次优政策；高估会诱使模型反复执行分布外动作，在缺乏环境反馈时直接放大实盘损失概率，外汇贵金属杠杆下风险更高。
小布盯盘的 AIGC 模块可内置密度边界诊断，实时标注智能体动作相对训练集支持集的偏移度，省去手动跑 VAE 估算的重复劳动。
CVAE 是条件变分自动编码器，在密度估算时额外 conditioning 于状态，从而学到状态依赖的行为策略密度，比无条件 VAE 更贴合离线数据集的支持定义。
不冲突。SPOT 从密度支持约束理论直接推导正则项，可叠加到现成 RL 算法；KL 惩罚若不符合基于密度的支持定义，避分布外效果会弱，两者可互补。
倾向先在线优调。SPOT 设计允许离线预训练后接在线微调，直接实盘可能因环境漂移导致政策偏移，贵金属高波动期尤甚。