🧠
神经网络变得简单（第 57 部分）：随机边际扮演者-评论者（SMAC）·进阶篇
（2/3）· 最大熵强化学习在部分可观测环境里常卡在熵估值失真，本篇拆解随机边际扮演者-评论者的工程解法
接上篇，我们继续深挖强化学习里那个绕不开的坑：用潜在变量提升政策表现力之后，熵没法简单算，朴素估值直接把优化带崩。很多自行改写的扮演者-评论者系统，在部分可观测行情里越训越窄，本质是随机更新方差没压住。
◍ 用奇异值散度给隐状态熵打分
在 MT5 的强化学习策略里，critic 网络第 2、3 层都按 LatentCount 个 defNeuronBaseOCL 神经元搭起，激活用 LReLU、优化用 ADAM；第 4 层把神经元数换成 NRewards 且激活置 None，直接输出原始价值估计。这几层若 Add 失败就 delete descr 并 return false，避免野指针拖垮整个模型。 衡量隐空间是否坍塌，可以借奇异值之和与矩阵元素模长总量的比值。下面这段 Python 原型用 20×10 的正态样本，按 2^(-k/2) 的尺度因子 k∈[0,19] 逐步收缩标准差，对每个缩放后矩阵做 SVD 取奇异值和，再除以元素平方和开方乘最大维数，得到 reward 序列。画出来横轴是 STD multiplier 的对数轴（底 2），纵轴 Reward，曲线随尺度变小而上扬，说明噪声压低后结构信号更集中。 MQL5 侧用 EntropyLatentState 把这套思路落到实盘前验证。先以 MathRandomNormal 抽 LatentCount*32 个标准正态填成 32 行隐状态矩阵，从 LatentLayer-1 层取均值与标准差做 CumSum 后 Vsplit 成两半，重构出 states = states*std + mean。 再把当前隐状态追到矩阵末行，做 SVD 后按 temp.Sum() / sqrt(pow(states,2).Sum() * max(33, LatentCount)) 算熵。外汇与贵金属杠杆高、跳空频繁，该值偏低可能预示策略隐空间过拟合，上真仓前建议在 MT5 策略测试器跑一轮观察分布。
MQL5 / C++
class="macro">#define LatentLayer class="num">7 # Import libraries class="kw">import numpy as np class="kw">import matplotlib.pyplot as plt mean = np.random.normal(size=[class="num">1,class="num">10]) std = np.random.rand(class="num">1,class="num">10) data = np.random.normal(size=[class="num">20,class="num">10]) reward=np.zeros([class="num">20]) scl = [class="num">2**(-k/class="num">2.0) for k in range(class="num">20)] for idx, k in enumerate(scl): new_data=mean+data*(std*k) _,S,_=np.linalg.svd(new_data) reward[idx]=S.sum()/(np.sqrt(new_data*new_data).sum()*max(new_data.shape)) # Draw results plt.plot(scl,reward) plt.gca().invert_xaxis() plt.ylabel(&class="macro">#x27;Reward&class="macro">#x27;) plt.xlabel(&class="macro">#x27;STD multiplier&class="macro">#x27;) plt.xscale(&class="macro">#x27;log&class="macro">#x27;,base=class="num">2) plt.savefig("graph.png") plt.show() class="macro">#define SamplLatentStates class="num">32 class="type">class="kw">float EntropyLatentState(CNet &net) { class=class="str">"cmt">//--- random values class="type">class="kw">double random[]; Math::MathRandomNormal(class="num">0,class="num">1,LatentCount * SamplLatentStates,random); matrix<class="type">class="kw">float> states; states.Assign(random); states.Reshape(SamplLatentStates,LatentCount); class=class="str">"cmt">//--- get means and std vector<class="type">class="kw">float> temp; matrix<class="type">class="kw">float> stats = matrix<class="type">class="kw">float>::Zeros(SamplLatentStates,class="num">2 * LatentCount); net.GetLayerOutput(LatentLayer - class="num">1,temp); stats.Row(temp,class="num">0); stats=stats.CumSum(class="num">0); matrix<class="type">class="kw">float> split[]; stats.Vsplit(class="num">2,split); class=class="str">"cmt">//--- calculate latent values states = states * split[class="num">1] + split[class="num">0]; class=class="str">"cmt">//--- add current latent value net.GetLayerOutput(LatentLayer,temp); states.Resize(SamplLatentStates + class="num">1,LatentCount); states.Row(temp,SamplLatentStates); class=class="str">"cmt">//--- calculate entropy states.SVD(split[class="num">0],split[class="num">1],temp); class="type">class="kw">float result = temp.Sum() / (MathSqrt(MathPow(states,class="num">2.0f).Sum() * MathMax(SamplLatentStates + class="num">1,LatentCount))); class=class="str">"cmt">//--- class="kw">return result; }
「状态向量里塞了哪些账户特征」
强化学习训练前，要把每笔历史轨迹的状态展开成定长向量。下面这段 Train 函数里，State 数组先承接原始 state，再追加 13 个由 account 数组算出的派生特征，维度直接决定后续嵌入矩阵列数。 账户余额变化率用 (account[0]-PrevBalance)/PrevBalance 表达，权益相对余额的比值用 account[1]/PrevBalance，权益变化率则除 PrevEquity。这几个量把资金曲线的一阶、二阶波动都压进了向量。 时间周期特征用了年、月、周、日四个尺度：account[7] 分别除以 2023 全年 tick 跨度、PERIOD_MN1/W1/D1 的秒数，再套 sin/cos 映射成循环编码。注意年化项里硬编码了 D'2024.01.01'-D'2023.01.01'，回测跨年数据时要改这处否则周期会偏。 别把正态当圣经：这些特征没做标准化，直接喂给 Zeros 初始化的 state_embedding 矩阵，训练初期梯度可能集中在余额量级大的维度上。开 MT5 把 Buffer 结构打印出来，确认 account 数组下标含义再改特征顺序。
MQL5 / C++
class="type">void Train(class="type">void) { class="type">int total_tr = ArraySize(Buffer); class="type">uint ticks = GetTickCount(); class="type">int total_states = Buffer[class="num">0].Total; for(class="type">int i = class="num">1; i < total_tr; i++) total_states += Buffer[i].Total; vector<class="type">class="kw">float> temp, next; Convolution.getResults(temp); matrix<class="type">class="kw">float> state_embedding = matrix<class="type">class="kw">float>::Zeros(total_states,temp.Size()); matrix<class="type">class="kw">float> rewards = matrix<class="type">class="kw">float>::Zeros(total_states,NRewards); class="type">int state = class="num">0; for(class="type">int tr = class="num">0; tr < total_tr; tr++) { for(class="type">int st = class="num">0; st < Buffer[tr].Total; st++) { State.AssignArray(Buffer[tr].States[st].state); class="type">class="kw">float PrevBalance = Buffer[tr].States[MathMax(st,class="num">0)].account[class="num">0]; class="type">class="kw">float PrevEquity = Buffer[tr].States[MathMax(st,class="num">0)].account[class="num">1]; State.Add((Buffer[tr].States[st].account[class="num">0] - PrevBalance) / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">1] / PrevBalance); State.Add((Buffer[tr].States[st].account[class="num">1] - PrevEquity) / PrevEquity); State.Add(Buffer[tr].States[st].account[class="num">2]); State.Add(Buffer[tr].States[st].account[class="num">3]); State.Add(Buffer[tr].States[st].account[class="num">4] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">5] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); State.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); State.Add((class="type">class="kw">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); State.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); State.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0));
卷积网络喂数据时的状态嵌入与采样细节
这段逻辑在做强化学习训练前的状态嵌入：先把当前状态喂给卷积网络做前向传播，失败时直接打印函数名与行号并移除 Expert，避免脏数据继续跑。 Convolution.getResults(temp) 拿到输出后写进 state_embedding 矩阵，奖励差用 temp - next * DiscFactor 计算并写入 rewards 行。DiscFactor 是折扣因子，调大一点会更看重远端奖励，调小则偏重眼前盈亏。 进度显示用 GetTickCount() 做节流，每超过 500 毫秒才刷新一次 Comment，显示 Embedding 完成百分比 = state * 100.0 / total_states。开 MT5 把 500 改成 100，能更密地观察嵌入进度，但会多占一点主线程时间。 外层循环按 MathRand() 随机抽轨迹 tr 和步 i，其中 i 用了 MathRand()*MathRand()/32767^2 的平方分布，让采样更集中在前半段状态。若 i<0 就 iter-- 重抽，保证每条样本有效。外汇与贵金属市场高波动，这类随机采样在实盘回测中可能因滑点产生偏差，结论仅作概率参考。 当 iter >= StartTargetIter 后切换目标网络逻辑，把下一状态的账户变化（余额差/净值比等）塞进 Account 向量，作为目标奖励的一部分。StartTargetIter 设得太早可能让目标不稳定，设太晚则收敛慢，建议从总迭代 10% 起调。
MQL5 / C++
if(!Convolution.feedForward(GetPointer(State),class="num">1,class="kw">false,NULL)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">return; } Convolution.getResults(temp); state_embedding.Row(temp,state); temp.Assign(Buffer[tr].States[st].rewards); next.Assign(Buffer[tr].States[st + class="num">1].rewards); rewards.Row(temp - next * DiscFactor,state); state++; if(GetTickCount() - ticks > class="num">500) { class="type">class="kw">string str = StringFormat("%-15s %class="num">6.2f%%", "Embedding ", state * class="num">100.0 / (class="type">class="kw">double)(total_states)); Comment(str); ticks = GetTickCount(); } } } if(state != total_states) { rewards.Resize(state,NRewards); state_embedding.Reshape(state,state_embedding.Cols()); total_states = state; } vector<class="type">class="kw">float> rewards1, rewards2; class="type">int bar = (HistoryBars - class="num">1) * BarDescr; for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++) { class="type">int tr = (class="type">int)((MathRand() / class="num">32767.0) * (total_tr - class="num">1)); class="type">int i = (class="type">int)((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">2)); if(i < class="num">0) { iter--; class="kw">continue; } target_reward = vector<class="type">class="kw">float>::Zeros(NRewards); reward.Assign(Buffer[tr].States[i].rewards); class=class="str">"cmt">//--- Target TargetState.AssignArray(Buffer[tr].States[i + class="num">1].state); if(iter >= StartTargetIter) { class="type">class="kw">float PrevBalance = Buffer[tr].States[i].account[class="num">0]; class="type">class="kw">float PrevEquity = Buffer[tr].States[i].account[class="num">1]; Account.Clear(); Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">0] - PrevBalance) / PrevBalance); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">1] / PrevBalance); Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">1] - PrevEquity) / PrevEquity); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">2]); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">3]); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">4] / PrevBalance);
◍ 账户特征里的周期相位编码
这段代码把账户状态压进神经网络的输入缓冲，核心是拿持仓天数除以不同时间粒度，再套正弦余弦做成周期性特征。账户索引 5、6 两项先按前一周期余额归一化入栈，相当于把净值波动锚定到基准规模上。 double x = (double)Buffer[tr].States[i + 1].account[7] / (double)(D'2024.01.01' - D'2023.01.01'); 这一行用一整年秒数（31536000 秒）做分母，把账户项 7 折算成「日/年」频率；若 x 非零则喂入 2πx 的正弦值，零则直接填 0，避免除零噪声。 后面三行换 PERIOD_MN1（2592000 秒）、PERIOD_W1（604800 秒）、PERIOD_D1（86400 秒）分别求余弦、正弦、正弦。同一账户指标在月/周/日三种尺度下给出不同相位，模型可能更容易分辨「慢变量」和「快扰动」。外汇与贵金属杠杆高，这类特征若直接接实盘策略，回测过拟合概率不低。 喂完 Account 缓冲后，代码走 Actor 前推、双 Critic 前推，取两者奖励和较小者当 target_reward，再逐元素减掉下一状态的已知奖励、乘折扣因子 DiscFactor，末位塞入潜变量熵。整段没有显式收益假设，只是把「账户节奏」转成可微输入。
MQL5 / C++
Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">5] / PrevBalance); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); Account.Add((class="type">class="kw">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); class=class="str">"cmt">//--- if(Account.GetIndex() >= class="num">0) Account.BufferWrite(); if(!Actor.feedForward(GetPointer(TargetState), class="num">1, class="kw">false, GetPointer(Account))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } class=class="str">"cmt">//--- if(!TargetCritic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) || !TargetCritic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } TargetCritic1.getResults(rewards1); TargetCritic2.getResults(rewards2); if(rewards1.Sum() <= rewards2.Sum()) target_reward = rewards1; else target_reward = rewards2; for(class="type">class="kw">ulong r = class="num">0; r < target_reward.Size(); r++) target_reward -= Buffer[tr].States[i + class="num">1].rewards[r]; target_reward *= DiscFactor; target_reward[NRewards - class="num">1] = EntropyLatentState(Actor); class=class="str">"cmt">//--- Q-function study
「账户特征怎么喂给强化学习网络」
这段逻辑把每根 K 线对应的账户状态，转成一组归一化特征向量再送入 Actor 网络。PrevBalance 和 PrevEquity 取上一根的状态（i-1 越界时回退到 0），用来算当前权益相对昨日的增减幅度，分母都用 PrevBalance 做缩放，避免绝对金额干扰梯度。 Account 容器里依次塞了 8 类数值：余额变化率、权益/前余、权益变化率，以及 account[2]~[6] 除以 PrevBalance 的四项；接着用 account[7]（推测为时间戳）分别除以年、月、周、日周期秒数，取 sin/cos 编码周期位相——例如 D'2024.01.01' - D'2023.01.01' 得到 31536000 秒作为年基准。 特征写满后调 Actor.feedForward 做前向推理，失败就 PrintFormat 断点并 break；Actions 同理缓冲写入。Critic1/Critic2 双网络接收 Actor 与 Actions 做价值估计，任一失败即退出循环，最后用 CAGrad 融合奖励写回 Result。外汇与贵金属杠杆高，这类特征工程只是训练闭环的一小步，实盘前务必在 MT5 策略测试器跑通整段逻辑。
MQL5 / C++
State.AssignArray(Buffer[tr].States[i].state); class="type">class="kw">float PrevBalance = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">0]; class="type">class="kw">float PrevEquity = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">1]; Account.Clear(); Account.Add((Buffer[tr].States[i].account[class="num">0] - PrevBalance) / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">1] / PrevBalance); Account.Add((Buffer[tr].States[i].account[class="num">1] - PrevEquity) / PrevEquity); Account.Add(Buffer[tr].States[i].account[class="num">2]); Account.Add(Buffer[tr].States[i].account[class="num">3]); Account.Add(Buffer[tr].States[i].account[class="num">4] / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">5] / PrevBalance); Account.Add(Buffer[tr].States[i].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[i].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); Account.Add((class="type">class="kw">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); Account.Add((class="type">class="kw">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); if(Account.GetIndex() >= class="num">0) Account.BufferWrite(); if(!Actor.feedForward(GetPointer(State), class="num">1, class="kw">false, GetPointer(Account))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } Actions.AssignArray(Buffer[tr].States[i].action); if(Actions.GetIndex() >= class="num">0) Actions.BufferWrite(); class=class="str">"cmt">//--- if(!Critic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actions)) || !Critic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actions))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } Critic1.getResults(rewards1); Result.AssignArray(CAGrad(reward + target_reward - rewards1) + rewards1);
双评论员网络下的策略回传与账户预测
这段代码跑在强化学习训练循环里，Critic1 与 Critic2 各自做反向传播，任一失败就打印函数名与行号并跳出。两个评论员用对称结构更新梯度，Actor 的策略梯度由 Account 与 LatentLayer 共同驱动，误差扩散路径和常规单评论员架构不一样。 选哪个评论员参与策略学习，看谁的最近平均误差更低：Critic1.getRecentAverageError() 小于等于 Critic2 就取前者，否则用后者。这个细节直接决定后续 feedForward 走的网络实例，回测时若发现策略收敛慢，先打印两者平均误差对比，大概率能定位偏向。 账户推演那步用了 SYMBOL_TRADE_TICK_VALUE_PROFIT 除以 SYMBOL_POINT 把点位换算成金额，再喂给 ForecastAccount 生成 5 维 forecast 向量。当 forecast[3] 和 forecast[4] 同时为 0.0f 时，target_reward[2] 会扣掉下一状态第 bar+6 维状态除以 PrevBalance 的值，相当于对空仓情形做余额占比惩罚。外汇与贵金属杠杆高，这类隐性惩罚项会放大回撤，实盘前务必在 MT5 策略测试器里把 PrevBalance 与状态索引对齐验证。 最后 critic 切到非训练模式做最终 backProp，若失败会重新打开 TrainMode(true) 再 break，保证网络状态不脏。整段没有显式学习率，超参都封在类内部，想调频率只能改上层调用周期。
MQL5 / C++
if(!Critic1.backProp(Result, GetPointer(Actions), GetPointer(Gradient)) || !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient), LatentLayer)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } Critic2.getResults(rewards2); Result.AssignArray(CAGrad(reward + target_reward - rewards2) + rewards2); if(!Critic2.backProp(Result, GetPointer(Actions), GetPointer(Gradient)) || !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient), LatentLayer)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } class=class="str">"cmt">//--- Policy study CNet *critic = NULL; if(Critic1.getRecentAverageError() <= Critic2.getRecentAverageError()) critic = GetPointer(Critic1); else critic = GetPointer(Critic2); Actor.getResults(rewards1); class="type">class="kw">double cl_op = Buffer[tr].States[i + class="num">1].state[bar]; class="type">class="kw">double prof_1l = SymbolInfoDouble(_Symbol, SYMBOL_TRADE_TICK_VALUE_PROFIT) * cl_op / SymbolInfoDouble(_Symbol, SYMBOL_POINT); vector<class="type">class="kw">float> forecast = ForecastAccount(Buffer[tr].States[i].account,rewards1,prof_1l, Buffer[tr].States[i + class="num">1].account[class="num">7]); TargetState.AddArray(forecast); if(!critic.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) || !Convolution.feedForward(GetPointer(TargetState))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); break; } next.Assign(Buffer[tr].States[i + class="num">1].rewards); Convolution.getResults(rewards1); target_reward += KNNReward(KNN,rewards1,state_embedding,rewards) + next * DiscFactor; if(forecast[class="num">3] == class="num">0.0f && forecast[class="num">4] == class="num">0.0f) target_reward[class="num">2] -= (Buffer[tr].States[i + class="num">1].state[bar + class="num">6] / PrevBalance); critic.getResults(reward); reward += CAGrad(target_reward - reward); Result.AssignArray(reward); critic.TrainMode(class="kw">false); if(!critic.backProp(Result, GetPointer(Actor)) || !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); critic.TrainMode(true); break; }
把多模态探索诊断交给小布
这些诊断小布盯盘的 AIGC 已内置，打开对应品种页即可看到潜在变量政策的发散度与局部收敛预警，你专注调参决策。把重复劳动交给小布，你专注决策。
常见问题
潜在变量令政策涵盖多模态场景，其分布无法用简单闭式表达熵；朴素估值会高估或低估随机度，导致灾难性政策退化。
可以，小布盯盘的品种页内置了 AIGC 诊断，能呈现政策在状态空间的发散与坍缩，辅助判断探索是否健康，外汇贵金属波动剧烈仍属高风险。
对扮演者政策和潜在变量函数都用分解高斯，抽样与密度估算成本低，只需在原有算法加一个随机节点 s_t，就能复用既有框架。
原文默认马尔可夫假设，但结构可扩展至非马尔可夫；实践中若观测滞后严重，需自行补历史窗口，训练稳定性可能下降。
它压低了局部随机效应与多模态探索混淆带来的高方差，使评论者梯度更新更稳，关于完整推导见《神经网络变得简单（第 57 部分）：基础概念·基础篇》。