📘

神经网络变得轻松（第四十九部分）：软性扮演者-评价者·进阶篇

第 2/3 篇

本章目录

SAC 中 Alpha 对数概率的内核装配
SAC 里 alpha 梯度的核函数拆解
SAC 网络里 Alpha 梯度与日志概率的取数路径
强化学习回放里的状态与奖励构造
双评论员下的奖励重算与账户向量回填
双评论家网络的反向传播分支

「SAC 中 Alpha 对数概率的内核装配」

这段前向逻辑属于 SAC 策略里 alpha 分支的 OpenCL 计算路径：先让基础网络 CNeuronFQF 跑通，再把分位数相关的 cAlphas 前向传播接上，任何一步失败立即返回 false，避免脏数据进 GPU。 global_work_size 直接取 Neurons() 数量，意味着每个神经元分配一个工作项；cSoftMax.Neurons() 除以该值后强转 int 作为分位数计数传参，若你改了网络宽度，这个比值会直接决定内核循环次数。所有 SetArgumentBuffer 调用都绑定了具体输出索引（如 cAlphas.getOutputIndex()、cQuantile2.getOutputIndex()），任一处缓冲区索引错配都会在 printf 里带出 __LINE__，开 MT5 跑时建议把 Experts 日志级别开到全部，方便定位是第几个参数挂了。最后 Execute 用一维偏移 {0} 和上面算出的 size 启动内核；外汇与贵金属市场高杠杆、GPU 推理出错可能导致信号翻转，实盘前务必在策略测试器用历史数据验证该分支返回 true 的稳定性。

MQL5 / C++

if(!CNeuronFQF::feedForward(NeuronOCL))
   class="kw">return class="kw">false;
if(!cAlphas.FeedForward(GetPointer(cQuantile0), cQuantile2.getOutput()))
   class="kw">return class="kw">false;
class="type">uint global_work_offset[class="num">1] = {class="num">0};
class="type">uint global_work_size[class="num">1] = {Neurons()};
if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_alphas, cAlphas.getOutputIndex()))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_log_probs, cLogProbs.GetIndex()))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_outputs, getOutputIndex()))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_probs, cSoftMax.getOutputIndex()))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_quantiles, cQuantile2.getOutputIndex()))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgument(def_k_SAC_AlphaLogProbs, def_k_sac_alp_count_quants, (class="type">int)(cSoftMax.Neurons() / global_work_size[class="num">0])))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.SetArgument(def_k_SAC_AlphaLogProbs, def_k_sac_alp_activation, (class="type">int)activation))
   {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
   }
if(!OpenCL.Execute(def_k_SAC_AlphaLogProbs, class="num">1, global_work_offset, global_work_size))
   {
     printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError());
     class="kw">return class="kw">false;
   }
class=class="str">"cmt">//---
   class="kw">return true;
   }
__kernel class="type">void SAC_AlphaGradients(__global class="type">class="kw">float *outputs,
                                 __global class="type">class="kw">float *gradient,

SAC 里 alpha 梯度的核函数拆解

在 MT5 的 OpenCL 训练路径里，alpha 参数的梯度不是在主线程里算的，而是丢进一个独立 kernel 并行处理。下面这段核函数接收 outputs、log_probs、梯度缓冲和激活类型，逐元素算出 alphas_grad。核函数开头取全局 ID 作为下标 i，先读出当前神经元输出 out，再算基础梯度 grad = -gradient[i] * log_probs[i]。这里的负号来自策略熵损失对 alpha 的反向传播方向。激活分支里，case 0 是 tanh 类：先把 out 夹到 [-1,1]，梯度做 clamp(out+grad, -1,1)-out 后再乘 (1-out^2) 的下界 1e-4；case 1 是 sigmoid 类，夹到 [0,1] 后乘 out*(1-out) 同样保底 1e-4；case 2 是带 0.01 泄漏的 ReLU，负区梯度直接乘 0.01f。外层 calcAlphaGradients 负责把 cAlphas 的输出缓冲和梯度缓冲绑定到 kernel 参数。Neurons() 决定全局工作项数量，任一处 SetArgumentBuffer 失败就 printf 出错函数名和行号并返回 false，开 MT5 跑训练时若终端突然报 kernel 参数错误，优先查这两处绑定。外汇与贵金属杠杆交易高风险，这类自定义神经网络模块仅用于策略研究，实盘前务必在模拟环境验证数值稳定性。

MQL5 / C++

 __global class="type">class="kw">float *log_probs,
 __global class="type">class="kw">float *alphas_grad,
 class="kw">const class="type">int activation
 )
{
 class="kw">const class="type">int i = get_global_id(class="num">0);
 class="type">class="kw">float out = outputs[i];
class=class="str">"cmt">//---
 class="type">class="kw">float grad = -gradient[i] * log_probs[i];
 class="kw">switch(activation)
   {
    case class="num">0:
      out = clamp(out, -class="num">1.0f, class="num">1.0f);
      grad = clamp(grad + out, -class="num">1.0f, class="num">1.0f) - out;
      grad = grad * max(class="num">1 - pow(out, class="num">2), class="num">1.0e-4f);
      class="kw">break;
    case class="num">1:
      out = clamp(out, class="num">0.0f, class="num">1.0f);
      grad = clamp(grad + out, class="num">0.0f, class="num">1.0f) - out;
      grad = grad * max(out * (class="num">1 - out), class="num">1.0e-4f);
      class="kw">break;
    case class="num">2:
      if(out < class="num">0)
        grad = grad * class="num">0.01f;
      class="kw">break;
    class="kw">default:
      class="kw">break;
   }
class=class="str">"cmt">//---
 alphas_grad[i] = grad;
}
class="type">bool CNeuronSoftActorCritic::calcAlphaGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(!OpenCL || !NeuronOCL || !NeuronOCL.getGradient() ||
      !NeuronOCL.getGradientIndex()<class="num">0)
     class="kw">return class="kw">false;
   class="type">uint global_work_offset[class="num">1] = {class="num">0};
   class="type">uint global_work_size[class="num">1] = {Neurons()};
   if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_outputs, cAlphas.getOutputIndex()))
    {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
    }
   if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_alphas_grad, cAlphas.getGradientIndex()))
    {
     printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
     class="kw">return class="kw">false;
    }
   if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_gradient, NeuronOCL.getGradientIndex()))
    {

◍ SAC 网络里 Alpha 梯度与日志概率的取数路径

在 Soft Actor-Critic 的 MT5 实现里，Alpha 梯度不是直接算的，而是借 OpenCL 内核把缓冲区参数逐一绑定后异步执行，再回灌到隐藏层梯度。任何一次 SetArgumentBuffer 或 Execute 返回 false，函数立即退出并打印错误行号，这种失败在显存紧张或参数索引错配时概率不低。 GetLogProbs 只认最后一层且神经元类型必须为 defNeuronSoftActorCritic，否则直接返回 false。也就是说如果你改了输出层结构却忘了同步类型标识，策略网络的 log_probs 会静默取不到，回测里表现为动作分布突然退化。 AlphasGradient 有个反直觉细节：当策略网最后一层不是 SAC 神经元时它返回 true 而非 false，相当于“无事发生”。这容易让调用方误以为梯度已更新，实际 cAlphas.calcHiddenGradients 根本没跑。建网描述里第 9 层用 defNeuronSoftActorCritic、count=NActions、window_out=32、optimization=ADAM，这是动作头的标准配置。想验证的话，把 window_out 从 32 改成 64 重跑，显存占用和梯度收敛速度会有可观测差异。

MQL5 / C++

if(!OpenCL.SetArgumentBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_log_probs, cLogProbs.GetIndex()))
  {
    printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
    class="kw">return class="kw">false;
  }
if(!OpenCL.SetArgument(def_k_SAC_AlphaGradients, def_k_sac_alg_activation, (class="type">int)cAlphas.Activation()))
  {
    printf("Error of set parameter kernel %s: %d; line %d", __FUNCTION__, GetLastError(), __LINE__);
    class="kw">return class="kw">false;
  }
if(!OpenCL.Execute(def_k_SAC_AlphaGradients, class="num">1, global_work_offset, global_work_size))
  {
    printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError());
    class="kw">return class="kw">false;
  }
class="kw">return cAlphas.calcHiddenGradients(GetPointer(cQuantile0), cQuantile2.getOutput(), cQuantile2.getGradient());

class="type">bool CNet::GetLogProbs(vectorf &log_probs)
  {
  if(!layers)
    class="kw">return class="kw">false;
  class="type">int total = layers.Total();
  if(total <= class="num">0 || !layers.At(total - class="num">1))
    class="kw">return class="kw">false;
  CLayer *layer = layers.At(total - class="num">1);
  if(!layer.At(class="num">0) || layer.At(class="num">0).Type() != defNeuronSoftActorCritic)
    class="kw">return class="kw">false;
  CNeuronSoftActorCritic *neuron = layer.At(class="num">0);
  class="kw">return neuron.GetAlphaLogProbs(log_probs);
  }

class="type">bool CNet::AlphasGradient(CNet *PolicyNet)
  {
  if(!PolicyNet || !PolicyNet.layers)
    class="kw">return class="kw">false;
  class="type">int total = PolicyNet.layers.Total();
  if(total <= class="num">0)
    class="kw">return class="kw">false;
  CLayer *layer = PolicyNet.layers.At(total - class="num">1);
  if(!layer || !layer.At(class="num">0))
    class="kw">return class="kw">false;
  if(layer.At(class="num">0).Type() != defNeuronSoftActorCritic)
    class="kw">return true;
  CNeuronSoftActorCritic *neuron = layer.At(class="num">0);
  if(!layers)
    class="kw">return class="kw">false;
  total = layers.Total();
  if(total <= class="num">0 || !layers.At(total - class="num">1))
    class="kw">return class="kw">false;
  layer = layers.At(total - class="num">1);
  class="kw">return neuron.calcAlphaGradients((CNeuronBaseOCL*) layer.At(class="num">0));
  }

class=class="str">"cmt">//--- layer class="num">9
if(!(descr = new CLayerDescription()))
  class="kw">return class="kw">false;
descr.type = defNeuronSoftActorCritic;
descr.count = NActions;
descr.window_out = class="num">32;
descr.optimization = ADAM;

「强化学习回放里的状态与奖励构造」

这段逻辑跑在 Train() 里，核心是从历史回放 Buffer 中随机抽取一条轨迹，再从中切一段 transition 喂给 Actor 做前向传播。抽轨迹用 MathRand()/32767.0 线性映射，抽步长则用 MathRand()*MathRand()/32767^2 的平方分布，偏向靠前的样本，老样本被翻到的概率更低。奖励侧先把下一状态的账户向量算成相对变化：余额变化率、权益/余额、权益变化率，以及第 2~6 号账户特征分别除以 PrevBalance 归一。注意第 0 项用的是 (next_balance-prev_balance)/prev_balance，直接把绝对收益压成比率，避免不同资金规模下梯度尺度漂移。如果 Actor.feedForward 失败，立刻 PrintFormat 打出函数名与行号并 ExpertRemove() 终止 EA，说明这一环不容忍静默降级。外汇与贵金属杠杆品种下，这类自归一奖励对爆仓敏感，回测前先把 Buffer 容量和 Iterations 调小，观察单次 Train 的 GetTickCount 耗时再放大。

MQL5 / C++

  descr.activation = SIGMOID;
  if(!actor.Add(descr))
    {
      class="kw">delete descr;
      class="kw">return class="kw">false;
    }
class=class="str">"cmt">//--- Critic
class=class="str">"cmt">//---
  class="kw">return true;
  }
class="type">void OnTick()
  {
class=class="str">"cmt">//---
  class="type">class="kw">float reward = Account[class="num">0];
  if((buy_value + sell_value) == class="num">0)
      reward -= (class="type">class="kw">float)(atr / PrevBalance);
  for(class="type">class="kw">ulong i = class="num">0; i < temp.Size(); i++)
      sState.action[i] = temp[i];
  temp.Clip(class="num">0.001f, class="num">0.999f);
  temp = MathLog((temp - class="num">1.0f) * (-class="num">1.0f) / temp) * (-class="num">1);
  Result.AssignArray(temp);
  if(Result.GetIndex() >= class="num">0)
      Result.BufferWrite();
  if(Actor.CalcLogProbs(Result))
    {
      Result.GetData(temp);
      reward += temp.Sum();
    }
  if(!Base.Add(sState, reward))
      ExpertRemove();
  }
class="type">void Train(class="type">void)
  {
  class="type">int total_tr = ArraySize(Buffer);
  class="type">uint ticks = GetTickCount();
class=class="str">"cmt">//---
  for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++)
    {
      class="type">int tr = (class="type">int)((MathRand() / class="num">32767.0) * (total_tr - class="num">1));
      class="type">int i = (class="type">int)((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">2));
      class=class="str">"cmt">//--- Target
      State.AssignArray(Buffer[tr].States[i + class="num">1].state);
      class="type">class="kw">float PrevBalance = Buffer[tr].States[i].account[class="num">0];
      class="type">class="kw">float PrevEquity = Buffer[tr].States[i].account[class="num">1];
      Account.Clear();
      Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">0] - PrevBalance) / PrevBalance);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">1] / PrevBalance);
      Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">1] - PrevEquity) / PrevEquity);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">2]);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">3]);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">4] / PrevBalance);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">5] / PrevBalance);
      Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">6] / PrevBalance);
      class=class="str">"cmt">//---
      if(Account.GetIndex() >= class="num">0)
          Account.BufferWrite();
      if(!Actor.feedForward(GetPointer(State), class="num">1, class="kw">false, GetPointer(Account)))
        {
          PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
          ExpertRemove();
          class="kw">break;

双评论员下的奖励重算与账户向量回填

这段训练循环里，先拿 TargetCritic1 和 TargetCritic2 对 Actor 做前向传播，任一失败就打印函数名加行号并 break，避免脏梯度进网络。奖励项用两个目标评论员的最小值加上策略 log_prob 之和，再减掉下一步即时奖励，公式里 DiscFactor 是折扣因子，外汇与贵金属行情跳空多，折扣因子设太大容易把噪声当趋势，概率上会放大过拟合风险。账户状态被拆成 8 个归一化分量：0~2 是余额/净值的环比变化，3 是裸仓位，4~6 是三类敞口相对前余额的比值，全部写进 Account 环形缓冲，供 Actor 下一帧感知权益曲线形状。每隔一次迭代（iter%2==0）才更新 Critic1 和 Critic2，其中 Critic2 的输入换成了 Actions 而非 Actor，这种错峰更新能压住 TD 误差共振，实盘前可在 MT5 里把模 2 改成模 4 看回测稳定性变化。

MQL5 / C++

   }
   class=class="str">"cmt">//---
   if(!TargetCritic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) ||
       !TargetCritic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)))
      {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
      }
   vector<class="type">class="kw">float> log_prob;
   if(!Actor.GetLogProbs(log_prob))
      {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
      }
   TargetCritic1.getResults(Result);
   class="type">class="kw">float reward = Result[class="num">0];
   TargetCritic2.getResults(Result);
   reward = Buffer[tr].Revards[i] + DiscFactor * (MathMin(reward, Result[class="num">0]) + log_prob.Sum() - Buffer[tr].Revards[i + class="num">1]);
   class=class="str">"cmt">//--- Q-function study
   State.AssignArray(Buffer[tr].States[i].state);
   PrevBalance = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">0];
   PrevEquity = Buffer[tr].States[MathMax(i - class="num">1, class="num">0)].account[class="num">1];
   Account.Update(class="num">0, (Buffer[tr].States[i].account[class="num">0] - PrevBalance) / PrevBalance);
   Account.Update(class="num">1, Buffer[tr].States[i].account[class="num">1] / PrevBalance);
   Account.Update(class="num">2, (Buffer[tr].States[i].account[class="num">1] - PrevEquity) / PrevEquity);
   Account.Update(class="num">3, Buffer[tr].States[i].account[class="num">2]);
   Account.Update(class="num">4, Buffer[tr].States[i].account[class="num">3] / PrevBalance);
   Account.Update(class="num">5, Buffer[tr].States[i].account[class="num">4] / PrevBalance);
   Account.Update(class="num">6, Buffer[tr].States[i].account[class="num">5] / PrevBalance);
   Account.Update(class="num">7, Buffer[tr].States[i].account[class="num">6] / PrevBalance);
   class=class="str">"cmt">//---
   Account.BufferWrite();
   if(!Actor.feedForward(GetPointer(State), class="num">1, class="kw">false, GetPointer(Account)))
      {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
      }
   class=class="str">"cmt">//---
   Actor.GetLogProbs(log_prob);
   Actions.AssignArray(Buffer[tr].States[i].action);
   if(Actions.GetIndex() >= class="num">0)
      Actions.BufferWrite();
class=class="str">"cmt">//---
   if((iter % class="num">2) == class="num">0)
      {
      if(!Critic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) ||
         !Critic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actions)))
         {

◍ 双评论家网络的反向传播分支

这段逻辑跑在强化学习训练循环里，按是否使用双评论家结构分成两支。若启用双评论家，先清 Result 并写入 reward 减 log_prob 之和，再依次对 Critic1 做 backProp、AlphasGradient，以及对 Actor 做两次 backPropGradient；任何一步返回 false 就打印函数名与行号并 break。 Critic2 的反向传播单独用 Result.Update(0, Buffer[tr].Revards[i]) 覆盖奖励值，再喂 Actions 和 Gradient 指针做 backProp，失败同样断点退出。这里 Revards 拼写沿用了原变量名，接 MT5 编译时别手改错成 Rewards 导致链接失败。未启用双评论家时走 else 分支：先 feedForward 让 Critic2、Critic1 拿到 Actor 与隐层输出，失败即断；随后 Critic2 做 backProp 与 AlphasGradient，Actor 两次反向，最后用 Buffer 奖励更新 Result 并让 Critic1 backProp。两个分支收尾都接 TargetCritic1.WeightsUpdate(GetPointer(Critic1), Tau) 做软更新，Tau 越小目标网跟踪越慢、训练可能更稳。外汇与贵金属杠杆高，这类自研智能体在实盘前务必用历史 tick 回测验证收敛性。

MQL5 / C++

PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
class="kw">break;
}
Result.Clear();
Result.Add(reward-log_prob.Sum());
if(!Critic1.backProp(Result, GetPointer(Actor)) ||
   !Critic1.AlphasGradient(GetPointer(Actor)) ||
   !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient), LatentLayer) ||
   !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient)))
  {
   PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
   class="kw">break;
  }
Result.Update(class="num">0,Buffer[tr].Revards[i]);
if(!Critic2.backProp(Result, GetPointer(Actions), GetPointer(Gradient)))
  {
   PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
   class="kw">break;
  }
}
else
  {
   if(!Critic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) ||
      !Critic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actions)))
     {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
     }
   Result.Clear();
   Result.Add(reward);
   if(!Critic2.backProp(Result, GetPointer(Actor)) ||
      !Critic2.AlphasGradient(GetPointer(Actor)) ||
      !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient), LatentLayer) ||
      !Actor.backPropGradient(GetPointer(Account), GetPointer(Gradient)))
     {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
     }
   Result.Update(class="num">0,Buffer[tr].Revards[i]);
   if(!Critic1.backProp(Result, GetPointer(Actions), GetPointer(Gradient)))
     {
      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);
      class="kw">break;
     }
  }
class=class="str">"cmt">//--- Update Target Nets
TargetCritic1.WeightsUpdate(GetPointer(Critic1), Tau);