神经网络变得轻松（第五十三部分）：奖励分解·进阶篇
（2/3）· 复合奖励信号模棱两可，代理者行为跑偏时你却分不清是哪个分量在捣乱
◍ SAC-D目标网络与CAGrad的初始化落点
在 SAC-D 这类离线与在线混合策略里，目标网络（target critic / target nu）的权重同步不能省。下面这段初始化先把 cTargetCritic1、cTargetCritic2、cTargetNu 用 Create 建出来，失败就直接 return false 并打印错误码，避免在 MT5 里跑出空指针导致 EA 静默崩掉。 随后所有网络对象（actor、critic、zeta、nu 及三个 target）统一绑定 OpenCL 上下文，这一步决定你能否用显卡加速矩阵运算。若机器无可用 GPU，SetOpenCL 虽不报错但后续训练会明显偏慢，外汇与贵金属 tick 级样本下延迟可能放大数倍，属高风险计算环境。 权重更新用 WeightsUpdate(GetPointer(cCritic1), 1.0) 把在线 critic 整份拷给目标网络，系数 1.0 即硬拷贝；若想做软更新须改这里。之后 fLambda 初始化为全 1e-5、fLambda_m / fLambda_v 为零向量，尺寸由 cZeta.getResults(fZeta) 的返回长度决定。 CAGrad 函数是多任务梯度冲突的缓和器：先算 grad 的外积 GG，把 NaN 置 0，若 GG 全零直接返回原梯度。scale 用对角线加 1e-4 后开方取均值做归一，迭代 iCAGrad_Iters 次用 softmax 搜最优 w，目标含约束项 c = sqrt(gg+1e-4)*fCAGrad_C。开 MT5 把这段贴进类方法，调 fCAGrad_C 与迭代次数能直接看冲突梯度是否收敛。
if(!cTargetCritic1.Create(critic) || !cTargetCritic2.Create(critic) || !cTargetNu.Create(nu)) { PrintFormat("Error of create target models: %d", GetLastError()); class="kw">return class="kw">false; } class=class="str">"cmt">//--- cActorExploer.SetOpenCL(opencl); cCritic1.SetOpenCL(opencl); cCritic2.SetOpenCL(opencl); cZeta.SetOpenCL(opencl); cNu.SetOpenCL(opencl); cTargetCritic1.SetOpenCL(opencl); cTargetCritic2.SetOpenCL(opencl); cTargetNu.SetOpenCL(opencl); class=class="str">"cmt">//--- if(!cTargetCritic1.WeightsUpdate(GetPointer(cCritic1), class="num">1.0) || !cTargetCritic2.WeightsUpdate(GetPointer(cCritic2), class="num">1.0) || !cTargetNu.WeightsUpdate(GetPointer(cNu), class="num">1.0)) { PrintFormat("Error of update target models: %d", GetLastError()); class="kw">return class="kw">false; } class=class="str">"cmt">//--- cZeta.getResults(fZeta); class="type">class="kw">ulong size = fZeta.Size(); fLambda = vector<class="type">class="kw">float>::Full(size,class="num">1.0e-5f); fLambda_m = vector<class="type">class="kw">float>::Zeros(size); fLambda_v = vector<class="type">class="kw">float>::Zeros(size); fQWeights = vector<class="type">class="kw">float>::Ones(size); iLatentLayer = latent_layer; class=class="str">"cmt">//--- class="kw">return true; } vector<class="type">class="kw">float> CNet_SAC_D_DICE::CAGrad(vector<class="type">class="kw">float> &grad) { matrix<class="type">class="kw">float> GG = grad.Outer(grad); GG.ReplaceNan(class="num">0); if(MathAbs(GG).Sum() == class="num">0) class="kw">return grad; class="type">class="kw">float scale = MathSqrt(GG.Diag() + class="num">1.0e-4f).Mean(); GG = GG / MathPow(scale,class="num">2); vector<class="type">class="kw">float> Gg = GG.Mean(class="num">1); class="type">class="kw">float gg = Gg.Mean(); vector<class="type">class="kw">float> w = vector<class="type">class="kw">float>::Zeros(grad.Size()); class="type">class="kw">float c = MathSqrt(gg + class="num">1.0e-4f) * fCAGrad_C; vector<class="type">class="kw">float> w_best = w; class="type">class="kw">float obj_best = FLT_MAX; vector<class="type">class="kw">float> moment = vector<class="type">class="kw">float>::Zeros(w.Size()); for(class="type">int i = class="num">0; i < iCAGrad_Iters; i++) { vector<class="type">class="kw">float> ww; w.Activation(ww,AF_SOFTMAX); class="type">class="kw">float obj = ww.Dot(Gg) + c * MathSqrt(ww.MatMul(GG).Dot(ww) + class="num">1.0e-4f); if(MathAbs(obj) < obj_best) { obj_best = MathAbs(obj); w_best = w; } if(i < (iCAGrad_Iters - class="num">1)) { class="type">class="kw">float loss = -obj; vector<class="type">class="kw">float> derev = Gg + GG.MatMul(ww) * c / (MathSqrt(ww.MatMul(GG).Dot(ww) + class="num">1.0e-4f) * class="num">2) +
「梯度耦合里的权重归一与动量更新」
这段逻辑处在策略网络反向修正的核心段：先用 ww 与 Gram 矩阵 GG 做矩阵乘，再除以带 1e-4 稳定项的范数，得到缩放系数 c 作用下的方向向量。 delta = derev * loss 拿到基础梯度后，代码构造了 size 阶单位阵 ident 与全 1 向量 ones，通过 ones.Outer(ww) 生成softmax 雅可比的近似结构 sm_der，转置后乘 (ident - sm_der) 完成链式展开。 sm_der.MatMul(delta) 把梯度压回参数流形；若 delta 的点积范数（Ptp）非零就就地归一，避免量级爆炸。动量项 moment = delta * 0.8f + moment * 0.5f 显示历史冲量权重仅 0.5，当前梯度占 0.8，偏向快变环境——外汇与贵金属的高波动行情下这种设定可能更跟手，但过拟合风险也偏高。 w += moment 后再次按 Ptp 归一，保证策略向量始终在单位球附近。末段 w_best 走 softmax 激活，用 gw_norm 与常数 c 算 lmbda，最终 result 融合 grad 与 1/size 偏置，分母含 fCAGrad_C 平方——调这个 C 值会直接改输出幅度，开 MT5 把 fCAGrad_C 从默认改到 0.5 和 2.0 各跑一轮回测就能看出梯度裁剪松紧的差别。 Study 函数入口先判空 Actions 与 NextState，随后用 feedForward 把下一状态送进目标网络；cTargetCritic1/2 与 cTargetNu 都从 iLatentLayer 拉特征到末层，任意一条前向失败立即返回 false，说明这套 SAC-DICE 实现把目标网络同步当作训练不可跳过的闸门。
ww.MatMul(GG) * c / (MathSqrt(ww.MatMul(GG).Dot(ww) + class="num">1.0e-4f) * class="num">2); vector<class="type">class="kw">float> delta = derev * loss; class="type">class="kw">ulong size = delta.Size(); matrix<class="type">class="kw">float> ident = matrix<class="type">class="kw">float>::Identity(size, size); vector<class="type">class="kw">float> ones = vector<class="type">class="kw">float>::Ones(size); matrix<class="type">class="kw">float> sm_der = ones.Outer(ww); sm_der = sm_der.Transpose() * (ident - sm_der); delta = sm_der.MatMul(delta); if(delta.Ptp() != class="num">0) delta = delta / delta.Ptp(); moment = delta * class="num">0.8f + moment * class="num">0.5f; w += moment; if(w.Ptp() != class="num">0) w = w / w.Ptp(); } } w_best.Activation(w,AF_SOFTMAX); class="type">class="kw">float gw_norm = MathSqrt(w.MatMul(GG).Dot(w) + class="num">1.0e-4f); class="type">class="kw">float lmbda = c / (gw_norm + class="num">1.0e-4f); vector<class="type">class="kw">float> result = ((w * lmbda + class="num">1.0f / (class="type">class="kw">float)grad.Size()) * grad) / (class="num">1 + MathPow(fCAGrad_C,class="num">2)); class=class="str">"cmt">//--- class="kw">return result; } class="type">bool CNet_SAC_D_DICE::Study(CArrayFloat *State, CArrayFloat *SecondInput, CBufferFloat *Actions, vector<class="type">class="kw">float> &Rewards, CBufferFloat *NextState, CBufferFloat *NextSecondInput, class="type">class="kw">float discount, class="type">class="kw">float tau) { class=class="str">"cmt">//--- if(!Actions) class="kw">return class="kw">false; if(!!NextState) if(!CNet::feedForward(NextState, class="num">1, class="kw">false, NextSecondInput)) class="kw">return class="kw">false; if(!cTargetCritic1.feedForward(GetPointer(this), iLatentLayer, GetPointer(this), layers.Total() - class="num">1) || !cTargetCritic2.feedForward(GetPointer(this), iLatentLayer, GetPointer(this), layers.Total() - class="num">1)) class="kw">return class="kw">false; class=class="str">"cmt">//--- if(!cTargetNu.feedForward(GetPointer(this), iLatentLayer, GetPointer(this), layers.Total() - class="num">1)) class="kw">return class="kw">false;
DualCVaR 里的 nu 与 zeta 梯度回传
这段逻辑跑在强化学习评判网络的反向更新里，核心是两个影子变量 nu 和 zeta 的梯度落地。nu 负责逼近状态价值，zeta 充当凸对偶变量，靠 fLambda 这个拉格朗日乘子把 CVaR 约束缝进策略梯度。 先看前向收尾：把策略输出拷进 output 缓冲，再依次喂给 cNu、cZeta 两个子网络拿回 nu 与 zeta 向量。若 NextState 为空，next_nu 直接置零向量，否则取目标网络的下一个 nu，这一步决定了贝尔曼残差是否带 bootstrap。 policy_ratio 用 log_prob 和 Rewards 算指数均值，再乘到 (next_nu*discount+Rewards) 上减 nu，得到 bellman_residuals。zeta_loss 和 nu_loss 都显式用了 MathAbs(bellman_residuals)，说明梯度对残差符号不敏感，只在幅度上 penalize——这是 DualCVaR 跟普通 TD 最不一样的点。 fLambda 自身走了一套简化 Adam：m 和 v 分别用 b1、b2 做动量与二阶矩衰减，更新步长被 lr/sqrt(v+lr/100) 缩放。把 lr/100 写死进 epsilon 项，意味着学习率若调过 100 量级，分母稳定项会失效，MT5 里跑时应把 lr 压在 1e-3 附近才稳。 nu 的梯度回传要手动把 nu_grad 写进最后一层神经元梯度缓冲，再调 cNu.backPropGradient 往浅层传。代码里 last_layer 取 layers.Total()-1 后做了两次空指针 return false，漏掉任何一层对象都会导致整轮训练静默中断，开 MT5 调试时建议在这两处下断点看 layer/neuron 是否真的构建成功。
if(!CNet::feedForward(State, class="num">1, class="kw">false, SecondInput)) class="kw">return class="kw">false; CBufferFloat *output = ((CNeuronBaseOCL*)((CLayer*)layers.At(layers.Total() - class="num">1)).At(class="num">0)).getOutput(); output.AssignArray(Actions); output.BufferWrite(); if(!cNu.feedForward(GetPointer(this), iLatentLayer, GetPointer(this))) class="kw">return class="kw">false; if(!cZeta.feedForward(GetPointer(this), iLatentLayer, GetPointer(this))) class="kw">return class="kw">false; vector<class="type">class="kw">float> nu, next_nu, zeta, ones; cNu.getResults(nu); cZeta.getResults(zeta); if(!!NextState) cTargetNu.getResults(next_nu); else next_nu = vector<class="type">class="kw">float>::Zeros(nu.Size()); ones = vector<class="type">class="kw">float>::Ones(zeta.Size()); vector<class="type">class="kw">float> log_prob = GetLogProbability(output); class="type">int shift = (class="type">int)(Rewards.Size() - log_prob.Size()); if(shift < class="num">0) class="kw">return class="kw">false; class="type">class="kw">float policy_ratio = class="num">0; for(class="type">class="kw">ulong i = class="num">0; i < log_prob.Size(); i++) policy_ratio += log_prob[i] - Rewards[shift + i] / LogProbMultiplier; policy_ratio = MathExp(policy_ratio / log_prob.Size()); vector<class="type">class="kw">float> bellman_residuals = (next_nu * discount + Rewards) * policy_ratio - nu; vector<class="type">class="kw">float> zeta_loss = MathPow(zeta, class="num">2.0f) / class="num">2.0f - zeta * (MathAbs(bellman_residuals) - fLambda) ; vector<class="type">class="kw">float> nu_loss = zeta * MathAbs(bellman_residuals) + MathPow(nu, class="num">2.0f) / class="num">2.0f; vector<class="type">class="kw">float> lambda_los = fLambda * (ones - zeta); vector<class="type">class="kw">float> grad_lambda = CAGrad((ones - zeta) * (lambda_los * (-class="num">1.0f))); fLambda_m = fLambda_m * b1 + grad_lambda * (class="num">1 - b1); fLambda_v = fLambda_v * b2 + MathPow(grad_lambda, class="num">2) * (class="num">1.0f - b2); fLambda += fLambda_m * lr / MathSqrt(fLambda_v + lr / class="num">100.0f); CBufferFloat temp; temp.BufferInit(MathMax(Actions.Total(), SecondInput.Total()), class="num">0); temp.BufferCreate(opencl); class=class="str">"cmt">//--- update nu class="type">int last_layer = cNu.layers.Total() - class="num">1; CLayer *layer = cNu.layers.At(last_layer); if(!layer) class="kw">return class="kw">false; CNeuronBaseOCL *neuron = layer.At(class="num">0); if(!neuron) class="kw">return class="kw">false; CBufferFloat *buffer = neuron.getGradient(); if(!buffer) class="kw">return class="kw">false; vector<class="type">class="kw">float> nu_grad = CAGrad(nu_loss * (zeta * bellman_residuals / MathAbs(bellman_residuals) - nu)); if(!buffer.AssignArray(nu_grad) || !buffer.BufferWrite()) class="kw">return class="kw">false; if(!cNu.backPropGradient(output, GetPointer(temp))) class="kw">return class="kw">false; class=class="str">"cmt">//--- update zeta last_layer = cZeta.layers.Total() - class="num">1; layer = cZeta.layers.At(last_layer);
◍ 分布critic的梯度回传与zeta归一
这段逻辑跑在智能体反向更新里，先取最后一层神经元梯度缓冲，任何一层指针为空就直接 return false，避免空指针把EA拖崩。zeta梯度用 CAGrad 算，系数里带了 fLambda 和 bellman_residuals 的绝对值，符号翻负，说明是在做分布风险项的反向惩罚。 前向部分把 cCritic1 / cCritic2 都喂一遍，输出给后续策略网。fZeta 的更新很有意思：首次用 MathAbs(zeta) 直接赋值，之后走 0.9 与 0.1 的滑动平均，相当于给 zeta 做了个低通。归一化那行把 zeta 开三次方再除以 fZeta 三次方根的 10 倍，压缩了量纲差异，让不同品种（含外汇、贵金属）的梯度尺度倾向一致——但杠杆品种跳空时残差可能爆，仍属高风险。 critic 损失是 zeta 加权的二次误差，fLoss1 用 0.999 惯性平滑，初值为均方根，后续滚动更新。梯度 grad = CAGrad(loss * zeta * (target - result) * 2.0f)，写进最后一层缓冲后触发 backPropGradient。你在 MT5 里改 fLambda 或把 0.9/0.1 换成 0.95/0.05，能直接观察回测曲线抖动变化。
if(!layer) class="kw">return class="kw">false; neuron = layer.At(class="num">0); if(!neuron) class="kw">return class="kw">false; buffer = neuron.getGradient(); if(!buffer) class="kw">return class="kw">false; vector<class="type">class="kw">float> zeta_grad = CAGrad(zeta_loss * (zeta - MathAbs(bellman_residuals) + fLambda) * (-class="num">1.0f)); if(!buffer.AssignArray(zeta_grad) || !buffer.BufferWrite()) class="kw">return class="kw">false; if(!cZeta.backPropGradient(output, GetPointer(temp))) class="kw">return class="kw">false; class=class="str">"cmt">//--- feed forward critics if(!cCritic1.feedForward(GetPointer(this), iLatentLayer, output) || !cCritic2.feedForward(GetPointer(this), iLatentLayer, output)) class="kw">return class="kw">false; vector<class="type">class="kw">float> result; if(fZeta.CompareByDigits(vector<class="type">class="kw">float>::Zeros(fZeta.Size()),class="num">8) == class="num">0) fZeta = MathAbs(zeta); else fZeta = fZeta * class="num">0.9f + MathAbs(zeta) * class="num">0.1f; zeta = MathPow(MathAbs(zeta), class="num">1.0f / class="num">3.0f) / (MathPow(fZeta, class="num">1.0f / class="num">3.0f) * class="num">10.0f); vector<class="type">class="kw">float> target = vector<class="type">class="kw">float>::Zeros(Rewards.Size()); if(!!NextState) { cTargetCritic1.getResults(target); cTargetCritic2.getResults(result); if(fQWeights.Dot(result) < fQWeights.Dot(target)) target = result; } target = (target * discount + Rewards); class="type">class="kw">ulong total = log_prob.Size(); for(class="type">class="kw">ulong i = class="num">0; i < total; i++) target[shift + i] = log_prob[i] * LogProbMultiplier; class=class="str">"cmt">//--- update critic1 cCritic1.getResults(result); vector<class="type">class="kw">float> loss = zeta * MathPow(result - target, class="num">2.0f); if(fLoss1 == class="num">0) fLoss1 = MathSqrt(fQWeights.Dot(loss) / fQWeights.Sum()); else fLoss1 = MathSqrt(class="num">0.999f * MathPow(fLoss1, class="num">2.0f) + class="num">0.001f * fQWeights.Dot(loss) / fQWeights.Sum()); vector<class="type">class="kw">float> grad = CAGrad(loss * zeta * (target - result) * class="num">2.0f); last_layer = cCritic1.layers.Total() - class="num">1; layer = cCritic1.layers.At(last_layer); if(!layer) class="kw">return class="kw">false; neuron = layer.At(class="num">0); if(!neuron) class="kw">return class="kw">false; buffer = neuron.getGradient(); if(!buffer) class="kw">return class="kw">false; if(!buffer.AssignArray(grad) || !buffer.BufferWrite()) class="kw">return class="kw">false; if(!cCritic1.backPropGradient(output, GetPointer(temp)) || !backPropGradient(SecondInput, GetPointer(temp), iLatentLayer)) class="kw">return class="kw">false; class=class="str">"cmt">//--- update critic2
「双评论家网络下的梯度回传与探索策略更新」
这段逻辑跑在强化学习训练循环里，用两个评论家网络 cCritic1 与 cCritic2 交叉验证误差，再决定拿谁的结果去回传。loss 用 zeta 乘均方误差，fLoss2 首次直接取根号加权误差，之后按 0.999 与 0.001 做指数滑动平均，老值占 99.9% 新值占 0.1%，让训练曲线倾向平滑。 梯度写入前先抓最后一层神经元，取它的梯度缓冲并赋值为 CAGrad 算出的 grad，任何一步 AssignArray 或 BufferWrite 失败就直接 return false，避免脏梯度污染网络。 选评论家时比 fLoss1 与 fLoss2：若 fLoss1 更小，用 cCritic1 的 result 和 cCritic2 的 mean；反之对调。var 取两者差绝对值的一半，mean 取平均，target 先被写成 discount*log_prob*LogProbMultiplier 的序列，再叠一层 CAGrad(zeta*(target - var*2.5) - result) + result 的修正。 回传完立刻切到探索策略：cActorExploer 前向吃 State，把输出强行换成 Actions 再写回缓冲，target 改用 var*2.0 的正向偏移。外汇与贵金属行情跳空频繁，这类 OCL 网络在实盘前务必用 MT5 策略测试器以 Tick 级数据跑通回传，否则显存或缓冲区报错只会在 live 阶段暴露。
cCritic2.getResults(result); loss = zeta * MathPow(result - target, class="num">2.0f); if(fLoss2 == class="num">0) fLoss2 = MathSqrt(fQWeights.Dot(loss) / fQWeights.Sum()); else fLoss2 = MathSqrt(class="num">0.999f * MathPow(fLoss2, class="num">2.0f) + class="num">0.001f * fQWeights.Dot(loss) / fQWeights.Sum()); grad = CAGrad(loss * zeta * (target - result) * class="num">2.0f); last_layer = cCritic2.layers.Total() - class="num">1; layer = cCritic2.layers.At(last_layer); if(!layer) class="kw">return class="kw">false; neuron = layer.At(class="num">0); if(!neuron) class="kw">return class="kw">false; buffer = neuron.getGradient(); if(!buffer) class="kw">return class="kw">false; if(!buffer.AssignArray(grad) || !buffer.BufferWrite()) class="kw">return class="kw">false; if(!cCritic2.backPropGradient(output, GetPointer(temp)) || !backPropGradient(SecondInput, GetPointer(temp), iLatentLayer)) class="kw">return class="kw">false; vector<class="type">class="kw">float> mean; CNet *critic = NULL; if(fLoss1 <= fLoss2) { cCritic1.getResults(result); cCritic2.getResults(mean); critic = GetPointer(cCritic1); } else { cCritic1.getResults(mean); cCritic2.getResults(result); critic = GetPointer(cCritic2); } vector<class="type">class="kw">float> var = MathAbs(mean - result) / class="num">2.0f; mean += result; mean /= class="num">2.0f; target = mean; for(class="type">class="kw">ulong i = class="num">0; i < log_prob.Size(); i++) target[shift + i] = discount * log_prob[i] * LogProbMultiplier; target = CAGrad(zeta * (target - var * class="num">2.5f) - result) + result; CBufferFloat bTarget; bTarget.AssignArray(target); critic.TrainMode(class="kw">false); if(!critic.backProp(GetPointer(bTarget), GetPointer(this)) || !backPropGradient(SecondInput, GetPointer(temp))) { critic.TrainMode(true); class="kw">return class="kw">false; } class=class="str">"cmt">//--- update exploration policy if(!cActorExploer.feedForward(State, class="num">1, class="kw">false, SecondInput)) { critic.TrainMode(true); class="kw">return class="kw">false; } output = ((CNeuronBaseOCL*)((CLayer*)cActorExploer.layers.At(layers.Total() - class="num">1)).At(class="num">0)).getOutput(); output.AssignArray(Actions); output.BufferWrite(); cActorExploer.GetLogProbs(log_prob); target = mean; for(class="type">class="kw">ulong i = class="num">0; i < log_prob.Size(); i++) target[shift + i] = discount * log_prob[i] * LogProbMultiplier; target = CAGrad(zeta * (target + var * class="num">2.0f) - result) + result; bTarget.AssignArray(target); if(!critic.backProp(GetPointer(bTarget), GetPointer(cActorExploer)) || !cActorExploer.backPropGradient(SecondInput, GetPointer(temp))) { critic.TrainMode(true); class="kw">return class="kw">false; } critic.TrainMode(true);
目标网络延迟更新与奖励向量定义
目标网络权重不是每帧都动。代码里用 iUpdateDelayCount 做递减闸门：当 NextState 有效且计数器大于 0 时，先自减并返回 true，跳过本次同步；归零后才把 iUpdateDelay 重新装入计数器，实现每隔 iUpdateDelay 步更新一次目标模型。这种延迟能缓和训练振荡，外汇与贵金属行情跳变频繁，过小的延迟值可能让策略对噪声过拟合。 若 cTargetCritic1 / cTargetCritic2 / cTargetNu 任一WeightsUpdate失败，立即打印错误码并返回 false，训练循环应捕获该信号停止。三个目标网络分别对应双评论家与 nu 网络，tau 为软更新系数。 奖励结构用宏写死：NActions=6 表示 6 种可执行动作，NRewards=3+NActions=9 即奖励向量长度。索引 0 为余额增量，1 为净值增减（负值回撤、正值盈利），2 为空仓惩罚，3 之后为各动作的对数概率。开 MT5 把 NActions 改成 4 试跑，能直接看到 NRewards 变为 7，影响 critic 第 4 层神经元数量。 CreateDescriptions 中 critic 末层 descr.count = NRewards、activation = None、optimization = ADAM，输出原始奖励估计值供后续计算。建好描述数组后返回 true，若 new CLayerDescription 失败则删对象并返 false，内存泄漏风险在此被截断。
if(!!NextState) { if(iUpdateDelayCount > class="num">0) { iUpdateDelayCount--; class="kw">return true; } iUpdateDelayCount = iUpdateDelay; } if(!cTargetCritic1.WeightsUpdate(GetPointer(cCritic1), tau) || !cTargetCritic2.WeightsUpdate(GetPointer(cCritic2), tau) || !cTargetNu.WeightsUpdate(GetPointer(cNu), tau)) { PrintFormat("Error of update target models: %d", GetLastError()); class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true; } class=class="str">"cmt">//+------------------------------------------------------------------+ class=class="str">"cmt">//| Rewards structure | class=class="str">"cmt">//| class="num">0 - Delta Balance | class=class="str">"cmt">//| class="num">1 - Delta Equity( "-" Drawdown / "+" Profit) | class=class="str">"cmt">//| class="num">2 - Penalty for no open positions | class=class="str">"cmt">//| class="num">3... - LogProbs vector | class=class="str">"cmt">//+------------------------------------------------------------------+ class="macro">#define NActions class="num">6 class=class="str">"cmt">//Number of possible Actions class="macro">#define NRewards class="num">3+NActions class=class="str">"cmt">//Number of rewards class="type">bool CreateDescriptions(CArrayObj *actor, CArrayObj *critic) { class=class="str">"cmt">//--- CLayerDescription *descr; class=class="str">"cmt">//--- if(!actor) { actor = new CArrayObj(); if(!actor) class="kw">return class="kw">false; } if(!critic) { critic = new CArrayObj(); if(!critic) class="kw">return class="kw">false; } class=class="str">"cmt">//--- Actor ........ ........ class=class="str">"cmt">//--- Critic critic.Clear(); class=class="str">"cmt">//--- Input layer ........ ........ class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = NRewards; descr.optimization = ADAM; descr.activation = None; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true; } class="kw">struct SState { class="type">class="kw">float state[HistoryBars * BarDescr];