神经网络变得轻松(第五十五部分):对比内在控制(CIC)·进阶篇
(2/3)· 离散技能算法在连续控制里失灵,CIC 如何靠对比密度估测补上这块短板
◍ 编码器与策略网络的分层堆法
状态编码器(state_encoder)在第五、第六层做了维度压缩与拼接:第五层用 256 个基础 OCL 神经元降到 128 个,激活函数选 LReLU,优化器走 ADAM;第六层切换为 defNeuronConcatenate 类型,把上层的 128 维窗口(prev_count)与账户描述(AccountDescr)按 NSkills 步长拼合,输出经 SIGMOID 压到 0~1 区间。 Actor 网络从 layer 0 起手就是 NSkills 个无激活基础神经元,随后 layer 1 用 Concatenate 把 LatentCount 与 NSkills 做窗口拼接,再叠两层 LReLU 的 BaseOCL(各 LatentCount 宽),末层直接映射到 NActions 并用 SIGMOID 给出动作概率——这套结构在 MT5 里跑强化学习策略时,改 LatentCount 会明显影响显存占用与收敛速度。 Critic 网络先 Clear 再建输入层,代码在 Input layer 处截断,但前面每一层都遵循同一防御式写法:new 失败立即 return false,Add 失败则 delete 描述符再返回,避免层描述对象泄漏。
descr.type = defNeuronBaseOCL; descr.count = class="num">256; descr.optimization = ADAM; descr.activation = LReLU; if(!state_encoder.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = class="num">128; descr.activation = LReLU; descr.optimization = ADAM; if(!state_encoder.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">6 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConcatenate; descr.count = NSkills; descr.window = prev_count; descr.step = AccountDescr; descr.optimization = ADAM; descr.activation = SIGMOID; if(!state_encoder.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Actor actor.Clear(); class=class="str">"cmt">//--- layer class="num">0 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = NSkills; descr.activation = None; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConcatenate; descr.count = LatentCount; descr.window = prev_count; descr.step = NSkills; descr.optimization = ADAM; descr.activation = SIGMOID; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = NActions; descr.activation = SIGMOID; descr.optimization = ADAM; if(!actor.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Critic critic.Clear(); class=class="str">"cmt">//--- Input layer if(!(descr = new CLayerDescription()))
「评论家与卷积网络的层定义细节」
这段逻辑在搭建强化学习智能体的两个子网络:critic 负责价值估计,convolution 负责状态特征提取。每一层都先 new 一个 CLayerDescription,填完参数再丢进对应容器,Add 失败就 delete 并回 false,避免内存泄漏。 critic 的网络从 base 层起步,LatentCount 个节点、None 激活、ADAM 优化;随后用 defNeuronConcatenate 把上层的 prev_count 窗口按 NActions 步长拼接,再叠三层 base 层(均用 LReLU),最后一层输出 NRewards 个节点且激活函数为 None,对应多奖励信号的价值头。 convolution 的输入层维度直接算死:2 * (HistoryBars * BarDescr + AccountDescr),也就是把历史 K 线描述和账户状态拼成双倍宽向量。第一层卷积给 512 个节点、SIGMOID 激活;第二层改用 defNeuronConvOCL,节点数 512/8=64,窗口与步长都是 8,window_out 固定为 2,LReLU 激活——这种 8 倍压缩率在小显存显卡上跑 MT5 的 OpenCL 后端时延迟可能更稳。 开 MT5 把 LatentCount、HistoryBars、NActions 这几个宏改成你自己的品种参数,重新编译就能看到网络规模对回测速度的影响,外汇与贵金属杠杆交易高风险,参数乱调可能直接爆仓。
class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = LatentCount; descr.activation = None; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConcatenate; descr.count = LatentCount; descr.window = prev_count; descr.step = NActions; descr.optimization = ADAM; descr.activation = LReLU; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = NRewards; descr.optimization = ADAM; descr.activation = None; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Convolution convolution.Clear(); class=class="str">"cmt">//--- Input layer if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = class="num">2 * (HistoryBars * BarDescr + AccountDescr); descr.activation = None; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = class="num">512; descr.window = prev_count; descr.step = NActions; descr.optimization = ADAM; descr.activation = SIGMOID; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConvOCL; prev_count = descr.count = class="num">512 / class="num">8; descr.window = class="num">8; descr.step = class="num">8; class="type">int prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) {
卷积与判别器的层描述装配
这段逻辑在生成器(convolution)里连续堆了三层卷积:第 3、4 层都是 defNeuronConvOCL,窗口和步长固定为 4,输出窗口 window_out 压到 2,每层神经元数按 (prev_count * prev_wout) / 4 收缩。第 5 层切到 defNeuronBaseOCL,数量直接取 EmbeddingSize,不再做卷积降维。 判别器(descriminator)先 Clear 再重建。第 0 层用 defNeuronBaseOCL 接 NSkills 个输入、无激活;第 1 层是 defNeuronConcatenate,把 window=prev_count、step=NSkills 拼成 LatentCount 宽,激活走 SIGMOID;后面两层 BaseOCL 各 LatentCount 宽、LReLU。 每层都先 new CLayerDescription,Add 失败就 delete 并返回 false——在 MT5 里跑这套,若某层参数算出来 prev_count 为 0,后续除法会静默产出空层,建议加 Print(prev_count) 自查。外汇与贵金属模型训练属高风险实验,过拟合概率不低。
class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConvOCL; prev_count = descr.count = (prev_count * prev_wout) / class="num">4; descr.window = class="num">4; descr.step = class="num">4; prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConvOCL; prev_count = descr.count = (prev_count * prev_wout) / class="num">4; descr.window = class="num">4; descr.step = class="num">4; prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = EmbeddingSize; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Descriminator descriminator.Clear(); class=class="str">"cmt">//--- layer class="num">0 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = NSkills; descr.activation = None; descr.optimization = ADAM; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronConcatenate; descr.count = LatentCount; descr.window = prev_count; descr.step = NSkills; descr.optimization = ADAM; descr.activation = SIGMOID; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4
◍ 判别器与技能投影网络的层定义
在 MT5 的 OpenCL 神经网络封装里,判别器(descriminator)先清掉旧结构,再从第 4 层开始堆。第 4 层用 defNeuronBaseOCL、LReLU 激活、ADAM 优化,节点数等于 EmbeddingSize;第 5 层切到 defNeuronSoftMaxOCL,step 写死为 1,同样走 ADAM。 技能投影(skill_project)走的是另一条 6 层通道:layer 0 接收 NSkills 个原始输入,激活设 None;layer 1 压缩到 LatentCount 并用 SIGMOID;layer 2~4 都在 LatentCount 与 EmbeddingSize 之间用 LReLU 做非线性映射。 每段都用 new CLayerDescription() 拿对象,Add 失败就 delete 并回 false,全过完才 return true。EmbeddingSize、LatentCount、NSkills 都是外部宏或常量,改它们会直接改变网络宽度,外汇与贵金属行情下模型过拟合概率会随之波动,属高风险调参。 下面这段是两层网络构建的核心代码片段,逐行看的就是对象生命周期和层参数挂钩方式。
if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = EmbeddingSize; descr.activation = LReLU; descr.optimization = ADAM; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronSoftMaxOCL; descr.count = EmbeddingSize; descr.step = class="num">1; descr.optimization = ADAM; if(!descriminator.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- Skills project skill_project.Clear(); class=class="str">"cmt">//--- layer class="num">0 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; prev_count = descr.count = NSkills; descr.activation = None; descr.optimization = ADAM; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.optimization = ADAM; descr.activation = SIGMOID; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = EmbeddingSize; descr.activation = LReLU; descr.optimization = ADAM; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronSoftMaxOCL; descr.count = EmbeddingSize; descr.step = class="num">1; descr.optimization = ADAM; if(!skill_project.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true;
「用代码堆出五层隐含网络的结构」
这段函数把调度器里的网络层一层层塞进数组,从输入到 softmax 输出共 6 层描述对象。第一层节点数等于 NSkills,激活函数设为 None,优化器统一用 ADAM,相当于把原始特征直接喂给后续层。 第二层开始引入 LatentCount 个隐节点,window 绑定上一层节点数 prev_count,激活换成 SIGMOID;第三、四层继续用 LatentCount 节点但改 LReLU,第五层回到 NSkills 节点数并保留 LReLU,最后一层用 defNeuronSoftMaxOCL 做分类输出,step 写死为 1。 每一层 new 出来后都立刻 Add 进 scheduler,失败就 delete 并返回 false,这种写法在 MT5 里跑若 LatentCount 设得过大,可能让显存占用陡增。开 MT5 把 NSkills 和 LatentCount 打印出来,能直接看到每层维度是否对得上。 OnInit 里先校验品种名再 Refresh,随后创建 RSI 与 CCI 指标句柄,任一失败即 INIT_FAILED,这说明网络输入特征依赖这两个振荡器的周期参数 RSIPeriod 与 CCIPeriod。
class="type">bool SchedulerDescriptions(CArrayObj *scheduler) { class=class="str">"cmt">//--- Scheduller if(!scheduler) { scheduler = new CArrayObj(); if(!scheduler) class="kw">return class="kw">false; } scheduler.Clear(); class=class="str">"cmt">//--- CLayerDescription *descr = NULL; class=class="str">"cmt">//--- layer class="num">0 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; class="type">int prev_count = descr.count = NSkills; descr.activation = None; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.window = prev_count; descr.optimization = ADAM; descr.activation = SIGMOID; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronBaseOCL; descr.count = NSkills; descr.activation = LReLU; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return class="kw">false; descr.type = defNeuronSoftMaxOCL; descr.count = NSkills; descr.step = class="num">1; descr.optimization = ADAM; if(!scheduler.Add(descr)) { class="kw">delete descr; class="kw">return class="kw">false; } class=class="str">"cmt">//--- class="kw">return true; } class="type">int OnInit() { class=class="str">"cmt">//--- if(!Symb.Name(_Symbol)) class="kw">return INIT_FAILED; Symb.Refresh(); class=class="str">"cmt">//--- if(!RSI.Create(Symb.Name(), TimeFrame, RSIPeriod, RSIPrice)) class="kw">return INIT_FAILED; class=class="str">"cmt">//--- if(!CCI.Create(Symb.Name(), TimeFrame, CCIPeriod, CCIPrice))
初始化阶段对模型与缓冲区的硬校验
EA 在 OnInit 收尾段把所有依赖项串起来做一次性校验:指标句柄创建失败直接返回 INIT_FAILED,避免后续 Tick 里空指针崩盘。 ATR、MACD 的 Create 调用若失败立即退出;RSI/CCI/ATR/MACD 四个缓冲区按 HistoryBars 统一 Resize,任意一项失败就打印函数名与行号并终止,这一步决定了回看窗口能拉多长。 神经网络模型从 "Enc.nnw" 与 "Act.nnw" 载入,文件缺失时现场用 CreateDescriptions 重建网络描述并 Create,仍失败则清掉三个 CArrayObj 后返回 INIT_FAILED。 载入后做维度对账:Actor 输出节点数必须等同 NActions,否则报错退出;Encoder 第 0 层输出总数必须等于 HistoryBars * BarDescr,Actor 第 0 层输入必须等于 Encoder 输出尺寸,三处任一不匹配就 INIT_FAILED。 Scheduler 模型可选,载入失败仅置 bRandomSkills=true 并跳过其 OpenCL 绑定;最后把 Encoder 的 OpenCL 上下文共享给 Actor,完成初始化。外汇与贵金属杠杆高,这类校验疏漏可能在实盘放大滑点与异常下单风险。
class="kw">return INIT_FAILED; class=class="str">"cmt">//--- if(!ATR.Create(Symb.Name(), TimeFrame, ATRPeriod)) class="kw">return INIT_FAILED; class=class="str">"cmt">//--- if(!MACD.Create(Symb.Name(), TimeFrame, FastPeriod, SlowPeriod, SignalPeriod, MACDPrice)) class="kw">return INIT_FAILED; if(!RSI.BufferResize(HistoryBars) || !CCI.BufferResize(HistoryBars) || !ATR.BufferResize(HistoryBars) || !MACD.BufferResize(HistoryBars)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- if(!Trade.SetTypeFillingBySymbol(Symb.Name())) class="kw">return INIT_FAILED; class=class="str">"cmt">//--- load models class="type">class="kw">float temp; if(!Encoder.Load(FileName + "Enc.nnw", temp, temp, temp, dtStudied, true) || !Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true)) { CArrayObj *actor = new CArrayObj(); CArrayObj *encoder = new CArrayObj(); CArrayObj *descr = new CArrayObj(); if(!CreateDescriptions(encoder,actor, descr,descr,descr,descr)) { class="kw">delete encoder; class="kw">delete actor; class="kw">delete descr; class="kw">return INIT_FAILED; } if(!Encoder.Create(encoder) || !Actor.Create(actor)) { class="kw">delete encoder; class="kw">delete actor; class="kw">delete descr; class="kw">return INIT_FAILED; } class="kw">delete encoder; class="kw">delete actor; class="kw">delete descr; class=class="str">"cmt">//--- } bRandomSkills = (!Scheduler.Load(FileName + "Sch.nnw", temp, temp, temp, dtStudied, true)); COpenCLMy *opcl = Encoder.GetOpenCL(); Actor.SetOpenCL(opcl); if(!bRandomSkills) Scheduler.SetOpenCL(opcl); Actor.getResults(ActorResult); if(ActorResult.Size() != NActions) { PrintFormat("The scope of the actor does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Encoder.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of State Encoder doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- vector<class="type">class="kw">float> EncoderResults; Actor.GetLayerOutput(class="num">0,Result); Encoder.getResults(EncoderResults); if(Result.Total() != class="type">int(EncoderResults.Size())) { PrintFormat("Input size of Actor doesn&class="macro">#x27;t match Encoder outputs(%d <> %d)", Result.Total(), EncoderResults.Size()); class="kw">return INIT_FAILED; }