神经网络变得轻松（第五十四部分）：利用随机编码器（RE3）进行高效研究·进阶篇
◍ 评论家与卷积网络的层描述装配
在 MT5 的自定义强化学习框架里，critic 与 convolution 两个网络对象都靠 CLayerDescription 逐个堆层。每一层先 new 一个描述符，填完类型、神经元数、窗口、步长、优化器与激活函数后，用 Add 方法挂到网络；任何一步失败就 delete 并 return false，避免野指针。 critic 网络共 4 层：首层用 defNeuronConcatenate，count 取 LatentCount、window 为上一层神经元数 prev_count、step 为 NActions；后三层均为 defNeuronBaseOCL，前三隐层激活 LReLU、末层 NRewards 个输出神经元且激活 None，全部走 ADAM。 convolution 网络先 Clear 再重建。输入层神经元数被算作 (HistoryBars * BarDescr) + AccountDescr + NActions，激活 None；第 1 层 512 个 SIGMOID 神经元，window=prev_count、step=NActions；第 2 层转 defNeuronConvOCL，count 与 prev_count 同为 512/8=64，窗口与步长均为 8，window_out=2，激活 LReLU。 直接把下面这段原结构拷进 EA 的初始化函数，就能在 MT5 里复现两层网络的骨架；改 LatentCount 或 512 这类常量，网络容量会立刻变化，外汇与贵金属行情下过拟合风险偏高，调参前先用历史数据小样本跑通。
class="kw">return false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronConcatenate; descr.count = LatentCount; descr.window = prev_count; descr.step = NActions; descr.optimization = ADAM; descr.activation = LReLU; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = LatentCount; descr.activation = LReLU; descr.optimization = ADAM; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = NRewards; descr.optimization = ADAM; descr.activation = None; if(!critic.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- Convolution convolution.Clear(); class=class="str">"cmt">//--- Input layer if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; prev_count = descr.count = (HistoryBars * BarDescr) + AccountDescr + NActions; descr.activation = None; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">1 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = class="num">512; descr.window = prev_count; descr.step = NActions; descr.optimization = ADAM; descr.activation = SIGMOID; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">2 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronConvOCL; prev_count = descr.count = class="num">512 / class="num">8; descr.window = class="num">8; descr.step = class="num">8; class="type">int prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">3 if(!(descr = new CLayerDescription())) class="kw">return false;
「卷积层堆叠与双评论家初始化」
这段构建逻辑里，第 4、5 层卷积都沿用 window=4、step=4、window_out=2 的设定，通道数按 prev_count*prev_wout/4 递推，激活统一用 LReLU、优化器走 ADAM。外汇与贵金属行情的高噪声特征下，这种 4×4 压缩节奏可能比更宽窗口更不容易过拟合，但实盘前务必在 MT5 策略测试器里跑一遍确认梯度稳定。 第 5 层突然切到 defNeuronBaseOCL 且 count 直接等于 EmbeddingSize，相当于把时空特征压进嵌入向量交给后续网络。注意这里没有再设 window/step，说明它已是全连接式收口层，调 EmbeddingSize 会直接改 Actor 输出维度。 OnInit 里先 LoadTotalBase 拉基准数据，失败就 INIT_FAILED；随后 Actor/Critic1/Critic2/Convolution 及两个 Target 网络分别从 Act.nnw、Crt1.nnw、Crt2.nnw、CNN.nnw 载入。若任一 .nnw 缺失，代码现场 new 出描述数组并调 CreateDescriptions 重建——这意味着你改了网络结构后，删掉旧 nnw 文件就能强制重训，不用动调用逻辑。 双评论家（Critic1/Critic2）加对应 Target 副本是 TD3 风格防过估的标准做法。黄金 1 小时图这类高杠杆品种，Q 值高估会带来爆仓级回撤概率，留两个独立 Critic 取最小能压住这个倾向。
descr.type = defNeuronConvOCL; prev_count = descr.count = (prev_count * prev_wout) / class="num">4; descr.window = class="num">4; descr.step = class="num">4; prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">4 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronConvOCL; prev_count = descr.count = (prev_count * prev_wout) / class="num">4; descr.window = class="num">4; descr.step = class="num">4; prev_wout = descr.window_out = class="num">2; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- layer class="num">5 if(!(descr = new CLayerDescription())) class="kw">return false; descr.type = defNeuronBaseOCL; descr.count = EmbeddingSize; descr.activation = LReLU; descr.optimization = ADAM; if(!convolution.Add(descr)) { class="kw">delete descr; class="kw">return false; } class=class="str">"cmt">//--- class="kw">return true; } CNet Actor; CNet Critic1; CNet Critic2; CNet TargetCritic1; CNet TargetCritic2; CNet Convolution; class="type">int OnInit() { class=class="str">"cmt">//--- ResetLastError(); if(!LoadTotalBase()) { PrintFormat("Error of load study data: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- load models class="type">float temp; if(!Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true) || !Critic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !Critic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true) || !Convolution.Load(FileName + "CNN.nnw", temp, temp, temp, dtStudied, true) || !TargetCritic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) || !TargetCritic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true)) { CArrayObj *actor = new CArrayObj(); CArrayObj *critic = new CArrayObj(); CArrayObj *convolution = new CArrayObj(); if(!CreateDescriptions(actor, critic, convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } if(!Actor.Create(actor) || !Critic1.Create(critic) || !Critic2.Create(critic) ||
TD3初始化里的张量对齐与失败回退
这段初始化逻辑干的事很硬核：先把 actor、critic、convolution 三个神经网络对象尝试挂到 OpenCL 上下文，一旦 Convolution.Create 失败就当场 delete 三者并返回 INIT_FAILED，避免半初始化状态污染后续训练。 双目标评论家（TargetCritic1/2）必须各自从 critic 拷贝权重，任意一处 Create 不成功同样清场退出；成功后用 WeightsUpdate(GetPointer(Critic1), 1.0f) 把在线评论家权重整份灌进目标网，StartTargetIter 设为 StartTargetIteration，否则归零。 维度校验是容易踩坑的地方：Actor 输出节点数必须严格等于 NActions，输入层展开尺寸要等于 HistoryBars * BarDescr；中间隐层 latent_state 还会被拿去比对 Critic1 输入维，不一致就 PrintFormat 报错并 INIT_FAILED。 最后用 EventChartCustom(ChartID(), 1, 0, 0, "Init") 向图表抛自定义事件，失败也直接 INIT_FAILED；全过才返回 INIT_SUCCEEDED。OnDeinit 里则把目标评论家按 Tau 软更新、并把 Actor 与 TargetCritic1 存成 .nnw 权重文件留底。 开 MT5 把这段贴进 EA 的 OnInit，故意把 HistoryBars 改错一位，就能在专家日志里看到 'Input size of Actor doesn't match state description' 的精确报错，比盲调省时间。
if(!Convolution.Create(convolution)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } if(!TargetCritic1.Create(critic) || !TargetCritic2.Create(critic)) { class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class="kw">return INIT_FAILED; } class="kw">delete actor; class="kw">delete critic; class="kw">delete convolution; class=class="str">"cmt">//--- TargetCritic1.WeightsUpdate(GetPointer(Critic1), class="num">1.0f); TargetCritic2.WeightsUpdate(GetPointer(Critic2), class="num">1.0f); StartTargetIter = StartTargetIteration; } else StartTargetIter = class="num">0; class=class="str">"cmt">//--- OpenCL = Actor.GetOpenCL(); Critic1.SetOpenCL(OpenCL); Critic2.SetOpenCL(OpenCL); TargetCritic1.SetOpenCL(OpenCL); TargetCritic2.SetOpenCL(OpenCL); Convolution.SetOpenCL(OpenCL); Actor.getResults(Result); if(Result.Total() != NActions) { PrintFormat("The scope of the actor does not match the actions count(%d <> %d)", NActions, Result.Total()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Actor.GetLayerOutput(class="num">0, Result); if(Result.Total() != (HistoryBars * BarDescr)) { PrintFormat("Input size of Actor doesn&class="macro">#x27;t match state description(%d <> %d)", Result.Total(), (HistoryBars * BarDescr)); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- Actor.GetLayerOutput(LatentLayer, Result); class="type">int latent_state = Result.Total(); Critic1.GetLayerOutput(class="num">0, Result); if(Result.Total() != latent_state) { PrintFormat("Input size of Critic doesn&class="macro">#x27;t match latent state Actor(%d <> %d)", Result.Total(), latent_state); class="kw">return INIT_FAILED; } Gradient.BufferInit(AccountDescr, class="num">0); class=class="str">"cmt">//--- if(!EventChartCustom(ChartID(), class="num">1, class="num">0, class="num">0, "Init")) { PrintFormat("Error of create study event: %d", GetLastError()); class="kw">return INIT_FAILED; } class=class="str">"cmt">//--- class="kw">return(INIT_SUCCEEDED); } class="type">void OnDeinit(class="kw">const class="type">int reason) { class=class="str">"cmt">//--- TargetCritic1.WeightsUpdate(GetPointer(Critic1), Tau); TargetCritic2.WeightsUpdate(GetPointer(Critic2), Tau); Actor.Save(FileName + "Act.nnw", class="num">0, class="num">0, class="num">0, TimeCurrent(), true); TargetCritic1.Save(FileName + "Crt1.nnw", Critic1.getRecentAverageError(), class="num">0, class="num">0, TimeCurrent(), true);
◍ 把账户状态压成神经网络能吃的向量
强化学习里 agent 看到的不是 K 线图，而是一串被手工构造的特征向量。下面这段 Train 函数里，每个 state 都先塞进账户原始字段，再追加十几个衍生比率，目的就是让卷积网络捕捉权益曲线与时间的周期关系。 注意那几个 MathSin / MathCos 构造：用账户时间戳 account[7] 分别除以一年、月线秒数、周线秒数、日线秒数，再乘 2π 取正余弦。这是把绝对时间映射成循环特征，避免模型把‘1月’和‘12月’当成远离的两点。D'2024.01.01' - D'2023.01.01' 在 MT5 里就是一年的秒数常量，直接写死比调 PeriodSeconds(PERIOD_Y1) 更省事。 State.Add 连续追加了余额变化率、权益比、浮盈浮亏占比等共 7 个归一化项，再加 4 个周期项，单 state 维度取决于 CNN 输出 temp.Size()。你在 MT5 里跑这套，先打印 temp.Size() 确认嵌入矩阵列数，否则 state_embedding 零矩阵会直接喂出垃圾梯度。外汇与贵金属杠杆高，这种特征若训练集含极端滑点，实盘可能倾向过拟合。
class="type">int total_tr = ArraySize(Buffer); class="type">uint ticks = GetTickCount(); class=class="str">"cmt">//--- class="type">int total_states = Buffer[class="num">0].Total; for(class="type">int i = class="num">1; i < total_tr; i++) total_states += Buffer[i].Total; vector<class="type">float> temp; Convolution.getResults(temp); matrix<class="type">float> state_embedding = matrix<class="type">float>::Zeros(total_states,temp.Size()); matrix<class="type">float> rewards = matrix<class="type">float>::Zeros(total_states,NRewards); for(class="type">int tr = class="num">0; tr < total_tr; tr++) { for(class="type">int st = class="num">0; st < Buffer[tr].Total; st++) { State.AssignArray(Buffer[tr].States[st].state); class="type">float PrevBalance = Buffer[tr].States[MathMax(st,class="num">0)].account[class="num">0]; class="type">float PrevEquity = Buffer[tr].States[MathMax(st,class="num">0)].account[class="num">1]; State.Add((Buffer[tr].States[st].account[class="num">0] - PrevBalance) / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">1] / PrevBalance); State.Add((Buffer[tr].States[st].account[class="num">1] - PrevEquity) / PrevEquity); State.Add(Buffer[tr].States[st].account[class="num">2]); State.Add(Buffer[tr].States[st].account[class="num">3]); State.Add(Buffer[tr].States[st].account[class="num">4] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">5] / PrevBalance); State.Add(Buffer[tr].States[st].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); State.Add((class="type">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[st].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1);
「把状态序列喂给卷积网络做嵌入」
这段逻辑干的事，是把每一条回测轨迹里的状态向量经过一层卷积前向传播，得到 state_embedding 与 rewards 矩阵，供后续强化学习迭代使用。注意 feedForward 一旦返回 false 会直接 ExpertRemove() 退出，实盘加载前务必确认 Convolution 对象已正确初始化，否则 EA 会无声撤出。 循环里用 GetTickCount() 做节流：每超过 500 毫秒才用 Comment 刷新一次 Embedding 进度百分比（state*100.0/total_states）。在 MT5 策略测试器里跑大规模轨迹时，这个节流能避免日志被刷屏，但如果你想要更细的进度，可以把 500 调小。 嵌入完成后若 state != total_states，会对 rewards 和 state_embedding 做 Resize / Reshape 裁剪掉尾部空位，total_states 同步更新。后面训练循环里用 MathRand()/32767.0 做轨迹与步长采样，其中步长 i 用了两次随机相乘再除以 32767 平方，偏向取靠前样本；若 i<0 则 iter-- 重抽。外汇与贵金属品种波动大，这类采样偏差可能让模型更关注早期样本，回测结论仅具概率意义，实盘前需在多品种上验证。
State.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); State.AddArray(Buffer[tr].States[st].action); if(!Convolution.feedForward(GetPointer(State),class="num">1,false,NULL)) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); ExpertRemove(); class="kw">return; } Convolution.getResults(temp); state_embedding.Row(temp,state); temp.Assign(Buffer[tr].States[st].rewards); rewards.Row(temp,state); state++; if(GetTickCount() - ticks > class="num">500) { class="type">class="kw">string str = StringFormat("%-15s %class="num">6.2f%%", "Embedding ", state * class="num">100.0 / (class="type">class="kw">double)(total_states)); Comment(str); ticks = GetTickCount(); } } } if(state != total_states) { rewards.Resize(state,NRewards); state_embedding.Reshape(state,state_embedding.Cols()); total_states = state; } vector<class="type">float> rewards1, rewards2; for(class="type">int iter = class="num">0; (iter < Iterations && !IsStopped()); iter ++) { class="type">int tr = (class="type">int)((MathRand() / class="num">32767.0) * (total_tr - class="num">1)); class="type">int i = (class="type">int)((MathRand() * MathRand() / MathPow(class="num">32767, class="num">2)) * (Buffer[tr].Total - class="num">2)); if(i < class="num">0) { iter--; class="kw">continue; } vector<class="type">float> reward, target_reward = vector<class="type">float>::Zeros(NRewards); reward.Assign(Buffer[tr].States[i].rewards); class=class="str">"cmt">//--- Target if(iter >= StartTargetIter) { State.AssignArray(Buffer[tr].States[i + class="num">1].state); class="type">float PrevBalance = Buffer[tr].States[i].account[class="num">0]; class="type">float PrevEquity = Buffer[tr].States[i].account[class="num">1]; Account.Clear(); Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">0] - PrevBalance) / PrevBalance); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">1] / PrevBalance); Account.Add((Buffer[tr].States[i + class="num">1].account[class="num">1] - PrevEquity) / PrevEquity); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">2]);
账户特征与双评论家目标奖励的拼接
这段逻辑把下一状态的账户信息灌进特征容器，再做周期归一化的三角函数编码。account[3] 直接入列，account[4]~[6] 都除以 PrevBalance，把绝对金额压成相对净值比例，避免不同账户规模喂出量纲混乱的向量。 时间维度用了四组分母：先拿 2024.01.01 减 2023.01.01 的秒数（年化 365 天对应约 31536000 秒）算正弦，再分别用 MN1、W1、D1 的 PeriodSeconds 做余弦或正弦。x 为 0 时强制相位 0，否则统一乘 2*M_PI 把周期折回单位圆，相当于把持仓时长映射成不同频率的循环特征。 写完特征后 Account.GetIndex()>=0 才 BufferWrite，随后 Actor 做前向推理。若任意 feedForward 失败就 PrintFormat 打函数名加行号并 break，这是典型的离线训练防御写法，开 MT5 跑时若日志频繁出现这类输出，说明状态指针或网络层维度对不上。 TargetCritic1 与 TargetCritic2 各前向一次，取两者 rewards 求和较小者作为 target_reward，再逐元素减去下一状态已知 rewards，最后乘 DiscFactor 折扣。双评论家取小能缓解 Q 值高估，但外汇与贵金属杠杆品种下回测奖励曲线仍可能剧烈摆动，实盘验证前务必用小资金测通道。
Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">3]); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">4] / PrevBalance); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">5] / PrevBalance); Account.Add(Buffer[tr].States[i + class="num">1].account[class="num">6] / PrevBalance); class="type">class="kw">double x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)(D&class="macro">#x27;class="num">2024.01.class="num">01&class="macro">#x27; - D&class="macro">#x27;class="num">2023.01.class="num">01&class="macro">#x27;); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_MN1); Account.Add((class="type">float)MathCos(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_W1); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); x = (class="type">class="kw">double)Buffer[tr].States[i + class="num">1].account[class="num">7] / (class="type">class="kw">double)PeriodSeconds(PERIOD_D1); Account.Add((class="type">float)MathSin(x != class="num">0 ? class="num">2.0 * M_PI * x : class="num">0)); class=class="str">"cmt">//--- if(Account.GetIndex() >= class="num">0) Account.BufferWrite(); if(!Actor.feedForward(GetPointer(State), class="num">1, false, GetPointer(Account))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } if(!TargetCritic1.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor)) || !TargetCritic2.feedForward(GetPointer(Actor), LatentLayer, GetPointer(Actor))) { PrintFormat("%s -> %d", __FUNCTION__, __LINE__); class="kw">break; } TargetCritic1.getResults(rewards1); TargetCritic2.getResults(rewards2); if(rewards1.Sum() <= rewards2.Sum()) target_reward = rewards1; else target_reward = rewards2; for(class="type">class="kw">ulong r = class="num">0; r < target_reward.Size(); r++) target_reward -= Buffer[tr].States[i + class="num">1].rewards[r]; target_reward *= DiscFactor;