在算法交易中 Kohonen 神经网络的实际应用。 第 I 部分 工具·进阶篇
（2/3）· 从索引错乱到输入常规化缺失，旧版 CSOM 类藏着哪些让回测失真的暗坑
SOM 节点维度与邻域拓扑的底层定义
自组织映射（SOM）在 MT5 里做价格形态聚类时，每个节点除了基础特征维度，还额外挂了 5 个辅助维度：命中计数、U 矩阵、节点量化误差（即标准差平方的平均方差）、聚类标签、输出值。这些通过 EXTRA_DIMENSIONS 宏固定为 5，并在 DIM_HITCOUNT 到 DIM_OUTPUT 里按 m_dimension 偏移寻址，调参时改一处宏就能整体平移。 CSOMNode::CalculateDistance 只做一件事：取出对方节点的代码向量，再委托重载版算欧氏距离。这意味着节点间相似度比较完全依赖代码向量，不碰辅助维度，回测时若想加权重过滤，得改这里而非邻域逻辑。 邻域类用模板实现，正方形拓扑固定 4 邻居（上 -1、下 +1、左 -m_ycells、右 +m_ycells），六边形拓扑预留 6 邻居，其中 4、5 位按行奇偶动态填左上/左下或右上/右下。NBH_SQUARE_SIZE=4、NBH_HEXAGONAL_SIZE=6 两个宏直接决定数组大小，黄金 1 小时图用六边形可能更贴合真实波动扩散，但 CPU 开销多 50%。 loop 函数里 j = ind % m_ycells 拿到纵坐标，六边形分支用 oddy 修正斜向偏移。实盘跑前建议在策略测试器里把 hex 切到 true/false 各跑一遍，看节点收敛速度差异——外汇与贵金属杠杆高，拓扑选错可能让聚类对跳空响应迟钝。
class="macro">#define EXTRA_DIMENSIONS class="num">5 class="macro">#define DIM_HITCOUNT(m_dimension + class="num">0) class="macro">#define DIM_UMATRIX(m_dimension + class="num">1) class="macro">#define DIM_NODEMSE(m_dimension + class="num">2) class=class="str">"cmt">// 每个节点的量化误差：平均方差（标准差的平方） class="macro">#define DIM_CLUSTERS(m_dimension + class="num">3) class="macro">#define DIM_OUTPUT(m_dimension + class="num">4) class="type">class="kw">double CSOMNode::CalculateDistance(class="kw">const CSOMNode *other) class="kw">const { class="type">class="kw">double vector[]; other.GetCodeVector(vector); class="kw">return CalculateDistance(vector); } class="macro">#define NBH_SQUARE_SIZE class="num">4 class="macro">#define NBH_HEXAGONAL_SIZE class="num">6 class="kw">template<class="kw">typename T> class Neighbourhood { class="kw">protected: class="type">int neighbours[]; class="type">int nbhsize; class="type">bool hex; class="type">int m_ycells; class="kw">public: Neighbourhood(class="kw">const class="type">bool _hex, class="kw">const class="type">int ysize) { hex = _hex; m_ycells = ysize; if(hex) { nbhsize = NBH_HEXAGONAL_SIZE; ArrayResize(neighbours, NBH_HEXAGONAL_SIZE); neighbours[class="num">0] = -class="num">1; class=class="str">"cmt">// 上 (可视) neighbours[class="num">1] = +class="num">1; class=class="str">"cmt">// 下 (可视) neighbours[class="num">2] = -m_ycells; class=class="str">"cmt">// 左 neighbours[class="num">3] = +m_ycells; class=class="str">"cmt">// 右 class=class="str">"cmt">/* 模板，在下面的循环中动态应用 class=class="str">"cmt">// 奇数行 neighbours[class="num">4] = -m_ycells - class="num">1; class=class="str">"cmt">// 左上 neighbours[class="num">5] = -m_ycells + class="num">1; class=class="str">"cmt">// 左下 class=class="str">"cmt">// 偶数行 neighbours[class="num">4] = +m_ycells - class="num">1; class=class="str">"cmt">// 右上 neighbours[class="num">5] = +m_ycells + class="num">1; class=class="str">"cmt">// 右下 */ } else { nbhsize = NBH_SQUARE_SIZE; ArrayResize(neighbours, NBH_SQUARE_SIZE); neighbours[class="num">0] = -class="num">1; class=class="str">"cmt">// 上 (可视) neighbours[class="num">1] = +class="num">1; class=class="str">"cmt">// 下 (可视) neighbours[class="num">2] = -m_ycells; class=class="str">"cmt">// 左 neighbours[class="num">3] = +m_ycells; class=class="str">"cmt">// 右 } } ~Neighbourhood() { ArrayResize(neighbours, class="num">0); } T loop(class="kw">const class="type">int ind, class="kw">const CSOMNode &p_node[]) { class="type">int nodes = ArraySize(p_node); class="type">int j = ind % m_ycells; if(hex) { class="type">int oddy = ((j % class="num">2) == class="num">1) ? -class="num">1 : +class="num">1; neighbours[class="num">4] = oddy * m_ycells - class="num">1; neighbours[class="num">5] = oddy * m_ycells + class="num">1; } reset();
「UMatrix 距离与节点命中统计的实现细节」
自组织映射里，单个神经元和周边邻居的离散程度用 UMatrix 衡量。代码里 UMatrixNeighbourhood 继承通用邻域模板，reset 把累加量 n 和 d 清零，iterate 每遇到一对相邻节点就累加两者权重空间距离并计数，getResult 返回 d/n 的平均邻距。 主流程 CalculateDistances 遍历全部 m_xcells*m_ycells 个节点，对每个节点调 umnh.loop 拿平均邻距，顺手刷新 m_max[DIM_UMATRIX] 并记录到节点自身。外汇与贵金属行情做 SOM 聚类时，这套邻距矩阵能暴露稀疏区与密集区，但样本不足时边界误差可能放大，属高风险建模。 节点侧 RegisterPatternHit 在每次命中时累加输入向量的和与平方和，并算权重与输入的差方和进 m_mse；AddPatternStats 则负责把外部数据拷进静态 vector 后喂给最佳匹配节点。直接把这段粘进 MT5 的 SOM 类，能在可视化的 UMatrix Heatmap 上验证聚类质量。
for(class="type">int k = class="num">0; k < nbhsize; k++) { if(ind + neighbours[k] >= class="num">0 && ind + neighbours[k] < nodes) { class=class="str">"cmt">// 跳过包边 if(j == class="num">0) class=class="str">"cmt">// 顶行 { if(k == class="num">0 || k == class="num">4) class="kw">continue; } else if(j == m_ycells - class="num">1) class=class="str">"cmt">// 底行 { if(k == class="num">1 || k == class="num">5) class="kw">continue; } iterate(p_node[ind], p_node[ind + neighbours[k]]); } } class="kw">return getResult(); } class="kw">virtual class="type">void reset() = class="num">0; class="kw">virtual class="type">void iterate(class="kw">const CSOMNode &node1, class="kw">const CSOMNode &node2) = class="num">0; class="kw">virtual T getResult() class="kw">const = class="num">0; }; class UMatrixNeighbourhood: class="kw">public Neighbourhood<class="type">class="kw">double> { class="kw">private: class="type">int n; class="type">class="kw">double d; class="kw">public: UMatrixNeighbourhood(class="kw">const class="type">bool _hex, class="kw">const class="type">int ysize): Neighbourhood(_hex, ysize) { } class="kw">virtual class="type">void reset() class="kw">override { n = class="num">0; d = class="num">0.0; } class="kw">virtual class="type">void iterate(class="kw">const CSOMNode &node1, class="kw">const CSOMNode &node2) class="kw">override { d += node1.CalculateDistance(&node2); n++; } class="kw">virtual class="type">class="kw">double getResult() class="kw">const class="kw">override { class="kw">return d / n; } }; class="type">void CSOM::CalculateDistances() { UMatrixNeighbourhood umnh(m_hexCells, m_ycells); for(class="type">int i = class="num">0; i < m_xcells * m_ycells; i++) { class="type">class="kw">double d = umnh.loop(i, m_node); if(d > m_max[DIM_UMATRIX]) { m_max[DIM_UMATRIX] = d; } m_node[i].SetDistance(d); } } class="type">void CSOMNode::RegisterPatternHit(class="kw">const class="type">class="kw">double &vector[]) { m_hitCount++; class="type">class="kw">double e = class="num">0; for(class="type">int i = class="num">0; i < m_dimension; i++) { m_sum[i] += vector[i]; m_sumP2[i] += vector[i] * vector[i]; e += (m_weights[i] - vector[i]) * (m_weights[i] - vector[i]); } m_mse += e / m_dimension; } class="type">class="kw">double CSOM::AddPatternStats(class="kw">const class="type">class="kw">double &data[]) { class="kw">static class="type">class="kw">double vector[]; ArrayCopy(vector, data);
◍ 自组织映射的误差统计与归一化实现
这段 CSOM 类的实现把模式匹配后的误差量化和输入归一化都收进了同一套逻辑。CalculateStats 会遍历训练集或验证集，对每个向量调用 AddPatternStats 累加 MSE，最后用 trainedMSE 除以 m_dataMSE 得到归一化均方误差 NMSE；当 complete 为 true 时直接在日志打印 Overall NMSE= 数值，方便你直观看模型拟合偏差。 AddPatternStats 里先取最佳匹配节点索引，把输入向量登记为该节点的命中，再取出该节点的码本向量做反归一化，逐维算 (data[i]-code[i]) 平方并除以维度得到单样本 mse。外汇与贵金属行情高波动、高杠杆，这类自组织映射的 NMSE 仅反映历史样本重构误差，对 unseen 行情的泛化能力可能偏弱，实盘前务必用验证集偏移 m_validationOffset 之后的数据重算。 CalculateDataMSE 只统计验证段（从 m_validationOffset 到 m_nSet）相对整体均值 m_mean 的分散度，作为 NMSE 的分母基准。InitNormalization 则逐维扫一遍全样本，记录 max/min 用于区间缩放，若开启 normalization 还会顺手算均值与标准差：m_sigma[j] 用 MathSqrt(sigma/n - mean^2) 得到，若样本数为 0 则强制均值 0、标准差 1 避免除零。 节点层只暴露 GetHitsCount 返回 m_hitCount，这意味着你能在跑完 CalculateStats 后直接读每个节点的命中数，判断哪些码本向量被过度激活、哪些几乎闲置。打开 MT5 把这套 CSOM 挂到 EURUSD 的 M15 上，调 m_dimension 从 10 改到 20，观察 NMSE 打印值的变化倾向，比空谈聚类有效得多。
class="type">int ind = GetBestMatchingIndex(vector); m_node[ind].RegisterPatternHit(vector); class="type">class="kw">double code[]; m_node[ind].GetCodeVector(code); Denormalize(code); class="type">class="kw">double mse = class="num">0; for(class="type">int i = class="num">0; i < m_dimension; i++) { mse += (data[i] - code[i]) * (data[i] - code[i]); } mse /= m_dimension; class="kw">return mse; } class="type">class="kw">double CSOM::CalculateStats(class="kw">const class="type">bool complete = true) { class="type">class="kw">double data[]; ArrayResize(data, m_dimension); class="type">class="kw">double trainedMSE = class="num">0.0; for(class="type">int i = complete ? class="num">0 : m_validationOffset; i < m_nSet; i++) { ArrayCopy(data, m_set, class="num">0, m_dimension * i, m_dimension); trainedMSE += AddPatternStats(data, complete); } class="type">class="kw">double nmse = trainedMSE / m_dataMSE; if(complete) Print("Overall NMSE=", nmse); class="kw">return nmse; } class="type">void CSOM::CalculateDataMSE() { class="type">class="kw">double data[]; m_dataMSE = class="num">0.0; for(class="type">int i = m_validationOffset; i < m_nSet; i++) { ArrayCopy(data, m_set, class="num">0, m_dimension * i, m_dimension); class="type">class="kw">double mse = class="num">0; for(class="type">int k = class="num">0; k < m_dimension; k++) { mse += (data[k] - m_mean[k]) * (data[k] - m_mean[k]); } mse /= m_dimension; m_dataMSE += mse; } } class="type">void CSOM::InitNormalization(class="kw">const class="type">bool normalization = true) { ArrayResize(m_max, m_dimension + EXTRA_DIMENSIONS); ArrayResize(m_min, m_dimension + EXTRA_DIMENSIONS); ArrayInitialize(m_max, class="num">0); ArrayInitialize(m_min, class="num">0); ArrayResize(m_mean, m_dimension); ArrayResize(m_sigma, m_dimension); for(class="type">int j = class="num">0; j < m_dimension; j++) { class="type">class="kw">double maxv = -DBL_MAX; class="type">class="kw">double minv = +DBL_MAX; if(normalization) { m_mean[j] = class="num">0; m_sigma[j] = class="num">0; } for(class="type">int i = class="num">0; i < m_nSet; i++) { class="type">class="kw">double v = m_set[m_dimension * i + j]; if(v > maxv) maxv = v; if(v < minv) minv = v; if(normalization) { m_mean[j] += v; m_sigma[j] += v * v; } } m_max[j] = maxv; m_min[j] = minv; if(normalization && m_nSet > class="num">0) { m_mean[j] /= m_nSet; m_sigma[j] = MathSqrt(m_sigma[j] / m_nSet - m_mean[j] * m_mean[j]); } else { m_mean[j] = class="num">0; m_sigma[j] = class="num">1; } } } class="type">int CSOMNode::GetHitsCount() class="kw">const { class="kw">return m_hitCount; }
自组织映射的命中统计与聚类落地
自组织映射（SOM）每个节点在训练后会积累命中次数与加权和，直接决定后续聚类的可信度。下面这段节点类方法给出了均值、标准差与 MSE 的最简实现：命中数为 0 时直接返回 0，避免除零；方差用「平方和均值减均值平方」再开方，和统计学公式一致。 double CSOMNode::GetHitsMean(const int plane) const { if(m_hitCount == 0) return 0; return m_sum[plane] / m_hitCount; } double CSOMNode::GetHitsDeviation(const int plane) const { if(m_hitCount == 0) return 0; double z = m_sumP2[plane] / m_hitCount - m_sum[plane] / m_hitCount * m_sum[plane] / m_hitCount; if(z < 0) return 0; return MathSqrt(z); } double CSOMNode::GetMSE() const { if(m_hitCount == 0) return 0; return m_mse / m_hitCount; } 逐行看：GetHitsMean 第 3 行用 m_sum[plane] 除以 m_hitCount 得该维度加权均值；GetHitsDeviation 第 6 行算 E(x²)-E(x)²，第 8 行对负值兜底返回 0（浮点误差可能让 z 微负）；GetMSE 直接拿累计误差 m_mse 除以命中数。 CSOM::CalculateOutput 把输入向量灌进所有节点，记录最小最大输出距，供归一化或阈值判断用。KMeans 聚类则把每个节点的码向量抽出来送进 Alglib 的 CKMeans::KMeansGenerate，clusterNumber 传几就分几堆；info==1 才把 membership 写回节点。外汇与贵金属行情用这套做状态划分时波动剧烈，聚类边界可能随品种跳空漂移，属高风险用法，结论仅具概率意义。 void CSOM::Clusterize(const int clusterNumber) { int count = m_xcells * m_ycells; CMatrixDouble xy(count, m_dimension); int info; CMatrixDouble clusters; int membership[]; double weights[]; for(int i = 0; i < count; i++) { m_node[i].GetCodeVector(weights); xy[i] = weights; } CKMeans::KMeansGenerate(xy, count, m_dimension, clusterNumber, KMEANS_RETRY_NUMBER, info, clusters, membership); Print("KMeans result: ", info); if(info == 1) { for(int i = 0; i < m_xcells * m_ycells; i++) m_node[i].SetCluster(membership[i]); ArrayResize(m_clusters, clusterNumber * m_dimension); for(int j = 0; j < clusterNumber; j++) for(int i = 0; i < m_dimension; i++) m_clusters[j * m_dimension + i] = clusters[i][j]; } } Clusterize 里第 16 行 KMEANS_RETRY_NUMBER 控制重试次数，MT5 默认头部宏通常设 5～10 次，重试多可降低局部最优概率但拖慢回测。
class="type">class="kw">double CSOMNode::GetHitsMean(class="kw">const class="type">int plane) class="kw">const { if(m_hitCount == class="num">0) class="kw">return class="num">0; class="kw">return m_sum[plane] / m_hitCount; } class="type">class="kw">double CSOMNode::GetHitsDeviation(class="kw">const class="type">int plane) class="kw">const { if(m_hitCount == class="num">0) class="kw">return class="num">0; class="type">class="kw">double z = m_sumP2[plane] / m_hitCount - m_sum[plane] / m_hitCount * m_sum[plane] / m_hitCount; if(z < class="num">0) class="kw">return class="num">0; class="kw">return MathSqrt(z); } class="type">class="kw">double CSOMNode::GetMSE() class="kw">const { if(m_hitCount == class="num">0) class="kw">return class="num">0; class="kw">return m_mse / m_hitCount; } class="type">class="kw">double CSOMNode::CalculateOutput(class="kw">const class="type">class="kw">double &vector[]) { m_output = CalculateDistance(vector); class="kw">return m_output; } class="type">void CSOM::CalculateOutput(class="kw">const class="type">class="kw">double &vector[], class="kw">const class="type">bool normalize = class="kw">false) { class="type">class="kw">double temp[]; ArrayCopy(temp, vector); if(normalize) Normalize(temp); m_min[DIM_OUTPUT] = DBL_MAX; m_max[DIM_OUTPUT] = -DBL_MAX; for(class="type">int i = class="num">0; i < ArraySize(m_node); i++) { class="type">class="kw">double x = m_node[i].CalculateOutput(temp); if(x < m_min[DIM_OUTPUT]) m_min[DIM_OUTPUT] = x; if(x > m_max[DIM_OUTPUT]) m_max[DIM_OUTPUT] = x; } } class="macro">#include <Math/Alglib/dataanalysis.mqh> class="type">void CSOM::Clusterize(class="kw">const class="type">int clusterNumber) { class="type">int count = m_xcells * m_ycells; CMatrixDouble xy(count, m_dimension); class="type">int info; CMatrixDouble clusters; class="type">int membership[]; class="type">class="kw">double weights[]; for(class="type">int i = class="num">0; i < count; i++) { m_node[i].GetCodeVector(weights); xy[i] = weights; } CKMeans::KMeansGenerate(xy, count, m_dimension, clusterNumber, KMEANS_RETRY_NUMBER, info, clusters, membership); Print("KMeans result: ", info); if(info == class="num">1) { for(class="type">int i = class="num">0; i < m_xcells * m_ycells; i++) { m_node[i].SetCluster(membership[i]); } ArrayResize(m_clusters, clusterNumber * m_dimension); for(class="type">int j = class="num">0; j < clusterNumber; j++) { for(class="type">int i = class="num">0; i < m_dimension; i++) { m_clusters[j * m_dimension + i] = clusters[i][j]; } } } }
「聚类落地的代码骨架」
自组织映射跑完训练后，真正难的是把网格节点归并成有意义的簇。上面这段 CSOM::Clusterize 给出了一种按命中率与距离排序、再查邻域归属的实现路径。 先建一个 n 行 2 列的数组，n 等于 m_xcells 乘 m_ycells，也就是全部网格节点数。有命中的节点用 GetDistance 乘根号下 GetMSE 当排序键，空节点直接填 DBL_MAX，随后 ArraySort 把“稠密且贴近”的节点顶到前面。 ClusterNeighbourhood 的 loop 负责看当前节点是否挨着已有聚类：返回大于 -1 就直接 SetCluster 挂到那个簇，返回 -1 才在 m_clusters 里扩出一维新向量并自增 count。 那段被 override 的 iterate 是邻域判定的核心——它取邻居的 cluster 值，用 MathMin 做向下合并，保证相邻节点最终落到同一个最小编号的簇里。外汇与贵金属行情下用这套做形态分区，高风险在于空节点占比高时聚类数可能剧烈跳变，建议开 MT5 把 m_hexCells 调小观察 count 收敛速度。
cluster = -class="num">1; } class="kw">virtual class="type">void iterate(class="kw">const CSOMNode &node1, class="kw">const CSOMNode &node2) class="kw">override { class="type">int x = node2.GetCluster(); if(x > -class="num">1) { if(cluster != -class="num">1) cluster = MathMin(cluster, x); else cluster = x; } } class="kw">virtual class="type">int getResult() class="kw">const class="kw">override { class="kw">return cluster; } }; class="type">void CSOM::Clusterize() { class="type">class="kw">double array[][class="num">2]; class="type">int n = m_xcells * m_ycells; ArrayResize(array, n); for(class="type">int i = class="num">0; i < n; i++) { if(m_node[i].GetHitsCount() > class="num">0) { array[i][class="num">0] = m_node[i].GetDistance() * MathSqrt(m_node[i].GetMSE()); } else { array[i][class="num">0] = DBL_MAX; } array[i][class="num">1] = i; m_node[i].SetCluster(-class="num">1); } ArraySort(array); ClusterNeighbourhood clnh(m_hexCells, m_ycells); class="type">int count = class="num">0; class=class="str">"cmt">// 聚类数量 ArrayResize(m_clusters, class="num">0); for(class="type">int i = class="num">0; i < n; i++) { class=class="str">"cmt">// 如果已经分配则跳过 if(m_node[(class="type">int)array[i][class="num">1]].GetCluster() > -class="num">1) class="kw">continue; class=class="str">"cmt">// 检查当前节点是否与任何现有聚类相邻 class="type">int r = clnh.loop((class="type">int)array[i][class="num">1], m_node); if(r > -class="num">1) class=class="str">"cmt">// 邻居已经属于聚类 { m_node[(class="type">int)array[i][class="num">1]].SetCluster(r); } else class=class="str">"cmt">// 我们需要新的聚类 { ArrayResize(m_clusters, (count + class="num">1) * m_dimension); class="type">class="kw">double vector[]; m_node[(class="type">int)array[i][class="num">1]].GetCodeVector(vector); ArrayCopy(m_clusters, vector, count * m_dimension, class="num">0, m_dimension); m_node[(class="type">int)array[i][class="num">1]].SetCluster(count++); } } }