📘

使用Python和MQL5进行特征工程（第四部分）：基于UMAP回归的K线模式识别·进阶篇

第 2/3 篇

本章目录

用 UMAP 把 10 列行情压成 3 维再喂模型
用迭代次数扫描掐住神经网络的误差颈
把 UMAP 回归链塞进 EA 的工程骨架
EA 生命周期里的句柄与模型装载
给 ONNX 模型钉死张量维度再开跑

◍ 用 UMAP 把 10 列行情压成 3 维再喂模型

先把当日「H - L」(最高减最低的有效区间) 和「O - C」(开盘减收盘的净变化) 画成散点，EURUSD 样本上两者关系明显非线性且杂乱，直接拿原始列训练统计模型很难切分涨跌两类。 UMAP 的接入很轻：建对象、fit_transform 原始矩阵，默认就吐出 2 列。原文示例里用 n_neighbors=100、metric="euclidean" 把 10 列压到 2 维，散点里已经能看到橙、蓝两类各自抱团的区域，说明降维后类别边界更可被学习。但 2 维是随手选的。我们在 1~9 之间做线性搜索，用 TimeSeriesSplit(5 折、gap=24) 做时间序列交叉验证，GradientBoostingRegressor 算 RMSE。结果在 n_components=3 时验证误差最低——注意这 3 列不是原 10 列里「挑最好的 3 列」，而是 10 列被非线性重投影成的 3 个新维度。同样的交叉验证下，UMAP 变换后的最小误差明显低于直接用原始价格列 (classic_error) 的误差水平，说明对外汇 EURUSD / EURGBP 这类高噪声品种，先做嵌入再建模可能比裸喂 OHLC 衍生列更稳。外汇与贵金属杠杆高、回测优不代表实盘概率占优，上线前务必用 MT5 真实 tick 重跑。下面这段是原文里做可视化与降维搜索的核心片段，逐行拆一下关键调用：sns.scatterplot 用 hue='Class' 把两类标色看区间与净变化关系；umap.UMAP(n_neighbors=100, metric='euclidean') 定邻域与距离尺度；return_transformed_data(n_components) 里 HORIZON=24 做 24 根 K 线后的收盘价差分目标，并切掉最后约 (365*5 - 31*5) 行防回测穿越；for i in range(LEVELS) 循环 1~8 维算交叉验证绝对误差，idxmin 抓最优维数。

MQL5 / C++

sns.scatterplot(
    data=data,
    y=&class="macro">#x27;O - C&class="macro">#x27;,
    x=&class="macro">#x27;H - L&class="macro">#x27;,
    hue=&class="macro">#x27;Class&class="macro">#x27;
)
plt.grid()
plt.title("Visualizing Our Custom Columns on EURUSD Market Data")
plt.axhline(class="num">0,class="type">class="kw">color=&class="macro">#x27;black&class="macro">#x27;,linestyle=&class="macro">#x27;--&class="macro">#x27;)
reducer = umap.UMAP(n_neighbors=class="num">100,metric="euclidean")
embedding = reducer.fit_transform(data.iloc[:,class="num">2:-class="num">2])
embedding = pd.DataFrame(embedding,columns=[&class="macro">#x27;X1&class="macro">#x27;,&class="macro">#x27;X2&class="macro">#x27;])
embedding[&class="macro">#x27;Class&class="macro">#x27;] = data[&class="macro">#x27;Class&class="macro">#x27;]
sns.scatterplot(
    data=embedding,
    x=&class="macro">#x27;X1&class="macro">#x27;,
    y=&class="macro">#x27;X2&class="macro">#x27;,
    hue=&class="macro">#x27;Class&class="macro">#x27;
)
plt.grid()
plt.title("Visualizing the effects of UMAP on our EURUSD Market Data")
def return_transformed_data(n_components):
    HORIZON = class="num">24
    data = pd.read_csv("..\EURGBP UMAP Candlestick Recognition.csv")
    data[&class="macro">#x27;Target&class="macro">#x27;] = data[&class="macro">#x27;True Close&class="macro">#x27;].shift(-HORIZON) - data[&class="macro">#x27;True Close&class="macro">#x27;]
    data.dropna(inplace=True)
    data = data.iloc[:(-(class="num">365 * class="num">5) + (class="num">31 * class="num">5)),:]
    reducer = umap.UMAP(n_neighbors=class="num">100,metric="euclidean",n_components=n_components,n_jobs=-class="num">1)
    embedding = reducer.fit_transform(data.iloc[:,class="num">2:-class="num">1])
    cols = []
    for i in np.arange(n_components):
        s = &class="macro">#x27;X&class="macro">#x27; + &class="macro">#x27; &class="macro">#x27; + str(i)
        cols.append(s)
    
    embedding = pd.DataFrame(embedding,columns=cols)
    class="kw">return embedding.copy()
from sklearn.ensemble class="kw">import GradientBoostingRegressor
from sklearn.model_selection class="kw">import TimeSeriesSplit,cross_val_score
tscv = TimeSeriesSplit(n_splits=class="num">5,gap=HORIZON)
LEVELS = class="num">8
res = pd.DataFrame(columns=[&class="macro">#x27;X&class="macro">#x27;],index=np.arange(LEVELS))
for i in range(LEVELS):
    new_data = return_transformed_data(i+class="num">1)
    res.iloc[i,class="num">0] = np.mean(np.abs(cross_val_score(GradientBoostingRegressor(),new_data.iloc[:,class="num">0:],data[&class="macro">#x27;Target&class="macro">#x27;],cv=tscv)))
res[&class="macro">#x27;X&class="macro">#x27;] = pd.to_numeric(res[&class="macro">#x27;X&class="macro">#x27;], errors=&class="macro">#x27;coerce&class="macro">#x27;)
min_value = min(res.iloc[:,class="num">0])
min_index = res[&class="macro">#x27;X&class="macro">#x27;].idxmin()
plt.plot(res,class="type">class="kw">color=&class="macro">#x27;black&class="macro">#x27;)
plt.grid()
plt.title(&class="macro">#x27;Finding The Optimal Number of U-MAP Components&class="macro">#x27;)
plt.ylabel(&class="macro">#x27;RMSE Validation Error&class="macro">#x27;)
plt.xlabel(&class="macro">#x27;Training Iteration&class="macro">#x27;)
plt.scatter(min_index,min_value,class="type">class="kw">color=&class="macro">#x27;red&class="macro">#x27;)
classic_error = np.mean(np.abs(cross_val_score(GradientBoostingRegressor(),data.iloc[:,class="num">2:-class="num">2],data[&class="macro">#x27;Target&class="macro">#x27;],cv=tscv)))
results = [min(res.iloc[:,class="num">0]),classic_error]
sns.barplot(results,class="type">class="kw">color=&class="macro">#x27;black&class="macro">#x27;)

「用迭代次数扫描掐住神经网络的误差颈」

上面的脚本先把 UMAP 降维前后的验证 RMSE 画成对比线，红虚线标出原始误差基准，直观看降维是否把误差往下拽。接着它没拍脑袋定训练量，而是用 2 的幂次从 2^0 扫到 2^17 共 18 档，喂给 MLPRegressor 做时间序列交叉验证。循环里 hidden_layer_sizes 固定成 (特征数,10,5)，solver 用 adam，max_iter 随档位指数增长；每档取 5 折 CV 的平均绝对误差写进 NN_ERROR。跑完用 idxmin 抓最小误差对应的迭代档，红点标在图上——这一步决定了后面两个模型的实际训练轮数。找到 min_idx 后，脚本训了两个网络：一个把原始 OHLC 特征压成 UMAP 嵌入，一个拿嵌入去预测 EURGBP 收益率。两者 max_iter 都锁成 2**min_idx，避免过拟合或欠拟合拍脑袋。最后用 skl2onnx 把俩模型转成 ONNX（target_opset=12），存成「EURGBP UMAP.onnx」和「EURGBP UMAP Forecast.onnx」。你在 MT5 外接 Python 跑这套，能直接 load 这两个文件做实时推断；外汇与贵金属杠杆高，信号失效时回撤可能很大，实盘前先用历史分时段验证。

MQL5 / C++

plt.axhline(results[class="num">0],class="type">class="kw">color=&class="macro">#x27;red&class="macro">#x27;,linestyle=&class="macro">#x27;--&class="macro">#x27;)
plt.ylabel(&class="macro">#x27;RMSE Validation Error&class="macro">#x27;)
plt.xlabel(&class="macro">#x27;class="num">0: UMAP Transformed Data | class="num">1: Original OHLC Data&class="macro">#x27;)
plt.title("UMAP Transformations Are Helping Us Reduce Our Error Rates")
from sklearn.neural_network class="kw">import MLPRegressor
new_data = return_transformed_data(class="num">3)
LEVELS = class="num">18
NN_ERROR = pd.DataFrame(columns=[&class="macro">#x27;Error&class="macro">#x27;],index=np.arange(LEVELS))
for i in range(LEVELS):
    model = MLPRegressor(hidden_layer_sizes=(data.iloc[:,class="num">2:-class="num">2].shape[class="num">1],class="num">10,class="num">5),max_iter=(class="num">2 ** i),solver=&class="macro">#x27;adam&class="macro">#x27;)
    NN_ERROR.iloc[i,class="num">0] = np.mean(np.abs(cross_val_score(model,new_data,data[&class="macro">#x27;Target&class="macro">#x27;],cv=tscv)))
NN_ERROR[&class="macro">#x27;Error&class="macro">#x27;] = pd.to_numeric(NN_ERROR[&class="macro">#x27;Error&class="macro">#x27;], errors=&class="macro">#x27;coerce&class="macro">#x27;)
min_idx   = NN_ERROR.idxmin()
min_value = NN_ERROR.min()
plt.plot(NN_ERROR,class="type">class="kw">color=&class="macro">#x27;black&class="macro">#x27;)
plt.grid()
plt.ylabel(&class="macro">#x27;class="num">5 Fold CV RMSE&class="macro">#x27;)
plt.xlabel(&class="macro">#x27;Max Iterations As Powers of class="num">2&class="macro">#x27;)
plt.scatter(min_idx,min_value,class="type">class="kw">color=&class="macro">#x27;red&class="macro">#x27;)
plt.title(&class="macro">#x27;Minimizing The Error of Our Neural Network&class="macro">#x27;)
class="macro">#The first model will transform the given market data into its UMAP embeddings
umap_transform_model = MLPRegressor(hidden_layer_sizes=(data.iloc[:,class="num">2:-class="num">2].shape[class="num">1],class="num">10,class="num">5),max_iter=class="type">int(class="num">2 ** min_idx),solver=&class="macro">#x27;adam&class="macro">#x27;)
umap_transform_model.fit(data.iloc[:,class="num">2:-class="num">2],new_data)
class="macro">#The second model will forecast the future EURGBP returns, given UMAP embeddings
forecast_model = MLPRegressor(hidden_layer_sizes=(new_data.shape[class="num">1],class="num">10,class="num">5),max_iter=class="type">int(class="num">2 ** min_idx),solver=&class="macro">#x27;adam&class="macro">#x27;)
forecast_model.fit(new_data,data[&class="macro">#x27;Target&class="macro">#x27;])
class="kw">import onnx
class="kw">import netron
from skl2onnx class="kw">import convert_sklearn
from skl2onnx.common.data_types class="kw">import FloatTensorType
umap_transform_shape = [("float_input",FloatTensorType([class="num">1,data.iloc[:,class="num">2:-class="num">2].shape[class="num">1]]))]
umap_transform_output_shape = [("float_output",FloatTensorType([new_data.shape[class="num">1],class="num">1]))]
forecast_shape = [("float_input",FloatTensorType([class="num">1,new_data.shape[class="num">1]]))]
umap_model_proto = convert_sklearn(umap_transform_model,initial_types=umap_transform_shape,final_types=umap_transform_output_shape,target_opset=class="num">12)
forecast_model_proto = convert_sklearn(forecast_model,initial_types=forecast_shape,target_opset=class="num">12)
onnx.save(umap_model_proto,"EURGBP UMAP.onnx")
onnx.save(forecast_model_proto,"EURGBP UMAP Forecast.onnx")

把 UMAP 回归链塞进 EA 的工程骨架

做这套策略的 MQL5 实现，核心是先通过 #resource 把两个 ONNX 模型（EURGBP UMAP.onnx 与 EURGBP UMAP Forecast.onnx）以 uchar 数组形式编进程序，运行时再用 ONNX 接口加载成模型句柄。全局变量刻意压到最少：只留两个模型句柄、两个输出向量（维度 3 和 1）以及一个止损变量，因为整套逻辑是算法驱动，不需要堆参数。技术指标侧只挂了两条移动平均的句柄与缓冲区（ma_o / ma_c），分别盯开盘与收盘的均线读数。代码组织上建议每个事件处理程序都拆成独立函数——setup 做初始化并返回 bool，update 每天把指标拷进缓冲区并触发交易，forecast 串起两级预测：第一级把 10 个市场输入压成 3 维嵌入，第二级用这 3 维算出 EURGBP 回报信号。release 负责 EA 停用时清资源。回测刻意选了样本外区间（删掉 2020 年至今的重叠数据），并在订单执行与成交间加随机延迟来模拟真实摩擦。日志里能看到第一级模型确实把 10 输入降到 3 输出，链路通畅。跑出来的样本外结果：夏普 0.42，预期回报 7.05，盈利交易占比 64%，共 25 笔；平均持仓 1274 小时（约 54 天），说明 EA 倾向抓中期趋势而非刷短线。外汇与贵金属属高风险品种，这类回测只是概率层面的正面迹象，实盘可能偏离。

MQL5 / C++

class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//|                                              UMAP Regression.mq5 |
class=class="str">"cmt">//|                                       Gamuchirai Ndawana |
class=class="str">"cmt">//|                         [MQL5官方文档] |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="macro">#class="kw">property copyright "Gamuchirai Ndawana"
class="macro">#class="kw">property link      "[MQL5官方文档]
class="macro">#class="kw">property version   "class="num">1.00"
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| System resources                                                        |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="macro">#resource "\Files\EURGBP UMAP.onnx" as class="type">uchar umap_onnx_buffer[];
class="macro">#resource "\Files\EURGBP UMAP Forecast.onnx" as class="type">uchar umap_forecast_onnx_buffer[];
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Global Variables                                                       |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">long umap_onnx_model,umap_forecast_onnx_model;
vectorf umap_onnx_output(class="num">3),umap_forecast_onnx_output(class="num">1);
class="type">class="kw">double trade_sl;
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Technical indicators                                                   |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">int ma_o_handler,ma_c_handler;
class="type">class="kw">double ma_o[],ma_c[];
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Technical indicators                                                   |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">int ma_o_handler,ma_c_handler;
class="type">class="kw">double ma_o[],ma_c[];

◍ EA 生命周期里的句柄与模型装载

MT5 的 Expert 只要跑起来，就绕不开 OnInit、OnDeinit、OnTick 这三层骨架。初始化阶段如果 setup() 返回 false，系统直接给 INIT_FAILED，EA 不会进入报价循环；这比在 OnTick 里报错要干净，也避免每跳tick都重复申请资源。下面这段把两个 ONNX 模型（UMAP 与 UMAP Forecast）从内存 buffer 建出来，同时挂了两条周期=当前图、周期数=2、模式=EMA 的均线句柄，分别吃收盘价与开盘价。任一模型句柄等于 INVALID_HANDLE 就弹注释并回 false，实盘里若看到 'Failed to create EURGBP UMAP Transformer ONNX model'，说明模型 buffer 没编译进 ex5 或版本不匹配。 OnDeinit 里调 release() 做收尾，把指标句柄和 ONNX 句柄全 Release 掉。外汇与贵金属杠杆高，EA 退出不释放句柄可能拖慢终端，尤其在多品种同时加载模型时。别在 OnInit 里偷懒不检查句柄很多新手把 iMA 和 OnnxCreateFromBuffer 的返回值当必然成功，结果模型加载失败却照常 OnTick，预测数组全是垃圾值。开 MT5 跑一遍，故意改坏 buffer 名看是否触发 COMMENT，比盲信'应该没问题'靠谱。

MQL5 / C++

class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Expert initialization function                                     |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">int OnInit()
  {
class=class="str">"cmt">//---
   if(!setup())
      class="kw">return(INIT_FAILED);
class=class="str">"cmt">//---
   class="kw">return(INIT_SUCCEEDED);
  }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Expert deinitialization function                                   |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">void OnDeinit(const class="type">int reason)
  {
class=class="str">"cmt">//---
   release();
  }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Expert tick function                                               |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">void OnTick()
  {
class=class="str">"cmt">//---
   update();
  }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Custom functions                                                   |
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Free up system memory                                              |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">void release(class="type">void)
  {
   IndicatorRelease(ma_c_handler);
   IndicatorRelease(ma_o_handler);
   OnnxRelease(umap_onnx_model);
   OnnxRelease(umap_forecast_onnx_model);
  }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Setup system variables                                             |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">bool setup(class="type">void)
  {
   umap_onnx_model = OnnxCreateFromBuffer(umap_onnx_buffer,ONNX_DATA_TYPE_FLOAT);
   umap_forecast_onnx_model = OnnxCreateFromBuffer(umap_forecast_onnx_buffer,ONNX_DATA_TYPE_FLOAT);
   ma_c_handler = iMA(_Symbol,PERIOD_CURRENT,class="num">2,class="num">0,MODE_EMA,PRICE_CLOSE);
   ma_o_handler = iMA(_Symbol,PERIOD_CURRENT,class="num">2,class="num">0,MODE_EMA,PRICE_OPEN);
   if(umap_onnx_model == INVALID_HANDLE)
     {
      Comment("Failed to create EURGBP UMAP Transformer ONNX model");
      class="kw">return(false);
     }
   if(umap_forecast_onnx_model == INVALID_HANDLE)
     {

「给 ONNX 模型钉死张量维度再开跑」

加载完 UMAP 降维模型和欧元英镑预测模型后，第一步不是急着推理，而是把输入输出张量的 shape 在 MT5 里显式锁死。原始代码里主模型输入设为 {1,10}、输出 {3,1}，预测模型输入 {1,3}、输出 {1,1}；任何一项 OnnxSetInputShape / OnnxSetOutputShape 返回 false，就直接 Comment 报错并 return(false)，终端同时 Print 出实际张量数量供你比对。这段防护很实在：ONNX 在 MT5 里不会自动推断动态轴，shape 对不上要么加载崩、要么推理出全零矩阵。外汇与贵金属杠杆高，模型静默出错比显式报错更危险，所以每个 Set 调用都该独立判错。初始化末尾把 trade_sl 固定成 2e-2（即 0.02），这是给后续下单用的止损距离基数，调参时建议先按品种波动率手动改这个值。 update() 函数用 static datetime 做新 K 线触发器：当前周期时间变了才 CopyBuffer 拉最新 MA 收盘价与开盘价，且只在 PositionsTotal()==0 时空仓状态下调用 GetModelForecast 与 FindSetup。这样避免每 tick 重算，也防止已有持仓时重复开单。

MQL5 / C++

   Comment("Failed to create EURGBP UMAP Forecast ONNX model");
   class="kw">return(false);
   }
   class="type">ulong umap_input_shape[]   = { class="num">1 , class="num">10 };
   class="type">ulong umap_forecast_input_shape[]   = { class="num">1 , class="num">3 };
   class="type">ulong umap_output_shape[] = { class="num">3 , class="num">1  };
   class="type">ulong umap_forecast_output_shape[] = { class="num">1 , class="num">1  };
   if(!OnnxSetInputShape(umap_onnx_model,class="num">0,umap_input_shape))
   {
      Comment("Failed to specify ONNX model input shape");
      Print("Actual shape: ",OnnxGetInputCount(umap_onnx_model));
      class="kw">return(false);
   }
   if(!OnnxSetInputShape(umap_forecast_onnx_model,class="num">0,umap_forecast_input_shape))
   {
      Comment("Failed to specify EURGBP Forecast ONNX model input shape");
      Print("Actual shape: ",OnnxGetInputCount(umap_onnx_model));
      class="kw">return(false);
   }
   if(!OnnxSetOutputShape(umap_onnx_model,class="num">0,umap_output_shape))
   {
      Comment("Failed to specify ONNX model output shape");
      Print("Actual shape: ",OnnxGetOutputCount(umap_onnx_model));
      class="kw">return(false);
   }
   if(!OnnxSetOutputShape(umap_forecast_onnx_model,class="num">0,umap_forecast_output_shape))
   {
      Comment("Failed to specify EURGBP Forecast ONNX model output shape");
      Print("Actual shape: ",OnnxGetOutputCount(umap_onnx_model));
      class="kw">return(false);
   }
   trade_sl = class="num">2e-2;
   class="kw">return(true);
   }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Update our system variables                                      |
class=class="str">"cmt">//+------------------------------------------------------------------+
class="type">void update(class="type">void)
  {
   class="kw">static class="type">class="kw">datetime time_stamp;
   class="type">class="kw">datetime current_time = iTime(_Symbol,PERIOD_CURRENT,class="num">0);
   if(current_time != time_stamp)
     {
      time_stamp = current_time;
      CopyBuffer(ma_c_handler,class="num">0,class="num">0,class="num">1,ma_c);
      CopyBuffer(ma_o_handler,class="num">0,class="num">0,class="num">1,ma_o);
      if(PositionsTotal() == class="num">0)
        {
         GetModelForecast();
         FindSetup();
        }
     }
  }
class=class="str">"cmt">//+------------------------------------------------------------------+
class=class="str">"cmt">//| Get a forecast from our models                                   |