From 4038d6d7af961d5120e5cb4ac31175cdaa91e77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E5=A4=95?= <11061622+lin-xi1@user.noreply.gitee.com> Date: Fri, 14 Aug 2026 02:27:47 +0800 Subject: [PATCH] research: add TSLM MindSpore models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 林夕 <11061622+lin-xi1@user.noreply.gitee.com> --- research/TSLM/.gitignore | 20 + research/TSLM/DPMSAN_MindSpore/.gitignore | 36 ++ research/TSLM/DPMSAN_MindSpore/CITATION.cff | 34 ++ research/TSLM/DPMSAN_MindSpore/README.md | 43 ++ .../DPMSAN_MindSpore/configs/default.json | 17 + research/TSLM/DPMSAN_MindSpore/pyproject.toml | 13 + .../TSLM/DPMSAN_MindSpore/requirements.txt | 3 + .../TSLM/DPMSAN_MindSpore/scripts/train.py | 132 +++++ .../src/dpmsan_ms/__init__.py | 22 + .../DPMSAN_MindSpore/src/dpmsan_ms/cli.py | 39 ++ .../DPMSAN_MindSpore/src/dpmsan_ms/config.py | 46 ++ .../DPMSAN_MindSpore/src/dpmsan_ms/data.py | 210 ++++++++ .../DPMSAN_MindSpore/src/dpmsan_ms/layers.py | 208 ++++++++ .../DPMSAN_MindSpore/src/dpmsan_ms/model.py | 181 +++++++ .../src/dpmsan_ms/training.py | 97 ++++ .../DPMSAN_MindSpore/src/dpmsan_ms/utils.py | 45 ++ .../TSLM/DPMSAN_MindSpore/tests/test_data.py | 59 +++ .../DPMSAN_MindSpore/tests/test_devices.py | 44 ++ .../DPMSAN_MindSpore/tests/test_layers.py | 34 ++ .../DPMSAN_MindSpore/tests/test_train_cli.py | 79 +++ .../DPMSAN_MindSpore/tests/test_training.py | 73 +++ .../TSLM/DualForecast_MindSpore/.gitignore | 10 + .../TSLM/DualForecast_MindSpore/README.md | 104 ++++ research/TSLM/DualForecast_MindSpore/eval.py | 81 +++ .../DualForecast_MindSpore/requirements.txt | 6 + .../DualForecast_MindSpore/scripts/eval_c5.sh | 11 + .../scripts/train_c5.sh | 12 + .../DualForecast_MindSpore/src/__init__.py | 4 + .../DualForecast_MindSpore/src/dataset.py | 95 ++++ .../src/dualforecast.py | 491 ++++++++++++++++++ .../src/prompt_bank/C5.txt | 1 + .../src/prompt_bank/hm.txt | 8 + research/TSLM/DualForecast_MindSpore/train.py | 99 ++++ research/TSLM/PAT_LLM_MindSpore/README.md | 225 ++++++++ research/TSLM/PAT_LLM_MindSpore/README_CN.md | 126 +++++ .../PAT_LLM_MindSpore/default_config.yaml | 46 ++ research/TSLM/PAT_LLM_MindSpore/eval.py | 26 + .../TSLM/PAT_LLM_MindSpore/requirements.txt | 6 + .../scripts/run_all_horizons_ascend.sh | 26 + .../scripts/run_eval_ascend.sh | 23 + .../scripts/run_standalone_train_ascend.sh | 21 + .../TSLM/PAT_LLM_MindSpore/src/__init__.py | 7 + research/TSLM/PAT_LLM_MindSpore/src/config.py | 138 +++++ .../TSLM/PAT_LLM_MindSpore/src/dataset.py | 164 ++++++ research/TSLM/PAT_LLM_MindSpore/src/engine.py | 187 +++++++ research/TSLM/PAT_LLM_MindSpore/src/layers.py | 361 +++++++++++++ .../TSLM/PAT_LLM_MindSpore/src/metrics.py | 27 + .../TSLM/PAT_LLM_MindSpore/src/pat_llm.py | 241 +++++++++ .../PAT_LLM_MindSpore/src/time_features.py | 64 +++ research/TSLM/PAT_LLM_MindSpore/train.py | 33 ++ research/TSLM/README.md | 17 + 51 files changed, 4095 insertions(+) create mode 100644 research/TSLM/.gitignore create mode 100644 research/TSLM/DPMSAN_MindSpore/.gitignore create mode 100644 research/TSLM/DPMSAN_MindSpore/CITATION.cff create mode 100644 research/TSLM/DPMSAN_MindSpore/README.md create mode 100644 research/TSLM/DPMSAN_MindSpore/configs/default.json create mode 100644 research/TSLM/DPMSAN_MindSpore/pyproject.toml create mode 100644 research/TSLM/DPMSAN_MindSpore/requirements.txt create mode 100644 research/TSLM/DPMSAN_MindSpore/scripts/train.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/__init__.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/cli.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/config.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/data.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/layers.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/model.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/training.py create mode 100644 research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/utils.py create mode 100644 research/TSLM/DPMSAN_MindSpore/tests/test_data.py create mode 100644 research/TSLM/DPMSAN_MindSpore/tests/test_devices.py create mode 100644 research/TSLM/DPMSAN_MindSpore/tests/test_layers.py create mode 100644 research/TSLM/DPMSAN_MindSpore/tests/test_train_cli.py create mode 100644 research/TSLM/DPMSAN_MindSpore/tests/test_training.py create mode 100644 research/TSLM/DualForecast_MindSpore/.gitignore create mode 100644 research/TSLM/DualForecast_MindSpore/README.md create mode 100644 research/TSLM/DualForecast_MindSpore/eval.py create mode 100644 research/TSLM/DualForecast_MindSpore/requirements.txt create mode 100644 research/TSLM/DualForecast_MindSpore/scripts/eval_c5.sh create mode 100644 research/TSLM/DualForecast_MindSpore/scripts/train_c5.sh create mode 100644 research/TSLM/DualForecast_MindSpore/src/__init__.py create mode 100644 research/TSLM/DualForecast_MindSpore/src/dataset.py create mode 100644 research/TSLM/DualForecast_MindSpore/src/dualforecast.py create mode 100644 research/TSLM/DualForecast_MindSpore/src/prompt_bank/C5.txt create mode 100644 research/TSLM/DualForecast_MindSpore/src/prompt_bank/hm.txt create mode 100644 research/TSLM/DualForecast_MindSpore/train.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/README.md create mode 100644 research/TSLM/PAT_LLM_MindSpore/README_CN.md create mode 100644 research/TSLM/PAT_LLM_MindSpore/default_config.yaml create mode 100644 research/TSLM/PAT_LLM_MindSpore/eval.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/requirements.txt create mode 100644 research/TSLM/PAT_LLM_MindSpore/scripts/run_all_horizons_ascend.sh create mode 100644 research/TSLM/PAT_LLM_MindSpore/scripts/run_eval_ascend.sh create mode 100644 research/TSLM/PAT_LLM_MindSpore/scripts/run_standalone_train_ascend.sh create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/__init__.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/config.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/dataset.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/engine.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/layers.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/metrics.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/pat_llm.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/src/time_features.py create mode 100644 research/TSLM/PAT_LLM_MindSpore/train.py create mode 100644 research/TSLM/README.md diff --git a/research/TSLM/.gitignore b/research/TSLM/.gitignore new file mode 100644 index 0000000..bd6fde2 --- /dev/null +++ b/research/TSLM/.gitignore @@ -0,0 +1,20 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.idea/ +.vscode/ +build/ +dist/ +*.egg-info/ +data/ +dataset/ +datasets/ +checkpoints/ +outputs/ +results/ +logs/ +*.ckpt +*.pth +*.pt +*.log diff --git a/research/TSLM/DPMSAN_MindSpore/.gitignore b/research/TSLM/DPMSAN_MindSpore/.gitignore new file mode 100644 index 0000000..05ac715 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/.gitignore @@ -0,0 +1,36 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +build/ +dist/ +*.egg-info/ +checkpoints/ +outputs/ +results/ +logs/ +*.ckpt +*.pth +*.pt +*.log +*.out +rank_*/ + +# This source-only repository must never contain datasets or data artifacts. +data/ +dataset/ +datasets/ +*.csv +*.tsv +*.xls +*.xlsx +*.npy +*.npz +*.pkl +*.pickle +*.parquet +*.feather +*.h5 +*.hdf5 +*.mat +*.arff diff --git a/research/TSLM/DPMSAN_MindSpore/CITATION.cff b/research/TSLM/DPMSAN_MindSpore/CITATION.cff new file mode 100644 index 0000000..7d7d3fe --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/CITATION.cff @@ -0,0 +1,34 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite the article below." +title: "DPMSAN: MindSpore implementation" +type: software +authors: + - family-names: Lin + given-names: Haifeng +preferred-citation: + type: article + title: "DPMSAN: A Dual-Path Multiscale Attention Network With Diverse Local–Global Interactions for Industrial Quality Prediction" + authors: + - family-names: Lin + given-names: Haifeng + - family-names: Wang + given-names: Kai + - family-names: Yuan + given-names: Xiaofeng + - family-names: Wang + given-names: Yalin + - family-names: Yang + given-names: Chunhua + - family-names: Gui + given-names: Weihua + - family-names: Shen + given-names: Feifan + - family-names: Ye + given-names: Lingjian + - family-names: Zhou + given-names: Le + journal: "IEEE Transactions on Industrial Informatics" + year: 2026 + start: 1 + end: 12 + doi: "10.1109/TII.2026.3695231" diff --git a/research/TSLM/DPMSAN_MindSpore/README.md b/research/TSLM/DPMSAN_MindSpore/README.md new file mode 100644 index 0000000..05e7c19 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/README.md @@ -0,0 +1,43 @@ +# DPMSAN (MindSpore) + +MindSpore implementation of **DPMSAN: A Dual-Path Multiscale Attention Network With Diverse Local–Global Interactions for Industrial Quality Prediction**. + +## Installation + +Install the MindSpore build for your hardware by following the [MindSpore installation guide](https://www.mindspore.cn/install/en), then run: + +```bash +pip install -r requirements.txt +pip install -e . +``` + +## Training + +```bash +python scripts/train.py \ + --data_path /path/to/process.csv \ + --target_column quality \ + --feature_columns sensor_a sensor_b sensor_c \ + --config configs/default.json \ + --epochs 20 \ + --batch_size 64 \ + --learning_rate 0.001 \ + --device_target CPU +``` + +Supported device targets are `CPU`, `GPU`, `NPU`, and `Ascend`. Model hyperparameters in `configs/default.json` can be overridden through command-line arguments. + +Run `python scripts/train.py --help` for all options. + +## Citation + +```bibtex +@ARTICLE{11558520, + author={Lin, Haifeng and Wang, Kai and Yuan, Xiaofeng and Wang, Yalin and Yang, Chunhua and Gui, Weihua and Shen, Feifan and Ye, Lingjian and Zhou, Le}, + journal={IEEE Transactions on Industrial Informatics}, + title={DPMSAN: A Dual-Path Multiscale Attention Network With Diverse Local–Global Interactions for Industrial Quality Prediction}, + year={2026}, + pages={1-12}, + doi={10.1109/TII.2026.3695231} +} +``` diff --git a/research/TSLM/DPMSAN_MindSpore/configs/default.json b/research/TSLM/DPMSAN_MindSpore/configs/default.json new file mode 100644 index 0000000..97c0f01 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/configs/default.json @@ -0,0 +1,17 @@ +{ + "seq_len": 15, + "pred_len": 1, + "enc_in": 44, + "dec_in": 44, + "c_out": 1, + "d_model": 256, + "d_ff": 256, + "n_heads": 3, + "e_layers": 4, + "d_layers": 2, + "scale_k": 4, + "kernel_sizes": [1, 3, 5, 7], + "factor": 5, + "dropout": 0.0, + "activation": "gelu" +} diff --git a/research/TSLM/DPMSAN_MindSpore/pyproject.toml b/research/TSLM/DPMSAN_MindSpore/pyproject.toml new file mode 100644 index 0000000..6d827b0 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "dpmsan-mindspore" +version = "1.0.0" +description = "MindSpore implementation of DPMSAN" +requires-python = ">=3.9" +dependencies = ["mindspore>=2.9", "numpy>=1.21"] + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/research/TSLM/DPMSAN_MindSpore/requirements.txt b/research/TSLM/DPMSAN_MindSpore/requirements.txt new file mode 100644 index 0000000..9760eb1 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/requirements.txt @@ -0,0 +1,3 @@ +mindspore>=2.9 +numpy>=1.21 +pytest>=7 diff --git a/research/TSLM/DPMSAN_MindSpore/scripts/train.py b/research/TSLM/DPMSAN_MindSpore/scripts/train.py new file mode 100644 index 0000000..07c7164 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/scripts/train.py @@ -0,0 +1,132 @@ +"""Train and evaluate DPMSAN on an external CSV file.""" + +import argparse +import json +import sys +from dataclasses import replace +from pathlib import Path + +import mindspore as ms +from mindspore import nn + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from dpmsan_ms import ( + DPMSAN, + add_model_arguments, + config_from_args, + evaluate, + load_csv_data, + set_context, + train_epoch, +) + + +def external_checkpoint_path(value: str) -> Path: + path = Path(value).expanduser().resolve() + try: + path.relative_to(PROJECT_ROOT.resolve()) + except ValueError: + return path + raise argparse.ArgumentTypeError("checkpoint_path must be outside the repository directory.") + + +def format_metrics(metrics, prefix: str = "") -> str: + return " ".join( + f"{prefix}{name}={value:.6f}" for name, value in metrics.as_dict().items() + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Train DPMSAN on an external CSV time series.") + add_model_arguments(parser) + parser.add_argument("--data_path", type=Path, required=True) + parser.add_argument("--target_column", required=True) + parser.add_argument("--feature_columns", nargs="+", default=None) + parser.add_argument("--delimiter", default=",") + parser.add_argument("--train_ratio", type=float, default=0.7) + parser.add_argument("--validation_ratio", type=float, default=0.15) + parser.add_argument("--epochs", type=int, default=20) + parser.add_argument("--batch_size", type=int, default=64) + parser.add_argument("--learning_rate", type=float, default=1e-3) + parser.add_argument("--weight_decay", type=float, default=1e-4) + parser.add_argument("--device_target", default="CPU", choices=["CPU", "GPU", "NPU", "Ascend"]) + parser.add_argument("--device_id", type=int, default=0) + parser.add_argument("--mode", default="PYNATIVE_MODE", choices=["GRAPH_MODE", "PYNATIVE_MODE"]) + parser.add_argument("--seed", type=int, default=2026) + parser.add_argument("--checkpoint_path", type=external_checkpoint_path, default=None) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.epochs <= 0: + raise ValueError("epochs must be positive.") + target = set_context( + args.device_target, mode=args.mode, seed=args.seed, device_id=args.device_id + ) + config = config_from_args(args) + data = load_csv_data( + path=args.data_path, + target_column=args.target_column, + seq_len=config.seq_len, + pred_len=config.pred_len, + feature_columns=args.feature_columns, + delimiter=args.delimiter, + train_ratio=args.train_ratio, + validation_ratio=args.validation_ratio, + ) + config = replace(config, enc_in=data.input_size, dec_in=data.input_size, c_out=1) + model = DPMSAN(config) + optimizer = nn.AdamWeightDecay( + model.trainable_params(), + learning_rate=args.learning_rate, + weight_decay=args.weight_decay, + ) + + print( + json.dumps( + { + "device_target": target, + "device_id": args.device_id, + "features": list(data.feature_names), + "target": data.target_name, + "input_size": data.input_size, + "train_windows": len(data.train), + "validation_windows": len(data.validation), + "test_windows": len(data.test), + }, + ensure_ascii=False, + ) + ) + best_rmse = float("inf") + for epoch in range(1, args.epochs + 1): + train_loss = train_epoch( + model, optimizer, data.train, args.batch_size, args.seed + epoch + ) + validation_metrics = evaluate( + model, data.validation, data.target_scaler, args.batch_size + ) + print( + f"epoch={epoch} train_mse={train_loss:.6f} " + f"{format_metrics(validation_metrics, 'validation_')}" + ) + if args.checkpoint_path is not None and validation_metrics.rmse < best_rmse: + args.checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + ms.save_checkpoint(model, str(args.checkpoint_path)) + best_rmse = validation_metrics.rmse + + if args.checkpoint_path is not None: + parameters = ms.load_checkpoint(str(args.checkpoint_path)) + ms.load_param_into_net(model, parameters) + validation_metrics = evaluate(model, data.validation, data.target_scaler, args.batch_size) + test_metrics = evaluate(model, data.test, data.target_scaler, args.batch_size) + print(format_metrics(validation_metrics, "final_validation_")) + print(format_metrics(test_metrics, "final_test_")) + if args.checkpoint_path is not None: + print(f"checkpoint={args.checkpoint_path}") + + +if __name__ == "__main__": + main() diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/__init__.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/__init__.py new file mode 100644 index 0000000..d761081 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/__init__.py @@ -0,0 +1,22 @@ +from .config import DPMSANConfig +from .cli import add_model_arguments, config_from_args +from .data import DataBundle, StandardScaler, WindowedSplit, load_csv_data, prepare_data +from .model import DPMSAN +from .training import RegressionMetrics, evaluate, train_epoch +from .utils import set_context + +__all__ = [ + "DPMSAN", + "DPMSANConfig", + "DataBundle", + "RegressionMetrics", + "StandardScaler", + "WindowedSplit", + "add_model_arguments", + "config_from_args", + "evaluate", + "load_csv_data", + "prepare_data", + "set_context", + "train_epoch", +] diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/cli.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/cli.py new file mode 100644 index 0000000..99b54f8 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/cli.py @@ -0,0 +1,39 @@ +"""Shared command-line configuration helpers.""" + +import argparse +import json +from dataclasses import fields +from pathlib import Path + +from .config import DPMSANConfig + + +def add_model_arguments(parser: argparse.ArgumentParser) -> None: + defaults = DPMSANConfig() + parser.add_argument("--config", type=Path, default=None, help="Optional JSON configuration file.") + for field in fields(DPMSANConfig): + name = field.name + value = getattr(defaults, name) + option = f"--{name}" + if name == "kernel_sizes": + parser.add_argument(option, type=int, nargs="+", default=None) + elif isinstance(value, bool): + parser.add_argument(option, action=argparse.BooleanOptionalAction, default=None) + else: + parser.add_argument(option, type=type(value), default=None) + + +def config_from_args(args: argparse.Namespace) -> DPMSANConfig: + values = {} + if args.config is not None: + with args.config.open("r", encoding="utf-8") as stream: + values.update(json.load(stream)) + for field in fields(DPMSANConfig): + value = getattr(args, field.name, None) + if value is not None: + values[field.name] = value + if "kernel_sizes" in values: + values["kernel_sizes"] = tuple(values["kernel_sizes"]) + if getattr(args, "scale_k", None) is None: + values["scale_k"] = len(values["kernel_sizes"]) + return DPMSANConfig.from_dict(values) diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/config.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/config.py new file mode 100644 index 0000000..e63f9dc --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/config.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass, fields +from typing import Any, Dict, Tuple + + +@dataclass +class DPMSANConfig: + seq_len: int = 15 + pred_len: int = 1 + enc_in: int = 44 + dec_in: int = 44 + c_out: int = 1 + d_model: int = 256 + d_ff: int = 256 + n_heads: int = 3 + e_layers: int = 4 + d_layers: int = 2 + scale_k: int = 4 + kernel_sizes: Tuple[int, ...] = (1, 3, 5, 7) + factor: int = 5 + dropout: float = 0.0 + activation: str = "gelu" + + def __post_init__(self): + self.kernel_sizes = tuple(int(k) for k in self.kernel_sizes) + if self.seq_len <= 0: + raise ValueError("seq_len must be positive.") + if self.pred_len != 1: + raise ValueError("DPMSAN quality prediction requires pred_len=1.") + if self.enc_in <= 0 or self.dec_in <= 0: + raise ValueError("enc_in and dec_in must be positive.") + if self.c_out != 1: + raise ValueError("DPMSAN quality prediction requires c_out=1.") + if len(self.kernel_sizes) != self.scale_k: + raise ValueError("scale_k must equal the number of kernel_sizes.") + if any(k <= 0 or k % 2 == 0 for k in self.kernel_sizes): + raise ValueError("kernel_sizes must contain positive odd integers.") + if self.n_heads <= 0 or self.n_heads > self.d_model: + raise ValueError("n_heads must be between 1 and d_model.") + + @classmethod + def from_dict(cls, values: Dict[str, Any]): + valid = {field.name for field in fields(cls)} + unknown = set(values) - valid + if unknown: + raise ValueError(f"Unknown configuration keys: {sorted(unknown)}") + return cls(**values) diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/data.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/data.py new file mode 100644 index 0000000..f190019 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/data.py @@ -0,0 +1,210 @@ +"""Data preparation utilities for multivariate soft-sensor time series.""" + +import csv +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Sequence, Tuple + +import numpy as np + + +@dataclass +class StandardScaler: + mean: np.ndarray + std: np.ndarray + + @classmethod + def fit(cls, values: np.ndarray) -> "StandardScaler": + mean = values.mean(axis=0, keepdims=True).astype(np.float32) + std = values.std(axis=0, keepdims=True).astype(np.float32) + std = np.where(std < 1e-8, 1.0, std).astype(np.float32) + return cls(mean, std) + + def transform(self, values: np.ndarray) -> np.ndarray: + return ((values - self.mean) / self.std).astype(np.float32) + + def inverse_transform(self, values: np.ndarray) -> np.ndarray: + return (values * self.std + self.mean).astype(np.float32) + + +@dataclass +class WindowedSplit: + inputs: np.ndarray + targets: np.ndarray + + def __len__(self) -> int: + return self.inputs.shape[0] + + +@dataclass +class DataBundle: + train: WindowedSplit + validation: WindowedSplit + test: WindowedSplit + feature_names: Tuple[str, ...] + target_name: str + feature_scaler: StandardScaler + target_scaler: StandardScaler + + @property + def input_size(self) -> int: + return len(self.feature_names) + 1 + + +def read_csv_series( + path: Path, + target_column: str, + feature_columns: Optional[Sequence[str]] = None, + delimiter: str = ",", +) -> Tuple[np.ndarray, np.ndarray, Tuple[str, ...]]: + path = Path(path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"CSV file not found: {path}") + with path.open("r", encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream, delimiter=delimiter) + if reader.fieldnames is None: + raise ValueError("CSV file must contain a header row.") + fieldnames = tuple(reader.fieldnames) + if target_column not in fieldnames: + raise ValueError(f"Target column '{target_column}' is not present in the CSV header.") + selected = tuple(feature_columns) if feature_columns else tuple( + name for name in fieldnames if name != target_column + ) + if not selected: + raise ValueError("At least one feature column is required.") + missing = [name for name in selected if name not in fieldnames] + if missing: + raise ValueError(f"Feature columns not present in the CSV header: {missing}") + if target_column in selected: + raise ValueError("target_column must not also appear in feature_columns.") + + feature_rows: List[List[float]] = [] + target_rows: List[float] = [] + for line_number, row in enumerate(reader, start=2): + try: + feature_row = [float(row[name]) for name in selected] + target_value = float(row[target_column]) + except (TypeError, ValueError) as exc: + raise ValueError(f"Non-numeric or missing value at CSV line {line_number}.") from exc + if not np.isfinite(feature_row).all() or not np.isfinite(target_value): + raise ValueError(f"Non-finite value at CSV line {line_number}.") + feature_rows.append(feature_row) + target_rows.append(target_value) + + if not feature_rows: + raise ValueError("CSV file does not contain any data rows.") + features = np.asarray(feature_rows, dtype=np.float32) + targets = np.asarray(target_rows, dtype=np.float32).reshape((-1, 1)) + return features, targets, selected + + +def _make_windows( + features: np.ndarray, + targets: np.ndarray, + seq_len: int, + pred_len: int, +) -> WindowedSplit: + count = len(features) - seq_len - pred_len + 1 + if count <= 0: + raise ValueError( + f"Each chronological split needs at least seq_len + pred_len rows; " + f"got {len(features)} rows for seq_len={seq_len}, pred_len={pred_len}." + ) + inputs = np.stack([ + np.concatenate( + (features[i + 1 : i + seq_len + 1], targets[i : i + seq_len]), + axis=1, + ) + for i in range(count) + ]).astype(np.float32) + labels = np.stack([ + targets[i + seq_len : i + seq_len + pred_len, 0] for i in range(count) + ]).astype(np.float32) + return WindowedSplit(inputs, labels) + + +def prepare_data( + features: np.ndarray, + targets: np.ndarray, + seq_len: int, + pred_len: int = 1, + train_ratio: float = 0.7, + validation_ratio: float = 0.15, + feature_names: Optional[Sequence[str]] = None, + target_name: str = "target", +) -> DataBundle: + features = np.asarray(features, dtype=np.float32) + targets = np.asarray(targets, dtype=np.float32).reshape((-1, 1)) + if features.ndim != 2: + raise ValueError("features must have shape [time, variables].") + if len(features) != len(targets): + raise ValueError("features and targets must contain the same number of rows.") + if not 0.0 < train_ratio < 1.0: + raise ValueError("train_ratio must be between 0 and 1.") + if not 0.0 < validation_ratio < 1.0 or train_ratio + validation_ratio >= 1.0: + raise ValueError("validation_ratio must be positive and leave a non-empty test ratio.") + + row_count = len(features) + train_end = int(row_count * train_ratio) + validation_end = int(row_count * (train_ratio + validation_ratio)) + minimum_rows = seq_len + pred_len + split_sizes = (train_end, validation_end - train_end, row_count - validation_end) + if any(size < minimum_rows for size in split_sizes): + raise ValueError( + f"Train, validation, and test splits each require at least {minimum_rows} rows; " + f"got split sizes {split_sizes}." + ) + + feature_scaler = StandardScaler.fit(features[:train_end]) + target_scaler = StandardScaler.fit(targets[:train_end]) + scaled_features = feature_scaler.transform(features) + scaled_targets = target_scaler.transform(targets) + names = tuple(feature_names) if feature_names else tuple( + f"feature_{index}" for index in range(features.shape[1]) + ) + if len(names) != features.shape[1]: + raise ValueError("feature_names length must equal the number of feature columns.") + + return DataBundle( + train=_make_windows( + scaled_features[:train_end], scaled_targets[:train_end], seq_len, pred_len + ), + validation=_make_windows( + scaled_features[train_end:validation_end], + scaled_targets[train_end:validation_end], + seq_len, + pred_len, + ), + test=_make_windows( + scaled_features[validation_end:], scaled_targets[validation_end:], seq_len, pred_len + ), + feature_names=names, + target_name=target_name, + feature_scaler=feature_scaler, + target_scaler=target_scaler, + ) + + +def load_csv_data( + path: Path, + target_column: str, + seq_len: int, + pred_len: int = 1, + feature_columns: Optional[Sequence[str]] = None, + delimiter: str = ",", + train_ratio: float = 0.7, + validation_ratio: float = 0.15, +) -> DataBundle: + features, targets, names = read_csv_series( + path, target_column, feature_columns, delimiter + ) + return prepare_data( + features, + targets, + seq_len, + pred_len, + train_ratio, + validation_ratio, + names, + target_column, + ) diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/layers.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/layers.py new file mode 100644 index 0000000..d874244 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/layers.py @@ -0,0 +1,208 @@ +"""Neural network layers used by DPMSAN.""" + +import math +from typing import List, Optional, Tuple + +import mindspore as ms +import numpy as np +from mindspore import Parameter, Tensor, nn, ops +from mindspore.common.initializer import Constant, initializer + + +class PositionalEmbedding(nn.Cell): + def __init__(self, d_model: int, max_len: int = 5000): + super().__init__() + pe = np.zeros((max_len, d_model), dtype=np.float32) + position = np.arange(max_len, dtype=np.float32).reshape(max_len, 1) + div_term = np.exp(np.arange(0, d_model, 2, dtype=np.float32) * (-(math.log(10000.0) / d_model))) + pe[:, 0::2] = np.sin(position * div_term) + if d_model > 1: + pe[:, 1::2] = np.cos(position * div_term[: pe[:, 1::2].shape[1]]) + self.pe = Tensor(pe.reshape(1, max_len, d_model), ms.float32) + + def construct(self, x: Tensor) -> Tensor: + return self.pe[:, : x.shape[1], :] + + +class DataEmbedding(nn.Cell): + def __init__(self, c_in: int, d_model: int, dropout: float = 0.1): + super().__init__() + self.value_embedding = nn.Dense(c_in, d_model) + self.position_embedding = PositionalEmbedding(d_model) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, x: Tensor, x_mark: Optional[Tensor] = None) -> Tensor: + return self.dropout(self.value_embedding(x) + self.position_embedding(x)) + + +class DeformableConv1d(nn.Cell): + """One-dimensional deformable convolution with linear interpolation.""" + + def __init__(self, in_channels: int, out_channels: int, kernel_size: int): + super().__init__() + self.kernel_size = kernel_size + self.padding = kernel_size // 2 + self.offset_conv = nn.Conv1d( + in_channels, kernel_size, kernel_size=kernel_size, + pad_mode="pad", padding=self.padding, has_bias=True, + ) + self.offset_conv.weight.set_data(initializer(Constant(0.0), self.offset_conv.weight.shape)) + self.offset_conv.bias.set_data(initializer(Constant(0.0), self.offset_conv.bias.shape)) + bound = 1.0 / math.sqrt(in_channels * kernel_size) + weight = np.random.uniform( + -bound, bound, (out_channels, in_channels, kernel_size) + ).astype(np.float32) + self.weight = Parameter(Tensor(weight), name="weight") + + def _sample(self, x_pad: Tensor, coordinate: Tensor) -> Tensor: + max_index = x_pad.shape[-1] - 1 + left_float = ops.floor(coordinate) + left = left_float.astype(ms.int32) + right = left + 1 + left_valid = ((left >= 0) & (left <= max_index)).astype(coordinate.dtype).expand_dims(1) + right_valid = ((right >= 0) & (right <= max_index)).astype(coordinate.dtype).expand_dims(1) + left = ops.minimum(ops.maximum(left, Tensor(0, ms.int32)), Tensor(max_index, ms.int32)) + right = ops.minimum(ops.maximum(right, Tensor(0, ms.int32)), Tensor(max_index, ms.int32)) + alpha = (coordinate - left_float).expand_dims(1) + channels = x_pad.shape[1] + left_idx = ops.broadcast_to(left.expand_dims(1), (-1, channels, -1)) + right_idx = ops.broadcast_to(right.expand_dims(1), (-1, channels, -1)) + left_value = ops.gather_elements(x_pad, 2, left_idx) * left_valid + right_value = ops.gather_elements(x_pad, 2, right_idx) * right_valid + return left_value * (1.0 - alpha) + right_value * alpha + + def construct(self, x: Tensor) -> Tensor: + offsets = self.offset_conv(x) # [B, K, L] + x_pad = ops.pad(x, (self.padding, self.padding)) + length = x.shape[-1] + base = ops.arange(0, length, 1, dtype=ms.float32).reshape((1, length)) + self.padding + output = None + for k in range(self.kernel_size): + grid_offset = k - self.padding + sampled = self._sample(x_pad, base + grid_offset + offsets[:, k, :]) + contribution = ops.transpose( + ops.matmul(ops.transpose(sampled, (0, 2, 1)), self.weight[:, :, k].T), + (0, 2, 1), + ) + output = contribution if output is None else output + contribution + return output + + +class TimeConv1dDeformableEmbedding(nn.Cell): + def __init__(self, enc_in: int, d_model: int, kernel_size: int): + super().__init__() + self.conv1d = DeformableConv1d(enc_in, d_model, kernel_size) + self.position_embedding = PositionalEmbedding(d_model) + + def construct(self, x: Tensor) -> Tensor: + x = ops.transpose(self.conv1d(ops.transpose(x, (0, 2, 1))), (0, 2, 1)) + return x + self.position_embedding(x) + + +class MultiScaleDeformableConv1D(nn.Cell): + def __init__(self, enc_in: int, d_model: int, kernel_sizes: Tuple[int, ...]): + super().__init__() + self.layers = nn.CellList([ + TimeConv1dDeformableEmbedding(enc_in, d_model, kernel) for kernel in kernel_sizes + ]) + + def construct(self, x: Tensor) -> List[Tensor]: + return [layer(x) for layer in self.layers] + + +class FullAttention(nn.Cell): + def __init__(self, mask_flag: bool = False, dropout: float = 0.1): + super().__init__() + self.mask_flag = mask_flag + self.dropout = nn.Dropout(p=dropout) + self.softmax = nn.Softmax(axis=-1) + + def construct(self, queries: Tensor, keys: Tensor, values: Tensor) -> Tuple[Tensor, Tensor]: + d_head = queries.shape[-1] + q = ops.transpose(queries, (0, 2, 1, 3)) + k = ops.transpose(keys, (0, 2, 3, 1)) + v = ops.transpose(values, (0, 2, 1, 3)) + scores = ops.matmul(q, k) / math.sqrt(float(d_head)) + if self.mask_flag: + lq, lk = scores.shape[-2:] + mask = ops.ones((lq, lk), ms.bool_).triu(1).reshape((1, 1, lq, lk)) + scores = ops.where(mask, Tensor(-1e9, scores.dtype), scores) + attn = self.dropout(self.softmax(scores)) + return ops.transpose(ops.matmul(attn, v), (0, 2, 1, 3)), attn + + +class ProbSparseAttention(nn.Cell): + """Top-u active-query ProbSparse attention used by the intra-scale path.""" + + def __init__(self, factor: int = 5, dropout: float = 0.1): + super().__init__() + self.factor = factor + self.dropout = nn.Dropout(p=dropout) + self.softmax = nn.Softmax(axis=-1) + + def construct(self, queries: Tensor, keys: Tensor, values: Tensor) -> Tuple[Tensor, Tensor]: + # [B, L, H, D] -> [B, H, L, D] + q = ops.transpose(queries, (0, 2, 1, 3)) + k = ops.transpose(keys, (0, 2, 1, 3)) + v = ops.transpose(values, (0, 2, 1, 3)) + length = q.shape[2] + sample_k = min(length, max(1, int(self.factor * math.ceil(math.log(length + 1))))) + top_u = sample_k + sample_indices = ops.randint(0, length, (length, sample_k), dtype=ms.int32) + sampled_k = ops.gather(k, sample_indices.reshape((-1,)), 2) + sampled_k = sampled_k.reshape((q.shape[0], q.shape[1], length, sample_k, q.shape[-1])) + sampled_scores = (q.expand_dims(3) * sampled_k).sum(axis=-1) + sparsity = sampled_scores.max(axis=-1) - sampled_scores.sum(axis=-1) / float(length) + _, indices = ops.topk(sparsity, top_u, dim=-1) + gather_idx = ops.broadcast_to(indices.expand_dims(-1), (-1, -1, -1, q.shape[-1])) + active_q = ops.gather_elements(q, 2, gather_idx) + active_scores = ops.matmul(active_q, ops.transpose(k, (0, 1, 3, 2))) / math.sqrt(float(q.shape[-1])) + active_attn = self.dropout(self.softmax(active_scores)) + active_context = ops.matmul(active_attn, v) + + mean_context = v.mean(axis=2, keep_dims=True) + one_hot = ops.one_hot(indices, length, Tensor(1.0, q.dtype), Tensor(0.0, q.dtype)) + delta = active_context - mean_context + context = ops.broadcast_to(mean_context, (-1, -1, length, -1)) + context = context + ops.matmul(ops.transpose(one_hot, (0, 1, 3, 2)), delta) + return ops.transpose(context, (0, 2, 1, 3)), active_attn + + +class AttentionLayer(nn.Cell): + def __init__(self, attention: nn.Cell, d_model: int, n_heads: int): + super().__init__() + self.inner_attention = attention + self.n_heads = n_heads + self.d_head = d_model // n_heads + self.inner_dim = self.d_head * n_heads + self.query_projection = nn.Dense(d_model, self.inner_dim) + self.key_projection = nn.Dense(d_model, self.inner_dim) + self.value_projection = nn.Dense(d_model, self.inner_dim) + self.out_projection = nn.Dense(self.inner_dim, d_model) + + def construct(self, queries: Tensor, keys: Tensor, values: Tensor) -> Tuple[Tensor, Tensor]: + batch, lq, _ = queries.shape + lk = keys.shape[1] + q = self.query_projection(queries).reshape((batch, lq, self.n_heads, self.d_head)) + k = self.key_projection(keys).reshape((batch, lk, self.n_heads, self.d_head)) + v = self.value_projection(values).reshape((batch, lk, self.n_heads, self.d_head)) + out, attn = self.inner_attention(q, k, v) + return self.out_projection(out.reshape((batch, lq, -1))), attn + + +class EncoderLayer(nn.Cell): + def __init__(self, attention: AttentionLayer, d_model: int, d_ff: int, dropout: float, activation: str): + super().__init__() + self.attention = attention + self.ffn1 = nn.Dense(d_model, d_ff) + self.ffn2 = nn.Dense(d_ff, d_model) + self.norm1 = nn.LayerNorm((d_model,)) + self.norm2 = nn.LayerNorm((d_model,)) + self.dropout = nn.Dropout(p=dropout) + self.activation = ops.GeLU() if activation == "gelu" else ops.ReLU() + + def construct(self, x: Tensor) -> Tuple[Tensor, Tensor]: + attended, attn = self.attention(x, x, x) + z = self.norm1(x + self.dropout(attended)) + ffn = self.ffn2(self.dropout(self.activation(self.ffn1(z)))) + return self.norm2(z + self.dropout(ffn)), attn diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/model.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/model.py new file mode 100644 index 0000000..346da39 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/model.py @@ -0,0 +1,181 @@ +"""DPMSAN model definition.""" + +from typing import List + +from mindspore import Tensor, nn, ops + +from .config import DPMSANConfig +from .layers import ( + AttentionLayer, + DataEmbedding, + EncoderLayer, + FullAttention, + MultiScaleDeformableConv1D, + ProbSparseAttention, +) + + +class GSFF(nn.Cell): + """Gated Scale Feature Fusion.""" + + def __init__(self, d_model: int): + super().__init__() + self.gate_linear = nn.Dense(2 * d_model, d_model) + self.sigmoid = ops.Sigmoid() + + def construct(self, f_intra: Tensor, f_inter: Tensor) -> Tensor: + gate = self.sigmoid(self.gate_linear(ops.concat((f_intra, f_inter), axis=-1))) + return gate * f_inter + (1.0 - gate) * f_intra + + +class FrequencyEnhancedInterScale(nn.Cell): + """Full attention over scales, guided by per-scale spectral anchors.""" + + def __init__(self, d_model: int, scale_k: int, seq_len: int, d_ff: int, dropout: float): + super().__init__() + self.scale_k = scale_k + self.frequency_bins = seq_len // 2 + 1 + self.freq_conv = nn.Conv1d( + 2, d_model, kernel_size=self.frequency_bins, pad_mode="valid", has_bias=True + ) + self.query_projection = nn.Dense(2 * d_model, 2 * d_model) + self.key_projection = nn.Dense(2 * d_model, 2 * d_model) + self.value_projection = nn.Dense(d_model, d_model) + self.output_projection = nn.Dense(d_model, d_model) + self.ffn1 = nn.Dense(d_model, d_ff) + self.ffn2 = nn.Dense(d_ff, d_model) + self.norm1 = nn.LayerNorm((d_model,)) + self.norm2 = nn.LayerNorm((d_model,)) + self.softmax = nn.Softmax(axis=-1) + self.dropout = nn.Dropout(p=dropout) + self.activation = ops.ReLU() + + def _spectral_anchor(self, x: Tensor) -> Tensor: + spectrum = ops.rfft(x.mean(axis=-1), dim=1, norm="ortho") + spectral = ops.stack((ops.real(spectrum), ops.imag(spectrum)), axis=1) + return self.freq_conv(spectral).squeeze(-1) + + def construct(self, x_multi_scale: List[Tensor]) -> List[Tensor]: + _, length, d_model = x_multi_scale[0].shape + time_features = ops.stack(x_multi_scale, axis=2) # [B, L, S, D] + anchors = ops.stack([self._spectral_anchor(x) for x in x_multi_scale], axis=1) + anchors = ops.broadcast_to(anchors.expand_dims(1), (-1, length, -1, -1)) + time_spectral = ops.concat((time_features, anchors), axis=-1) + + q = self.query_projection(time_spectral) + k = ops.transpose(self.key_projection(time_spectral), (0, 1, 3, 2)) + v = self.value_projection(time_features) + scores = ops.matmul(q, k) / (float(2 * d_model) ** 0.5) + context = ops.matmul(self.dropout(self.softmax(scores)), v) + context = self.output_projection(context) + x = self.norm1(time_features + self.dropout(context)) + ffn = self.ffn2(self.dropout(self.activation(self.ffn1(x)))) + x = self.norm2(x + self.dropout(ffn)) + return [x[:, :, i, :] for i in range(self.scale_k)] + + +class UnifiedEncoderLayer(nn.Cell): + def __init__(self, config: DPMSANConfig): + super().__init__() + self.scale_k = config.scale_k + self.intra_scale_layers = nn.CellList([ + EncoderLayer( + AttentionLayer( + ProbSparseAttention(config.factor, config.dropout), + config.d_model, + config.n_heads, + ), + config.d_model, + config.d_ff, + config.dropout, + config.activation, + ) + for _ in range(config.scale_k) + ]) + self.inter_scale_attn = FrequencyEnhancedInterScale( + config.d_model, config.scale_k, config.seq_len, config.d_ff, config.dropout + ) + self.gsff_layers = nn.CellList([GSFF(config.d_model) for _ in range(config.scale_k)]) + + def construct(self, x_list: List[Tensor]) -> List[Tensor]: + intra = [self.intra_scale_layers[i](x_list[i])[0] for i in range(self.scale_k)] + inter = self.inter_scale_attn(x_list) + return [self.gsff_layers[i](intra[i], inter[i]) for i in range(self.scale_k)] + + +class ScaleAwareWeighting(nn.Cell): + def __init__(self, scale_k: int, d_model: int): + super().__init__() + self.scale_k = scale_k + self.mlp = nn.SequentialCell( + nn.Dense(scale_k * d_model, d_model), + nn.Tanh(), + nn.Dense(d_model, scale_k), + nn.Softmax(axis=-1), + ) + + def construct(self, scale_features: List[Tensor]) -> Tensor: + descriptors = [feature.mean(axis=1) for feature in scale_features] + return self.mlp(ops.concat(tuple(descriptors), axis=-1)) + + +class ScaleAwareDecoderLayer(nn.Cell): + def __init__(self, config: DPMSANConfig): + super().__init__() + self.scale_k = config.scale_k + self.cross_attentions = nn.CellList([ + AttentionLayer(FullAttention(False, config.dropout), config.d_model, config.n_heads) + for _ in range(config.scale_k) + ]) + + def construct(self, x: Tensor, scale_features: List[Tensor], scale_weights: Tensor) -> Tensor: + context = ops.zeros_like(x) + for i in range(self.scale_k): + scale_context = self.cross_attentions[i](x, scale_features[i], scale_features[i])[0] + context = context + scale_weights[:, i].reshape((-1, 1, 1)) * scale_context + return x + context + + +class ScaleAwareDecoder(nn.Cell): + def __init__(self, config: DPMSANConfig): + super().__init__() + self.layers = nn.CellList([ScaleAwareDecoderLayer(config) for _ in range(config.d_layers)]) + self.projection = nn.Dense(config.d_model, config.c_out) + self.output_activation = ops.GeLU() + + def construct(self, x: Tensor, scale_features: List[Tensor], scale_weights: Tensor) -> Tensor: + for layer in self.layers: + x = layer(x, scale_features, scale_weights) + return self.output_activation(self.projection(x)) + + +class DPMSAN(nn.Cell): + """Dual-Path Multiscale Attention Network.""" + + def __init__(self, config: DPMSANConfig): + super().__init__() + self.config = config + self.pred_len = config.pred_len + self.scale_k = config.scale_k + self.ms_conv = MultiScaleDeformableConv1D( + config.enc_in, config.d_model, config.kernel_sizes + ) + self.encoder_layers = nn.CellList([UnifiedEncoderLayer(config) for _ in range(config.e_layers)]) + self.decoder_embedding = DataEmbedding(config.dec_in, config.d_model, config.dropout) + self.scale_weighting = ScaleAwareWeighting(config.scale_k, config.d_model) + self.decoder = ScaleAwareDecoder(config) + + def forecast(self, x_enc: Tensor, x_mark_enc=None, x_dec: Tensor = None, x_mark_dec=None) -> Tensor: + if x_dec is None: + x_dec = x_enc + scale_features = self.ms_conv(x_enc) + for layer in self.encoder_layers: + scale_features = layer(scale_features) + scale_weights = self.scale_weighting(scale_features) + decoder_state = self.decoder_embedding(x_dec, x_mark_dec) + output = self.decoder(decoder_state, scale_features, scale_weights) + return output[:, -self.pred_len :, -1] + + def construct(self, x_enc: Tensor, x_mark_enc=None, x_dec: Tensor = None, + x_mark_dec=None, mask=None) -> Tensor: + return self.forecast(x_enc, x_mark_enc, x_dec, x_mark_dec) diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/training.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/training.py new file mode 100644 index 0000000..3e37096 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/training.py @@ -0,0 +1,97 @@ +"""Training and evaluation utilities.""" + +from dataclasses import dataclass +from typing import Dict, Iterator, Tuple + +import mindspore as ms +import numpy as np +from mindspore import Tensor, nn + +from .data import StandardScaler, WindowedSplit + + +@dataclass +class RegressionMetrics: + r2: float + rmse: float + mae: float + + def as_dict(self) -> Dict[str, float]: + return {"R2": self.r2, "RMSE": self.rmse, "MAE": self.mae} + + +def iterate_batches( + split: WindowedSplit, + batch_size: int, + shuffle: bool, + seed: int, +) -> Iterator[Tuple[Tensor, Tensor]]: + if batch_size <= 0: + raise ValueError("batch_size must be positive.") + indices = np.arange(len(split)) + if shuffle: + np.random.default_rng(seed).shuffle(indices) + for start in range(0, len(indices), batch_size): + batch_indices = indices[start : start + batch_size] + yield Tensor(split.inputs[batch_indices], ms.float32), Tensor( + split.targets[batch_indices], ms.float32 + ) + + +def train_epoch( + model: nn.Cell, + optimizer: nn.Optimizer, + split: WindowedSplit, + batch_size: int, + seed: int, +) -> float: + model.set_train(True) + loss_fn = nn.MSELoss() + + def forward_fn(inputs, labels): + predictions = model(inputs, None, inputs, None) + return loss_fn(predictions, labels) + + grad_fn = ms.value_and_grad(forward_fn, None, optimizer.parameters) + losses = [] + for inputs, labels in iterate_batches(split, batch_size, True, seed): + loss, gradients = grad_fn(inputs, labels) + optimizer(gradients) + losses.append(float(loss.asnumpy())) + if not losses: + raise ValueError("Training split does not contain any windows.") + return float(np.mean(losses)) + + +def regression_metrics(targets: np.ndarray, predictions: np.ndarray) -> RegressionMetrics: + targets = np.asarray(targets, dtype=np.float64).reshape((-1,)) + predictions = np.asarray(predictions, dtype=np.float64).reshape((-1,)) + if targets.shape != predictions.shape or targets.size == 0: + raise ValueError("targets and predictions must have matching non-empty shapes.") + residual = targets - predictions + mse = float(np.mean(residual ** 2)) + mae = float(np.mean(np.abs(residual))) + total = float(np.sum((targets - targets.mean()) ** 2)) + r2 = 0.0 if total <= 1e-12 else 1.0 - float(np.sum(residual ** 2)) / total + return RegressionMetrics(r2=r2, rmse=mse ** 0.5, mae=mae) + + +def evaluate( + model: nn.Cell, + split: WindowedSplit, + target_scaler: StandardScaler, + batch_size: int, +) -> RegressionMetrics: + model.set_train(False) + predictions = [] + targets = [] + for inputs, labels in iterate_batches(split, batch_size, False, 0): + predictions.append(model(inputs, None, inputs, None).asnumpy()) + targets.append(labels.asnumpy()) + if not predictions: + raise ValueError("Evaluation split does not contain any windows.") + scaled_predictions = np.concatenate(predictions, axis=0) + scaled_targets = np.concatenate(targets, axis=0) + predictions_raw = target_scaler.inverse_transform(scaled_predictions.reshape((-1, 1))) + targets_raw = target_scaler.inverse_transform(scaled_targets.reshape((-1, 1))) + return regression_metrics(targets_raw, predictions_raw) diff --git a/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/utils.py b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/utils.py new file mode 100644 index 0000000..a58f30c --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/src/dpmsan_ms/utils.py @@ -0,0 +1,45 @@ +import random +from typing import Literal + +import numpy as np +import mindspore as ms + + +DeviceTarget = Literal["CPU", "GPU", "NPU", "Ascend", "cpu", "gpu", "npu", "ascend"] + + +def normalize_device_target(device_target: DeviceTarget) -> str: + target = str(device_target).strip().lower() + if target == "npu": + return "Ascend" + if target == "ascend": + return "Ascend" + if target == "gpu": + return "GPU" + if target == "cpu": + return "CPU" + raise ValueError(f"Unsupported device target: {device_target}. Use CPU, GPU, NPU, or Ascend.") + + +def set_context( + device_target: DeviceTarget = "CPU", + mode: str = "GRAPH_MODE", + seed: int = 2026, + device_id: int = 0, +) -> str: + target = normalize_device_target(device_target) + if device_id < 0: + raise ValueError("device_id must be non-negative.") + exec_mode = ms.GRAPH_MODE if mode.upper() == "GRAPH_MODE" else ms.PYNATIVE_MODE + ms.set_context(mode=exec_mode) + try: + ms.set_device(target, device_id) + except (RuntimeError, ValueError) as exc: + raise RuntimeError( + f"Unable to initialize {target} device {device_id}. Install the matching " + "MindSpore build and verify that the requested device is available." + ) from exc + ms.set_seed(seed) + random.seed(seed) + np.random.seed(seed) + return target diff --git a/research/TSLM/DPMSAN_MindSpore/tests/test_data.py b/research/TSLM/DPMSAN_MindSpore/tests/test_data.py new file mode 100644 index 0000000..5078dcb --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/tests/test_data.py @@ -0,0 +1,59 @@ +import csv +import sys +from pathlib import Path + +import numpy as np + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from dpmsan_ms import load_csv_data, prepare_data + + +def test_csv_pipeline_builds_chronological_windows(tmp_path): + path = tmp_path / "external_series.csv" + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["temperature", "pressure", "quality"]) + for index in range(100): + writer.writerow([index, index * 2.0, index * 0.5]) + + bundle = load_csv_data( + path, + target_column="quality", + feature_columns=["temperature", "pressure"], + seq_len=5, + train_ratio=0.6, + validation_ratio=0.2, + ) + + assert bundle.input_size == 3 + assert bundle.train.inputs.shape == (55, 5, 3) + assert bundle.validation.inputs.shape == (15, 5, 3) + assert bundle.test.inputs.shape == (15, 5, 3) + assert np.allclose(bundle.feature_scaler.mean, [[29.5, 59.0]]) + assert np.allclose(bundle.target_scaler.mean, [[14.75]]) + expected_features = bundle.feature_scaler.transform( + np.asarray([[index, index * 2.0] for index in range(1, 6)], dtype=np.float32) + ) + expected_quality_history = bundle.target_scaler.transform( + np.asarray([[index * 0.5] for index in range(5)], dtype=np.float32) + ) + expected_target = bundle.target_scaler.transform(np.asarray([[2.5]], dtype=np.float32)) + assert np.allclose(bundle.train.inputs[0, :, :2], expected_features) + assert np.allclose(bundle.train.inputs[0, :, 2:], expected_quality_history) + assert np.allclose(bundle.train.targets[0], expected_target[0]) + + +def test_scalers_are_fitted_only_on_training_rows(): + features = np.arange(100, dtype=np.float32).reshape((-1, 1)) + targets = features.copy() + bundle = prepare_data( + features, + targets, + seq_len=5, + train_ratio=0.6, + validation_ratio=0.2, + ) + assert np.allclose(bundle.feature_scaler.mean, [[29.5]]) + assert bundle.validation.inputs[..., 0].mean() > bundle.train.inputs[..., 0].mean() diff --git a/research/TSLM/DPMSAN_MindSpore/tests/test_devices.py b/research/TSLM/DPMSAN_MindSpore/tests/test_devices.py new file mode 100644 index 0000000..87c8f71 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/tests/test_devices.py @@ -0,0 +1,44 @@ +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +from dpmsan_ms.utils import normalize_device_target, set_context + + +@pytest.mark.parametrize( + ("requested", "expected"), + [("CPU", "CPU"), ("GPU", "GPU"), ("NPU", "Ascend"), ("Ascend", "Ascend")], +) +def test_device_target_mapping(requested, expected): + assert normalize_device_target(requested) == expected + + +@pytest.mark.parametrize( + ("requested", "expected"), + [("GPU", "GPU"), ("NPU", "Ascend"), ("Ascend", "Ascend")], +) +def test_device_initialization_uses_target_and_device_id(requested, expected): + with patch("dpmsan_ms.utils.ms.set_context"), patch( + "dpmsan_ms.utils.ms.set_device" + ) as set_device: + target = set_context(requested, mode="GRAPH_MODE", seed=2026, device_id=2) + assert target == expected + set_device.assert_called_once_with(expected, 2) + + +def test_negative_device_id_is_rejected(): + with pytest.raises(ValueError, match="non-negative"): + set_context("CPU", device_id=-1) + + +def test_unavailable_backend_has_clear_error(): + with patch("dpmsan_ms.utils.ms.set_context"), patch( + "dpmsan_ms.utils.ms.set_device", side_effect=RuntimeError("backend unavailable") + ): + with pytest.raises(RuntimeError, match="matching MindSpore build"): + set_context("GPU", device_id=0) diff --git a/research/TSLM/DPMSAN_MindSpore/tests/test_layers.py b/research/TSLM/DPMSAN_MindSpore/tests/test_layers.py new file mode 100644 index 0000000..25b51cd --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/tests/test_layers.py @@ -0,0 +1,34 @@ +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +mindspore_available = importlib.util.find_spec("mindspore") is not None + + +@pytest.mark.skipif(not mindspore_available, reason="MindSpore is not installed.") +def test_deformable_convolution_matches_regular_convolution_at_zero_offset(): + import mindspore as ms + from mindspore import Tensor + + from dpmsan_ms.layers import DeformableConv1d + from dpmsan_ms import set_context + + set_context("CPU", mode="PYNATIVE_MODE", seed=2026) + layer = DeformableConv1d(2, 3, kernel_size=3) + input_array = np.arange(20, dtype=np.float32).reshape((1, 2, 10)) + inputs = Tensor(input_array, ms.float32) + + actual = layer(inputs).asnumpy() + padded = np.pad(input_array, ((0, 0), (0, 0), (1, 1))) + expected = sum( + np.einsum("bcl,oc->bol", padded[:, :, index : index + 10], layer.weight.asnumpy()[:, :, index]) + for index in range(3) + ) + + assert np.allclose(actual, expected, rtol=1e-5, atol=1e-5) diff --git a/research/TSLM/DPMSAN_MindSpore/tests/test_train_cli.py b/research/TSLM/DPMSAN_MindSpore/tests/test_train_cli.py new file mode 100644 index 0000000..522f821 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/tests/test_train_cli.py @@ -0,0 +1,79 @@ +import csv +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +mindspore_available = importlib.util.find_spec("mindspore") is not None + + +@pytest.mark.skipif(not mindspore_available, reason="MindSpore is not installed.") +def test_training_cli_with_external_csv(tmp_path): + path = tmp_path / "new_process.csv" + checkpoint = tmp_path / "dpmsan.ckpt" + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(["sensor_a", "sensor_b", "quality"]) + for index in range(100): + writer.writerow([index / 10.0, (index % 7) / 5.0, index / 20.0]) + + command = [ + sys.executable, + "scripts/train.py", + "--data_path", + str(path), + "--target_column", + "quality", + "--epochs", + "1", + "--batch_size", + "16", + "--seq_len", + "5", + "--d_model", + "9", + "--d_ff", + "16", + "--n_heads", + "2", + "--e_layers", + "1", + "--d_layers", + "1", + "--kernel_sizes", + "1", + "3", + "--device_target", + "CPU", + "--mode", + "PYNATIVE_MODE", + "--checkpoint_path", + str(checkpoint), + ] + completed = subprocess.run( + command, + cwd=PROJECT_ROOT, + check=True, + capture_output=True, + text=True, + timeout=60, + ) + + assert "validation_R2=" in completed.stdout + assert "validation_RMSE=" in completed.stdout + assert "validation_MAE=" in completed.stdout + assert "final_test_R2=" in completed.stdout + assert checkpoint.is_file() + assert f"checkpoint={checkpoint}" in completed.stdout + assert not list(PROJECT_ROOT.glob("*.ckpt")) + + +def test_checkpoint_path_must_be_external(): + sys.path.insert(0, str(PROJECT_ROOT / "scripts")) + from train import external_checkpoint_path + + with pytest.raises(Exception, match="outside the repository"): + external_checkpoint_path(str(PROJECT_ROOT / "model.ckpt")) diff --git a/research/TSLM/DPMSAN_MindSpore/tests/test_training.py b/research/TSLM/DPMSAN_MindSpore/tests/test_training.py new file mode 100644 index 0000000..752fda5 --- /dev/null +++ b/research/TSLM/DPMSAN_MindSpore/tests/test_training.py @@ -0,0 +1,73 @@ +import importlib.util +import sys +from pathlib import Path + +import numpy as np +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT / "src")) + +mindspore_available = importlib.util.find_spec("mindspore") is not None + + +@pytest.mark.skipif(not mindspore_available, reason="MindSpore is not installed.") +def test_one_epoch_training_and_validation(): + from mindspore import nn + + from dpmsan_ms import ( + DPMSAN, + DPMSANConfig, + evaluate, + prepare_data, + set_context, + train_epoch, + ) + + set_context("CPU", mode="PYNATIVE_MODE", seed=2026) + time = np.arange(100, dtype=np.float32) + features = np.stack((np.sin(time / 5.0), np.cos(time / 7.0)), axis=1) + targets = (0.4 * features[:, 0] - 0.2 * features[:, 1]).reshape((-1, 1)) + bundle = prepare_data( + features, + targets, + seq_len=5, + train_ratio=0.6, + validation_ratio=0.2, + ) + config = DPMSANConfig( + seq_len=5, + enc_in=bundle.input_size, + dec_in=bundle.input_size, + d_model=9, + d_ff=16, + n_heads=2, + e_layers=1, + d_layers=1, + scale_k=2, + kernel_sizes=(1, 3), + ) + model = DPMSAN(config) + optimizer = nn.AdamWeightDecay(model.trainable_params(), learning_rate=1e-3) + + loss = train_epoch(model, optimizer, bundle.train, batch_size=16, seed=2026) + metrics = evaluate(model, bundle.validation, bundle.target_scaler, batch_size=16) + + assert np.isfinite(loss) + assert np.isfinite(metrics.r2) + assert np.isfinite(metrics.rmse) + assert np.isfinite(metrics.mae) + assert metrics.rmse >= 0.0 + assert metrics.mae >= 0.0 + + +def test_regression_metrics(): + from dpmsan_ms.training import regression_metrics + + targets = np.asarray([1.0, 2.0, 3.0], dtype=np.float32) + predictions = np.asarray([1.0, 2.0, 4.0], dtype=np.float32) + metrics = regression_metrics(targets, predictions) + + assert metrics.r2 == pytest.approx(0.5) + assert metrics.rmse == pytest.approx(np.sqrt(1.0 / 3.0)) + assert metrics.mae == pytest.approx(1.0 / 3.0) diff --git a/research/TSLM/DualForecast_MindSpore/.gitignore b/research/TSLM/DualForecast_MindSpore/.gitignore new file mode 100644 index 0000000..cb52698 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/.gitignore @@ -0,0 +1,10 @@ +__pycache__/ +*.pyc +.idea/ +.vscode/ +checkpoints/ +results/ +*.ckpt +*.mindir +dataset/C5/*.csv +dataset/hm/*.csv diff --git a/research/TSLM/DualForecast_MindSpore/README.md b/research/TSLM/DualForecast_MindSpore/README.md new file mode 100644 index 0000000..9039070 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/README.md @@ -0,0 +1,104 @@ +# DualForecast (MindSpore) + +DualForecast is a dual-stream industrial time-series forecasting model. This directory is a MindSpore migration of the original PyTorch research implementation and is prepared for contribution to the MindSpore/OpenI ecosystem. + +## Architecture + +The MindSpore implementation preserves the original DualForecast design: + +1. **Adaptive stream gate**: sample statistics (mean/std/min/max) estimate the contribution of the LLM-enhanced stream and numerical stream. +2. **Numerical stream**: raw/trend/residual/difference signals are projected and processed by local/mid/long dilated depthwise temporal convolutions, followed by temporal attention, variable attention and variable-level gated fusion. +3. **Textual stream**: micro patches are reprogrammed into the embedding space of a frozen pretrained language model. MindNLP is used to run HuggingFace-compatible LLMs on MindSpore. +4. **Cross-stream fusion**: numerical features query textual features through single-direction cross attention. A feature gate mixes direct textual representations and text-enhanced numerical representations. +5. **Output head**: sample-level stream weights softly combine the two streams, and a temporal Conv1d head predicts the future horizon. + +Default short-term configuration: `seq_len=48`, `patch_len=4`, `stride=2`, `d_model=64`, `d_ff=256`, `n_heads=8`. + +## Environment + +Recommended environment: + +```bash +python -m pip install -r requirements.txt +``` + +MindSpore installation is hardware/CANN dependent. On Ascend, install the MindSpore package that matches the CANN version of the runtime image before installing the remaining Python dependencies. + +## Dataset + +Industrial C5/hm datasets are **not distributed in this community package**. Put your authorized CSV file under the expected directory, for example: + +```text +dataset/ +└── C5/ + └── C5_new.csv +``` + +The CSV should contain a `date` column (optional), numerical process variables, and the target column. The loader moves the target variable to the last feature position and uses an 80%/5%/15% train/validation/test split, matching the current industrial experiment code. + +## Train + +```bash +bash scripts/train_c5.sh +``` + +Or: + +```bash +python train.py \ + --root_path ./dataset/C5/ --data_path C5_new.csv \ + --target C5 --features MS \ + --seq_len 48 --label_len 3 --pred_len 3 \ + --enc_in 8 --d_model 64 --d_ff 256 --n_heads 8 \ + --batch_size 24 --learning_rate 1e-4 --epochs 30 \ + --llm_model openai-community/gpt2 --llm_dim 768 --llm_layers 2 \ + --device_target Ascend +``` + +## Evaluation + +```bash +bash scripts/eval_c5.sh +``` + +`--fusion_weight -1` uses the learned adaptive gate. A value in `[0,1]` fixes the LLM-enhanced stream ratio for fusion-weight ablation experiments. + +## Smoke tests + +The numerical stream, fusion block and output head can be shape-tested without downloading an LLM: + +```bash +pytest -q tests/test_shapes.py +``` + +## PyTorch → MindSpore mapping + +| PyTorch implementation | MindSpore implementation | +|---|---| +| `nn.Module.forward` | `nn.Cell.construct` | +| `nn.Linear` | `nn.Dense` | +| `nn.Conv1d` | `nn.Conv1d` | +| `torch.stack/cat/reshape/permute` | `ops.stack/concat/reshape/transpose` | +| `torch.softmax` | `ops.softmax` | +| `torch.einsum` | `ops.einsum` | +| `torch.no_grad / requires_grad=False` | frozen MindSpore parameters | +| `torch.optim.Adam` | `mindspore.nn.Adam` | +| autograd backward | `mindspore.value_and_grad` | +| PyTorch DataLoader | `mindspore.dataset.GeneratorDataset` | +| HuggingFace PyTorch LLM | MindNLP Transformers on MindSpore | + +## Notes on equivalence + +The mathematical data flow and tensor dimensions are kept aligned with the current DualForecast source. The migration intentionally removes PyTorch/Accelerate/DeepSpeed dependencies. Tokenization remains a Python-side operation because the frozen language-model prompt is static per batch; numerical model training and gradients are handled by MindSpore. + +The original project builds on ideas/code from Time-LLM and time-series libraries. Preserve upstream attribution and the repository license when publishing derivative code. + +## Community submission + +Suggested directory name under `mindspore-lab/models/research`: + +```text +research/hpc/dualforecast/ +``` + +If maintainers prefer application-based categorization, use the category requested during review. For OpenI, create a public repository named `DualForecast-MindSpore` and upload this directory as the repository root. diff --git a/research/TSLM/DualForecast_MindSpore/eval.py b/research/TSLM/DualForecast_MindSpore/eval.py new file mode 100644 index 0000000..825de75 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/eval.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Evaluate a MindSpore DualForecast checkpoint.""" +import argparse +import numpy as np +import mindspore as ms + +from src.dataset import create_dataset +from src.dualforecast import DualForecast, DualForecastConfig + + +def metrics(pred, true): + err = pred - true + mse = float(np.mean(err ** 2)) + mae = float(np.mean(np.abs(err))) + rmse = float(np.sqrt(mse)) + denom = np.sum((true - true.mean()) ** 2) + r2 = float(1.0 - np.sum(err ** 2) / denom) if denom > 0 else float("nan") + return mse, mae, rmse, r2 + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--root_path", required=True) + p.add_argument("--data_path", required=True) + p.add_argument("--target", default="C5") + p.add_argument("--features", default="MS", choices=["M", "MS", "S"]) + p.add_argument("--seq_len", type=int, default=48) + p.add_argument("--label_len", type=int, default=3) + p.add_argument("--pred_len", type=int, default=3) + p.add_argument("--enc_in", type=int, default=8) + p.add_argument("--d_model", type=int, default=64) + p.add_argument("--d_ff", type=int, default=256) + p.add_argument("--n_heads", type=int, default=8) + p.add_argument("--llm_model", default="openai-community/gpt2") + p.add_argument("--llm_dim", type=int, default=768) + p.add_argument("--llm_layers", type=int, default=2) + p.add_argument("--batch_size", type=int, default=24) + p.add_argument("--checkpoint", required=True) + p.add_argument("--fusion_weight", type=float, default=-1.0) + p.add_argument("--device_target", default="Ascend", choices=["Ascend", "GPU", "CPU"]) + p.add_argument("--device_id", type=int, default=0) + args = p.parse_args() + + ms.set_device(args.device_target, args.device_id) + _, test_ds = create_dataset( + args.root_path, args.data_path, "test", args.batch_size, + args.seq_len, args.label_len, args.pred_len, args.features, args.target, + shuffle=False + ) + cfg = DualForecastConfig( + seq_len=args.seq_len, pred_len=args.pred_len, enc_in=args.enc_in, + d_model=args.d_model, d_ff=args.d_ff, n_heads=args.n_heads, + llm_model=args.llm_model, llm_dim=args.llm_dim, llm_layers=args.llm_layers, + fusion_weight=args.fusion_weight + ) + net = DualForecast(cfg) + ms.load_param_into_net(net, ms.load_checkpoint(args.checkpoint)) + net.set_train(False) + + preds, trues, gates = [], [], [] + for batch in test_ds.create_dict_iterator(): + pred = net(batch["x"]) + true = batch["y"][:, -args.pred_len:, :] + if args.features == "MS": + pred = pred[:, :, -1:] + true = true[:, :, -1:] + preds.append(pred.asnumpy()) + trues.append(true.asnumpy()) + if net.last_gate_weight is not None: + gates.append(net.last_gate_weight.asnumpy()) + pred = np.concatenate(preds) + true = np.concatenate(trues) + mse, mae, rmse, r2 = metrics(pred, true) + print(f"mse:{mse:.8f}, mae:{mae:.8f}, rmse:{rmse:.8f}, r2:{r2:.8f}") + if gates: + gate = np.concatenate(gates) + print(f"fusion mean: llm={gate[:,0].mean():.4f}, numerical={gate[:,1].mean():.4f}") + + +if __name__ == "__main__": + main() diff --git a/research/TSLM/DualForecast_MindSpore/requirements.txt b/research/TSLM/DualForecast_MindSpore/requirements.txt new file mode 100644 index 0000000..ef5d96f --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/requirements.txt @@ -0,0 +1,6 @@ +mindspore>=2.9.0 +mindnlp[transformers]>=0.5.0 +numpy>=1.25.0 +pandas>=1.5.3 +scikit-learn>=1.2.2 +pytest>=7.0 diff --git a/research/TSLM/DualForecast_MindSpore/scripts/eval_c5.sh b/research/TSLM/DualForecast_MindSpore/scripts/eval_c5.sh new file mode 100644 index 0000000..0553734 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/scripts/eval_c5.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +python eval.py \ + --root_path ./dataset/C5/ \ + --data_path C5_new.csv \ + --target C5 --features MS \ + --seq_len 48 --label_len 3 --pred_len 3 \ + --enc_in 8 --d_model 64 --d_ff 256 --n_heads 8 \ + --checkpoint ./checkpoints/dualforecast_best.ckpt \ + --llm_model openai-community/gpt2 --llm_dim 768 --llm_layers 2 \ + --device_target Ascend diff --git a/research/TSLM/DualForecast_MindSpore/scripts/train_c5.sh b/research/TSLM/DualForecast_MindSpore/scripts/train_c5.sh new file mode 100644 index 0000000..1b79199 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/scripts/train_c5.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +python train.py \ + --root_path ./dataset/C5/ \ + --data_path C5_new.csv \ + --target C5 \ + --features MS \ + --seq_len 48 --label_len 3 --pred_len 3 \ + --enc_in 8 --d_model 64 --d_ff 256 --n_heads 8 \ + --batch_size 24 --learning_rate 1e-4 --epochs 30 \ + --llm_model openai-community/gpt2 --llm_dim 768 --llm_layers 2 \ + --device_target Ascend diff --git a/research/TSLM/DualForecast_MindSpore/src/__init__.py b/research/TSLM/DualForecast_MindSpore/src/__init__.py new file mode 100644 index 0000000..8687c64 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/src/__init__.py @@ -0,0 +1,4 @@ +from .dualforecast import DualForecast +from .dataset import IndustrialForecastDataset, create_dataset + +__all__ = ["DualForecast", "IndustrialForecastDataset", "create_dataset"] diff --git a/research/TSLM/DualForecast_MindSpore/src/dataset.py b/research/TSLM/DualForecast_MindSpore/src/dataset.py new file mode 100644 index 0000000..3e8d390 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/src/dataset.py @@ -0,0 +1,95 @@ +"""Dataset utilities for industrial multivariate forecasting with MindSpore.""" +from __future__ import annotations + +import os +from typing import Tuple + +import numpy as np +import pandas as pd +import mindspore.dataset as ds +from sklearn.preprocessing import StandardScaler + + +class IndustrialForecastDataset: + """Sliding-window dataset matching the original Dataset_C5 split logic. + + Split ratios: train 80%, validation 5%, test 15%. + Target column is moved to the last position to preserve MS evaluation behavior. + """ + + def __init__(self, root_path: str, data_path: str, flag: str = "train", + seq_len: int = 48, label_len: int = 3, pred_len: int = 3, + features: str = "MS", target: str = "C5", scale: bool = True): + if flag not in ("train", "val", "test"): + raise ValueError("flag must be train/val/test") + self.seq_len = seq_len + self.label_len = label_len + self.pred_len = pred_len + self.features = features + self.target = target + self.scale = scale + self.flag = flag + self.scaler = StandardScaler() + self._read(os.path.join(root_path, data_path)) + + def _read(self, path: str): + df = pd.read_csv(path) + if self.target not in df.columns: + raise KeyError(f"Target column '{self.target}' is not present in {path}") + date_col = "date" if "date" in df.columns else None + feature_cols = [c for c in df.columns if c not in ([date_col] if date_col else []) + [self.target]] + ordered = feature_cols + [self.target] + values = df[ordered].values.astype(np.float32) + + n = len(values) + n_train = int(n * 0.80) + n_test = int(n * 0.15) + n_val = n - n_train - n_test + border1 = { + "train": 0, + "val": n_train - self.seq_len, + "test": n - n_test - self.seq_len, + }[self.flag] + border2 = { + "train": n_train, + "val": n_train + n_val, + "test": n, + }[self.flag] + + if self.features == "S": + values = values[:, -1:] + if self.scale: + train = values[:n_train] + self.scaler.fit(train) + values = self.scaler.transform(values).astype(np.float32) + self.data = values[border1:border2] + self.enc_in = self.data.shape[-1] + self.length = len(self.data) - self.seq_len - self.pred_len + 1 + + def __getitem__(self, index: int): + s0 = index + s1 = s0 + self.seq_len + r0 = s1 - self.label_len + r1 = r0 + self.label_len + self.pred_len + x = self.data[s0:s1] + y = self.data[r0:r1] + return x.astype(np.float32), y.astype(np.float32) + + def __len__(self): + return self.length + + def inverse_transform(self, x): + return self.scaler.inverse_transform(x) + + +def create_dataset(root_path: str, data_path: str, flag: str, batch_size: int, + seq_len: int, label_len: int, pred_len: int, features: str, + target: str, shuffle: bool | None = None) -> Tuple[IndustrialForecastDataset, ds.Dataset]: + source = IndustrialForecastDataset( + root_path, data_path, flag, seq_len, label_len, pred_len, features, target + ) + if shuffle is None: + shuffle = flag == "train" + dataset = ds.GeneratorDataset(source, column_names=["x", "y"], shuffle=shuffle) + dataset = dataset.batch(batch_size, drop_remainder=True) + return source, dataset diff --git a/research/TSLM/DualForecast_MindSpore/src/dualforecast.py b/research/TSLM/DualForecast_MindSpore/src/dualforecast.py new file mode 100644 index 0000000..27c62f0 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/src/dualforecast.py @@ -0,0 +1,491 @@ +"""MindSpore implementation of DualForecast. + +Core architecture preserved from the original PyTorch implementation: +- raw/trend/residual/difference numerical decomposition +- multi-scale dilated depthwise temporal convolution +- temporal and variable self-attention +- micro-patch textual reprogramming into a frozen LLM +- numerical-query/text-key-value cross attention +- feature-level gated fusion and sample-level adaptive stream fusion +- temporal convolution prediction head +""" +from __future__ import annotations + +from dataclasses import dataclass +from math import sqrt +from typing import Optional + +import mindspore as ms +from mindspore import Tensor, Parameter, nn, ops +from mindspore.common.initializer import Normal, initializer + + +class MultiHeadAttention(nn.Cell): + """Batch-first multi-head attention with an optional cross-attention input.""" + + def __init__(self, d_model: int, n_heads: int, dropout: float = 0.1): + super().__init__() + if d_model % n_heads != 0: + raise ValueError(f"d_model ({d_model}) must be divisible by n_heads ({n_heads})") + self.d_model = d_model + self.n_heads = n_heads + self.d_head = d_model // n_heads + self.scale = 1.0 / sqrt(self.d_head) + self.q_proj = nn.Dense(d_model, d_model) + self.k_proj = nn.Dense(d_model, d_model) + self.v_proj = nn.Dense(d_model, d_model) + self.o_proj = nn.Dense(d_model, d_model) + self.dropout = nn.Dropout(p=dropout) + + def _split(self, x: Tensor) -> Tensor: + b, l, _ = x.shape + x = ops.reshape(x, (b, l, self.n_heads, self.d_head)) + return ops.transpose(x, (0, 2, 1, 3)) + + def construct(self, query: Tensor, key: Tensor, value: Tensor) -> Tensor: + q = self._split(self.q_proj(query)) + k = self._split(self.k_proj(key)) + v = self._split(self.v_proj(value)) + score = ops.matmul(q, ops.transpose(k, (0, 1, 3, 2))) * self.scale + attn = self.dropout(ops.softmax(score, axis=-1)) + out = ops.matmul(attn, v) + out = ops.transpose(out, (0, 2, 1, 3)) + b, l, _, _ = out.shape + return self.o_proj(ops.reshape(out, (b, l, self.d_model))) + + +class FeedForward(nn.Cell): + def __init__(self, d_model: int, dropout: float): + super().__init__() + self.fc1 = nn.Dense(d_model, d_model * 4) + self.fc2 = nn.Dense(d_model * 4, d_model) + self.gelu = nn.GELU() + self.dropout = nn.Dropout(p=dropout) + + def construct(self, x: Tensor) -> Tensor: + return self.fc2(self.dropout(self.gelu(self.fc1(x)))) + + +class TemporalBranch(nn.Cell): + def __init__(self, d_model: int, dilation: int, dropout: float): + super().__init__() + self.depthwise = nn.Conv1d( + d_model, d_model, kernel_size=3, stride=1, pad_mode="pad", + padding=dilation, dilation=dilation, group=d_model, has_bias=True + ) + self.pointwise = nn.Conv1d(d_model, d_model, kernel_size=1, has_bias=True) + self.gelu = nn.GELU() + self.dropout = nn.Dropout(p=dropout) + + def construct(self, x: Tensor) -> Tensor: + return self.dropout(self.gelu(self.pointwise(self.depthwise(x)))) + + +class MicroPatchNumericalStream(nn.Cell): + """Multi-scale temporal-variable numerical stream. + + Input: [B, T, N] + Output: [B, N, D, P] + """ + + def __init__(self, seq_len: int, enc_in: int, d_model: int, patch_len: int = 4, + stride: int = 2, n_heads: int = 4, dropout: float = 0.1): + super().__init__() + self.seq_len = seq_len + self.enc_in = enc_in + self.d_model = d_model + self.patch_len = patch_len + self.stride = stride + self.patch_nums = int((seq_len - patch_len) / stride + 2) + + self.channel_projection = nn.Conv1d(4, d_model, kernel_size=1, has_bias=True) + self.local_branch = TemporalBranch(d_model, 1, dropout) + self.mid_branch = TemporalBranch(d_model, 2, dropout) + self.long_branch = TemporalBranch(d_model, 4, dropout) + self.scale_gate = nn.Dense(d_model, 3) + self.patch_projection = nn.Dense(d_model * patch_len, d_model) + self.pos_embedding = Parameter( + initializer(Normal(sigma=0.02), (1, self.patch_nums, d_model), ms.float32), + name="pos_embedding" + ) + self.temporal_attn = MultiHeadAttention(d_model, n_heads, dropout) + self.temporal_norm1 = nn.LayerNorm((d_model,)) + self.temporal_ffn = FeedForward(d_model, dropout) + self.temporal_norm2 = nn.LayerNorm((d_model,)) + self.variable_embedding = Parameter( + initializer(Normal(sigma=0.02), (1, enc_in, d_model), ms.float32), + name="variable_embedding" + ) + self.variable_attn = MultiHeadAttention(d_model, n_heads, dropout) + self.variable_norm = nn.LayerNorm((d_model,)) + self.variable_gate = nn.SequentialCell(nn.Dense(d_model * 2, d_model), nn.Sigmoid()) + self.dropout = nn.Dropout(p=dropout) + + @staticmethod + def _moving_average(x: Tensor) -> Tensor: + # replicate padding, kernel_size=3, stride=1 + left = x[..., :1] + right = x[..., -1:] + x_pad = ops.concat((left, x, right), axis=-1) + return (x_pad[..., :-2] + x_pad[..., 1:-1] + x_pad[..., 2:]) / 3.0 + + def _build_patch_tokens(self, x: Tensor) -> Tensor: + # Original implementation pads `stride` points at both ends using replication. + left = ops.tile(x[..., :1], (1, 1, self.stride)) + right = ops.tile(x[..., -1:], (1, 1, self.stride)) + x = ops.concat((left, x, right), axis=-1) + patches = [] + for i in range(self.patch_nums): + start = i * self.stride + patch = x[..., start:start + self.patch_len] + patches.append(patch) + patches = ops.stack(patches, axis=1) # [BN, P, D, patch_len] + bn = patches.shape[0] + patches = ops.reshape(patches, (bn, self.patch_nums, self.d_model * self.patch_len)) + return self.patch_projection(patches) + + def construct(self, x_enc: Tensor) -> Tensor: + b, _, n = x_enc.shape + if n != self.enc_in: + raise ValueError(f"Expected {self.enc_in} input variables, got {n}") + + x_raw = ops.transpose(x_enc, (0, 2, 1)) + x_trend = self._moving_average(x_raw) + x_residual = x_raw - x_trend + x_diff = ops.concat((ops.zeros_like(x_raw[..., :1]), x_raw[..., 1:] - x_raw[..., :-1]), axis=-1) + x_multi = ops.stack((x_raw, x_trend, x_residual, x_diff), axis=2) + + base = self.channel_projection(ops.reshape(x_multi, (b * n, 4, -1))) + local_feat = self.local_branch(base) + mid_feat = self.mid_branch(base) + long_feat = self.long_branch(base) + scale_weight = ops.softmax(self.scale_gate(ops.mean(base, axis=-1)), axis=-1) + scale_weight = ops.reshape(scale_weight, (b * n, 3, 1, 1)) + multi_scale = ops.stack((local_feat, mid_feat, long_feat), axis=1) + temporal_feature = ops.sum(scale_weight * multi_scale, axis=1) + + patch_feat = self.dropout(self._build_patch_tokens(temporal_feature) + self.pos_embedding) + temporal_out = self.temporal_attn(patch_feat, patch_feat, patch_feat) + patch_feat = self.temporal_norm1(patch_feat + self.dropout(temporal_out)) + temporal_only = self.temporal_norm2(patch_feat + self.dropout(self.temporal_ffn(patch_feat))) + + temporal_only = ops.reshape(temporal_only, (b, n, self.patch_nums, self.d_model)) + variable_input = ops.transpose(temporal_only, (0, 2, 1, 3)) + variable_input = ops.reshape(variable_input, (b * self.patch_nums, n, self.d_model)) + variable_input = variable_input + self.variable_embedding + variable_out = self.variable_attn(variable_input, variable_input, variable_input) + variable_feat = self.variable_norm(variable_input + self.dropout(variable_out)) + variable_feat = ops.reshape(variable_feat, (b, self.patch_nums, n, self.d_model)) + variable_feat = ops.transpose(variable_feat, (0, 2, 1, 3)) + + gate = self.variable_gate(ops.concat((temporal_only, variable_feat), axis=-1)) + fused = gate * variable_feat + (1.0 - gate) * temporal_only + return ops.transpose(fused, (0, 1, 3, 2)) + + +class CircularPatchEmbedding(nn.Cell): + """TimeLLM-style micro-patch embedding with circular Conv1d across patches.""" + + def __init__(self, d_model: int, patch_len: int, stride: int, dropout: float): + super().__init__() + self.patch_len = patch_len + self.stride = stride + self.conv = nn.Conv1d(patch_len, d_model, kernel_size=3, pad_mode="valid", has_bias=False) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, x: Tensor): + # x [B, N, T]; right replication pad by stride, then extract patches. + n_vars = x.shape[1] + right = ops.tile(x[..., -1:], (1, 1, self.stride)) + x = ops.concat((x, right), axis=-1) + p = (x.shape[-1] - self.patch_len) // self.stride + 1 + patches = [] + for i in range(p): + start = i * self.stride + patches.append(x[..., start:start + self.patch_len]) + patches = ops.stack(patches, axis=2) # [B,N,P,L] + b = patches.shape[0] + patches = ops.reshape(patches, (b * n_vars, p, self.patch_len)) + patches = ops.transpose(patches, (0, 2, 1)) # [BN,L,P] + # circular padding of patch axis by one on each side + patches = ops.concat((patches[..., -1:], patches, patches[..., :1]), axis=-1) + out = self.conv(patches) + out = ops.transpose(out, (0, 2, 1)) + return self.dropout(out), n_vars + + +class ReprogrammingLayer(nn.Cell): + def __init__(self, d_model: int, n_heads: int, d_keys: Optional[int] = None, + d_llm: int = 768, attention_dropout: float = 0.1): + super().__init__() + d_keys = d_keys or (d_model // n_heads) + self.n_heads = n_heads + self.d_keys = d_keys + self.query_projection = nn.Dense(d_model, d_keys * n_heads) + self.key_projection = nn.Dense(d_llm, d_keys * n_heads) + self.value_projection = nn.Dense(d_llm, d_keys * n_heads) + self.out_projection = nn.Dense(d_keys * n_heads, d_llm) + self.dropout = nn.Dropout(p=attention_dropout) + + def construct(self, target_embedding: Tensor, source_embedding: Tensor, + value_embedding: Tensor) -> Tensor: + b, l, _ = target_embedding.shape + s = source_embedding.shape[0] + h = self.n_heads + q = ops.reshape(self.query_projection(target_embedding), (b, l, h, self.d_keys)) + k = ops.reshape(self.key_projection(source_embedding), (s, h, self.d_keys)) + v = ops.reshape(self.value_projection(value_embedding), (s, h, self.d_keys)) + score = ops.einsum("blhe,she->bhls", q, k) / sqrt(self.d_keys) + attn = self.dropout(ops.softmax(score, axis=-1)) + out = ops.einsum("bhls,she->blhe", attn, v) + out = ops.reshape(out, (b, l, h * self.d_keys)) + return self.out_projection(out) + + +class CrossAttentionFusion(nn.Cell): + def __init__(self, d_model: int, n_heads: int, d_ff_text: int, dropout: float = 0.1): + super().__init__() + self.text_proj = nn.Dense(d_ff_text, d_model) + self.num_proj = nn.Dense(d_model, d_model) + self.cross_attn = MultiHeadAttention(d_model, n_heads, dropout) + self.gate = nn.SequentialCell(nn.Dense(d_model * 2, d_model), nn.Sigmoid()) + self.fusion_proj = nn.Dense(d_model, d_model) + self.norm = nn.LayerNorm((d_model,)) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, text_features: Tensor, num_features: Tensor) -> Tensor: + text_feat = self.text_proj(text_features) + num_feat = self.num_proj(num_features) + enhanced = self.cross_attn(num_feat, text_feat, text_feat) + enhanced = self.norm(num_feat + self.dropout(enhanced)) + gate = self.gate(ops.concat((text_feat, enhanced), axis=-1)) + fused = gate * text_feat + (1.0 - gate) * enhanced + return self.fusion_proj(fused) + + +class TemporalConvHead(nn.Cell): + def __init__(self, d_model: int, patch_nums: int, pred_len: int, dropout: float = 0.1): + super().__init__() + self.d_model = d_model + self.patch_nums = patch_nums + self.pred_len = pred_len + self.conv = nn.Conv1d(d_model, d_model, kernel_size=3, pad_mode="pad", padding=1, has_bias=True) + self.proj = nn.Dense(d_model * patch_nums, pred_len) + self.relu = nn.ReLU() + self.dropout = nn.Dropout(p=dropout) + + def construct(self, x: Tensor) -> Tensor: + b, n, d, p = x.shape + x = ops.reshape(x, (b * n, d, p)) + x = self.relu(self.conv(x)) + x = ops.reshape(x, (b * n, d * p)) + x = self.dropout(self.proj(x)) + return ops.reshape(x, (b, n, self.pred_len)) + + +class MindNLPBackbone: + """Frozen LLM/tokenizer wrapper for the MindSpore ecosystem. + + MindNLP provides HuggingFace-compatible transformers implemented on MindSpore. + This wrapper keeps tokenization outside the graph while returning MindSpore tensors. + """ + + def __init__(self, model_name: str, llm_layers: Optional[int] = None, + local_files_only: bool = False): + try: + from mindnlp.transformers import AutoConfig, AutoModel, AutoTokenizer + except ImportError as exc: + raise ImportError( + "MindNLP is required for the textual stream. Install `mindnlp[transformers]`." + ) from exc + + cfg = AutoConfig.from_pretrained(model_name, local_files_only=local_files_only) + if llm_layers is not None: + for attr in ("num_hidden_layers", "n_layer", "num_layers"): + if hasattr(cfg, attr): + setattr(cfg, attr, llm_layers) + break + if hasattr(cfg, "output_hidden_states"): + cfg.output_hidden_states = True + self.model = AutoModel.from_pretrained( + model_name, config=cfg, local_files_only=local_files_only + ) + self.tokenizer = AutoTokenizer.from_pretrained( + model_name, local_files_only=local_files_only + ) + if self.tokenizer.pad_token is None: + if self.tokenizer.eos_token is not None: + self.tokenizer.pad_token = self.tokenizer.eos_token + else: + self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + for p in self.model.get_parameters(): + p.requires_grad = False + self.model.set_train(False) + + @property + def embeddings(self): + embedding_layer = self.model.get_input_embeddings() + if hasattr(embedding_layer, "embedding_table"): + return embedding_layer.embedding_table + if hasattr(embedding_layer, "weight"): + return embedding_layer.weight + raise AttributeError("Cannot locate the LLM input embedding parameter") + + def encode_prompt(self, prompts, max_length: int = 256) -> Tensor: + # Use NumPy as the tokenizer interchange format. This avoids depending on + # tokenizer-version-specific support for return_tensors="ms". + encoded = self.tokenizer( + prompts, padding=True, truncation=True, max_length=max_length, + return_tensors="np" + ) + ids = Tensor(encoded["input_ids"], ms.int64) + return self.model.get_input_embeddings()(ids) + + def forward_embeddings(self, embeddings: Tensor) -> Tensor: + out = self.model(inputs_embeds=embeddings) + if hasattr(out, "last_hidden_state"): + return out.last_hidden_state + return out[0] + + +@dataclass +class DualForecastConfig: + seq_len: int = 48 + pred_len: int = 3 + enc_in: int = 8 + d_model: int = 64 + d_ff: int = 256 + n_heads: int = 8 + dropout: float = 0.1 + llm_dim: int = 768 + llm_layers: int = 2 + llm_model: str = "openai-community/gpt2" + prompt_domain: bool = True + content: str = "Industrial multivariate process time series." + fusion_weight: float = -1.0 + local_files_only: bool = False + + +class DualForecast(nn.Cell): + """MindSpore DualForecast network.""" + + def __init__(self, config: DualForecastConfig, llm_backbone: Optional[MindNLPBackbone] = None): + super().__init__() + self.config = config + self.seq_len = config.seq_len + self.pred_len = config.pred_len + self.enc_in = config.enc_in + self.d_model = config.d_model + self.d_ff = config.d_ff + self.d_llm = config.llm_dim + self.patch_len = 4 + self.stride = 2 + self.patch_nums = int((self.seq_len - self.patch_len) / self.stride + 2) + self.fixed_llm_weight = float(config.fusion_weight) + if self.fixed_llm_weight != -1.0 and not 0.0 <= self.fixed_llm_weight <= 1.0: + raise ValueError("fusion_weight must be -1 or in [0, 1]") + + self.llm = llm_backbone or MindNLPBackbone( + config.llm_model, config.llm_layers, config.local_files_only + ) + embeddings = self.llm.embeddings + actual_llm_dim = embeddings.shape[-1] + if actual_llm_dim != self.d_llm: + raise ValueError( + f"llm_dim={self.d_llm}, but backbone embedding dimension is {actual_llm_dim}. " + "Please set --llm_dim to match the selected LLM." + ) + self.word_embeddings = embeddings + self.vocab_size = embeddings.shape[0] + self.num_tokens = 1000 + self.mapping_layer = nn.Dense(self.vocab_size, self.num_tokens) + + self.text_patch_embedding = CircularPatchEmbedding( + config.d_model, self.patch_len, self.stride, config.dropout + ) + self.reprogramming_layer = ReprogrammingLayer( + config.d_model, config.n_heads, config.d_ff, self.d_llm, config.dropout + ) + self.num_stream = MicroPatchNumericalStream( + config.seq_len, config.enc_in, config.d_model, + self.patch_len, self.stride, config.n_heads, config.dropout + ) + self.stream_gate = nn.SequentialCell( + nn.Dense(4, 16), nn.ReLU(), nn.Dense(16, 2) + ) + # Equivalent initial gate bias [0.4, -0.4]. + final_dense = self.stream_gate[2] + final_dense.bias.set_data(Tensor([0.4, -0.4], ms.float32)) + + self.cross_attn_fusion = CrossAttentionFusion( + config.d_model, config.n_heads, config.d_ff, config.dropout + ) + self.output_projection = TemporalConvHead( + config.d_model, self.patch_nums, config.pred_len, config.dropout + ) + self.description = config.content if config.prompt_domain else ( + "The Electricity Transformer Temperature is a time-series forecasting benchmark." + ) + self.last_gate_weight = None + + def _text_stream(self, x_enc: Tensor, b: int, n: int) -> Tensor: + prompt_text = ( + f"<|start_prompt|>Task: forecast next {self.pred_len} steps." + f"Dataset: {self.description}.Historical time-series representation only." + f"<|end_prompt|>" + ) + prompt_embeddings = self.llm.encode_prompt([prompt_text] * (b * n)) + + # [vocab,d_llm] -> [d_llm,vocab] -> Dense(vocab,1000) -> [1000,d_llm] + source_embeddings = ops.transpose(self.word_embeddings, (1, 0)) + source_embeddings = self.mapping_layer(source_embeddings) + source_embeddings = ops.transpose(source_embeddings, (1, 0)) + + x_patch = ops.transpose(x_enc, (0, 2, 1)) + enc_out, _ = self.text_patch_embedding(x_patch) + enc_out = self.reprogramming_layer(enc_out, source_embeddings, source_embeddings) + llm_input = ops.concat((prompt_embeddings, enc_out), axis=1) + llm_output = self.llm.forward_embeddings(llm_input) + llm_output = llm_output[:, -self.patch_nums:, :self.d_ff] + text_features = ops.reshape(llm_output, (b, n, self.patch_nums, self.d_ff)) + return ops.transpose(text_features, (0, 1, 3, 2)) + + def construct(self, x_enc: Tensor) -> Tensor: + b, _, n = x_enc.shape + gate_input = ops.stack(( + ops.mean(x_enc, axis=(1, 2)), + ops.mean(ops.std(x_enc, axis=1), axis=1), + ops.mean(ops.amin(x_enc, axis=1), axis=1), + ops.mean(ops.amax(x_enc, axis=1), axis=1), + ), axis=1) + gate_weight = ops.softmax(self.stream_gate(gate_input), axis=-1) + if self.fixed_llm_weight >= 0.0: + llm_fixed = ops.ones_like(gate_weight[:, 0]) * self.fixed_llm_weight + gate_weight = ops.stack((llm_fixed, 1.0 - llm_fixed), axis=-1) + self.last_gate_weight = gate_weight + + means = ops.stop_gradient(ops.mean(x_enc, axis=1, keep_dims=True)) + var = ops.mean((x_enc - means) ** 2, axis=1, keep_dims=True) + stdev = ops.sqrt(var + 1e-5) + x_norm = (x_enc - means) / stdev + + text_features = self._text_stream(x_norm, b, n) + num_features = self.num_stream(x_norm) + + text_flat = ops.reshape( + ops.transpose(text_features, (0, 1, 3, 2)), + (b * n, self.patch_nums, self.d_ff) + ) + num_flat = ops.reshape( + ops.transpose(num_features, (0, 1, 3, 2)), + (b * n, self.patch_nums, self.d_model) + ) + fused_llm = self.cross_attn_fusion(text_flat, num_flat) + fused_llm = ops.reshape(fused_llm, (b, n, self.patch_nums, self.d_model)) + fused_llm = ops.transpose(fused_llm, (0, 1, 3, 2)) + + llm_w = ops.reshape(gate_weight[:, 0], (b, 1, 1, 1)) + fused_features = llm_w * fused_llm + (1.0 - llm_w) * num_features + out = self.output_projection(fused_features) + out = ops.transpose(out, (0, 2, 1)) + return out * stdev + means diff --git a/research/TSLM/DualForecast_MindSpore/src/prompt_bank/C5.txt b/research/TSLM/DualForecast_MindSpore/src/prompt_bank/C5.txt new file mode 100644 index 0000000..78e790f --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/src/prompt_bank/C5.txt @@ -0,0 +1 @@ +The C5 Dataset is a crucial indicator in the chemical industry, specifically for monitoring and controlling the purification process of the distillation column C5. This dataset consists of approximately 4 years of data from chemical plant operations, which might be from different production units. To explore the multi-variable time-series forecasting (MVTSF) problem in an industrial context, different subsets are created. Each data point consists of the target value “C5” and 7 process features. The time interval is irregular, with data points occurring approximately every 4 or 8 hours. The entire dataset covers the operating conditions and fluid dynamics within the column. \ No newline at end of file diff --git a/research/TSLM/DualForecast_MindSpore/src/prompt_bank/hm.txt b/research/TSLM/DualForecast_MindSpore/src/prompt_bank/hm.txt new file mode 100644 index 0000000..92457b7 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/src/prompt_bank/hm.txt @@ -0,0 +1,8 @@ +"<|start_prompt|>Background: This is a multivariate time series dataset from an industrial petrochemical refining process, specifically a hydrocracking unit. The data is sampled every 4 hours. The process involves complex thermodynamic and fluid dynamic transitions across a multi-bed reactor and a fractionation tower. +Variable Semantics: +1. Reaction Section: Features include 'Reactor Inlet Temperature', 'Reactor Differential Pressure', and multi-bed temperatures (Bed 1 to Bed 4 top/bottom). These dictate the chemical conversion rate. +2. Fractionation Section: Features include 'Tower Top Pressure', 'Tower Bottom Temperature', 'Reflux Flow', and side-draw flows (e.g., Heavy Naphtha, Jet Fuel, Diesel). These determine product separation efficiency. +3. Key Flows: 'Total Outlet Flow', 'Cyclic Hydrogen Flow', 'Stripping Steam Flow', and 'Water Injection'. +Task: You are required to perform time series forecasting for the 'hm' variable, which represents a specific light hydrocarbon stream flow or yield. +Industrial Dynamics: The system exhibits strong temporal delays (thermal lag), non-linear phase changes, and multivariable coupling. An increase in reactor temperature typically increases light ends (like C5) yield, but shifts in tower pressure or reflux can drastically alter the draw rate. +Instruction: Based on the provided historical statistics and temporal trends of these sensor channels, infer the underlying operational regime (e.g., stable, transition, or upset) and predict the future trajectory of the C5 variable.<|end_prompt|>" diff --git a/research/TSLM/DualForecast_MindSpore/train.py b/research/TSLM/DualForecast_MindSpore/train.py new file mode 100644 index 0000000..7084a62 --- /dev/null +++ b/research/TSLM/DualForecast_MindSpore/train.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Train DualForecast with MindSpore.""" +import argparse +import os + +import mindspore as ms +from mindspore import nn, ops + +from src.dataset import create_dataset +from src.dualforecast import DualForecast, DualForecastConfig + + +def parse_args(): + p = argparse.ArgumentParser("DualForecast MindSpore training") + p.add_argument("--root_path", required=True) + p.add_argument("--data_path", required=True) + p.add_argument("--target", default="C5") + p.add_argument("--features", default="MS", choices=["M", "MS", "S"]) + p.add_argument("--seq_len", type=int, default=48) + p.add_argument("--label_len", type=int, default=3) + p.add_argument("--pred_len", type=int, default=3) + p.add_argument("--enc_in", type=int, default=8) + p.add_argument("--d_model", type=int, default=64) + p.add_argument("--d_ff", type=int, default=256) + p.add_argument("--n_heads", type=int, default=8) + p.add_argument("--dropout", type=float, default=0.1) + p.add_argument("--llm_model", default="openai-community/gpt2") + p.add_argument("--llm_dim", type=int, default=768) + p.add_argument("--llm_layers", type=int, default=2) + p.add_argument("--content", default="Industrial multivariate process time series") + p.add_argument("--batch_size", type=int, default=24) + p.add_argument("--epochs", type=int, default=30) + p.add_argument("--learning_rate", type=float, default=1e-4) + p.add_argument("--device_target", default="Ascend", choices=["Ascend", "GPU", "CPU"]) + p.add_argument("--device_id", type=int, default=0) + p.add_argument("--save_dir", default="./checkpoints") + return p.parse_args() + + +def main(): + args = parse_args() + ms.set_seed(2021) + ms.set_device(args.device_target, args.device_id) + _, train_ds = create_dataset( + args.root_path, args.data_path, "train", args.batch_size, + args.seq_len, args.label_len, args.pred_len, args.features, args.target + ) + _, val_ds = create_dataset( + args.root_path, args.data_path, "val", args.batch_size, + args.seq_len, args.label_len, args.pred_len, args.features, args.target, + shuffle=False + ) + cfg = DualForecastConfig( + seq_len=args.seq_len, pred_len=args.pred_len, enc_in=args.enc_in, + d_model=args.d_model, d_ff=args.d_ff, n_heads=args.n_heads, + dropout=args.dropout, llm_model=args.llm_model, llm_dim=args.llm_dim, + llm_layers=args.llm_layers, content=args.content + ) + net = DualForecast(cfg) + optimizer = nn.Adam(net.trainable_params(), learning_rate=args.learning_rate) + + def loss_fn(x, y): + pred = net(x) + target = y[:, -args.pred_len:, :] + if args.features == "MS": + pred = pred[:, :, -1:] + target = target[:, :, -1:] + return ops.mean((pred - target) ** 2) + + grad_fn = ms.value_and_grad(loss_fn, None, optimizer.parameters) + + @ms.jit + def train_step(x, y): + loss, grads = grad_fn(x, y) + optimizer(grads) + return loss + + os.makedirs(args.save_dir, exist_ok=True) + best_val = float("inf") + for epoch in range(1, args.epochs + 1): + net.set_train(True) + train_losses = [] + for batch in train_ds.create_dict_iterator(): + train_losses.append(float(train_step(batch["x"], batch["y"]).asnumpy())) + + net.set_train(False) + val_losses = [] + for batch in val_ds.create_dict_iterator(): + val_losses.append(float(loss_fn(batch["x"], batch["y"]).asnumpy())) + train_mse = sum(train_losses) / max(len(train_losses), 1) + val_mse = sum(val_losses) / max(len(val_losses), 1) + print(f"Epoch {epoch:03d} | train_mse={train_mse:.6f} | val_mse={val_mse:.6f}") + if val_mse < best_val: + best_val = val_mse + ms.save_checkpoint(net, os.path.join(args.save_dir, "dualforecast_best.ckpt")) + + +if __name__ == "__main__": + main() diff --git a/research/TSLM/PAT_LLM_MindSpore/README.md b/research/TSLM/PAT_LLM_MindSpore/README.md new file mode 100644 index 0000000..4588965 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/README.md @@ -0,0 +1,225 @@ +# PAT-LLM + +> MindSpore implementation of **PAT-LLM: Textualized Multi-Patch Attention +> for Multivariate Time Series Forecasting with LLMs**. + +[中文](README_CN.md) + +## Contents + +- [Overview](#overview) +- [Authors and Affiliations](#authors-and-affiliations) +- [Model Architecture](#model-architecture) +- [Dataset](#dataset) +- [Environment](#environment) +- [Quick Start](#quick-start) +- [Configuration](#configuration) +- [Limitations](#limitations) +- [Acknowledgements](#acknowledgements) +- [Citation](#citation) + +## Overview + +PAT-LLM forecasts strongly coupled industrial multivariate time series using +a frozen large language model. It keeps the complete multivariate input window +and combines two learned structural components: + +1. **Multi-Patch Attention (MPA)** aggregates global cross-variable + interactions and then refines patch-level temporal dependencies. +2. **Attention-as-Prompt (AaP)** converts influential variables and important + temporal regions from the attention maps into natural-language context. + +The prompts and reprogrammed time-series embeddings are concatenated and sent +to a frozen LLM. The patch encoder, MPA, prototype mapping, reprogramming layer, +and forecast head are trainable. + +## Authors and Affiliations + +**Xiaofeng Yuan**, **Zhiyuan Liu**, **Zijian Xu**, **Kai Wang**, +**Yalin Wang**, **Chunhua Yang**, **Weihua Gui**, **Feifan Shen**, and +**Lingjian Ye**. + +- Xiaofeng Yuan, Zhiyuan Liu, Zijian Xu, Kai Wang, Yalin Wang, Chunhua Yang, + and Weihua Gui are with the School of Automation, Central South University, + Changsha 410083, China. +- Lingjian Ye is with the School of Engineering, Huzhou University, + Huzhou 313000, China. +- Feifan Shen is with the School of Information Science and Engineering, + NingboTech University, Ningbo 315000, China. + +IEEE membership: Xiaofeng Yuan and Yalin Wang are Senior Members; Chunhua Yang +is a Fellow; Kai Wang, Weihua Gui, Feifan Shen, and Lingjian Ye are Members. + +Contact: yuanxf@csu.edu.cn, 254612061@csu.edu.cn, 254603027@csu.edu.cn, +219083@csu.edu.cn, ylwang@csu.edu.cn, ychh@csu.edu.cn, gwh@csu.edu.cn, +ffshen@nbt.edu.cn, and lingjian.ye@gmail.com. + +## Model Architecture + +```text +multivariate history + | +patch embedding + | +two-stage Multi-Patch Attention + |-----------------------------| + | | +reprogramming tokens Attention-as-Prompt + | | + | tokenizer + embeddings + | | + +-------------+---------------+ + | + frozen LLM + | + forecast head +``` + +The manuscript uses a 96-step look-back window, patch length 8, +`d_model=64`, three attention heads, and forecast horizons 8, 18, 32, and 72. +The custom projected multi-head attention supports the non-divisible +`64 / 3` setting by projecting each head to `floor(d_model / n_heads)`. + +## Dataset + +The manuscript evaluates two industrial hydrocracking datasets for C4 and C5 +quality forecasting. These datasets are not distributed in this directory. +Prepare each dataset as a CSV file with this schema: + +```text +date,process_variable_1,...,process_variable_n,C5 +2025-01-01 00:00:00,...,...,... +``` + +- `date` is required and must be parseable by pandas. +- The target name is selected with `target` in the YAML configuration. +- Other numeric columns are treated as process variables. +- Missing numeric values are interpolated, forward-filled, then back-filled. +- Samples are split chronologically into 70% train, 20% validation, and 10% + test partitions. Scaling statistics are fitted only on the training data. + +For C4 forecasting, change `data_path` and `target` to the corresponding CSV +and target column. Do not commit confidential industrial data to the model +repository. + +## Environment + +The code targets Python 3.10/3.11, MindSpore 2.7.x, and MindNLP 0.6.x. +Install the MindSpore package suitable for the target hardware, then install +the remaining dependencies: + +```bash +pip install -r requirements.txt +``` + +Download or convert a MindNLP-compatible Llama 2 model locally. The default +configuration disables remote downloads and remote model code. + +## Quick Start + +Run from the `research/nlp/pat_llm` directory. + +### Standalone training on Ascend + +```bash +bash scripts/run_standalone_train_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b +``` + +Optional third and fourth arguments select the relative CSV path and target: + +```bash +bash scripts/run_standalone_train_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b \ + C4/C4.csv C4 +``` + +### Four manuscript horizons + +```bash +bash scripts/run_all_horizons_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b +``` + +### Evaluation + +```bash +bash scripts/run_eval_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b \ + /absolute/path/to/pat_llm.ckpt +``` + +The evaluation command writes `metrics.json` and `predictions.npz` under +`output_dir`. + +## Configuration + +Defaults are stored in `default_config.yaml`. Override individual options +without editing the file: + +```bash +python train.py --config_path default_config.yaml \ + --set device_target=Ascend \ + --set device_id=0 \ + --set llm_model_path=/models/llama2-7b \ + --set pred_len=32 +``` + +`features=MS` trains from all variables and evaluates the final target +channel. `features=M` forecasts all channels, while `features=S` uses only the +target series. + +## Limitations + +- Attention-as-Prompt performs tensor-to-text conversion and tokenisation on + the Python host. The network therefore uses `PYNATIVE_MODE` and currently + does not support MindSpore static graph export. +- The LLM backbone and tokenizer must be compatible with MindNLP. +- Distributed training and Ascend 310 offline inference are not yet included. +- Before a model-zoo pull request, add reproducible MindSpore performance, + execution logs, hardware details, and a public or reviewable dataset access + procedure. + +## Acknowledgements + +This work was supported in part by the National Natural Science Foundation of +China under Grants 92267205 and 62573432; in part by the Foundation of Hunan, +China under Grants 2024RC1020 and 2025JJ10007; and in part by the CAAI-MindSpore +Open Fund under Grant CAAIXSJLJJ 2025 MindSpore 10. The work was developed on +the OpenI Community. + +## Citation + +PAT-LLM is currently provided as a manuscript. Volume, issue, page, and DOI +information should be added after publication. + +```bibtex +@article{yuan2026patllm, + title={{PAT-LLM}: Textualized Multi-Patch Attention for Multivariate + Time Series Forecasting with LLMs}, + author={Yuan, Xiaofeng and Liu, Zhiyuan and Xu, Zijian and Wang, Kai and + Wang, Yalin and Yang, Chunhua and Gui, Weihua and Shen, Feifan and + Ye, Lingjian}, + journal={IEEE Transactions on Industrial Informatics}, + year={2026}, + note={Manuscript} +} +``` + +The implementation also builds on the LLM reprogramming paradigm from +Time-LLM: + +```bibtex +@inproceedings{jin2023time, + title={{Time-LLM}: Time Series Forecasting by Reprogramming Large Language Models}, + author={Jin, Ming and Wang, Shiyu and Ma, Lintao and Chu, Zhixuan and + Zhang, James Y. and Shi, Xiaoming and Chen, Pin-Yu and + Liang, Yuxuan and Li, Yuan-Fang and Pan, Shirui and Wen, Qingsong}, + booktitle={International Conference on Learning Representations}, + year={2024} +} +``` diff --git a/research/TSLM/PAT_LLM_MindSpore/README_CN.md b/research/TSLM/PAT_LLM_MindSpore/README_CN.md new file mode 100644 index 0000000..2014c13 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/README_CN.md @@ -0,0 +1,126 @@ +# PAT-LLM + +> 论文 **PAT-LLM: Textualized Multi-Patch Attention for Multivariate Time +> Series Forecasting with LLMs** 的 MindSpore 实现。 + +[English](README.md) + +## 模型简介 + +PAT-LLM 面向强耦合工业多变量时间序列预测。模型保留完整的多变量输入， +通过两阶段 Multi-Patch Attention 建模变量间关系和跨时间片依赖，再通过 +Attention-as-Prompt 将重要变量与时间区域转换成自然语言提示。提示嵌入与 +重编程后的时间序列嵌入共同输入冻结的大语言模型，最后由预测头输出结果。 + +论文默认设置为：历史窗口 96、patch 长度 8、`d_model=64`、注意力头数 3， +预测长度为 8、18、32 和 72。实现中的自定义投影式多头注意力支持 64 无法被 +3 整除的参数组合。 + +## 作者与单位 + +作者依次为：Xiaofeng Yuan、Zhiyuan Liu、Zijian Xu、Kai Wang、Yalin Wang、 +Chunhua Yang、Weihua Gui、Feifan Shen 和 Lingjian Ye。 + +- Xiaofeng Yuan、Zhiyuan Liu、Zijian Xu、Kai Wang、Yalin Wang、Chunhua Yang + 和 Weihua Gui：中南大学自动化学院，长沙 410083。 +- Lingjian Ye：湖州学院工学院，湖州 313000。 +- Feifan Shen：浙大宁波理工学院信息科学与工程学院，宁波 315000。 + +IEEE 会籍：Xiaofeng Yuan、Yalin Wang 为 Senior Member，Chunhua Yang 为 +Fellow，Kai Wang、Weihua Gui、Feifan Shen、Lingjian Ye 为 Member。 + +联系邮箱：yuanxf@csu.edu.cn、254612061@csu.edu.cn、254603027@csu.edu.cn、 +219083@csu.edu.cn、ylwang@csu.edu.cn、ychh@csu.edu.cn、gwh@csu.edu.cn、 +ffshen@nbt.edu.cn、lingjian.ye@gmail.com。 + +## 目录结构 + +```text +pat_llm/ +├── default_config.yaml +├── train.py +├── eval.py +├── requirements.txt +├── scripts/ +└── src/ + ├── config.py + ├── dataset.py + ├── engine.py + ├── layers.py + ├── metrics.py + └── pat_llm.py +``` + +## 数据集 + +代码要求 CSV 至少包含 `date`、若干数值过程变量及目标列。例如： + +```text +date,variable_1,...,variable_n,C5 +2025-01-01 00:00:00,...,...,... +``` + +数据按照时间顺序划分为 70% 训练、20% 验证和 10% 测试；标准化统计量只从 +训练集计算。论文使用的工业 C4/C5 数据没有放入投稿目录，禁止将保密工业 +数据直接提交到公共仓库。 + +## 环境与训练 + +先安装与硬件匹配的 MindSpore 2.7.x，再执行： + +```bash +pip install -r requirements.txt +bash scripts/run_standalone_train_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b +``` + +运行四个论文预测长度： + +```bash +bash scripts/run_all_horizons_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b +``` + +评估已有权重： + +```bash +bash scripts/run_eval_ascend.sh \ + /absolute/path/to/dataset \ + /absolute/path/to/llama2-7b \ + /absolute/path/to/pat_llm.ckpt +``` + +## 当前限制 + +- AaP 在 Python 侧完成张量转文字和分词，因此使用 `PYNATIVE_MODE`，暂不支持 + 静态图导出。 +- 尚未加入分布式训练与 Ascend 310 离线推理。 +- 当前电脑不具备运行环境，不能把论文表格数值标记为该 MindSpore 实现的 + 已复现结果。 +- 正式发起模型仓 Pull Request 前，还需要补充 MindSpore 实测结果、运行日志、 + 硬件信息和可审查的数据获取方案。 + +## 致谢 + +本工作得到国家自然科学基金项目 92267205、62573432，湖南省相关基金项目 +2024RC1020、2025JJ10007，以及 CAAI-MindSpore Open Fund 项目 +CAAIXSJLJJ 2025 MindSpore 10 的部分资助，并基于 OpenI 启智社区完成开发。 + +## 引用 + +论文目前为 manuscript，正式发表后需要补充卷期、页码和 DOI。 + +```bibtex +@article{yuan2026patllm, + title={{PAT-LLM}: Textualized Multi-Patch Attention for Multivariate + Time Series Forecasting with LLMs}, + author={Yuan, Xiaofeng and Liu, Zhiyuan and Xu, Zijian and Wang, Kai and + Wang, Yalin and Yang, Chunhua and Gui, Weihua and Shen, Feifan and + Ye, Lingjian}, + journal={IEEE Transactions on Industrial Informatics}, + year={2026}, + note={Manuscript} +} +``` diff --git a/research/TSLM/PAT_LLM_MindSpore/default_config.yaml b/research/TSLM/PAT_LLM_MindSpore/default_config.yaml new file mode 100644 index 0000000..81e6af8 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/default_config.yaml @@ -0,0 +1,46 @@ +# Runtime. Attention-as-Prompt requires PYNATIVE_MODE. +device_target: Ascend +device_id: 0 +seed: 2021 + +# Dataset. The target column is moved to the final channel internally. +root_path: ./dataset +data_path: C5/C5-2.csv +features: MS +target: C5 +freq: h +percent: 100 +num_workers: 1 + +# Forecasting setup from the PAT-LLM manuscript. +seq_len: 96 +label_len: 48 +pred_len: 32 +patch_len: 8 +stride: 8 +d_model: 64 +n_heads: 3 +d_ff: 256 +dropout: 0.1 +aap_top_k: 5 +aap_top_m: 3 +aap_recent_fraction: 0.2 + +# Frozen LLM. Use an existing local MindNLP-compatible model directory. +llm_model_path: /path/to/llama2-7b +llm_layers: 6 +llm_dtype: float16 +num_prototypes: 1000 +allow_download: false +trust_remote_code: false +description: >- + Industrial hydrocracking process data with strongly coupled process + variables and delayed light-naphtha quality measurements. + +# Optimisation and outputs. +batch_size: 16 +train_epochs: 10 +learning_rate: 0.0001 +patience: 5 +checkpoint_path: ./checkpoints/pat_llm_c5_32.ckpt +output_dir: ./results/pat_llm_c5_32 diff --git a/research/TSLM/PAT_LLM_MindSpore/eval.py b/research/TSLM/PAT_LLM_MindSpore/eval.py new file mode 100644 index 0000000..4636306 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/eval.py @@ -0,0 +1,26 @@ +"""Evaluate a trained PAT-LLM MindSpore checkpoint.""" + +from src import PATLLM, create_dataset, load_config +from src.engine import ( + attach_dataset_metadata, + configure_runtime, + load_trainable_checkpoint, + predict, + save_evaluation, +) + + +def main(): + """Load a trainable-parameter checkpoint and evaluate it.""" + config = load_config("Evaluate PAT-LLM with MindSpore") + configure_runtime(config) + test_source, test_dataset = create_dataset(config, "test") + attach_dataset_metadata(config, test_source) + model = PATLLM(config) + load_trainable_checkpoint(model, config.checkpoint_path) + predictions, targets, histories = predict(config, model, test_dataset) + save_evaluation(config, predictions, targets, histories) + + +if __name__ == "__main__": + main() diff --git a/research/TSLM/PAT_LLM_MindSpore/requirements.txt b/research/TSLM/PAT_LLM_MindSpore/requirements.txt new file mode 100644 index 0000000..2011783 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/requirements.txt @@ -0,0 +1,6 @@ +mindspore>=2.7.1,<2.8 +mindnlp>=0.6,<0.7 +numpy>=1.23.5,<2.0 +pandas>=1.5.3,<3.0 +PyYAML>=6.0,<7.0 +sentencepiece>=0.2 diff --git a/research/TSLM/PAT_LLM_MindSpore/scripts/run_all_horizons_ascend.sh b/research/TSLM/PAT_LLM_MindSpore/scripts/run_all_horizons_ascend.sh new file mode 100644 index 0000000..0e27030 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/scripts/run_all_horizons_ascend.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 2 || $# -gt 4 ]]; then + echo "Usage: bash run_all_horizons_ascend.sh DATA_DIR LLM_DIR [DATA_PATH] [TARGET]" + exit 1 +fi + +DATA_DIR=$(realpath "$1") +LLM_DIR=$(realpath "$2") +DATA_PATH=${3:-C5/C5-2.csv} +TARGET=${4:-C5} +PROJECT_DIR=$(cd "$(dirname "$0")/.." && pwd) + +cd "${PROJECT_DIR}" +for PRED_LEN in 8 18 32 72; do + python train.py \ + --config_path default_config.yaml \ + --set "root_path=${DATA_DIR}" \ + --set "data_path=${DATA_PATH}" \ + --set "target=${TARGET}" \ + --set "llm_model_path=${LLM_DIR}" \ + --set "pred_len=${PRED_LEN}" \ + --set "checkpoint_path=./checkpoints/pat_llm_${TARGET}_${PRED_LEN}.ckpt" \ + --set "output_dir=./results/pat_llm_${TARGET}_${PRED_LEN}" +done diff --git a/research/TSLM/PAT_LLM_MindSpore/scripts/run_eval_ascend.sh b/research/TSLM/PAT_LLM_MindSpore/scripts/run_eval_ascend.sh new file mode 100644 index 0000000..7aeace1 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/scripts/run_eval_ascend.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 3 || $# -gt 5 ]]; then + echo "Usage: bash run_eval_ascend.sh DATA_DIR LLM_DIR CKPT [DATA_PATH] [TARGET]" + exit 1 +fi + +DATA_DIR=$(realpath "$1") +LLM_DIR=$(realpath "$2") +CHECKPOINT=$(realpath "$3") +DATA_PATH=${4:-C5/C5-2.csv} +TARGET=${5:-C5} +PROJECT_DIR=$(cd "$(dirname "$0")/.." && pwd) + +cd "${PROJECT_DIR}" +python eval.py \ + --config_path default_config.yaml \ + --set "root_path=${DATA_DIR}" \ + --set "data_path=${DATA_PATH}" \ + --set "target=${TARGET}" \ + --set "llm_model_path=${LLM_DIR}" \ + --set "checkpoint_path=${CHECKPOINT}" diff --git a/research/TSLM/PAT_LLM_MindSpore/scripts/run_standalone_train_ascend.sh b/research/TSLM/PAT_LLM_MindSpore/scripts/run_standalone_train_ascend.sh new file mode 100644 index 0000000..22ee12c --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/scripts/run_standalone_train_ascend.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 2 || $# -gt 4 ]]; then + echo "Usage: bash run_standalone_train_ascend.sh DATA_DIR LLM_DIR [DATA_PATH] [TARGET]" + exit 1 +fi + +DATA_DIR=$(realpath "$1") +LLM_DIR=$(realpath "$2") +DATA_PATH=${3:-C5/C5-2.csv} +TARGET=${4:-C5} +PROJECT_DIR=$(cd "$(dirname "$0")/.." && pwd) + +cd "${PROJECT_DIR}" +python train.py \ + --config_path default_config.yaml \ + --set "root_path=${DATA_DIR}" \ + --set "data_path=${DATA_PATH}" \ + --set "target=${TARGET}" \ + --set "llm_model_path=${LLM_DIR}" diff --git a/research/TSLM/PAT_LLM_MindSpore/src/__init__.py b/research/TSLM/PAT_LLM_MindSpore/src/__init__.py new file mode 100644 index 0000000..440d831 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/__init__.py @@ -0,0 +1,7 @@ +"""PAT-LLM MindSpore implementation.""" + +from .config import load_config +from .dataset import create_dataset +from .pat_llm import PATLLM + +__all__ = ["PATLLM", "create_dataset", "load_config"] diff --git a/research/TSLM/PAT_LLM_MindSpore/src/config.py b/research/TSLM/PAT_LLM_MindSpore/src/config.py new file mode 100644 index 0000000..a6a805c --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/config.py @@ -0,0 +1,138 @@ +"""YAML configuration loader for PAT-LLM entry points.""" + +import argparse +from types import SimpleNamespace + +import yaml + + +def _parse_scalar(value): + """Parse an override value with YAML scalar semantics.""" + parsed = yaml.safe_load(value) + if isinstance(parsed, (dict, list)): + raise ValueError("configuration overrides must be scalar values") + return parsed + + +def _build_parser(description): + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + "--config_path", + type=str, + default="default_config.yaml", + help="Path to the YAML configuration file.", + ) + parser.add_argument( + "--set", + dest="overrides", + action="append", + default=[], + metavar="KEY=VALUE", + help="Override one YAML value; this option can be repeated.", + ) + return parser + + +def load_config(description="PAT-LLM"): + """Load YAML and apply command-line ``--set KEY=VALUE`` overrides.""" + arguments = _build_parser(description).parse_args() + with open(arguments.config_path, "r", encoding="utf-8") as stream: + values = yaml.safe_load(stream) or {} + if not isinstance(values, dict): + raise ValueError("the top level of the configuration must be a mapping") + + for item in arguments.overrides: + if "=" not in item: + raise ValueError(f"invalid override {item!r}; expected KEY=VALUE") + key, raw_value = item.split("=", 1) + if key not in values: + raise KeyError(f"unknown configuration key: {key}") + values[key] = _parse_scalar(raw_value) + + _validate(values) + return SimpleNamespace(**values) + + +def _validate(values): + required = { + "aap_recent_fraction", + "aap_top_k", + "aap_top_m", + "allow_download", + "batch_size", + "checkpoint_path", + "d_ff", + "device_target", + "device_id", + "description", + "dropout", + "features", + "freq", + "label_len", + "learning_rate", + "llm_dtype", + "llm_layers", + "root_path", + "data_path", + "target", + "seq_len", + "pred_len", + "d_model", + "n_heads", + "llm_model_path", + "num_prototypes", + "num_workers", + "output_dir", + "patch_len", + "patience", + "percent", + "seed", + "stride", + "train_epochs", + "trust_remote_code", + } + missing = sorted(required.difference(values)) + if missing: + raise KeyError(f"missing required configuration values: {missing}") + if values["device_target"] not in {"CPU", "GPU", "Ascend"}: + raise ValueError("device_target must be CPU, GPU, or Ascend") + if values.get("features", "MS") not in {"M", "MS", "S"}: + raise ValueError("features must be M, MS, or S") + positive_values = ( + "seq_len", + "label_len", + "pred_len", + "patch_len", + "stride", + "d_model", + "n_heads", + "d_ff", + "batch_size", + "train_epochs", + "llm_layers", + "num_prototypes", + "aap_top_k", + "aap_top_m", + "patience", + ) + for name in positive_values: + if int(values[name]) <= 0: + raise ValueError(f"{name} must be positive") + if values["label_len"] > values["seq_len"]: + raise ValueError("label_len cannot exceed seq_len") + if values["patch_len"] > values["seq_len"] + values["stride"]: + raise ValueError("patch_len exceeds the padded sequence length") + if not 1 <= int(values.get("percent", 100)) <= 100: + raise ValueError("percent must be between 1 and 100") + if float(values["learning_rate"]) <= 0: + raise ValueError("learning_rate must be positive") + if not 0 <= float(values["dropout"]) < 1: + raise ValueError("dropout must be in the interval [0, 1)") + if not 0 < float(values["aap_recent_fraction"]) <= 1: + raise ValueError("aap_recent_fraction must be in the interval (0, 1]") + if values.get("llm_dtype", "float16") not in { + "float16", + "bfloat16", + "float32", + }: + raise ValueError("llm_dtype must be float16, bfloat16, or float32") diff --git a/research/TSLM/PAT_LLM_MindSpore/src/dataset.py b/research/TSLM/PAT_LLM_MindSpore/src/dataset.py new file mode 100644 index 0000000..af13904 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/dataset.py @@ -0,0 +1,164 @@ +"""Chronological multivariate dataset and MindSpore input pipeline.""" + +# Dataset objects intentionally retain split metadata used to configure the +# model after the CSV schema is known. +# pylint: disable=too-many-instance-attributes + +import os + +import mindspore.dataset as ds +import numpy as np +import pandas as pd + +from .time_features import time_features + + +COLUMN_NAMES = ["batch_x", "batch_y", "batch_x_mark", "batch_y_mark"] + + +class StandardScaler: + """Feature-wise standardisation fitted only on the training partition.""" + + def __init__(self): + self.mean_ = None + self.scale_ = None + + def fit(self, values): + """Fit feature statistics and return this scaler.""" + self.mean_ = np.mean(values, axis=0, keepdims=True) + self.scale_ = np.std(values, axis=0, keepdims=True) + self.scale_ = np.where(self.scale_ < 1e-8, 1.0, self.scale_) + return self + + def transform(self, values): + """Standardise an array using fitted feature statistics.""" + return (values - self.mean_) / self.scale_ + + def inverse_transform(self, values): + """Restore values to their original feature scale.""" + return values * self.scale_ + self.mean_ + + +class PATDataset: + """Return full multivariate windows for Multi-Patch Attention.""" + + def __init__(self, config, split): + if split not in {"train", "val", "test"}: + raise ValueError("split must be train, val, or test") + self.config = config + self.split = split + self.scaler = StandardScaler() + csv_path = os.path.join(config.root_path, config.data_path) + self._read_data(csv_path) + + def _read_data(self, csv_path): + frame = pd.read_csv(csv_path) + if "date" not in frame.columns: + raise ValueError(f"CSV must contain a date column: {csv_path}") + if self.config.target not in frame.columns: + raise ValueError( + f"target {self.config.target!r} not found in {csv_path}" + ) + + feature_columns = [ + column + for column in frame.columns + if column not in {"date", self.config.target} + ] + frame = frame[["date", *feature_columns, self.config.target]] + if self.config.features == "S": + self.variable_names = [self.config.target] + else: + self.variable_names = [*feature_columns, self.config.target] + self.target_index = self.variable_names.index(self.config.target) + + values = frame[self.variable_names].apply( + pd.to_numeric, errors="coerce" + ) + if values.isna().any().any(): + values = values.interpolate(limit_direction="both").ffill().bfill() + values = values.to_numpy(dtype=np.float32) + + total = len(frame) + train_end = int(total * 0.70) + validation_end = train_end + int(total * 0.20) + boundaries = { + "train": (0, train_end), + "val": (train_end - self.config.seq_len, validation_end), + "test": (validation_end - self.config.seq_len, total), + } + start, end = boundaries[self.split] + start = max(0, start) + if self.split == "train" and self.config.percent < 100: + available = end - start - self.config.seq_len + end = ( + start + + self.config.seq_len + + available * self.config.percent // 100 + ) + + self.scaler.fit(values[:train_end]) + values = self.scaler.transform(values).astype(np.float32) + dates = self._parse_dates(frame["date"]) + stamps = time_features(dates, self.config.freq).transpose(1, 0) + self.data_x = values[start:end] + self.data_y = values[start:end] + self.data_stamp = stamps[start:end].astype(np.float32) + self.length = ( + len(self.data_x) + - self.config.seq_len + - self.config.pred_len + + 1 + ) + if self.length <= 0: + raise ValueError( + f"{self.split} split is too short for seq_len=" + f"{self.config.seq_len} and pred_len={self.config.pred_len}" + ) + + @staticmethod + def _parse_dates(date_column): + cleaned = date_column.astype(str).str.strip().str.strip("'\"") + try: + dates = pd.to_datetime(cleaned, errors="coerce", format="mixed") + except (TypeError, ValueError): + dates = pd.to_datetime(cleaned, errors="coerce") + if dates.isna().any(): + reparsed = cleaned[dates.isna()].map( + lambda value: pd.to_datetime(value, errors="coerce") + ) + dates.loc[dates.isna()] = reparsed + if dates.isna().any(): + raise ValueError("date column contains unparseable values") + return pd.DatetimeIndex(dates) + + def __getitem__(self, index): + sequence_end = index + self.config.seq_len + target_start = sequence_end - self.config.label_len + target_end = sequence_end + self.config.pred_len + return ( + self.data_x[index:sequence_end].astype(np.float32), + self.data_y[target_start:target_end].astype(np.float32), + self.data_stamp[index:sequence_end].astype(np.float32), + self.data_stamp[target_start:target_end].astype(np.float32), + ) + + def __len__(self): + return self.length + + +def create_dataset(config, split): + """Create a source dataset and its batched MindSpore pipeline.""" + source = PATDataset(config, split) + pipeline = ds.GeneratorDataset( + source=source, + column_names=COLUMN_NAMES, + shuffle=split == "train", + num_parallel_workers=max(1, config.num_workers), + python_multiprocessing=False, + ) + pipeline = pipeline.batch( + batch_size=config.batch_size, + drop_remainder=split == "train", + ) + return source, pipeline diff --git a/research/TSLM/PAT_LLM_MindSpore/src/engine.py b/research/TSLM/PAT_LLM_MindSpore/src/engine.py new file mode 100644 index 0000000..029fa0f --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/engine.py @@ -0,0 +1,187 @@ +"""Training, evaluation, checkpoint, and result helpers.""" + +# A training entry point naturally owns the loss, optimiser, gradient +# function, early-stopping state, and per-epoch statistics. +# pylint: disable=too-many-locals + +import json +import os +import random +import time + +import mindspore as ms +from mindspore import nn, ops +import numpy as np + +from .metrics import calculate_metrics + + +def configure_runtime(config): + """Configure deterministic PYNATIVE execution for dynamic prompts.""" + context = { + "mode": ms.PYNATIVE_MODE, + "device_target": config.device_target, + } + if config.device_target != "CPU": + context["device_id"] = config.device_id + ms.set_context(**context) + ms.set_seed(config.seed) + np.random.seed(config.seed) + random.seed(config.seed) + + +def attach_dataset_metadata(config, source): + """Add data-dependent model dimensions to the configuration.""" + config.variable_names = source.variable_names + config.enc_in = len(source.variable_names) + config.target_index = source.target_index + return config + + +def select_target(config, prediction, target): + """Select all channels for M/S and only the target channel for MS.""" + feature_start = -1 if config.features == "MS" else 0 + prediction = prediction[:, -config.pred_len :, feature_start:] + target = target[:, -config.pred_len :, feature_start:] + return prediction, target + + +def evaluate_loss(config, model, dataset): + """Return mean validation MSE and MAE.""" + loss_fn = nn.MSELoss() + model.set_train(False) + losses = [] + absolute_errors = [] + for batch_x, batch_y, _, _ in dataset.create_tuple_iterator( + num_epochs=1 + ): + prediction, target = select_target( + config, model(batch_x), batch_y + ) + losses.append(float(loss_fn(prediction, target).asnumpy())) + absolute_errors.append( + float(ops.mean(ops.abs(prediction - target)).asnumpy()) + ) + return float(np.mean(losses)), float(np.mean(absolute_errors)) + + +def predict(config, model, dataset): + """Collect predictions, targets, and historical input windows.""" + model.set_train(False) + predictions = [] + targets = [] + histories = [] + for batch_x, batch_y, _, _ in dataset.create_tuple_iterator( + num_epochs=1 + ): + prediction, target = select_target( + config, model(batch_x), batch_y + ) + predictions.append(prediction.asnumpy()) + targets.append(target.asnumpy()) + histories.append(batch_x.asnumpy()) + return ( + np.concatenate(predictions, axis=0), + np.concatenate(targets, axis=0), + np.concatenate(histories, axis=0), + ) + + +def load_trainable_checkpoint(model, checkpoint_path): + """Load a checkpoint containing only trainable PAT-LLM parameters.""" + if not os.path.isfile(checkpoint_path): + raise FileNotFoundError(f"checkpoint not found: {checkpoint_path}") + parameters = ms.load_checkpoint(checkpoint_path) + not_loaded, unused = ms.load_param_into_net(model, parameters) + trainable_names = {parameter.name for parameter in model.trainable_params()} + missing_trainable = [ + name for name in not_loaded if name in trainable_names + ] + if missing_trainable or unused: + raise RuntimeError( + "checkpoint mismatch: missing trainable parameters=" + f"{missing_trainable}, unused parameters={unused}" + ) + + +def train_model(config, model, train_dataset, validation_dataset): + """Train PAT-LLM and save the best validation checkpoint.""" + loss_fn = nn.MSELoss() + optimizer = nn.Adam( + model.trainable_params(), learning_rate=config.learning_rate + ) + + def forward_fn(batch_x, batch_y): + prediction, target = select_target( + config, model(batch_x), batch_y + ) + return loss_fn(prediction, target), prediction + + gradient_fn = ms.value_and_grad( + forward_fn, + grad_position=None, + weights=optimizer.parameters, + has_aux=True, + ) + best_loss = None + best_epoch = 0 + stale_epochs = 0 + checkpoint_dir = os.path.dirname(config.checkpoint_path) + if checkpoint_dir: + os.makedirs(checkpoint_dir, exist_ok=True) + + for epoch in range(1, config.train_epochs + 1): + model.set_train(True) + epoch_start = time.time() + train_losses = [] + for batch_x, batch_y, _, _ in train_dataset.create_tuple_iterator( + num_epochs=1 + ): + (loss, _), gradients = gradient_fn(batch_x, batch_y) + optimizer(gradients) + train_losses.append(float(loss.asnumpy())) + + validation_loss, validation_mae = evaluate_loss( + config, model, validation_dataset + ) + train_loss = float(np.mean(train_losses)) + improved = best_loss is None or validation_loss < best_loss + if improved: + ms.save_checkpoint( + model.trainable_params(), config.checkpoint_path + ) + best_loss = validation_loss + best_epoch = epoch + stale_epochs = 0 + else: + stale_epochs += 1 + print( + f"epoch={epoch:03d} train_mse={train_loss:.7f} " + f"val_mse={validation_loss:.7f} " + f"val_mae={validation_mae:.7f} " + f"seconds={time.time() - epoch_start:.1f}" + ) + if stale_epochs >= config.patience: + print(f"early stopping at epoch {epoch}") + break + return best_epoch + + +def save_evaluation(config, predictions, targets, histories): + """Write reproducible numeric outputs without plotting dependencies.""" + os.makedirs(config.output_dir, exist_ok=True) + metrics = calculate_metrics(predictions, targets) + with open( + os.path.join(config.output_dir, "metrics.json"), + "w", + encoding="utf-8", + ) as stream: + json.dump(metrics, stream, indent=2) + np.savez_compressed( + os.path.join(config.output_dir, "predictions.npz"), + predictions=predictions, + targets=targets, + histories=histories, + ) + print(json.dumps(metrics, indent=2)) + return metrics diff --git a/research/TSLM/PAT_LLM_MindSpore/src/layers.py b/research/TSLM/PAT_LLM_MindSpore/src/layers.py new file mode 100644 index 0000000..8ef7e49 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/layers.py @@ -0,0 +1,361 @@ +"""Core MindSpore layers used by PAT-LLM.""" + +# Neural-network cells expose several hyperparameters and child cells by +# design; keeping them explicit makes the architecture auditable. +# pylint: disable=too-many-arguments,too-many-instance-attributes + +import math +from math import sqrt + +import mindspore.common.dtype as mstype +from mindspore import Parameter, Tensor, nn, ops +import numpy as np + + +class PatchEmbedding(nn.Cell): + """Tokenise each variable into overlapping local patches.""" + + def __init__(self, seq_len, d_model, patch_len, stride, dropout=0.1): + super().__init__() + if patch_len <= 0 or stride <= 0: + raise ValueError("patch_len and stride must be positive") + if patch_len > seq_len + stride: + raise ValueError("patch_len exceeds the padded sequence length") + self.patch_len = patch_len + self.stride = stride + self.n_patches = (seq_len - patch_len) // stride + 2 + self.projection = nn.Dense(patch_len, d_model) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, values): + """Embed a batch shaped [batch, variables, time].""" + if values.ndim != 3: + raise ValueError("PatchEmbedding expects [batch, variables, time]") + padding = ops.tile(values[:, :, -1:], (1, 1, self.stride)) + padded = ops.concat((values, padding), axis=-1) + patches = tuple( + padded[:, :, start : start + self.patch_len] + for start in range( + 0, self.n_patches * self.stride, self.stride + ) + ) + return self.dropout(self.projection(ops.stack(patches, axis=2))) + + +class ProjectedMultiheadAttention(nn.Cell): + """Attention with projected heads for non-divisible model dimensions.""" + + def __init__(self, d_model, n_heads, dropout=0.1): + super().__init__() + if d_model <= 0 or n_heads <= 0: + raise ValueError("d_model and n_heads must be positive") + self.n_heads = n_heads + self.d_head = d_model // n_heads + if self.d_head == 0: + raise ValueError("n_heads cannot exceed d_model") + projected_dim = n_heads * self.d_head + self.query_projection = nn.Dense(d_model, projected_dim) + self.key_projection = nn.Dense(d_model, projected_dim) + self.value_projection = nn.Dense(d_model, projected_dim) + self.output_projection = nn.Dense(projected_dim, d_model) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, query, key, value): + """Apply scaled dot-product attention and average head weights.""" + batch_size, query_length, _ = query.shape + key_length = key.shape[1] + query = ops.reshape( + self.query_projection(query), + (batch_size, query_length, self.n_heads, self.d_head), + ) + key = ops.reshape( + self.key_projection(key), + (batch_size, key_length, self.n_heads, self.d_head), + ) + value = ops.reshape( + self.value_projection(value), + (batch_size, key_length, self.n_heads, self.d_head), + ) + query = ops.transpose(query, (0, 2, 1, 3)) + key = ops.transpose(key, (0, 2, 3, 1)) + value = ops.transpose(value, (0, 2, 1, 3)) + weights = ops.softmax( + ops.matmul(query, key) / sqrt(self.d_head), axis=-1 + ) + output = ops.matmul(self.dropout(weights), value) + output = ops.transpose(output, (0, 2, 1, 3)) + output = ops.reshape( + output, + (batch_size, query_length, self.n_heads * self.d_head), + ) + return self.output_projection(output), ops.mean(weights, axis=1) + + +class FeedForward(nn.Cell): + """Position-wise feed-forward block.""" + + def __init__(self, d_model, d_ff, dropout): + super().__init__() + self.net = nn.SequentialCell( + nn.Dense(d_model, d_ff), + nn.GELU(), + nn.Dropout(p=dropout), + nn.Dense(d_ff, d_model), + nn.Dropout(p=dropout), + ) + + def construct(self, values): + """Apply the position-wise network.""" + return self.net(values) + + +class MultiPatchAttention(nn.Cell): + """Two-stage cross-variable and cross-temporal patch attention.""" + + def __init__(self, d_model, n_heads, d_ff, dropout=0.1): + super().__init__() + self.stage1_attention = ProjectedMultiheadAttention( + d_model, n_heads, dropout + ) + self.stage1_attention_norm = nn.LayerNorm((d_model,)) + self.stage1_ffn = FeedForward(d_model, d_ff, dropout) + self.stage1_ffn_norm = nn.LayerNorm((d_model,)) + self.stage2_attention = ProjectedMultiheadAttention( + d_model, n_heads, dropout + ) + self.stage2_attention_norm = nn.LayerNorm((d_model,)) + self.stage2_ffn = FeedForward(d_model, d_ff, dropout) + self.stage2_ffn_norm = nn.LayerNorm((d_model,)) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, patch_embeddings): + """Refine patch embeddings and return both attention maps.""" + if patch_embeddings.ndim != 4: + raise ValueError( + "patch embeddings must be [batch, variables, patches, hidden]" + ) + batch_size, n_vars, n_patches, d_model = patch_embeddings.shape + flattened = ops.reshape( + patch_embeddings, (batch_size, n_vars * n_patches, d_model) + ) + latest = patch_embeddings[:, :, -1, :] + variable_context, stage1_weights = self.stage1_attention( + latest, flattened, flattened + ) + variable_context = self.stage1_attention_norm( + latest + self.dropout(variable_context) + ) + variable_context = self.stage1_ffn_norm( + variable_context + self.stage1_ffn(variable_context) + ) + refined, stage2_weights = self.stage2_attention( + flattened, variable_context, variable_context + ) + refined = self.stage2_attention_norm( + flattened + self.dropout(refined) + ) + refined = self.stage2_ffn_norm(refined + self.stage2_ffn(refined)) + refined = ops.reshape( + refined, (batch_size, n_vars, n_patches, d_model) + ) + return refined, stage1_weights, stage2_weights + + +class AttentionPromptFeatures(nn.Cell): + """Extract important variables and patches from MPA attention maps.""" + + def __init__( + self, n_vars, n_patches, top_k=5, top_m=3, recent_fraction=0.2 + ): + super().__init__() + self.n_vars = n_vars + self.n_patches = n_patches + self.top_k = min(top_k, n_vars) + self.top_m = min(top_m, n_patches) + self.recent_count = max( + 1, math.ceil(n_patches * recent_fraction) + ) + cross_mask = np.ones((n_vars, n_vars), dtype=np.float32) + np.fill_diagonal(cross_mask, 0.0) + self.cross_mask = Tensor(cross_mask, mstype.float32) + + def construct(self, stage1_attention, stage2_attention): + """Select important variables, patches, and recent attention.""" + batch_size = stage1_attention.shape[0] + stage1 = ops.reshape( + stage1_attention, + (batch_size, self.n_vars, self.n_vars, self.n_patches), + ) + stage1 = ops.transpose(stage1, (0, 1, 3, 2)) + cross_variable = ops.sum(stage1, dim=2) + cross_variable = cross_variable * ops.cast( + self.cross_mask, cross_variable.dtype + ) + variable_importance = ops.sum(cross_variable, dim=1) + + stage2 = ops.reshape( + stage2_attention, + (batch_size, self.n_vars, self.n_patches, self.n_vars), + ) + temporal = ops.diagonal(stage2, dim1=1, dim2=3) + temporal = ops.transpose(temporal, (0, 2, 1)) + temporal_importance = ops.mean(temporal, dim=1) + recent_weight = ops.mean( + temporal_importance[:, -self.recent_count :], dim=1 + ) + _, top_variables = ops.topk( + variable_importance, self.top_k, dim=-1 + ) + _, top_patches = ops.topk( + temporal_importance, self.top_m, dim=-1 + ) + return top_variables, top_patches, recent_weight + + +class AttentionAsPrompt: + """Textualise attention features on the Python host.""" + + def __init__(self, variable_names, n_patches): + self.variable_names = list(variable_names) + self.n_patches = n_patches + + def __call__(self, top_variables, top_patches, recent_weight): + variable_indices = top_variables.asnumpy().tolist() + patch_indices = top_patches.asnumpy().tolist() + recent_values = recent_weight.asnumpy().astype(float).tolist() + prompts = [] + for batch_index, selected_variables in enumerate(variable_indices): + influential = [ + self.variable_names[index] for index in selected_variables + ] + periods = [] + for patch_index in patch_indices[batch_index]: + label = self._period_label(patch_index) + if label not in periods: + periods.append(label) + prompts.append( + "Attention-derived structure: top influential variables: " + f"{', '.join(influential)}; focus periods: " + f"{', '.join(periods)} (recent weight: " + f"{recent_values[batch_index]:.4f})." + ) + return prompts + + def _period_label(self, patch_index): + if self.n_patches == 1: + return "very-recent" + relative_position = patch_index / (self.n_patches - 1) + if relative_position >= 0.8: + return "very-recent" + if relative_position >= 0.6: + return "recent" + if relative_position >= 0.4: + return "mid-term" + if relative_position >= 0.2: + return "early" + return "initial" + + +class ReprogrammingLayer(nn.Cell): + """Map temporal tokens into the frozen LLM embedding space.""" + + def __init__( + self, d_model, n_heads, d_llm, d_keys=None, dropout=0.1 + ): + super().__init__() + self.d_keys = d_keys or d_model // n_heads + self.n_heads = n_heads + projected_dim = self.d_keys * n_heads + self.query_projection = nn.Dense(d_model, projected_dim) + self.key_projection = nn.Dense(d_llm, projected_dim) + self.value_projection = nn.Dense(d_llm, projected_dim) + self.output_projection = nn.Dense(projected_dim, d_llm) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, target, source, value): + """Cross-attend temporal queries to vocabulary prototypes.""" + batch_size, target_length, _ = target.shape + source_length = source.shape[0] + query = ops.reshape( + self.query_projection(target), + (batch_size, target_length, self.n_heads, self.d_keys), + ) + key = ops.reshape( + self.key_projection(source), + (source_length, self.n_heads, self.d_keys), + ) + value = ops.reshape( + self.value_projection(value), + (source_length, self.n_heads, self.d_keys), + ) + query = ops.transpose(query, (0, 2, 1, 3)) + key = ops.transpose(key, (1, 2, 0)) + attention = ops.softmax( + ops.matmul(query, key) / sqrt(self.d_keys), axis=-1 + ) + value = ops.transpose(value, (1, 0, 2)) + output = ops.matmul(self.dropout(attention), value) + output = ops.transpose(output, (0, 2, 1, 3)) + output = ops.reshape( + output, + (batch_size, target_length, self.n_heads * self.d_keys), + ) + return self.output_projection(output) + + +class FlattenHead(nn.Cell): + """Project LLM features from every variable to the forecast horizon.""" + + def __init__(self, n_patches, d_ff, target_window, dropout=0.0): + super().__init__() + self.input_size = n_patches * d_ff + self.projection = nn.Dense(self.input_size, target_window) + self.dropout = nn.Dropout(p=dropout) + + def construct(self, values): + """Flatten per-variable features and predict the horizon.""" + batch_size, n_vars, _, _ = values.shape + values = ops.reshape( + values, (batch_size, n_vars, self.input_size) + ) + return self.dropout(self.projection(values)) + + +class Normalize(nn.Cell): + """Reversible instance normalisation.""" + + def __init__(self, num_features, eps=1e-5, affine=False): + super().__init__() + self.eps = eps + self.affine = affine + if affine: + self.affine_weight = Parameter( + ops.ones((num_features,), mstype.float32), + name="affine_weight", + ) + self.affine_bias = Parameter( + ops.zeros((num_features,), mstype.float32), + name="affine_bias", + ) + + def construct(self, values): + """Normalise each sample along its temporal axis.""" + mean = ops.stop_gradient(ops.mean(values, axis=1, keep_dims=True)) + variance = ops.mean( + ops.square(values - mean), axis=1, keep_dims=True + ) + stdev = ops.stop_gradient(ops.sqrt(variance + self.eps)) + normalised = (values - mean) / stdev + if self.affine: + normalised = ( + normalised * self.affine_weight + self.affine_bias + ) + return normalised, mean, stdev + + def denormalize(self, values, mean, stdev): + """Restore the prediction to the original per-sample scale.""" + if self.affine: + values = (values - self.affine_bias) / ( + self.affine_weight + self.eps * self.eps + ) + return values * stdev + mean diff --git a/research/TSLM/PAT_LLM_MindSpore/src/metrics.py b/research/TSLM/PAT_LLM_MindSpore/src/metrics.py new file mode 100644 index 0000000..8c99f84 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/metrics.py @@ -0,0 +1,27 @@ +"""Forecasting metrics with stable handling around zero targets.""" + +import numpy as np + + +def calculate_metrics(prediction, target, eps=1e-8): + """Return MAE, MSE, RMSE, MAPE, MSPE, and R2.""" + prediction = np.asarray(prediction) + target = np.asarray(target) + error = prediction - target + mse = np.mean(np.square(error)) + valid = np.abs(target) > eps + denominator = np.sum(np.square(target - target.mean())) + return { + "MAE": float(np.mean(np.abs(error))), + "MSE": float(mse), + "RMSE": float(np.sqrt(mse)), + "MAPE": float(np.mean(np.abs(error[valid] / target[valid]))) + if valid.any() + else float("nan"), + "MSPE": float(np.mean(np.square(error[valid] / target[valid]))) + if valid.any() + else float("nan"), + "R2": float( + 1.0 - np.sum(np.square(error)) / max(denominator, eps) + ), + } diff --git a/research/TSLM/PAT_LLM_MindSpore/src/pat_llm.py b/research/TSLM/PAT_LLM_MindSpore/src/pat_llm.py new file mode 100644 index 0000000..4eca089 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/pat_llm.py @@ -0,0 +1,241 @@ +"""PAT-LLM network implemented with MindSpore and MindNLP.""" + +# PAT-LLM explicitly exposes its encoder, prompt, reprogramming, LLM, and +# output components. The forward pass also keeps named intermediate tensors +# to mirror the method described in the manuscript. +# pylint: disable=too-many-instance-attributes,too-many-locals + +import mindspore.common.dtype as mstype +from mindspore import Tensor, nn, ops +from mindnlp.transformers import AutoConfig, AutoModel, AutoTokenizer +import numpy as np + +from .layers import ( + AttentionAsPrompt, + AttentionPromptFeatures, + FlattenHead, + MultiPatchAttention, + Normalize, + PatchEmbedding, + ReprogrammingLayer, +) + + +class PATLLM(nn.Cell): + """Textualized Multi-Patch Attention forecasting network.""" + + def __init__(self, config): + super().__init__() + self.seq_len = config.seq_len + self.pred_len = config.pred_len + self.n_vars = config.enc_in + self.d_ff = config.d_ff + self.description = config.description + self.target_index = config.target_index + self.variable_names = list(config.variable_names) + + self.patch_embedding = PatchEmbedding( + config.seq_len, + config.d_model, + config.patch_len, + config.stride, + config.dropout, + ) + self.n_patches = self.patch_embedding.n_patches + self.multi_patch_attention = MultiPatchAttention( + config.d_model, + config.n_heads, + config.d_ff, + config.dropout, + ) + self.prompt_features = AttentionPromptFeatures( + self.n_vars, + self.n_patches, + config.aap_top_k, + config.aap_top_m, + config.aap_recent_fraction, + ) + self.attention_as_prompt = AttentionAsPrompt( + self.variable_names, self.n_patches + ) + self.normalizer = Normalize(self.n_vars, affine=False) + + llm_config = AutoConfig.from_pretrained( + config.llm_model_path, + local_files_only=not config.allow_download, + trust_remote_code=config.trust_remote_code, + ) + if hasattr(llm_config, "num_hidden_layers"): + llm_config.num_hidden_layers = config.llm_layers + elif hasattr(llm_config, "n_layer"): + llm_config.n_layer = config.llm_layers + llm_config.output_hidden_states = True + self.llm_model = AutoModel.from_pretrained( + config.llm_model_path, + config=llm_config, + ms_dtype=getattr(mstype, config.llm_dtype), + local_files_only=not config.allow_download, + trust_remote_code=config.trust_remote_code, + ) + self.tokenizer = AutoTokenizer.from_pretrained( + config.llm_model_path, + local_files_only=not config.allow_download, + trust_remote_code=config.trust_remote_code, + ) + if self.tokenizer.pad_token is None: + if self.tokenizer.eos_token is not None: + self.tokenizer.pad_token = self.tokenizer.eos_token + else: + self.tokenizer.add_special_tokens({"pad_token": "[PAD]"}) + self.llm_model.resize_token_embeddings(len(self.tokenizer)) + for parameter in self.llm_model.get_parameters(): + parameter.requires_grad = False + + embedding_layer = self.llm_model.get_input_embeddings() + word_embeddings = self._embedding_weight(embedding_layer) + self.vocab_size, self.d_llm = word_embeddings.shape + if self.d_ff > self.d_llm: + raise ValueError("d_ff cannot exceed the LLM hidden dimension") + self.mapping_layer = nn.Dense( + self.vocab_size, config.num_prototypes + ) + self.reprogramming_layer = ReprogrammingLayer( + config.d_model, + config.n_heads, + self.d_llm, + dropout=config.dropout, + ) + self.output_projection = FlattenHead( + self.n_patches, + self.d_ff, + self.pred_len, + config.dropout, + ) + + def set_train(self, mode=True): + """Keep the frozen LLM in evaluation mode during PAT-LLM training.""" + super().set_train(mode) + self.llm_model.set_train(False) + return self + + def construct(self, x_enc): + """Forecast a multivariate batch shaped [batch, time, variables].""" + x_enc, mean, stdev = self.normalizer(x_enc) + patches = self.patch_embedding(ops.transpose(x_enc, (0, 2, 1))) + refined, stage1_attention, stage2_attention = ( + self.multi_patch_attention(patches) + ) + top_variables, top_patches, recent_weight = self.prompt_features( + stage1_attention, stage2_attention + ) + structure_prompts = self.attention_as_prompt( + top_variables, top_patches, recent_weight + ) + prompt_embeddings, prompt_attention_mask = self._embed_prompts( + self._build_prompts(x_enc, structure_prompts) + ) + + embedding_layer = self.llm_model.get_input_embeddings() + word_embeddings = ops.cast( + self._embedding_weight(embedding_layer), mstype.float32 + ) + prototypes = self.mapping_layer( + ops.transpose(word_embeddings, (1, 0)) + ) + prototypes = ops.transpose(prototypes, (1, 0)) + batch_size = refined.shape[0] + temporal_tokens = ops.reshape( + refined, + (batch_size, self.n_vars * self.n_patches, -1), + ) + reprogrammed = self.reprogramming_layer( + temporal_tokens, prototypes, prototypes + ) + reprogrammed = ops.cast(reprogrammed, prompt_embeddings.dtype) + llm_input = ops.concat((prompt_embeddings, reprogrammed), axis=1) + temporal_attention_mask = ops.ones( + (batch_size, self.n_vars * self.n_patches), mstype.int32 + ) + attention_mask = ops.concat( + (prompt_attention_mask, temporal_attention_mask), axis=1 + ) + llm_output = self.llm_model( + inputs_embeds=llm_input, + attention_mask=attention_mask, + return_dict=True, + ) + hidden_state = ( + llm_output.last_hidden_state + if hasattr(llm_output, "last_hidden_state") + else llm_output[0] + ) + hidden_state = hidden_state[ + :, -self.n_vars * self.n_patches :, : self.d_ff + ] + hidden_state = ops.reshape( + hidden_state, + (batch_size, self.n_vars, self.n_patches, self.d_ff), + ) + prediction = self.output_projection(hidden_state) + prediction = ops.transpose(prediction, (0, 2, 1)) + return self.normalizer.denormalize(prediction, mean, stdev) + + def _build_prompts(self, x_enc, structure_prompts): + values = x_enc.asnumpy() + target = values[:, :, self.target_index] + prompts = [] + for batch_index, series in enumerate(target): + trend = "upward" if np.diff(series).sum() > 0 else "downward" + lags = self._top_lags(series, top_k=5) + prompts.append( + "<|start_prompt|>" + f"Dataset description: {self.description} " + f"Task description: forecast the next {self.pred_len} steps " + f"from the previous {self.seq_len} steps. " + f"Target statistics: minimum {series.min():.6g}, " + f"maximum {series.max():.6g}, " + f"median {np.median(series):.6g}, trend {trend}, " + f"top lags {lags}. {structure_prompts[batch_index]}" + "<|end_prompt|>" + ) + return prompts + + def _embed_prompts(self, prompts): + tokenized = self.tokenizer( + prompts, + padding=True, + truncation=True, + max_length=2048, + return_tensors="ms", + ) + input_ids = tokenized["input_ids"] + if not isinstance(input_ids, Tensor): + input_ids = Tensor(np.asarray(input_ids), mstype.int32) + attention_mask = tokenized.get("attention_mask") + if attention_mask is None: + attention_mask = ops.ones(input_ids.shape, mstype.int32) + elif not isinstance(attention_mask, Tensor): + attention_mask = Tensor( + np.asarray(attention_mask), mstype.int32 + ) + else: + attention_mask = ops.cast(attention_mask, mstype.int32) + embeddings = self.llm_model.get_input_embeddings()(input_ids) + return embeddings, attention_mask + + @staticmethod + def _embedding_weight(embedding_layer): + if hasattr(embedding_layer, "weight"): + return embedding_layer.weight + if hasattr(embedding_layer, "embedding_table"): + return embedding_layer.embedding_table + raise AttributeError("LLM input embedding has no accessible weight") + + @staticmethod + def _top_lags(series, top_k): + spectrum = np.fft.rfft(series) + correlation = np.fft.irfft( + spectrum * np.conj(spectrum), n=len(series) + ) + count = min(top_k, len(correlation)) + return np.argsort(correlation)[-count:][::-1].tolist() diff --git a/research/TSLM/PAT_LLM_MindSpore/src/time_features.py b/research/TSLM/PAT_LLM_MindSpore/src/time_features.py new file mode 100644 index 0000000..314c29c --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/src/time_features.py @@ -0,0 +1,64 @@ +"""Calendar features for time-series CSV input.""" + +import numpy as np +from pandas.tseries import offsets +from pandas.tseries.frequencies import to_offset + + +def _second(index): + return index.second / 59.0 - 0.5 + + +def _minute(index): + return index.minute / 59.0 - 0.5 + + +def _hour(index): + return index.hour / 23.0 - 0.5 + + +def _weekday(index): + return index.dayofweek / 6.0 - 0.5 + + +def _day(index): + return (index.day - 1) / 30.0 - 0.5 + + +def _day_of_year(index): + return (index.dayofyear - 1) / 365.0 - 0.5 + + +def _month(index): + return (index.month - 1) / 11.0 - 0.5 + + +def _week(index): + return (index.isocalendar().week.to_numpy() - 1) / 52.0 - 0.5 + + +def time_features(dates, freq="h"): + """Return normalised calendar features for a pandas DatetimeIndex.""" + feature_map = { + offsets.YearEnd: [], + offsets.QuarterEnd: [_month], + offsets.MonthEnd: [_month], + offsets.Week: [_day, _week], + offsets.Day: [_weekday, _day, _day_of_year], + offsets.BusinessDay: [_weekday, _day, _day_of_year], + offsets.Hour: [_hour, _weekday, _day, _day_of_year], + offsets.Minute: [_minute, _hour, _weekday, _day, _day_of_year], + offsets.Second: [ + _second, + _minute, + _hour, + _weekday, + _day, + _day_of_year, + ], + } + offset = to_offset(freq) + for offset_type, functions in feature_map.items(): + if isinstance(offset, offset_type): + return np.vstack([function(dates) for function in functions]) + raise ValueError(f"unsupported time frequency: {freq}") diff --git a/research/TSLM/PAT_LLM_MindSpore/train.py b/research/TSLM/PAT_LLM_MindSpore/train.py new file mode 100644 index 0000000..48cd832 --- /dev/null +++ b/research/TSLM/PAT_LLM_MindSpore/train.py @@ -0,0 +1,33 @@ +"""Train PAT-LLM with MindSpore.""" + +from src import PATLLM, create_dataset, load_config +from src.engine import ( + attach_dataset_metadata, + configure_runtime, + load_trainable_checkpoint, + predict, + save_evaluation, + train_model, +) + + +def main(): + """Run standalone training followed by test-set evaluation.""" + config = load_config("Train PAT-LLM with MindSpore") + configure_runtime(config) + train_source, train_dataset = create_dataset(config, "train") + _, validation_dataset = create_dataset(config, "val") + _, test_dataset = create_dataset(config, "test") + attach_dataset_metadata(config, train_source) + model = PATLLM(config) + best_epoch = train_model( + config, model, train_dataset, validation_dataset + ) + print(f"best validation checkpoint: epoch {best_epoch}") + load_trainable_checkpoint(model, config.checkpoint_path) + predictions, targets, histories = predict(config, model, test_dataset) + save_evaluation(config, predictions, targets, histories) + + +if __name__ == "__main__": + main() diff --git a/research/TSLM/README.md b/research/TSLM/README.md new file mode 100644 index 0000000..67cbd30 --- /dev/null +++ b/research/TSLM/README.md @@ -0,0 +1,17 @@ +# TSLM + +**中国人工智能学会** + +**昇思MindSpore学术基金项目“面向工业多模态数据场景的AI时序大模型”的代码仓库** + +TSLM 面向工业多模态数据场景，包含三个基于 MindSpore 实现的时序建模项目。 + +## 项目代码 + +| 目录 | 模型 | +|---|---| +| `DPMSAN_MindSpore/` | DPMSAN 的 MindSpore 实现 | +| `DualForecast_MindSpore/` | DualForecast 的 MindSpore 实现 | +| `PAT_LLM_MindSpore/` | PAT-LLM 的 MindSpore 实现 | + +各模型的环境配置、训练方法和使用说明请参阅对应目录中的 `README.md`。 -- Gitee