Upload 3 files

pipeline/evaluate_pipeline.py

@@ -0,0 +1,86 @@
+import sys
+import os
+import json
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import torch
+import torch.nn as nn
+from torch.utils.data import TensorDataset, DataLoader
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
+from models.forecasting.lstm import LSTMForecaster
+
+
+def load_test_data(csv_path="../data/processed/merged_features.csv", seq_len=30, test_ratio=0.2):
+    df = pd.read_csv(csv_path)
+    df = df.select_dtypes(include=[np.number]).dropna()
+    data = df.values
+
+    split = int(len(data) * (1 - test_ratio))
+    test_data = data[split:]
+
+    X_test, y_test = [], []
+    for i in range(len(test_data) - seq_len - 1):
+        X_test.append(test_data[i:i+seq_len])
+        y_test.append(test_data[i+seq_len][0])  
+
+    X_test = torch.tensor(np.array(X_test), dtype=torch.float32)
+    y_test = torch.tensor(np.array(y_test), dtype=torch.float32).unsqueeze(1)
+
+    return DataLoader(TensorDataset(X_test, y_test), batch_size=32), X_test.shape[2]
+
+
+def evaluate_model(test_loader, model_path="./trained_models/lstm_forecaster.pt", config_path="./trained_models/config.json"):
+    with open(config_path, "r") as f:
+        config = json.load(f)
+
+    model = LSTMForecaster(
+        input_size=config["input_size"],
+        hidden_size=config["hidden_size"],
+        num_layers=config["num_layers"],
+        output_size=config["output_size"]
+    )
+
+    model.load_state_dict(torch.load(model_path))
+    model.eval()
+
+    preds, targets = [], []
+
+    with torch.no_grad():
+        for X, y in test_loader:
+            out = model(X)
+            preds.append(out.numpy())
+            targets.append(y.numpy())
+
+    preds = np.concatenate(preds)
+    targets = np.concatenate(targets)
+
+    mse = mean_squared_error(targets, preds)
+    mae = mean_absolute_error(targets, preds)
+    r2 = r2_score(targets, preds)
+
+    print("\n📊 Evaluation Metrics:")
+    print(f"➡️ MSE: {mse:.4f}")
+    print(f"➡️ MAE: {mae:.4f}")
+    print(f"➡️ R2 Score: {r2:.4f}")
+
+    os.makedirs("outputs", exist_ok=True)
+    plt.figure(figsize=(12, 5))
+    plt.plot(targets, label='Actual', color='blue')
+    plt.plot(preds, label='Predicted', color='orange')
+    plt.title("📈 LSTM Forecast vs Actual")
+    plt.xlabel("Time Step")
+    plt.ylabel("Value")
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+    plt.savefig("outputs/evaluation_plot.png")
+    plt.show()
+
+
+if __name__ == "__main__":
+    test_loader, input_size = load_test_data()
+    print(f"🧪 Detected input feature size: {input_size}")
+    evaluate_model(test_loader)

pipeline/inference_pipeline.py

@@ -0,0 +1,71 @@
+import os
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import json
+import torch
+import numpy as np
+import pandas as pd
+import argparse
+from models.forecasting.lstm import LSTMForecaster
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+def load_model(model_path="pipeline/trained_models/lstm_forecaster.pt", config_path="pipeline/trained_models/config.json"):
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"❌ Missing config file: {config_path}")
+    if not os.path.exists(model_path):
+        raise FileNotFoundError(f"❌ Missing model file: {model_path}")
+
+    with open(config_path, "r") as f:
+        config = json.load(f)
+
+    model = LSTMForecaster(
+        input_size=config["input_size"],
+        hidden_size=config["hidden_size"],
+        num_layers=config["num_layers"],
+        output_size=config["output_size"]
+    ).to(device)
+
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.eval()
+    return model
+
+def predict_batch(input_data, model=None):
+    """
+    input_data: numpy array of shape [batch_size, seq_len, input_size]
+    returns: list of predictions
+    """
+    if model is None:
+        model = load_model()
+
+    if isinstance(input_data, list):
+        input_data = np.array(input_data)
+
+    input_tensor = torch.tensor(input_data, dtype=torch.float32).to(device)
+
+    with torch.no_grad():
+        outputs = model(input_tensor)
+
+    return outputs.cpu().numpy().flatten().tolist()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Batch Inference for LSTM Forecasting")
+    parser.add_argument("--input", type=str, required=True, help="Path to .npy or .csv input file")
+    args = parser.parse_args()
+
+    if args.input.endswith(".npy"):
+        input_data = np.load(args.input)
+    elif args.input.endswith(".csv"):
+        df = pd.read_csv(args.input).dropna().select_dtypes(include=[np.number])
+        data = df.values
+        if len(data.shape) == 2:
+            input_data = np.expand_dims(data, axis=0) 
+        else:
+            input_data = data
+    else:
+        raise ValueError("Input must be a .npy or .csv file")
+
+    model = load_model()
+    predictions = predict_batch(input_data, model)
+    print("📈 Predictions:", predictions)

pipeline/train_pipeline.py

@@ -0,0 +1,70 @@
+import sys
+import os
+import json
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import torch
+import torch.nn as nn
+import pandas as pd
+import numpy as np
+from torch.utils.data import TensorDataset, DataLoader
+from sklearn.model_selection import train_test_split
+from models.forecasting.lstm import LSTMForecaster
+
+df = pd.read_csv("../data/processed/merged_features.csv")
+df = df.select_dtypes(include=[np.number]).dropna()
+data = df.values
+
+seq_len = 30
+X, y = [], []
+for i in range(len(data) - seq_len - 1):
+    X.append(data[i:i+seq_len])
+    y.append(data[i+seq_len][0])  
+
+X = torch.tensor(np.array(X), dtype=torch.float32)
+y = torch.tensor(np.array(y), dtype=torch.float32).unsqueeze(1)
+
+X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)
+train_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=32, shuffle=True)
+val_loader = DataLoader(TensorDataset(X_val, y_val), batch_size=32)
+
+input_size = X.shape[2]
+hidden_size = 256
+num_layers = 2
+output_size = 1
+
+model = LSTMForecaster(
+    input_size=input_size,
+    hidden_size=hidden_size,
+    num_layers=num_layers,
+    output_size=output_size
+)
+
+criterion = nn.MSELoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
+
+for epoch in range(10):
+    model.train()
+    total_loss = 0
+    for xb, yb in train_loader:
+        optimizer.zero_grad()
+        loss = criterion(model(xb), yb)
+        loss.backward()
+        optimizer.step()
+        total_loss += loss.item()
+    avg_loss = total_loss / len(train_loader)
+    print(f"Epoch {epoch+1}: Train Loss = {avg_loss:.4f}")
+
+os.makedirs("trained_models", exist_ok=True)
+torch.save(model.state_dict(), "trained_models/lstm_forecaster.pt")
+
+config = {
+    "input_size": input_size,
+    "hidden_size": hidden_size,
+    "num_layers": num_layers,
+    "output_size": output_size
+}
+with open("trained_models/config.json", "w") as f:
+    json.dump(config, f)
+
+print("✅ Model trained and saved.")