{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Reproducing results of the paper: Distributional regression\n", "===========================================================\n", "\n", "This notebook contains the code used to reproduce the results presented in our paper comparing different methods for distributional regression." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "np.random.seed(1)\n", "import os\n", "import sys\n", "import matplotlib.pyplot as plt\n", "# Get the directory where the current notebook is located\n", "NOTEBOOK_DIR = os.getcwd()\n", "\n", "# Go up two levels (adjust the '..' count as needed)\n", "REPO_ROOT = os.path.abspath(os.path.join(NOTEBOOK_DIR, '../../../../'))\n", "\n", "# Add to sys.path if not already there\n", "if REPO_ROOT not in sys.path:\n", " sys.path.insert(0, REPO_ROOT)\n", "from DisTreebution.UQ.UQ import UQ\n", "import pickle\n", "import random\n", "import argparse\n", "from DisTreebution.UQ.utils import compute_crps,load_dataset" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1. Computing the CRPS metric for the following different methods: CRPS-RF, PMQRF and SKLearn" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def process_dataset(data_path, name_dataset):\n", " for ite in range(100):\n", " ntrain = 1000\n", " if name_dataset=='red_wine':\n", " ntest = 1000\n", " ntest = 500\n", " \n", " np.random.seed(ite)\n", " random.seed(ite)\n", " X, y = load_dataset(data_path, name_dataset)\n", " N = X.shape[0]\n", " N = min(N,ntrain+3000)\n", " idxs = np.arange(len(y))\n", " np.random.shuffle(idxs)\n", " print(f'Processing {name_dataset}')\n", " x_train = X[idxs[:N][:ntrain], :]\n", " y_train = y[idxs[:N][:ntrain]]\n", " x_test = X[idxs[:N][(ntrain):], :]\n", " y_test = y[idxs[:N][(ntrain):]]\n", " \n", " nTrees = 100\n", " alpha = 0.1\n", " ls_q = [0.05 * i for i in range(1, 20)]\n", " results = []\n", "\n", "\n", " # Baseline SKLearn Quantile Regressor\n", " from sklearn.linear_model import QuantileRegressor\n", " marginal_level = np.zeros(len(ls_q))\n", " sample2quantiles = np.zeros((len(y_test), len(ls_q)))\n", " for i_q, q in enumerate(ls_q):\n", " model = QuantileRegressor(quantile=q, alpha=0)\n", " model.fit(x_train, y_train)\n", " predictions = model.predict(x_test)\n", " sample2quantiles[:,i_q] = predictions\n", " \n", " \n", " crps_value = compute_crps(sample2quantiles, y_test, ls_q)\n", " results.append({\n", " 'dataset': name_dataset,\n", " 'type_tree': 'SKLearn',\n", " 'type_conformal': None,\n", " 'type_aggregation_trees': None,\n", " 'nested_set': None,\n", " 'groupconf': None,\n", " 'IG_biais_correction': None,\n", " 'split': None,\n", " 'crps_value': crps_value,\n", " 'ntrain': ntrain,\n", " 'alpha':alpha\n", " })\n", " \n", " \n", " model_configs = []\n", " for type_tree in ['CRPS','PMQRT']:\n", " for type_aggregation_trees in ['vr']:\n", " for split in [False, True]:\n", " model_configs.append(\n", " {\n", " 'type_tree': type_tree,\n", " 'type_conformal': None,\n", " 'type_aggregation_trees': type_aggregation_trees,\n", " 'nested_set': None,\n", " 'groupconf': False,\n", " \"IG_biais_correction\": \"LOO\",\n", " \"split\":split\n", " }\n", " )\n", " nested_set = None\n", " groupconf = False\n", " for config in model_configs:\n", " type_tree, type_conformal = config['type_tree'], config['type_conformal']\n", " type_aggregation_trees = config['type_aggregation_trees']\n", " IG_biais_correction = config[\"IG_biais_correction\"]\n", " split = config[\"split\"]\n", " if split and type_tree!=\"PMQRT\":\n", " pass\n", " else:\n", " print(type_tree, type_conformal, nested_set, \"IG_biais_correction\", IG_biais_correction)\n", " \n", " params = {'nTrees': nTrees, 'max_depth': 10, 'min_samples_split': 10, \n", " 'IG_biais_correction':IG_biais_correction, 'list_distri_low_quantiles': [0.01 * i for i in range(1, 20)]}\n", " \n", " treeID2quantiles_train = None\n", " if type_tree == \"PMQRT\":\n", " if not (split):\n", " treeID2quantiles_train = {ID:[0.05 * i for i in range(1, 20)] for ID in range(nTrees)}\n", " else:\n", " treeID2quantiles_train = {ID: [0.05 * i for i in range(1, 10)] for ID in range(nTrees//2)}\n", " treeID2quantiles_train.update({ID: [0.05 * i for i in range(10,20)] for ID in range(nTrees//2,nTrees)})\n", " params.update({'treeID2quantiles_train': treeID2quantiles_train})\n", " model = UQ(type_tree=type_tree, nested_set=nested_set, type_conformal=type_conformal, group_coverage=groupconf, type_aggregation_trees=type_aggregation_trees, params=params)\n", " \n", " print('Training') \n", " trees, sample2calib_trees = model.train_trees(x_train, y_train)\n", " \n", " print('Inference') \n", " sample2quantiles = model.get_quantile_estimate(trees, x_test, quantiles=ls_q)\n", " crps_value = compute_crps(sample2quantiles, y_test, ls_q)\n", " \n", " results.append({\n", " 'dataset': name_dataset,\n", " 'type_tree': type_tree,\n", " 'type_conformal': type_conformal,\n", " 'type_aggregation_trees': type_aggregation_trees,\n", " 'nested_set': nested_set,\n", " 'groupconf': groupconf,\n", " 'IG_biais_correction': IG_biais_correction,\n", " 'split': split,\n", " 'crps_value': crps_value,\n", " 'ntrain': ntrain,\n", " 'alpha':alpha\n", " })\n", " \n", " # Save all\n", " import pandas as pd\n", " results_df = pd.DataFrame(results)\n", " #results_df.to_csv(os.path.join(save_path, f'results_final_crps_{name_dataset}_ite_{ite}.csv'), index=False)\n", " print(f\"Saved {len(results)} results to results_{name_dataset}_{ite}.csv\")\n", "\n", "\n", "\n", "if __name__ == '__main__':\n", " parser = argparse.ArgumentParser(description=\"Configure regression tree and conformalization options.\") \n", " parser.add_argument(\"--datasetID\", type=int, default=None,\n", " help=\"datasetID\")\n", " parser.add_argument(\"--data_path\", type=int, default=None,\n", " help=\"data path\")\n", " args = parser.parse_args()\n", " datasets = [\"abalone\", \"gpu\", \"gas_turbine\", \"combined_cycle_power_plant\", \"red_wine\", \"white_wine\"]\n", " name_dataset = datasets[args.datasetID]\n", " data_path = args.data_path\n", " process_dataset(data_path, name_dataset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2. Computing the CRPS metric for QRF" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def process_dataset(name_dataset):\n", " for ite in range(100):\n", " ntrain = 1000\n", " if name_dataset=='red_wine':\n", " ntest = 1000\n", " ntest = 500\n", " \n", " np.random.seed(ite)\n", " random.seed(ite)\n", " X, y = load_dataset(name_dataset)\n", " N = X.shape[0]\n", " N = min(N,ntrain+3000)\n", " idxs = np.arange(len(y))\n", " np.random.shuffle(idxs)\n", " print(f'Processing {name_dataset}')\n", " x_train = X[idxs[:N][:ntrain], :]\n", " y_train = y[idxs[:N][:ntrain]]\n", " x_test = X[idxs[:N][(ntrain):], :]\n", " y_test = y[idxs[:N][(ntrain):]]\n", " \n", " nTrees = 100\n", " alpha = 0.1\n", " ls_q = [0.05 * i for i in range(1, 20)]\n", " results = []\n", "\n", "\n", " from sklearn.linear_model import QuantileRegressor\n", " marginal_level = np.zeros(len(ls_q))\n", " sample2quantiles = np.zeros((len(y_test), len(ls_q)))\n", " for i_q, q in enumerate(ls_q):\n", " model = QuantileRegressor(quantile=q, alpha=0)\n", " model.fit(x_train, y_train)\n", " predictions = model.predict(x_test)\n", " sample2quantiles[:,i_q] = predictions\n", " \n", " \n", " crps_value = compute_crps(sample2quantiles, y_test, ls_q)\n", " results.append({\n", " 'dataset': name_dataset,\n", " 'type_tree': 'SKLearn',\n", " 'type_conformal': None,\n", " 'type_aggregation_trees': None,\n", " 'nested_set': None,\n", " 'groupconf': None,\n", " 'IG_biais_correction': None,\n", " 'split': None,\n", " 'crps_value': crps_value,\n", " 'ntrain': ntrain,\n", " 'alpha':alpha\n", " })\n", " \n", " \n", " model_configs = []\n", " for type_tree in ['RT']:\n", " for type_aggregation_trees in ['vr','vr-avg']:\n", " model_configs.append(\n", " {\n", " 'type_tree': type_tree,\n", " 'type_conformal': None,\n", " 'type_aggregation_trees': type_aggregation_trees,\n", " 'nested_set': None,\n", " 'groupconf': False,\n", " \"IG_biais_correction\": None,\n", " \"split\":False\n", " }\n", " )\n", " nested_set = None\n", " groupconf = False\n", " for config in model_configs:\n", " type_tree, type_conformal = config['type_tree'], config['type_conformal']\n", " type_aggregation_trees = config['type_aggregation_trees']\n", " IG_biais_correction = config[\"IG_biais_correction\"]\n", " split = config[\"split\"]\n", " print(type_tree, type_conformal, nested_set, \"IG_biais_correction\", IG_biais_correction)\n", " \n", " params = {'nTrees': nTrees, 'max_depth': 10, 'min_samples_split': 10, \n", " 'IG_biais_correction':IG_biais_correction, 'list_distri_low_quantiles': [0.01 * i for i in range(1, 20)]}\n", " \n", " model = UQ(type_tree=type_tree, nested_set=nested_set, type_conformal=type_conformal, group_coverage=groupconf, type_aggregation_trees=type_aggregation_trees, params=params)\n", " \n", " print('Training') \n", " trees, sample2calib_trees = model.train_trees(x_train, y_train)\n", " \n", " print('Inference') \n", " sample2quantiles = model.get_quantile_estimate(trees, x_test, quantiles=ls_q)\n", " crps_value = compute_crps(sample2quantiles, y_test, ls_q)\n", " \n", " results.append({\n", " 'dataset': name_dataset,\n", " 'type_tree': type_tree,\n", " 'type_conformal': type_conformal,\n", " 'type_aggregation_trees': type_aggregation_trees,\n", " 'nested_set': nested_set,\n", " 'groupconf': groupconf,\n", " 'IG_biais_correction': IG_biais_correction,\n", " 'split': split,\n", " 'crps_value': crps_value,\n", " 'ntrain': ntrain,\n", " 'alpha':alpha\n", " })\n", " \n", " # Save all\n", " import pandas as pd\n", " results_df = pd.DataFrame(results)\n", " results_df.to_csv(os.path.join(save_path, f'results_final_qrf_{name_dataset}_ite_{ite}.csv'), index=False)\n", " print(f\"Saved {len(results)} results to results_{name_dataset}_{ite}.csv\")\n", "\n", "\n", "\n", "if __name__ == '__main__':\n", " parser = argparse.ArgumentParser(description=\"Configure regression tree and conformalization options.\") \n", " parser.add_argument(\"--datasetID\", type=int, default=None,\n", " help=\"datasetID\")\n", " parser.add_argument(\"--data_path\", type=int, default=None,\n", " help=\"data path\")\n", " args = parser.parse_args()\n", " datasets = [\"abalone\", \"gpu\", \"gas_turbine\", \"combined_cycle_power_plant\", \"red_wine\", \"white_wine\"]\n", " name_dataset = datasets[args.datasetID]\n", " data_path = args.data_path\n", " process_dataset(data_path, name_dataset)\n", "\n", "\n", " \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 4 }