{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Quick start example\n", "\n", "This script shows a minimal pipeline to train a forest of PMQRTs to predict confidence intervals targeting a marginal coverage of $1-\\alpha$:\n", "\n", "- either by predicting the quantiles of order $\\alpha/2$ and $1-\\alpha/2$, i.e. confidence intervals are $$|\\hat q_{\\alpha/2}, \\hat q_{1-\\alpha/2}]$$\n", "\n", "- or using conformal prediction with CQR nested sets with a norminal quantile level equal to $2\\alpha$, meaning that the conformal prediction intervals are of the form:\n", "$$ [\\hat q_{2\\alpha}-t, \\hat q_{1-2\\alpha}+t]$$\n", "where $t$ is a real parameter ajdusted at the conformalization step. For more information on CQR, one can check this paper : https://arxiv.org/abs/1905.03222. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training trees...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 40/40 [00:11<00:00, 3.61it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Done training. 40 trees\n", "Conformalizing...\n", "Conformalization done.\n", "Predicting conformal sets...\n", "Average width: 1.8486468901945303\n", "Empirical coverage: 0.9133333333333333\n", "Example finished.\n" ] } ], "source": [ "import os\n", "import sys\n", "import numpy as np\n", "\n", "# Get the directory where the current notebook is located\n", "NOTEBOOK_DIR = os.getcwd()\n", "\n", "# Go up two levels (adjust the '..' count as needed)\n", "REPO_ROOT = os.path.abspath(os.path.join(NOTEBOOK_DIR, '../../../../'))\n", "\n", "# Add to sys.path if not already there\n", "if REPO_ROOT not in sys.path:\n", " sys.path.insert(0, REPO_ROOT)\n", "\n", "from DisTreebution.UQ.UQ import UQ\n", "\n", "# Create synthetic data\n", "n = 900\n", "alpha = 0.1\n", "X = np.random.randn(n, 3)\n", "y = X[:, 0]*2.0 + 0.5*np.random.randn(n)\n", "\n", "# Split train / calib / test\n", "idx = np.arange(n)\n", "np.random.shuffle(idx)\n", "train_idx = idx[:300]\n", "calib_idx = idx[300:600]\n", "test_idx = idx[600:]\n", "\n", "X_train, y_train = X[train_idx], y[train_idx]\n", "X_calib, y_calib = X[calib_idx], y[calib_idx]\n", "X_test, y_test = X[test_idx], y[test_idx]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 1. Getting confidence intervals by predicting empirical quantiles without conformalization" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training trees...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 40/40 [00:10<00:00, 3.99it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Done training. 40 trees\n", "Predicting conformal sets...\n", "Average width: 1.5073783052203367\n", "Empirical coverage: 0.8333333333333334\n", "Example finished.\n" ] } ], "source": [ "nTrees = 40\n", "treeID2quantiles = {treeID: [alpha/2, 1-alpha/2] for treeID in range(nTrees)}\n", "params = {'nTrees': nTrees, 'max_depth': 6, 'min_samples_split': 5, 'treeID2quantiles_train': treeID2quantiles}\n", "\n", "# Instantiate UQ: use simple PQRT + CQR conformalization for the demo\n", "uq = UQ(type_tree='PMQRT', type_conformal=None, params=params)\n", "\n", "# Train trees\n", "print('Training trees...')\n", "trees, sample2calib = uq.train_trees(X_train, y_train)\n", "print('Done training.', len(trees), 'trees')\n", "\n", "# Predict confidence interval on the test set\n", "print('Predicting conformal sets...')\n", "sample2predset = uq.get_quantile_estimate(trees, X_test, quantiles=[alpha/2, 1-alpha/2])\n", "\n", "# Evaluate widths and coverage\n", "widths, coverages = uq.compute_width_coverage(sample2predset, y_test)\n", "print('Average width:', float(np.mean(widths)))\n", "print('Empirical coverage:', float(np.mean(coverages)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# 2. Getting confidence intervals with conformalization" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Training trees...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 40/40 [00:11<00:00, 3.54it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Done training. 40 trees\n", "Conformalizing...\n", "Conformalization done.\n", "Predicting conformal sets...\n", "Average width: 1.8653316403691773\n", "Empirical coverage: 0.9166666666666666\n", "Example finished.\n" ] } ], "source": [ "nTrees = 40\n", "treeID2quantiles = {treeID: [2*alpha, 1-2*alpha] for treeID in range(nTrees)}\n", "params = {'nTrees': nTrees, 'max_depth': 6, 'min_samples_split': 5, 'treeID2quantiles_train': treeID2quantiles}\n", "\n", "# Instantiate UQ: use simple PQRT + CQR conformalization for the demo\n", "uq = UQ(type_tree='PMQRT', type_conformal=\"split\", nested_set='CQR', params=params)\n", "\n", "# Train trees\n", "print('Training trees...')\n", "trees, sample2calib = uq.train_trees(X_train, y_train)\n", "print('Done training.', len(trees), 'trees')\n", "\n", "# Conformalize on calibration set with alpha = 0.1\n", "print('Conformalizing...')\n", "uq.conformalize(trees, X_calib, y_calib, alpha, nominal_quantiles=[2*alpha])\n", "print('Conformalization done.')\n", "\n", "# Predict conformal sets on the test set\n", "print('Predicting conformal sets...')\n", "sample2predset = uq.predict_conformal_set(trees, X_test)\n", "\n", "# Evaluate widths and coverage\n", "widths, coverages = uq.compute_width_coverage(sample2predset[0], y_test)\n", "print('Average width:', float(np.mean(widths)))\n", "print('Empirical coverage:', float(np.mean(coverages)))" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 4 }