{
"cells": [
{
"cell_type": "markdown",
"id": "6c4d4faf-ab84-4a72-a80e-535b211747cd",
"metadata": {
"tags": []
},
"source": [
"# Morris Sensitivity Analysis Explainer Demo"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "69f414e3-bc88-478b-bed5-890352b1041a",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import logging\n",
"\n",
"import datatable\n",
"import daimojo\n",
"\n",
"from h2o_sonar import interpret\n",
"from h2o_sonar.lib.api import commons\n",
"from h2o_sonar.lib.api import explainers\n",
"from h2o_sonar.explainers import morris_sa_explainer as explainer\n",
"from h2o_sonar.lib.api.models import ModelApi"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "bbe0ca51",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'id': 'h2o_sonar.explainers.morris_sa_explainer.MorrisSensitivityAnalysisExplainer',\n",
" 'name': 'MorrisSensitivityAnalysisExplainer',\n",
" 'display_name': 'Morris Sensitivity Analysis',\n",
" 'tagline': 'MorrisSensitivityAnalysisExplainer.',\n",
" 'description': 'Morris sensitivity analysis (SA) explainer provides Morris sensitivity analysis based feature importance which is a measure of the contribution of an input variable to the overall predictions of the model. In applied statistics, the Morris method for global sensitivity analysis is a so-called one-step-at-a-time method (OAT), meaning that in each run only one input parameter is given a new value. This explainer is based based on InterpretML library - see http://interpret.ml',\n",
" 'brief_description': 'MorrisSensitivityAnalysisExplainer.',\n",
" 'model_types': ['iid'],\n",
" 'can_explain': ['regression', 'binomial'],\n",
" 'explanation_scopes': ['global_scope'],\n",
" 'explanations': [{'explanation_type': 'global-feature-importance',\n",
" 'name': 'GlobalFeatImpExplanation',\n",
" 'category': '',\n",
" 'scope': 'global',\n",
" 'has_local': '',\n",
" 'formats': []}],\n",
" 'keywords': ['explains-original-feature-importance', 'h2o-sonar'],\n",
" 'parameters': [{'name': 'leakage_warning_threshold',\n",
" 'description': 'The threshold above which to report a potentially detectedfeature importance leak problem.',\n",
" 'comment': '',\n",
" 'type': 'float',\n",
" 'val': 0.95,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''}],\n",
" 'metrics_meta': []}"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# explainer description\n",
"interpret.describe_explainer(explainer.MorrisSensitivityAnalysisExplainer)"
]
},
{
"cell_type": "markdown",
"id": "90d401d2-14cd-4686-982f-3cac9e9f5eb7",
"metadata": {
"tags": []
},
"source": [
"## Interpretation"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "15201d08-873b-45c3-82ad-052266f0526c",
"metadata": {},
"outputs": [],
"source": [
"# dataset\n",
"dataset_path = \"../../data/predictive/pd_ice_creditcard_train.csv\"\n",
"target_col = \"LIMIT_BAL\"\n",
"\n",
"# model\n",
"mojo_path = \"../../data/predictive/models/creditcard-regression.mojo\"\n",
"mojo_model = daimojo.model(mojo_path)\n",
"model = ModelApi().create_model(\n",
" model_src=mojo_model,\n",
" target_col=target_col,\n",
" used_features=list(mojo_model.feature_names),\n",
")\n",
"\n",
"# results\n",
"results_location = \"./results\"\n",
"os.makedirs(results_location, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "kfcalnxpg0g",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✓ Morris Sensitivity Analysis explainer is compatible with this model/dataset\n"
]
}
],
"source": [
"# check explainer compatibility before running interpretation\n",
"morris_explainer = explainer.MorrisSensitivityAnalysisExplainer()\n",
"morris_explainer.logger = logging.getLogger(\"morris_compatibility_check\")\n",
"\n",
"params = commons.CommonInterpretationParams(\n",
" model=model,\n",
" models=[],\n",
" dataset=dataset_path,\n",
" target_col=target_col,\n",
")\n",
"\n",
"is_compatible = morris_explainer.check_compatibility(params=params, model=model)\n",
"\n",
"if not is_compatible:\n",
" error_msg = \"Morris Sensitivity Analysis explainer is NOT compatible with this model/dataset.\\n\\n\"\n",
" error_msg += \"Possible reasons:\\n\"\n",
" error_msg += \"1. Missing dependencies: interpret==0.1.20 or gevent==1.5.0\\n\"\n",
" error_msg += \" Install with: pip install interpret==0.1.20 gevent==1.5.0\\n\"\n",
" error_msg += \"2. Model type not supported (H2O-3 models are not compatible)\\n\"\n",
" error_msg += \"3. Model must be regression or binary classification\\n\\n\"\n",
" error_msg += \"Cannot continue with interpretation. Please resolve compatibility issues first.\"\n",
" raise RuntimeError(error_msg)\n",
"else:\n",
" print(\"✓ Morris Sensitivity Analysis explainer is compatible with this model/dataset\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "0ba8f0aa-2e0e-4a0a-93ab-77ce9e968fa0",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"%%capture\n",
"interpretation = interpret.run_interpretation(\n",
" dataset=dataset_path,\n",
" model=model,\n",
" target_col=target_col,\n",
" results_location=results_location,\n",
" explainers=[explainer.MorrisSensitivityAnalysisExplainer.explainer_id()],\n",
" log_level=logging.INFO,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "ff9df4be-d4da-44db-a479-7d8d7f45c29d",
"metadata": {
"tags": []
},
"source": [
"## Explainer Result"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "25556ca5-8239-4201-8a23-1ace2b3a46d4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# retrieve the result\n",
"result = interpretation.get_explainer_result(\n",
" explainer.MorrisSensitivityAnalysisExplainer.explainer_id()\n",
")"
]
},
{
"cell_type": "markdown",
"id": "490d132b-b7e2-48a2-8ec4-dbd71886edf9",
"metadata": {
"tags": []
},
"source": [
"### Display Data"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "2aa6274e-79d5-49b1-b29a-2263db5cb8a8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" | feature | importance |
|---|
\n",
" | ▪▪▪▪ | ▪▪▪▪▪▪▪▪ |
\n",
" \n",
" \n",
" | 0 | PAY_AMT2 | 576582 |
\n",
" | 1 | BILL_AMT1 | 483290 |
\n",
" | 2 | PAY_AMT6 | 315424 |
\n",
" | 3 | PAY_AMT1 | 262665 |
\n",
" | 4 | PAY_AMT3 | 250634 |
\n",
" | 5 | PAY_AMT5 | 229284 |
\n",
" | 6 | BILL_AMT3 | 227349 |
\n",
" | 7 | PAY_AMT4 | 226623 |
\n",
" | 8 | BILL_AMT4 | 216030 |
\n",
" | 9 | BILL_AMT5 | 144456 |
\n",
" | 10 | BILL_AMT6 | 140663 |
\n",
" | 11 | AGE | 85044.2 |
\n",
" | 12 | PAY_3 | 70326.1 |
\n",
" | 13 | PAY_5 | 50648.5 |
\n",
" | 14 | PAY_2 | 36126.8 |
\n",
" | 15 | EDUCATION | 28226.3 |
\n",
" | 16 | PAY_1 | 16431.9 |
\n",
" | 17 | MARRIAGE | 15073.3 |
\n",
" | 18 | PAY_4 | 14440.6 |
\n",
" | 19 | PAY_6 | 13579.2 |
\n",
" | 20 | SEX | 5472.43 |
\n",
" | 21 | BILL_AMT2 | 4649.54 |
\n",
" \n",
"
\n",
" \n",
"