{
"cells": [
{
"cell_type": "markdown",
"id": "6c4d4faf-ab84-4a72-a80e-535b211747cd",
"metadata": {
"tags": []
},
"source": [
"# Shapley Values for Transformed Features of MOJO Models Demo\n",
"\n",
"This example demonstrates how to interpret a Driverless AI MOJO model using the H2O Sonar library and retrieve the data and plot with transformed features importances."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "69f414e3-bc88-478b-bed5-890352b1041a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import os\n",
"import logging\n",
"\n",
"import datatable\n",
"import daimojo\n",
"import webbrowser\n",
"\n",
"from h2o_sonar import interpret\n",
"from h2o_sonar.lib.api import commons\n",
"from h2o_sonar.lib.api import explainers\n",
"from h2o_sonar.explainers import transformed_fi_shapley_explainer as explainer\n",
"from h2o_sonar.lib.api.models import ModelApi"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "bbe0ca51",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'id': 'h2o_sonar.explainers.transformed_fi_shapley_explainer.ShapleyMojoTransformedFeatureImportanceExplainer',\n",
" 'name': 'ShapleyMojoTransformedFeatureImportanceExplainer',\n",
" 'display_name': 'Shapley Values for Transformed Features of MOJO Models',\n",
" 'tagline': 'ShapleyMojoTransformedFeatureImportanceExplainer.',\n",
" 'description': 'Shapley explanations are a technique with credible theoretical support that presents consistent global and local variable contributions. Local numeric Shapley values are calculated by tracing single rows of data through a trained tree ensemble and aggregating the contribution of each input variable as the row of data moves through the trained ensemble. For regression tasks Shapley values sum to the prediction of the (Driverless AI) MOJO model. For classification problems, Shapley values sum to the prediction of the MOJO model before applying the link function. Global Shapley values are the average of the absolute local Shapley values over every row of a data set.',\n",
" 'brief_description': 'ShapleyMojoTransformedFeatureImportanceExplainer.',\n",
" 'model_types': ['iid', 'time_series'],\n",
" 'can_explain': ['regression', 'binomial', 'multinomial'],\n",
" 'explanation_scopes': ['global_scope', 'local_scope'],\n",
" 'explanations': [{'explanation_type': 'global-feature-importance',\n",
" 'name': 'GlobalFeatImpExplanation',\n",
" 'category': '',\n",
" 'scope': 'global',\n",
" 'has_local': '',\n",
" 'formats': []},\n",
" {'explanation_type': 'local-feature-importance',\n",
" 'name': 'LocalFeatImpExplanation',\n",
" 'category': '',\n",
" 'scope': 'local',\n",
" 'has_local': '',\n",
" 'formats': []}],\n",
" 'keywords': ['run-by-default',\n",
" 'explains-transformed-feature-importance',\n",
" 'is_fast',\n",
" 'h2o-sonar'],\n",
" 'parameters': [{'name': 'sample_size',\n",
" 'description': 'Sample size.',\n",
" 'comment': '',\n",
" 'type': 'int',\n",
" 'val': 100000,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''},\n",
" {'name': 'calculate_predictions',\n",
" 'description': 'Score dataset and include predictions in the explanation (local explanations speed-up cache).',\n",
" 'comment': '',\n",
" 'type': 'bool',\n",
" 'val': False,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''},\n",
" {'name': 'fast_approx_contrib',\n",
" 'description': 'Speed up predictions with fast contributions predictions approximation.',\n",
" 'comment': '',\n",
" 'type': 'bool',\n",
" 'val': True,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''}],\n",
" 'metrics_meta': []}"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# explainer description\n",
"interpret.describe_explainer(explainer.ShapleyMojoTransformedFeatureImportanceExplainer)"
]
},
{
"cell_type": "markdown",
"id": "90d401d2-14cd-4686-982f-3cac9e9f5eb7",
"metadata": {
"tags": []
},
"source": [
"## Interpretation"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "15201d08-873b-45c3-82ad-052266f0526c",
"metadata": {},
"outputs": [],
"source": [
"# dataset\n",
"dataset_path = \"../../data/predictive/creditcard.csv\"\n",
"target_col = \"default payment next month\"\n",
"\n",
"# model\n",
"mojo_path = \"../../data/predictive/models/creditcard-binomial.mojo\"\n",
"mojo_model = daimojo.model(mojo_path)\n",
"model = ModelApi().create_model(\n",
" model_src=mojo_model,\n",
" target_col=target_col,\n",
" used_features=list(mojo_model.feature_names),\n",
")\n",
"\n",
"# results\n",
"results_location = \"./results\"\n",
"os.makedirs(results_location, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0ba8f0aa-2e0e-4a0a-93ab-77ce9e968fa0",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/user/h/mli/git/h2o-sonar-FLOSS/.venv/lib/python3.11/site-packages/ragas/metrics/__init__.py:1: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain_core.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n",
"\n",
"For example, replace imports like: `from langchain_core.pydantic_v1 import BaseModel`\n",
"with: `from pydantic import BaseModel`\n",
"or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n",
"\n",
" from ragas.metrics._answer_correctness import AnswerCorrectness, answer_correctness\n",
"/home/user/h/mli/git/h2o-sonar-FLOSS/.venv/lib/python3.11/site-packages/ragas/metrics/__init__.py:4: LangChainDeprecationWarning: As of langchain-core 0.3.0, LangChain uses pydantic v2 internally. The langchain.pydantic_v1 module was a compatibility shim for pydantic v1, and should no longer be used. Please update the code to import from Pydantic directly.\n",
"\n",
"For example, replace imports like: `from langchain.pydantic_v1 import BaseModel`\n",
"with: `from pydantic import BaseModel`\n",
"or the v1 compatibility namespace if you are working in a code base that has not been fully upgraded to pydantic 2 yet. \tfrom pydantic.v1 import BaseModel\n",
"\n",
" from ragas.metrics._context_entities_recall import (\n"
]
}
],
"source": [
"interpretation = interpret.run_interpretation(\n",
" dataset=dataset_path,\n",
" model=model,\n",
" target_col=target_col,\n",
" results_location=results_location,\n",
" explainers=[explainer.ShapleyMojoTransformedFeatureImportanceExplainer.explainer_id()],\n",
" log_level=logging.INFO,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "ff9df4be-d4da-44db-a479-7d8d7f45c29d",
"metadata": {
"tags": []
},
"source": [
"## Explainer Result"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "25556ca5-8239-4201-8a23-1ace2b3a46d4",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# retrieve the result\n",
"result = interpretation.get_explainer_result(\n",
" explainer.ShapleyMojoTransformedFeatureImportanceExplainer.explainer_id()\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "49cb45cc-1482-4535-9663-c8d316775223",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# open interpretation HTML report in web browser\n",
"webbrowser.open(interpretation.result.get_html_report_location())"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "375b9bd6-e424-4f8e-a73f-aa7fda7f7606",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'id': 'h2o_sonar.explainers.transformed_fi_shapley_explainer.ShapleyMojoTransformedFeatureImportanceExplainer',\n",
" 'name': 'ShapleyMojoTransformedFeatureImportanceExplainer',\n",
" 'display_name': 'Shapley Values for Transformed Features of MOJO Models',\n",
" 'tagline': 'ShapleyMojoTransformedFeatureImportanceExplainer.',\n",
" 'description': 'Shapley explanations are a technique with credible theoretical support that presents consistent global and local variable contributions. Local numeric Shapley values are calculated by tracing single rows of data through a trained tree ensemble and aggregating the contribution of each input variable as the row of data moves through the trained ensemble. For regression tasks Shapley values sum to the prediction of the (Driverless AI) MOJO model. For classification problems, Shapley values sum to the prediction of the MOJO model before applying the link function. Global Shapley values are the average of the absolute local Shapley values over every row of a data set.',\n",
" 'brief_description': 'ShapleyMojoTransformedFeatureImportanceExplainer.',\n",
" 'model_types': ['iid', 'time_series'],\n",
" 'can_explain': ['regression', 'binomial', 'multinomial'],\n",
" 'explanation_scopes': ['global_scope', 'local_scope'],\n",
" 'explanations': [{'explanation_type': 'global-feature-importance',\n",
" 'name': 'Shapley Values for Transformed Features of MOJO Models',\n",
" 'category': 'DAI MODEL',\n",
" 'scope': 'global',\n",
" 'has_local': 'local-feature-importance',\n",
" 'formats': ['application/vnd.h2oai.json+datatable.jay',\n",
" 'application/vnd.h2oai.json+csv',\n",
" 'application/json']},\n",
" {'explanation_type': 'local-feature-importance',\n",
" 'name': 'Shapley Values for Transformed Features of MOJO Models',\n",
" 'category': 'DAI MODEL',\n",
" 'scope': 'local',\n",
" 'has_local': None,\n",
" 'formats': ['application/vnd.h2oai.json+datatable.jay']},\n",
" {'explanation_type': 'global-html-fragment',\n",
" 'name': 'Shapley Values for Transformed Features of MOJO Models',\n",
" 'category': 'DAI MODEL',\n",
" 'scope': 'global',\n",
" 'has_local': None,\n",
" 'formats': ['text/html']}],\n",
" 'keywords': ['run-by-default',\n",
" 'explains-transformed-feature-importance',\n",
" 'is_fast',\n",
" 'h2o-sonar'],\n",
" 'parameters': [{'name': 'sample_size',\n",
" 'description': 'Sample size.',\n",
" 'comment': '',\n",
" 'type': 'int',\n",
" 'val': 100000,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''},\n",
" {'name': 'calculate_predictions',\n",
" 'description': 'Score dataset and include predictions in the explanation (local explanations speed-up cache).',\n",
" 'comment': '',\n",
" 'type': 'bool',\n",
" 'val': False,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''},\n",
" {'name': 'fast_approx_contrib',\n",
" 'description': 'Speed up predictions with fast contributions predictions approximation.',\n",
" 'comment': '',\n",
" 'type': 'bool',\n",
" 'val': True,\n",
" 'predefined': [],\n",
" 'tags': [],\n",
" 'min_': 0.0,\n",
" 'max_': 0.0,\n",
" 'category': ''}],\n",
" 'metrics_meta': []}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# summary\n",
"result.summary()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "046dadb3-4472-4068-9bfe-d8ba85b2489d",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'sample_size': 100000,\n",
" 'calculate_predictions': False,\n",
" 'fast_approx_contrib': True}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# parameters\n",
"result.params()"
]
},
{
"cell_type": "markdown",
"id": "490d132b-b7e2-48a2-8ec4-dbd71886edf9",
"metadata": {
"tags": []
},
"source": [
"### Display Data"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "2aa6274e-79d5-49b1-b29a-2263db5cb8a8",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" | feature | importance |
|---|
\n",
" | ▪▪▪▪ | ▪▪▪▪▪▪▪▪ |
\n",
" \n",
" \n",
" | 0 | 10_PAY_0 | 0.484314 |
\n",
" | 1 | 11_PAY_2 | 0.177288 |
\n",
" | 2 | 8_LIMIT_BAL | 0.14293 |
\n",
" | 3 | 19_PAY_AMT4 | 0.125988 |
\n",
" | 4 | 17_PAY_AMT2 | 0.109839 |
\n",
" | 5 | 1_BILL_AMT1 | 0.0856685 |
\n",
" | 6 | 12_PAY_3 | 0.0460469 |
\n",
" | 7 | 18_PAY_AMT3 | 0.0405643 |
\n",
" | 8 | 15_PAY_6 | 0.035129 |
\n",
" | 9 | 2_BILL_AMT2 | 0.0307533 |
\n",
" | 10 | 13_PAY_4 | 0.0292232 |
\n",
" | 11 | 6_BILL_AMT6 | 0.0230935 |
\n",
" | 12 | 14_PAY_5 | 0.0203799 |
\n",
" | 13 | 16_PAY_AMT1 | 0.014774 |
\n",
" | 14 | 7_EDUCATION | 0.00983341 |
\n",
" | 15 | 0_AGE | 0.00831316 |
\n",
" | 16 | 9_MARRIAGE | 0.00725285 |
\n",
" | 17 | 21_PAY_AMT6 | 0.00588243 |
\n",
" | 18 | 20_PAY_AMT5 | 0.00541516 |
\n",
" | 19 | 5_BILL_AMT5 | 0.00464642 |
\n",
" | 20 | 4_BILL_AMT4 | 0.00144066 |
\n",
" | 21 | 3_BILL_AMT3 | 0.000818275 |
\n",
" \n",
"
\n",
" \n",
"