{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7de3eb73-75e0-43b3-887e-329773a72bf0",
   "metadata": {},
   "source": [
    "# Ignoring Exceptions with Failure Modes\n",
    "\n",
    "`rubicon-ml` is often used for logging in scenarios that require high availability, like model inference pipelines\n",
    "running in production environments. If something were to go wrong with `rubicon-ml` during live model inference,\n",
    "we could end up halting predictions just for a logging issue. `rubicon-ml`'s configurable failure modes allow users\n",
    "to choose what to do with `rubicon-ml` exceptions!\n",
    "\n",
    "First, let's try to get a project that we haven't yet created. This will show the default failure behavior - raising\n",
    "a `RubiconException` that halts execution of the code it originated from."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b650c884-51cd-41bb-b035-bddfd3de15f4",
   "metadata": {},
   "outputs": [
    {
     "ename": "RubiconException",
     "evalue": "No project with name 'failure modes' found.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[0;32m~/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/implementations/memory.py:213\u001b[0m, in \u001b[0;36mMemoryFileSystem.cat_file\u001b[0;34m(self, path, start, end, **kwargs)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 213\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mbytes\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstore\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mgetbuffer()[start:end])\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n",
      "\u001b[0;31mKeyError\u001b[0m: '/root/failure-modes/metadata.json'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py:102\u001b[0m, in \u001b[0;36mBaseRepository.get_project\u001b[0;34m(self, project_name)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 102\u001b[0m     project \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfilesystem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcat\u001b[49m\u001b[43m(\u001b[49m\u001b[43mproject_metadata_path\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m:\n",
      "File \u001b[0;32m~/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/spec.py:755\u001b[0m, in \u001b[0;36mAbstractFileSystem.cat\u001b[0;34m(self, path, recursive, on_error, **kwargs)\u001b[0m\n\u001b[1;32m    754\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 755\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcat_file\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpaths\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/implementations/memory.py:215\u001b[0m, in \u001b[0;36mMemoryFileSystem.cat_file\u001b[0;34m(self, path, start, end, **kwargs)\u001b[0m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n\u001b[0;32m--> 215\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(path)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: /root/failure-modes/metadata.json",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mRubiconException\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [1], line 5\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mrubicon_ml\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Rubicon\n\u001b[1;32m      4\u001b[0m rb \u001b[38;5;241m=\u001b[39m Rubicon(persistence\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmemory\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 5\u001b[0m \u001b[43mrb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_project\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfailure modes\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py:46\u001b[0m, in \u001b[0;36mfailsafe.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraise\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 46\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwarn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m     48\u001b[0m         warnings\u001b[38;5;241m.\u001b[39mwarn(traceback\u001b[38;5;241m.\u001b[39mformat_exc(limit\u001b[38;5;241m=\u001b[39mTRACEBACK_LIMIT, chain\u001b[38;5;241m=\u001b[39mTRACEBACK_CHAIN))\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py:43\u001b[0m, in \u001b[0;36mfailsafe.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 43\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     44\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     45\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraise\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/rubicon.py:123\u001b[0m, in \u001b[0;36mRubicon.get_project\u001b[0;34m(self, name, id)\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`name` OR `id` required.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    122\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 123\u001b[0m     project \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrepository\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_project\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    124\u001b[0m     project \u001b[38;5;241m=\u001b[39m Project(project, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig)\n\u001b[1;32m    125\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py:104\u001b[0m, in \u001b[0;36mBaseRepository.get_project\u001b[0;34m(self, project_name)\u001b[0m\n\u001b[1;32m    102\u001b[0m     project \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfilesystem\u001b[38;5;241m.\u001b[39mcat(project_metadata_path))\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m:\n\u001b[0;32m--> 104\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m RubiconException(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo project with name \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mproject_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m found.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m domain\u001b[38;5;241m.\u001b[39mProject(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mproject)\n",
      "\u001b[0;31mRubiconException\u001b[0m: No project with name 'failure modes' found."
     ]
    }
   ],
   "source": [
    "from rubicon_ml import Rubicon\n",
    "\n",
    "\n",
    "rb = Rubicon(persistence=\"memory\")\n",
    "rb.get_project(name=\"failure modes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b1fb098-c7c3-4210-b7a1-c1ea5a0e4317",
   "metadata": {},
   "source": [
    "But, let's say we're far more concerned with keeping our code running than we are with whether or\n",
    "not our logs get logged.\n",
    "\n",
    "We can `set_failure_mode` to \"warn\" to instead raise warnings (via the builtin `warnings.warn`)\n",
    "whenever `rubicon-ml` encounters an exception and continue execution of the offending code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "457871b9-cb20-49b8-847e-bf563fbb255f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py:48: UserWarning: Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/rubicon.py\", line 123, in get_project\n",
      "    project = self.repository.get_project(name)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 104, in get_project\n",
      "    raise RubiconException(f\"No project with name '{project_name}' found.\")\n",
      "rubicon_ml.exceptions.RubiconException: No project with name 'failure modes' found.\n",
      "\n",
      "  warnings.warn(traceback.format_exc(limit=TRACEBACK_LIMIT, chain=TRACEBACK_CHAIN))\n"
     ]
    }
   ],
   "source": [
    "from rubicon_ml import set_failure_mode\n",
    "\n",
    "\n",
    "set_failure_mode(\"warn\")\n",
    "\n",
    "rb.get_project(name=\"failure modes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd274187-68b4-4a2f-9734-aa84e2b77f87",
   "metadata": {},
   "source": [
    "We can also `set_failure_mode` to \"log\" to log the error with the builtin `logging.error`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbbbdb5d-3d90-45b6-8ab0-df70cab60074",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/rubicon.py\", line 123, in get_project\n",
      "    project = self.repository.get_project(name)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 104, in get_project\n",
      "    raise RubiconException(f\"No project with name '{project_name}' found.\")\n",
      "rubicon_ml.exceptions.RubiconException: No project with name 'failure modes' found.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_failure_mode(\"log\")\n",
    "\n",
    "rb.get_project(name=\"failure modes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec648cde-0d9d-4d29-83f5-e58d090eb150",
   "metadata": {},
   "source": [
    "`set_failure_mode` back to the default - \"raise\" - to return to raising the exceptions.\n",
    "\n",
    "## Log and warning verbosity\n",
    "\n",
    "The \"log\" and \"warn\" failure modes leverage the builtin `traceback.exc_info()` in order to print an error's\n",
    "traceback. `set_failure_mode`'s `traceback_limit` and `traceback_chain` are passed directly through to the\n",
    "underlying call to `traceback.exc_info()` as the `limit` and `chain` arguments.\n",
    "\n",
    "`limit` is an integer between 0 and the depth of the stack trace that controls the verbosity of the trace."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "214a0d51-89bd-4c10-833f-60463517af7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:rubicon_ml.exceptions.RubiconException: No project with name 'failure modes' found.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_failure_mode(\"log\", traceback_limit=0)\n",
    "\n",
    "rb.get_project(name=\"failure modes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd12b2c0-6873-4f44-b751-58c572dbbfd8",
   "metadata": {},
   "source": [
    "`chain` can be set to `True` to see the full chain of exceptions rather than just the final exception in the\n",
    "chain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "86516e95-02e7-40a8-b474-672bdabe19f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/implementations/memory.py\", line 213, in cat_file\n",
      "    return bytes(self.store[path].getbuffer()[start:end])\n",
      "KeyError: '/root/failure-modes/metadata.json'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 102, in get_project\n",
      "    project = json.loads(self.filesystem.cat(project_metadata_path))\n",
      "  File \"/Users/nvd215/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/spec.py\", line 755, in cat\n",
      "    return self.cat_file(paths[0], **kwargs)\n",
      "  File \"/Users/nvd215/mambaforge/envs/rubicon-ml-dev/lib/python3.10/site-packages/fsspec/implementations/memory.py\", line 215, in cat_file\n",
      "    raise FileNotFoundError(path)\n",
      "FileNotFoundError: /root/failure-modes/metadata.json\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/rubicon.py\", line 123, in get_project\n",
      "    project = self.repository.get_project(name)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 104, in get_project\n",
      "    raise RubiconException(f\"No project with name '{project_name}' found.\")\n",
      "rubicon_ml.exceptions.RubiconException: No project with name 'failure modes' found.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_failure_mode(\"log\", traceback_chain=True)\n",
    "\n",
    "rb.get_project(name=\"failure modes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dca6c089-997c-4394-adbc-5629d91594c8",
   "metadata": {},
   "source": [
    "## Caution with return values\n",
    "\n",
    "Some workflows require the returned `rubicon-ml` objects to be leveraged for future logging in the\n",
    "same process. For example, let's finally create the \"failure modes\" project and take a look at the\n",
    "returned `rubicon-ml` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd1815bf-3482-47a1-820c-ecfb5d82d6e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Project(name='failure modes', id='8d6ac09b-45c5-4097-9728-302c481a1665', description=None, github_url=None, training_metadata=None, created_at=datetime.datetime(2022, 11, 15, 17, 5, 2, 510729))\n"
     ]
    }
   ],
   "source": [
    "rb.create_project(name=\"failure modes\")\n",
    "project = rb.get_project(name=\"failure modes\")\n",
    "\n",
    "print(project)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b96df86-9926-4d2c-bcd0-55c9288a0623",
   "metadata": {},
   "source": [
    "Now we can take any standard action on the returned `rubicon-ml` object, like inspecting its ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "76ebe66a-cb47-4de5-9cae-4cab134c0050",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8d6ac09b-45c5-4097-9728-302c481a1665\n"
     ]
    }
   ],
   "source": [
    "print(project.id)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "026f68bf-1709-46d3-96dc-d4206cb9bbc9",
   "metadata": {},
   "source": [
    "If we were to leverage either the \"log\" or \"warn\" failure mode, which does not stop execution for `rubicon-ml`\n",
    "errors, we need to be cautious of returned `rubicon-ml` objects.\n",
    "\n",
    "Now we'll try to get another project that doesn't exist. Even though this code will not stop execution after the\n",
    "failed `get_project` call, we need to be aware that a `rubicon-ml` project **will not be returned in this case**.\n",
    "A `None` will be returned in its place, thus any action taken on this returned `None` may fail if only `rubicon-ml`\n",
    "objects are expected downstream."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c603cc65-ca2f-4877-ae89-bde97420af34",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/rubicon.py\", line 123, in get_project\n",
      "    project = self.repository.get_project(name)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 104, in get_project\n",
      "    raise RubiconException(f\"No project with name '{project_name}' found.\")\n",
      "rubicon_ml.exceptions.RubiconException: No project with name 'failure modes v2' found.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "set_failure_mode(\"log\")\n",
    "\n",
    "project = rb.get_project(name=\"failure modes v2\")\n",
    "print(project)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d637dec-3725-4df5-aa1c-e0e08c2b45a9",
   "metadata": {},
   "source": [
    "When leveraging failure modes that do not interrupt execution, it is important to check the types of the objects\n",
    "returned from `rubicon-ml`. Simply trying to access the `id` attribute in this case would result in an `AttributeError`\n",
    "as a `NoneType` object hs no attribute `id`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5d9e7a53-b30e-4941-8657-3a9f338194a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if project is not None:\n",
    "    print(project.id)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da65cc2f-711e-45df-a597-7bd834574ccd",
   "metadata": {},
   "source": [
    "## A more practical use case\n",
    "\n",
    "Let's take a look at how this may work in a more practical machine learning workflow. For\n",
    "this example, we'll train a k-neighbors classifier from Scikit-learn and attempt to log\n",
    "the input parameters and the score the trained model produces on a test dataset.\n",
    "\n",
    "First, we'll create a new project and experiment to attempt to log our inputs and outputs to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e76935dd-a711-4c30-b37a-a3a209410d30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<rubicon_ml.client.experiment.Experiment at 0x15cacce20>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project = rb.create_project(name=\"failure modes v3\")\n",
    "experiment = project.log_experiment()\n",
    "\n",
    "experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06c4e03a-dc2c-4b06-a90c-8369e5eecc48",
   "metadata": {},
   "source": [
    "Now that we've got an experiment, lets replace `rubicon-ml`'s `filesystem` (the part\n",
    "of the library that handles actual filesystem operations) with a no-op class representing\n",
    "a broken filesystem. Imagine this simulating something like losing connection to S3.\n",
    "\n",
    "We'll also set the failure mode back to raise for this first execution of our workflow\n",
    "with the broken filesystem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1d4dd196-e640-4df8-8718-e4187e5cefe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BrokenFilesystem:\n",
    "    pass\n",
    "\n",
    "rb.config.repository.filesystem = BrokenFilesystem()\n",
    "\n",
    "set_failure_mode(\"raise\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fea9d4cc-1e51-4aec-8fb2-222c55b59a2c",
   "metadata": {},
   "source": [
    "Now, notice that when we run the cell below, we only fit the model before execution is halted due\n",
    "to `experiment.log_parameter` raising an exception because of the broken filesystem. We never get\n",
    "a `score`, and it is never displayed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7cac1030-71d2-4c57-a76b-16d50778ee06",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'BrokenFilesystem' object has no attribute 'exists'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [12], line 8\u001b[0m\n\u001b[1;32m      5\u001b[0m X_train, y_train, X_test, y_test \u001b[38;5;241m=\u001b[39m [[\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m3\u001b[39m]], [\u001b[38;5;241m0\u001b[39m], [[\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m3\u001b[39m]], [\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m      7\u001b[0m knn\u001b[38;5;241m.\u001b[39mfit(X_train, y_train)\n\u001b[0;32m----> 8\u001b[0m \u001b[43mexperiment\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlog_parameter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mn_neighbors\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     10\u001b[0m score \u001b[38;5;241m=\u001b[39m knn\u001b[38;5;241m.\u001b[39mscore(X_test, y_test)\n\u001b[1;32m     11\u001b[0m experiment\u001b[38;5;241m.\u001b[39mlog_metric(name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mscore\u001b[39m\u001b[38;5;124m\"\u001b[39m, value\u001b[38;5;241m=\u001b[39mscore)\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py:46\u001b[0m, in \u001b[0;36mfailsafe.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraise\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 46\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28;01melif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwarn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m     48\u001b[0m         warnings\u001b[38;5;241m.\u001b[39mwarn(traceback\u001b[38;5;241m.\u001b[39mformat_exc(limit\u001b[38;5;241m=\u001b[39mTRACEBACK_LIMIT, chain\u001b[38;5;241m=\u001b[39mTRACEBACK_CHAIN))\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py:43\u001b[0m, in \u001b[0;36mfailsafe.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 43\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     44\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     45\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m FAILURE_MODE \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraise\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/client/experiment.py:237\u001b[0m, in \u001b[0;36mExperiment.log_parameter\u001b[0;34m(self, name, value, description, tags)\u001b[0m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;124;03m\"\"\"Create a parameter under the experiment.\u001b[39;00m\n\u001b[1;32m    215\u001b[0m \n\u001b[1;32m    216\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    234\u001b[0m \u001b[38;5;124;03m    The created parameter.\u001b[39;00m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    236\u001b[0m parameter \u001b[38;5;241m=\u001b[39m domain\u001b[38;5;241m.\u001b[39mParameter(name, value\u001b[38;5;241m=\u001b[39mvalue, description\u001b[38;5;241m=\u001b[39mdescription, tags\u001b[38;5;241m=\u001b[39mtags)\n\u001b[0;32m--> 237\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrepository\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_parameter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparameter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mproject\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    239\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m Parameter(parameter, \u001b[38;5;28mself\u001b[39m)\n",
      "File \u001b[0;32m~/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py:852\u001b[0m, in \u001b[0;36mBaseRepository.create_parameter\u001b[0;34m(self, parameter, project_name, experiment_id)\u001b[0m\n\u001b[1;32m    836\u001b[0m \u001b[38;5;124;03m\"\"\"Persist a parameter to the configured filesystem.\u001b[39;00m\n\u001b[1;32m    837\u001b[0m \n\u001b[1;32m    838\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    846\u001b[0m \u001b[38;5;124;03m    The ID of the experiment this parameter belongs to.\u001b[39;00m\n\u001b[1;32m    847\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    848\u001b[0m parameter_metadata_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_parameter_metadata_path(\n\u001b[1;32m    849\u001b[0m     project_name, experiment_id, parameter\u001b[38;5;241m.\u001b[39mname\n\u001b[1;32m    850\u001b[0m )\n\u001b[0;32m--> 852\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfilesystem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexists\u001b[49m(parameter_metadata_path):\n\u001b[1;32m    853\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m RubiconException(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mA parameter with name \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparameter\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m already exists.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    855\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_persist_domain(parameter, parameter_metadata_path)\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'BrokenFilesystem' object has no attribute 'exists'"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "X_train, y_train, X_test, y_test = [[0, 1, 2, 3]], [0], [[0, 1, 2, 3]], [0]\n",
    "\n",
    "knn.fit(X_train, y_train)\n",
    "experiment.log_parameter(name=\"n_neighbors\", value=1)\n",
    "\n",
    "score = knn.score(X_test, y_test)\n",
    "experiment.log_metric(name=\"score\", value=score)\n",
    "\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28669f56-aaae-4112-b284-e4f51f5c56e1",
   "metadata": {},
   "source": [
    "By setting the failure mode to \"log\" and ensuring we are not using any unchecked objects returned\n",
    "by `rubicon-ml`, we can ensure that the entire workflow is completed regardless of whether or not\n",
    "the filesystem is working."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2c2556ce-e11f-4173-9e3a-f041dda56e98",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/experiment.py\", line 237, in log_parameter\n",
      "    self.repository.create_parameter(parameter, self.project.name, self.id)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 852, in create_parameter\n",
      "    if self.filesystem.exists(parameter_metadata_path):\n",
      "AttributeError: 'BrokenFilesystem' object has no attribute 'exists'\n",
      "\n",
      "ERROR:root:Traceback (most recent call last):\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/utils/exception_handling.py\", line 43, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/client/experiment.py\", line 76, in log_metric\n",
      "    self.repository.create_metric(metric, self.project.name, self.id)\n",
      "  File \"/Users/nvd215/github/capitalone/rubicon-ml/rubicon_ml/repository/base.py\", line 746, in create_metric\n",
      "    if self.filesystem.exists(metric_metadata_path):\n",
      "AttributeError: 'BrokenFilesystem' object has no attribute 'exists'\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_failure_mode(\"log\")\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "X_train, y_train, X_test, y_test = [[0, 1, 2, 3]], [0], [[0, 1, 2, 3]], [0]\n",
    "\n",
    "knn.fit(X_train, y_train)\n",
    "experiment.log_parameter(name=\"n_neighbors\", value=1)\n",
    "\n",
    "score = knn.score(X_test, y_test)\n",
    "experiment.log_metric(name=\"score\", value=score)\n",
    "\n",
    "score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}