pipelines/samples/simple/mlpipeline_sample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca91b440",
   "metadata": {},
   "source": [
    "### OCI Data Science - ML Pipelines\n",
    "<details>\n",
    "<summary><font size=\"2\">Check for Public Internet Access</font></summary>\n",
    "\n",
    "```python\n",
    "import requests\n",
    "response = requests.get(\"https://oracle.com\")\n",
    "assert response.status_code==200, \"Internet connection failed\"\n",
    "```\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Helpful Documentation </font></summary>\n",
    "<ul><li><a href=\"https://docs.cloud.oracle.com/en-us/iaas/data-science/using/data-science.htm\">Data Science Service Documentation</a></li>\n",
    "<li><a href=\"https://docs.cloud.oracle.com/iaas/tools/ads-sdk/latest/index.html\">ADS documentation</a></li>\n",
    "</ul>\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Typical Cell Imports and Settings for ADS</font></summary>\n",
    "\n",
    "```python\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)\n",
    "\n",
    "import ads\n",
    "from ads.dataset.factory import DatasetFactory\n",
    "from ads.automl.provider import OracleAutoMLProvider\n",
    "from ads.automl.driver import AutoML\n",
    "from ads.evaluations.evaluator import ADSEvaluator\n",
    "from ads.common.data import ADSData\n",
    "from ads.explanations.explainer import ADSExplainer\n",
    "from ads.explanations.mlx_global_explainer import MLXGlobalExplainer\n",
    "from ads.explanations.mlx_local_explainer import MLXLocalExplainer\n",
    "from ads.catalog.model import ModelCatalog\n",
    "from ads.common.model_artifact import ModelArtifact\n",
    "```\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Useful Environment Variables</font></summary>\n",
    "\n",
    "```python\n",
    "import os\n",
    "print(os.environ[\"NB_SESSION_COMPARTMENT_OCID\"])\n",
    "print(os.environ[\"PROJECT_OCID\"])\n",
    "print(os.environ[\"USER_OCID\"])\n",
    "print(os.environ[\"TENANCY_OCID\"])\n",
    "print(os.environ[\"NB_REGION\"])\n",
    "```\n",
    "</details>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c2ee01a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import oci\n",
    "import os\n",
    "from os import environ\n",
    "import ads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f00538c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'NB_SESSION_COMPARTMENT_OCID' in os.environ:\n",
    "    # using the notebook's compartment if running on OCI Data Science Notebook Session\n",
    "    compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']\n",
    "else:\n",
    "    # set the compartment OCID if you are working locally\n",
    "    compartment_id = '<YOUR_COMPARTMENT_OCID>'\n",
    "print('compartment OCID = ', compartment_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97fc8e47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# this will work whether working on a local machine (using API key) or in OCI Notebook Session (using Resource Principal)\n",
    "if \"OCI_RESOURCE_PRINCIPAL_VERSION\" in os.environ:\n",
    "    # Use resource principal\n",
    "    print(\"using Resource Principal for auth\")\n",
    "    ads.set_auth(auth=\"resource_principal\")\n",
    "else:\n",
    "    # Use api_key with config file\n",
    "    print(\"using API key for auth\")\n",
    "    ads.set_auth(auth=\"api_key\")    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cc038dc6",
   "metadata": {},
   "source": [
    "Fill in your resources details:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f0f647f",
   "metadata": {},
   "outputs": [],
   "source": [
    "project_id = \"<YOUR_PROJECT_OCID>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ef78404",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_group_id = \"<YOUR_LOGGROUP_OCID>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9009da31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a data science client to communicate with the service\n",
    "config = oci.config.from_file()\n",
    "data_science_client = oci.data_science.DataScienceClient(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b0e8058",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list all pipelines in the project. This tests the new APIs are available and working. If you recieve an error, check that you installed the latest Beta OCI Python SDK and CLI\n",
    "res = data_science_client.list_pipelines(compartment_id=compartment_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f986bcb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(res.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d0a8fe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_name = \"pipeline_sample_1\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8600b58e",
   "metadata": {},
   "source": [
    "## create a new pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a739101",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_payload = {\n",
    "    \"projectId\": project_id,\n",
    "    \"compartmentId\": compartment_id,\n",
    "    \"displayName\": pipeline_name,\n",
    "    \"infrastructureConfigurationDetails\": {\n",
    "        \"shapeName\": \"VM.Standard2.4\",\n",
    "        \"blockStorageSizeInGBs\": \"50\"\n",
    "    },\n",
    "    \"logConfigurationDetails\": {\n",
    "        \"enableLogging\": True,\n",
    "        \"logGroupId\": log_group_id,\n",
    "        \"enableAutoLogCreation\": True   # log will be automatically created\n",
    "    },\n",
    "    \"configurationDetails\": {\n",
    "        \"type\": \"DEFAULT\",\n",
    "        \"maximumRuntimeInMinutes\": 30,\n",
    "        \"environmentVariables\": {\n",
    "            \"CONDA_ENV_TYPE\": \"service\",\n",
    "            \"CONDA_ENV_SLUG\": \"onnx110_p37_cpu_v1\"            \n",
    "        }\n",
    "    },\n",
    "    \"stepDetails\": [\n",
    "        {\n",
    "            \"stepName\": \"step1\",\n",
    "            \"description\": \"Pass data by value\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 30,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"mlpipeline_step1.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"onnx110_p37_cpu_v1\"\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"step2\",\n",
    "            \"description\": \"pass data by reference\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 30,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"mlpipeline_step2.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"onnx110_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"step1\"]\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"step3\",\n",
    "            \"description\": \"read data by reference\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 30,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"mlpipeline_step3.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"onnx110_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"step2\"]\n",
    "        }\n",
    "    ],\n",
    "    \"freeformTags\": {\n",
    "        \"freeTags\": \"testing pipeline\"\n",
    "    }\n",
    "}\n",
    "pipeline_res = data_science_client.create_pipeline(pipeline_payload)\n",
    "pipeline_id = pipeline_res.data.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f5975f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pipeline_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2ca57d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the pipeline will be in CREATING state until all steps have their artifacts uploaded\n",
    "print(pipeline_res.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5bcf217",
   "metadata": {},
   "outputs": [],
   "source": [
    "# upload steps artifacts\n",
    "file1 = open(\"mlpipeline_step1.zip\", \"rb\")\n",
    "ret1 = data_science_client.create_step_artifact(pipeline_id, \"step1\", file1, content_disposition=f\"attachment; filename={'mlpipeline_step1.zip'}\")\n",
    "print(\"OK\" if ret1.status==204 else ret1.status)  # 204 is ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb96f5bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "file2 = open(\"mlpipeline_step2.zip\", \"rb\")\n",
    "ret2 = data_science_client.create_step_artifact(pipeline_id, \"step2\", file2, content_disposition=f\"attachment; filename={'mlpipeline_step2.zip'}\")\n",
    "print(\"OK\" if ret2.status==204 else ret2.status)  # 204 is ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d5ca3dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "file3 = open(\"mlpipeline_step3.zip\", \"rb\")\n",
    "ret3 = data_science_client.create_step_artifact(pipeline_id, \"step3\", file3, content_disposition=f\"attachment; filename={'mlpipeline_step3.zip'}\")\n",
    "print(\"OK\" if ret3.status==204 else ret3.status)  # 204 is ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "260cd8ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pipeline should be in ACTIVE state now\n",
    "res = data_science_client.get_pipeline(pipeline_id)\n",
    "print(res.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58c70cae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#enable service logs - to catch issues during step provisioning\n",
    "logging_client = oci.logging.LoggingManagementClient(config)\n",
    "service_log_name = pipeline_name + \"-service_log\"\n",
    "create_log_response = logging_client.create_log(\n",
    "    log_group_id=log_group_id,\n",
    "    create_log_details=oci.logging.models.CreateLogDetails(\n",
    "        display_name=service_log_name,\n",
    "        log_type=\"SERVICE\",\n",
    "        is_enabled=True,\n",
    "        configuration=oci.logging.models.Configuration(\n",
    "            source=oci.logging.models.OciService(\n",
    "                source_type=\"OCISERVICE\",\n",
    "                service=\"datascience\",\n",
    "                resource=pipeline_id,\n",
    "                category=\"pipelinerunlog\"),\n",
    "            compartment_id=compartment_id)))\n",
    "\n",
    "# Check the response\n",
    "print(create_log_response.status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fddef232",
   "metadata": {},
   "source": [
    "## Run the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b88b3bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_run_name = \"pipeline-run-1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cfed42f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment the next line and set to the OCI Object Storage location to use for passing data. Make sure you have proper permissions.\n",
    "#data_location = \"<YOUR_OBJECT_STORAGE_BUCKET>\"  # use: 'oci://<bucket>@<workspace>/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "400b94fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pipeline_run_payload = {\n",
    "    \"projectId\": project_id,\n",
    "    \"displayName\": pipeline_run_name,\n",
    "    \"pipelineId\": pipeline_id,\n",
    "    \"compartmentId\": compartment_id,\n",
    "    \"configurationOverrideDetails\": {\n",
    "        \"type\": \"DEFAULT\",\n",
    "        \"environmentVariables\": {\n",
    "            \"DATA_LOCATION\": data_location\n",
    "        }\n",
    "    }\n",
    "}\n",
    "run_res = data_science_client.create_pipeline_run(pipeline_run_payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91ece4fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check pipeline run status\n",
    "run_status = data_science_client.get_pipeline_run(run_res.data.id)\n",
    "print(run_status.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39d228e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#check pipeline step run status\n",
    "print(run_status.data.step_runs[\"step_name\"==\"step1\"].lifecycle_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55b836aa",
   "metadata": {},
   "source": [
    "## view logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "369cfe7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# wait a few seconds after the pipeline run is created to make sure the log was generated\n",
    "run1 = data_science_client.get_pipeline_run(run_res.data.id)\n",
    "log_id = run1.data.log_details.log_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "881d5c30",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "now = datetime.now()\n",
    "loggingsearch_client = oci.loggingsearch.LogSearchClient(config)\n",
    "search_logs_response = loggingsearch_client.search_logs(\n",
    "    search_logs_details=oci.loggingsearch.models.SearchLogsDetails(\n",
    "        time_start= now - timedelta(days=1),\n",
    "        time_end=now,\n",
    "        search_query=f\"search \\\"{compartment_id}/{log_group_id}/{log_id}\\\"\",\n",
    "        is_return_field_info=False,\n",
    "    ),    \n",
    "    limit=10\n",
    ")\n",
    " \n",
    "# Get the data from response\n",
    "print(search_logs_response.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4403539b",
   "metadata": {},
   "source": [
    "## View the pipeline run form the OCI console UI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e77c057",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Ctrl-Click the hyperlink to open the pipeline run page in the OCI console UI\")\n",
    "print(\"https://cloud.oracle.com/data-science/pipeline-runs/{}\".format(run_res.data.id))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97c9271b",
   "metadata": {},
   "source": [
    "## Delete the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4e7db49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete the pipeline when done\n",
    "data_science_client.delete_pipeline(pipeline_id)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}