config_template.json

{
  "kernel_python_credentials": {
    "username": "",
    "password": "",
    "url": "http://${conf.livyIp}:8998"
  },
  "kernel_scala_credentials": {
    "username": "",
    "password": "",
    "url": "http://${conf.livyIp}:8998"
  },
  "kernel_r_credentials": {
    "username": "",
    "password": "",
    "url": "http://${conf.livyIp}:8998"
  },
  "logging_config": {
    "version": 1,
    "formatters": {
      "magicsFormatter": {
        "format": "%(asctime)s\t%(levelname)s\t%(message)s",
        "datefmt": ""
      }
    },
    "handlers": {
      "magicsHandler": {
        "class": "hdijupyterutils.filehandler.MagicsFileHandler",
        "formatter": "magicsFormatter",
        "home_path": "${conf.jupyterHome}/.sparkmagic"
      }
    },
    "loggers": {
      "magicsLogger": {
        "handlers": ["magicsHandler"],
        "level": "WARN",
        "propagate": 0
      }
    }
  },
  "wait_for_idle_timeout_seconds": 15,
  "status_sleep_seconds": 2,
  "statement_sleep_seconds": 2,
  "livy_session_startup_timeout_seconds": ${conf.livyStartupTimeout},
  "fatal_error_suggestion": "The code failed because of a fatal error:\n\t{}.\n\nSome things to try:\na) Make sure Spark has enough available resources for Jupyter to create a Spark context.\nb) Contact your Jupyter administrator to make sure the Spark magics library is configured correctly.\nc) Restart the kernel.",
  "ignore_ssl_errors": false,
  "session_configs": {
    "driverCores": ${conf.driverCores?c},
    "driverMemory": "${conf.driverMemory}",
    "numExecutors": ${conf.numExecutors?c},
    "executorCores": ${conf.executorCores?c},
    "executorMemory": "${conf.executorMemory}",
    "proxyUser": "${conf.hdfsUser}",
    "queue": "${conf.yarnQueue}",
    "conf": {
${conf.sparkConfiguration}
    }
  },
  "use_auto_viz": true,
  "max_results_sql": 2500,
  "pyspark_dataframe_encoding": "utf-8",
  "heartbeat_refresh_seconds": 30,
  "livy_server_heartbeat_timeout_seconds": 0,
  "heartbeat_retry_seconds": 10,
  "server_extension_default_kernel_name": "pysparkkernel",
  "env": {
    "HADOOP_HOME": "${conf.hadoopHome}",
    "HADOOP_HDFS_HOME": "${conf.hadoopHome}",
    "HADOOP_CONF_DIR": "${conf.hadoopHome}/etc/hadoop",
    "HADOOP_USER_NAME": "${conf.hdfsUser}",
    "HADOOP_VERSION": "${conf.hadoopVersion}"
  }
}