ch16.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 16: Visualizing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "pl.__version__  # The book is built with Polars version 1.20.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NYC Bike Trips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips = pl.read_parquet(\"data/citibike/*.parquet\")\n",
    "\n",
    "print(trips[:, :4])\n",
    "print(trips[:, 4:7])\n",
    "print(trips[:, 7:11])\n",
    "print(trips[:, 11:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Built-In Plotting with Altair"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing Altair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import altair as alt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Methods in the Plot Namespaces"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed = trips.select(\n",
    "    pl.col(\"distance\"),\n",
    "    pl.col(\"duration\").dt.total_seconds() / 3600,  \n",
    "    pl.col(\"bike_type\"),\n",
    ").with_columns(speed=pl.col(\"distance\") / pl.col(\"duration\"))\n",
    "\n",
    "trips_speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This raises a MaxRowsError:\n",
    "# trips_speed.plot.scatter(\n",
    "#    x=\"distance\",\n",
    "#    y=\"duration\",\n",
    "#    color=\"bike_type:N\",\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed = (\n",
    "    trips.filter(pl.col(\"station_start\") == \"W 70 St & Amsterdam Ave\")\n",
    "    .select(\n",
    "        pl.col(\"distance\"),\n",
    "        pl.col(\"duration\").dt.total_seconds() / 3600,\n",
    "        pl.col(\"bike_type\"),\n",
    "    )\n",
    "    .with_columns(speed=pl.col(\"distance\") / pl.col(\"duration\"))\n",
    ")\n",
    "\n",
    "trips_speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed.plot.scatter(\n",
    "    x=\"distance\",\n",
    "    y=\"duration\",\n",
    "    color=\"bike_type:N\",  \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Too Large to Handle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import altair as alt\n",
    "\n",
    "alt.data_transformers.disable_max_rows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alt.data_transformers.enable(\"vegafusion\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_type_counts = trips.group_by(\"rider_type\", \"bike_type\").len()\n",
    "trips_type_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display\n",
    "\n",
    "chart = trips_type_counts.plot.bar(\n",
    "    x=\"rider_type\", y=\"len\", fill=\"bike_type:N\"\n",
    ").properties(\n",
    "    width=300,\n",
    ")\n",
    "display(chart)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed[\"distance\"].plot.kde()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed[\"distance\"].plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_hour_num_speed = (\n",
    "    trips.sort(\"datetime_start\")\n",
    "    .group_by_dynamic(\"datetime_start\", every=\"1h\")\n",
    "    .agg(\n",
    "        num_trips=pl.len(),\n",
    "        speed=(\n",
    "            pl.col(\"distance\") / (pl.col(\"duration\").dt.total_seconds() / 3600)\n",
    "        ).median(),\n",
    "    )\n",
    "    .filter(pl.col(\"datetime_start\") > pl.date(2024, 3, 26))\n",
    ")\n",
    "\n",
    "trips_hour_num_speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_hour_num_speed.plot.line(x=\"datetime_start\", y=\"num_trips\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pandas-Like Plotting with hvPlot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing hvPlot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hvplot.polars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A First Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed.hvplot.scatter(\n",
    "    x=\"distance\",\n",
    "    y=\"duration\",\n",
    "    color=\"bike_type\",  \n",
    "    xlabel=\"distance (km)\",\n",
    "    ylabel=\"duration (h)\",  \n",
    "    ylim=(0, 2),  \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Methods in the hvPlot Namespace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### pandas as Backup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_per_day_hour = (\n",
    "    trips.sort(\"datetime_start\")\n",
    "    .group_by_dynamic(\"datetime_start\", every=\"1h\")\n",
    "    .agg(pl.len())\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This raises a ValueError:\n",
    "# trips_per_day_hour.hvplot.heatmap(\n",
    "#     x=\"datetime_start.hour\", y=\"datetime_start.day\", C=\"len\", cmap=\"reds\"\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hvplot.pandas\n",
    "\n",
    "trips_per_day_hour.to_pandas().hvplot.heatmap(\n",
    "    x=\"datetime_start.hour\", y=\"datetime_start.day\", C=\"len\", cmap=\"reds\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Manual Transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_type_counts.hvplot.bar(\n",
    "    x=\"rider_type\",\n",
    "    y=\"len\",\n",
    "    by=\"bike_type\",\n",
    "    ylabel=\"count\",\n",
    "    stacked=True,\n",
    "    color=[\"orange\", \"green\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Changing the Plotting Backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hvplot\n",
    "\n",
    "hvplot.extension(\"matplotlib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_type_counts.hvplot.bar(\n",
    "    x=\"rider_type\",\n",
    "    y=\"len\",\n",
    "    by=\"bike_type\",\n",
    "    ylabel=\"count\",\n",
    "    stacked=True,\n",
    "    color=[\"orange\", \"green\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hvplot.extension(\"bokeh\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting Points on a Map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips.hvplot.points(\n",
    "    x=\"lon_start\",\n",
    "    y=\"lat_start\",\n",
    "    datashade=True,\n",
    "    geo=True,\n",
    "    tiles=\"CartoLight\",\n",
    "    width=800,\n",
    "    height=600,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Composing Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    trips_hour_num_speed.hvplot.line(x=\"datetime_start\", y=\"num_trips\")\n",
    "    + trips_hour_num_speed.hvplot.line(x=\"datetime_start\", y=\"speed\")\n",
    ").cols(  \n",
    "    1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    trips_hour_num_speed.hvplot.line(x=\"datetime_start\", y=\"num_trips\")\n",
    "    * trips_hour_num_speed.filter(pl.col(\"num_trips\") > 9000).hvplot.scatter(\n",
    "        x=\"datetime_start\", y=\"num_trips\", c=\"red\", s=50\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding Interactive Widgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_per_hour = (\n",
    "    trips.sort(\"datetime_start\")\n",
    "    .group_by_dynamic(\"datetime_start\", group_by=\"borough_start\", every=\"1h\")\n",
    "    .agg(pl.len())\n",
    "    .with_columns(date=pl.col(\"datetime_start\").dt.date())\n",
    ")\n",
    "trips_per_hour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_per_hour.hvplot.line(\n",
    "    x=\"datetime_start\",\n",
    "    by=\"borough_start\",\n",
    "    groupby=\"date\",\n",
    "    widget_location=\"left_top\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Publication-Quality Graphics with plotnine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing plotnine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from plotnine import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plots for Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(trips_speed, aes(x=\"distance\", y=\"duration\", color=\"bike_type\"))\n",
    "    + geom_point()  \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ggplot(trips_speed, aes(x=\"distance\")) + geom_histogram()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ggplot(trips, aes(x=\"rider_type\", fill=\"bike_type\")) + geom_bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ggplot(trips_speed, aes(x=\"distance\", fill=\"bike_type\")) + geom_density(\n",
    "    alpha=0.7, color=\"none\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(trips_hour_num_speed, aes(x=\"datetime_start\", y=\"num_trips\"))\n",
    "    + geom_line(size=1, color=\"steelblue\")\n",
    "    + geom_point(\n",
    "        data=trips_hour_num_speed.filter(pl.col(\"num_trips\") > 9000),\n",
    "        color=\"red\",\n",
    "        size=4,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plots for Communication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trips_speed = (\n",
    "    trips.group_by(\"neighborhood_start\", \"neighborhood_end\")\n",
    "    .agg(\n",
    "        pl.col(\"duration\").dt.total_seconds().median() / 60,\n",
    "        pl.col(\"distance\").median(),\n",
    "        pl.col(\"borough_start\").first(),\n",
    "        pl.col(\"borough_end\").first(),\n",
    "        pl.len(),\n",
    "    )\n",
    "    .filter(\n",
    "        (pl.col(\"len\") > 30)\n",
    "        & (pl.col(\"distance\") > 0.2)\n",
    "        & (pl.col(\"neighborhood_start\") != pl.col(\"neighborhood_end\")),\n",
    "    )\n",
    "    .with_columns(speed=pl.col(\"distance\") / pl.col(\"duration\"))\n",
    "    .sort(\"borough_start\")\n",
    ")\n",
    "trips_speed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(\n",
    "        data=trips_speed.filter(\n",
    "            pl.col(\"borough_start\") == pl.col(\"borough_end\")\n",
    "        ),\n",
    "        mapping=aes(x=\"distance\", y=\"duration\", color=\"borough_end\"),\n",
    "    )\n",
    "    + geom_point(size=0.25, alpha=0.5)\n",
    "    + geom_smooth(method=\"lowess\", size=2, se=False, alpha=0.8)\n",
    "    + xlim(0, 15)  \n",
    "    + ylim(0, 60)\n",
    "    + scale_color_brewer(type=\"qualitative\", palette=\"Set1\")\n",
    "    + labs(\n",
    "        title=\"Trip distance and duration within each borough\",\n",
    "        x=\"Distance (km)\",\n",
    "        y=\"Duration (min)\",\n",
    "        color=\"Borough\",\n",
    "    )\n",
    "    + theme_tufte(base_size=14)\n",
    "    + theme(\n",
    "        figure_size=(8, 6),\n",
    "        plot_background=element_rect(color=\"#ffffff\"),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(\n",
    "        data=trips_speed.filter(\n",
    "            pl.col(\"borough_start\") != pl.col(\"borough_end\")\n",
    "        ).with_columns(\n",
    "            (\"From \" + pl.col(\"borough_start\")).alias(\"borough_start\")\n",
    "        ),\n",
    "        mapping=aes(x=\"distance\", y=\"duration\", color=\"borough_end\"),\n",
    "    )\n",
    "    + geom_point(size=0.25, alpha=0.5)\n",
    "    + geom_smooth(method=\"lowess\", size=2, se=False, alpha=0.8)\n",
    "    + xlim(0, 15)\n",
    "    + ylim(0, 60)\n",
    "    + scale_color_brewer(type=\"qualitative\", palette=\"Set1\")\n",
    "    + facet_wrap(\"borough_start\")\n",
    "    + labs(\n",
    "        title=\"Trip distance and duration cross borough\",\n",
    "        x=\"Distance (km)\",\n",
    "        y=\"Duration (min)\",\n",
    "        color=\"To Borough\",\n",
    "    )\n",
    "    + theme_linedraw(base_size=14)\n",
    "    + theme(figure_size=(8, 6))\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Styling DataFrames With Great Tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars.selectors as cs\n",
    "\n",
    "busiest_stations = (\n",
    "    trips.group_by(  \n",
    "        station=pl.col(\"station_start\"), date=pl.col(\"datetime_start\").dt.date()\n",
    "    )\n",
    "    .agg(\n",
    "        borough=pl.col(\"borough_start\").first(),\n",
    "        neighborhood=pl.col(\"neighborhood_start\").first(),\n",
    "        num_rides=pl.len(),\n",
    "        percent_member=(pl.col(\"rider_type\") == \"member\").mean(),\n",
    "        percent_electric=(pl.col(\"bike_type\") == \"electric\").mean(),\n",
    "    )\n",
    "    .sort(\"date\")\n",
    "    .group_by(\"station\")\n",
    "    .agg(\n",
    "        cs.string().first(),\n",
    "        cs.numeric().mean(),\n",
    "        pl.col(\"num_rides\").alias(\"rides_per_day\"),  \n",
    "    )\n",
    "    .sort(\"num_rides\", descending=True)\n",
    "    .group_by(\"borough\", maintain_order=True)\n",
    "    .head(3)\n",
    ")\n",
    "\n",
    "busiest_stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from great_tables import GT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GT(busiest_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from great_tables import style, md\n",
    "\n",
    "(\n",
    "    GT(busiest_stations)\n",
    "    .tab_stub(rowname_col=\"station\", groupname_col=\"borough\")  \n",
    "    .cols_label(  \n",
    "        neighborhood=\"Neighborhood\",\n",
    "        num_rides=\"Mean Daily Rides\",\n",
    "        percent_member=\"Members\",\n",
    "        percent_electric=\"E-Bikes\",\n",
    "        rides_per_day=\"Rides Per Day\",\n",
    "    )\n",
    "    .tab_header(\n",
    "        title=\"Busiest Bike Stations in NYC\",\n",
    "        subtitle=\"In March 2024, Per Borough\",\n",
    "    )\n",
    "    .tab_stubhead(label=\"Station\")\n",
    "    .fmt_number(columns=\"num_rides\", decimals=1)\n",
    "    .fmt_percent(columns=cs.starts_with(\"percent_\"), decimals=0)  \n",
    "    .fmt_nanoplot(columns=\"rides_per_day\", reference_line=\"mean\")\n",
    "    .data_color(columns=\"num_rides\", palette=\"Blues\")\n",
    "    .tab_options(row_group_font_weight=\"bold\")\n",
    "    .tab_source_note(\n",
    "        source_note=md(\n",
    "            \"Source: [NYC Citi Bike](https://citibikenyc.com/system-data)\"\n",
    "        )\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Takeaways"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}