From 4b256c680e853e9f1f34707717bc82f7f939a483 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 12 Jul 2023 12:10:33 -0400 Subject: [PATCH 01/33] ENH: eyetracking plot_heatmap function --- doc/visualization.rst | 15 ++ mne/viz/__init__.py | 1 + mne/viz/eyetracking/__init__.py | 5 + mne/viz/eyetracking/heatmap.py | 160 ++++++++++++++++++++++ mne/viz/eyetracking/tests/__init__.py | 0 mne/viz/eyetracking/tests/test_heatmap.py | 28 ++++ 6 files changed, 209 insertions(+) create mode 100644 mne/viz/eyetracking/__init__.py create mode 100644 mne/viz/eyetracking/heatmap.py create mode 100644 mne/viz/eyetracking/tests/__init__.py create mode 100644 mne/viz/eyetracking/tests/test_heatmap.py diff --git a/doc/visualization.rst b/doc/visualization.rst index 62fa54a8cee..1af32d21015 100644 --- a/doc/visualization.rst +++ b/doc/visualization.rst @@ -86,3 +86,18 @@ Visualization set_browser_backend get_browser_backend use_browser_backend + + + +.. currentmodule:: mne.viz.eyetracking + +:py:mod:`mne.viz.eyetracking`: + +.. automodule:: mne.viz.eyetracking + :no-members: + :no-inherited-members: + +.. autosummary:: + :toctree: generated/ + + plot_gaze diff --git a/mne/viz/__init__.py b/mne/viz/__init__.py index fd9a60faed7..ad4e33f5b33 100644 --- a/mne/viz/__init__.py +++ b/mne/viz/__init__.py @@ -87,3 +87,4 @@ from ._brain import Brain from ._figure import get_browser_backend, set_browser_backend, use_browser_backend from ._proj import plot_projs_joint +from . import eyetracking diff --git a/mne/viz/eyetracking/__init__.py b/mne/viz/eyetracking/__init__.py new file mode 100644 index 00000000000..7de13fd8900 --- /dev/null +++ b/mne/viz/eyetracking/__init__.py @@ -0,0 +1,5 @@ +"""Eye-tracking visualization routines.""" +# +# License: BSD-3-Clause + +from .heatmap import plot_gaze diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py new file mode 100644 index 00000000000..35535836fca --- /dev/null +++ b/mne/viz/eyetracking/heatmap.py @@ -0,0 +1,160 @@ +# Authors: Scott Huberty +# +# License: BSD-3-Clause + +import matplotlib.pyplot as plt +import numpy as np + +from ..utils import plt_show +from ...utils import _validate_type, logger + + +def plot_gaze( + epochs, + width, + height, + n_bins=25, + sigma=1, + cmap=None, + vmin=None, + vmax=None, + make_transparent=True, + axes=None, + show=True, +): + """Plot a heatmap of eyetracking gaze data. + + Parameters + ---------- + epochs : mne.Epochs + The epochs object containing the gaze data. + width : int + The width dimension of the plot canvas. For example, if the eyegaze data units + are pixels, and the display screens resolution was 1920x1080, then the width + should be 1920. + height : int + The height dimension of the plot canvas. For example, if the eyegaze data units + are pixels, and the display screens resolution was 1920x1080, then the height + should be 1080. + n_bins : int + The number of bins to use for the heatmap. Default is 25, which means the + heatmap will be a 25x25 grid. + sigma : float | None + The sigma value for the gaussian kernel used to smooth the heatmap. + If ``None``, no smoothing is applied. Default is 1. + cmap : matplotlib colormap | str | None + The matplotlib colormap to use. Defaults to None, which means the + colormap will default to matplotlib's default. + vmin : float | None + The minimum value for the colormap. The unit is seconds, for dwell time + to the pixel coordinate. If ``None``, the minimum value is set to the + minimum value of the heatmap. Default is ``None``. + vmax : float | None + The maximum value for the colormap. The unit is seconds, for dwell time + to the pixel coordinate. If ``None``, the maximum value is set to the + maximum value of the heatmap. Default is ``None``. + make_transparent : bool + Whether to make the background transparent. Default is ``True``. + axes : matplotlib.axes.Axes | None + The axes to plot on. If ``None``, a new figure and axes are created. + Default is ``None``. + show : bool + Whether to show the plot. Default is ``True``. + + Returns + ------- + fig : instance of matplotlib.figure.Figure + The resulting figure object for the heatmap plot. + """ + import matplotlib.colors as mcolors + from scipy.ndimage import gaussian_filter + from mne import BaseEpochs + + _validate_type(epochs, BaseEpochs, "epochs") + + # Find xpos and ypos channels: + # In principle we could check the coil_type for eyetrack position channels, + # which could be more robust if future readers use different channel names? + xpos_indices = np.where(np.char.startswith(epochs.ch_names, "xpos"))[0] + ypos_indices = np.where(np.char.startswith(epochs.ch_names, "ypos"))[0] + + data = epochs.get_data() + x_data = data[:, xpos_indices, :] + y_data = data[:, ypos_indices, :] + if xpos_indices.size > 1: # binocular recording. Average across eyes + logger.info("Detected binocular recording. Averaging positions across eyes.") + x_data = np.nanmean(x_data, axis=1) # shape (n_epochs, n_samples) + y_data = np.nanmean(y_data, axis=1) + x_data = x_data.flatten() + y_data = y_data.flatten() + gaze_data = np.vstack((x_data, y_data)).T # shape (n_samples, 2) + # Make sure gaze position data is within screen bounds + mask = ( + (gaze_data[:, 0] > 0) + & (gaze_data[:, 1] > 0) + & (gaze_data[:, 0] < width) + & (gaze_data[:, 1] < height) + ) + canvas = gaze_data[mask].astype(float) + + # Create heatmap + heatmap, _, _ = np.histogram2d( + canvas[:, 0], + canvas[:, 1], + bins=n_bins, + range=[[0, width], [0, height]], + ) + heatmap = heatmap.T # transpose to match screen coordinates + # Convert density from samples to seconds + heatmap /= epochs.info["sfreq"] + if sigma: + # Smooth heatmap + heatmap = gaussian_filter(heatmap, sigma=sigma) + + # Prepare axes + if axes is not None: + from matplotlib.axes import Axes + + _validate_type(axes, Axes, "axes") + ax = axes + fig = ax.get_figure() + else: + fig, ax = plt.subplots() + + ax.set_title("Gaze heatmap") + ax.set_xlabel("X position") + ax.set_ylabel("Y position") + + if make_transparent: + # Make heatmap transparent + norm = mcolors.Normalize(vmin=0, vmax=np.nanmax(heatmap)) + alphas = norm(heatmap) + else: + alphas = 1.0 + + # Prepare the heatmap + vmin = np.nanmin(heatmap) if vmin is None else vmin + vmax = np.nanmax(heatmap) if vmax is None else vmax + extent = [0, width, height, 0] # origin is the top left of the screen + # Plot heatmap + im = ax.imshow( + heatmap, + aspect="equal", + interpolation="gaussian", + alpha=alphas, + cmap=cmap, + extent=extent, + origin="upper", + vmin=vmin, + vmax=vmax, + ) + + # Prepare the colorbar + cbar = fig.colorbar(im, ax=ax, label="Dwell time (seconds)") + # Prepare the colorbar transparency + if make_transparent: + cbar.set_alpha(1.0) + cbar.solids.set(alpha=np.linspace(0, np.max(alphas), 256)) + + plt_show(show) + return fig diff --git a/mne/viz/eyetracking/tests/__init__.py b/mne/viz/eyetracking/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py new file mode 100644 index 00000000000..87b3d9df7d2 --- /dev/null +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -0,0 +1,28 @@ +import matplotlib.pyplot as plt +import pytest + +from mne import make_fixed_length_epochs +from mne.datasets.testing import data_path, requires_testing_data +from mne.io import read_raw_eyelink +from mne.preprocessing.eyetracking import interpolate_blinks +from mne.utils import requires_pandas +from mne.viz.eyetracking import plot_gaze + +fname = data_path(download=False) / "eyetrack" / "test_eyelink.asc" + + +@requires_testing_data +@requires_pandas +@pytest.mark.parametrize( + "fname, axes", + [(fname, None), (fname, True)], +) +def test_plot_heatmap(fname, axes): + """Test plot_gaze.""" + raw = read_raw_eyelink(fname, find_overlaps=True) + interpolate_blinks(raw, interpolate_gaze=True, buffer=(0.05, 0.2)) + epochs = make_fixed_length_epochs(raw, duration=5) + + if axes: + axes = plt.subplot() + plot_gaze(epochs, width=1920, height=1080, axes=axes) From 318d974fa3b482e5aeda4e5b9448e4b021a301a9 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 12 Jul 2023 12:23:10 -0400 Subject: [PATCH 02/33] FIX: next matplotlib --- mne/viz/eyetracking/heatmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 35535836fca..fd75bad5636 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -2,7 +2,6 @@ # # License: BSD-3-Clause -import matplotlib.pyplot as plt import numpy as np from ..utils import plt_show @@ -66,6 +65,7 @@ def plot_gaze( fig : instance of matplotlib.figure.Figure The resulting figure object for the heatmap plot. """ + import matplotlib.pyplot as plt import matplotlib.colors as mcolors from scipy.ndimage import gaussian_filter from mne import BaseEpochs From 895f2ea7b24a6a3bd3dc6db5d6c6a680c57232be Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 19 Jul 2023 12:05:50 -0400 Subject: [PATCH 03/33] WIP: refactor heatmap code --- doc/visualization.rst | 2 - mne/viz/eyetracking/__init__.py | 2 +- mne/viz/eyetracking/heatmap.py | 163 ++++++++++++++++++++------------ 3 files changed, 104 insertions(+), 63 deletions(-) diff --git a/doc/visualization.rst b/doc/visualization.rst index 1af32d21015..01035574c8d 100644 --- a/doc/visualization.rst +++ b/doc/visualization.rst @@ -87,8 +87,6 @@ Visualization get_browser_backend use_browser_backend - - .. currentmodule:: mne.viz.eyetracking :py:mod:`mne.viz.eyetracking`: diff --git a/mne/viz/eyetracking/__init__.py b/mne/viz/eyetracking/__init__.py index 7de13fd8900..907c1d041e3 100644 --- a/mne/viz/eyetracking/__init__.py +++ b/mne/viz/eyetracking/__init__.py @@ -2,4 +2,4 @@ # # License: BSD-3-Clause -from .heatmap import plot_gaze +from .heatmap import plot_heatmap_array, plot_gaze diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index fd75bad5636..7808f0e8c36 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -8,18 +8,103 @@ from ...utils import _validate_type, logger +def plot_heatmap_array( + data, + width, + height, + cmap=None, + alpha=None, + vmin=None, + vmax=None, + axes=None, + show=True, +): + """Plot a heatmap of eyetracking gaze data from a numpy array. + + Parameters + ---------- + data : numpy array + The heatmap data to plot. + width : int + The width dimension of the plot canvas. For example, if the eyegaze data units + are pixels, and the display screens resolution was 1920x1080, then the width + should be 1920. + height : int + The height dimension of the plot canvas. For example, if the eyegaze data units + are pixels, and the display screens resolution was 1920x1080, then the height + should be 1080. + cmap : matplotlib colormap | str | None + The matplotlib colormap to use. Defaults to None, which means the colormap will + default to matplotlib's default. + alpha : float | array-like | None + The alpha value(s) to use for the colormap. If ``None``, the alpha value is set + to 1. Default is ``None``. If an array-like object is passed, the shape must + match the shape of the data array. + vmin : float | None + The minimum value for the colormap. The unit is seconds, for the dwell time to + the pixel coordinate. If ``None``, the minimum value is set to the minimum value + of the heatmap. Default is ``None``. + vmax : float | None + The maximum value for the colormap. The unit is seconds, for the dwell time to + the pixel coordinate. If ``None``, the maximum value is set to the maximum value + of the heatmap. Default is ``None``. + axes : matplotlib.axes.Axes | None + The axes to plot on. If ``None``, a new figure and axes are created. + show : bool + Whether to show the plot. Default is ``True``. + """ + import matplotlib.pyplot as plt + + # Prepare axes + if axes is not None: + from matplotlib.axes import Axes + + _validate_type(axes, Axes, "axes") + ax = axes + fig = ax.get_figure() + else: + fig, ax = plt.subplots() + + ax.set_title("Gaze heatmap") + ax.set_xlabel("X position") + ax.set_ylabel("Y position") + + # Prepare the heatmap + alphas = 1 if alpha is None else alpha + vmin = np.nanmin(data) if vmin is None else vmin + vmax = np.nanmax(data) if vmax is None else vmax + extent = [0, width, height, 0] # origin is the top left of the screen + # Plot heatmap + im = ax.imshow( + data, + aspect="equal", + interpolation="none", + cmap=cmap, + alpha=alphas, + extent=extent, + origin="upper", + vmin=vmin, + vmax=vmax, + ) + + # Prepare the colorbar + fig.colorbar(im, ax=ax, label="Dwell time (seconds)") + plt_show(show) + return fig + + def plot_gaze( epochs, width, height, - n_bins=25, + bin_width, sigma=1, cmap=None, vmin=None, vmax=None, - make_transparent=True, axes=None, show=True, + return_array=False, ): """Plot a heatmap of eyetracking gaze data. @@ -35,7 +120,7 @@ def plot_gaze( The height dimension of the plot canvas. For example, if the eyegaze data units are pixels, and the display screens resolution was 1920x1080, then the height should be 1080. - n_bins : int + bin_width : int The number of bins to use for the heatmap. Default is 25, which means the heatmap will be a 25x25 grid. sigma : float | None @@ -65,16 +150,15 @@ def plot_gaze( fig : instance of matplotlib.figure.Figure The resulting figure object for the heatmap plot. """ - import matplotlib.pyplot as plt - import matplotlib.colors as mcolors from scipy.ndimage import gaussian_filter from mne import BaseEpochs _validate_type(epochs, BaseEpochs, "epochs") # Find xpos and ypos channels: - # In principle we could check the coil_type for eyetrack position channels, - # which could be more robust if future readers use different channel names? + # We could check the channeltype for eyegaze channels, which could be more robust if + # future readers use different channel names? However channel type will not + # differentiate between x-position and y-position. xpos_indices = np.where(np.char.startswith(epochs.ch_names, "xpos"))[0] ypos_indices = np.where(np.char.startswith(epochs.ch_names, "ypos"))[0] @@ -88,7 +172,7 @@ def plot_gaze( x_data = x_data.flatten() y_data = y_data.flatten() gaze_data = np.vstack((x_data, y_data)).T # shape (n_samples, 2) - # Make sure gaze position data is within screen bounds + # filter out gaze data that is outside screen bounds mask = ( (gaze_data[:, 0] > 0) & (gaze_data[:, 1] > 0) @@ -97,64 +181,23 @@ def plot_gaze( ) canvas = gaze_data[mask].astype(float) - # Create heatmap - heatmap, _, _ = np.histogram2d( + # Create 2D histogram + x_bins = np.linspace(0, width, width // bin_width) + y_bins = np.linspace(0, height, height // bin_width) + hist, _, _ = np.histogram2d( canvas[:, 0], canvas[:, 1], - bins=n_bins, + bins=(x_bins, y_bins), range=[[0, width], [0, height]], ) - heatmap = heatmap.T # transpose to match screen coordinates + hist = hist.T # transpose to match screen coordinates. i.e. width > height # Convert density from samples to seconds - heatmap /= epochs.info["sfreq"] + hist /= epochs.info["sfreq"] if sigma: # Smooth heatmap - heatmap = gaussian_filter(heatmap, sigma=sigma) - - # Prepare axes - if axes is not None: - from matplotlib.axes import Axes + hist = gaussian_filter(hist, sigma=sigma) - _validate_type(axes, Axes, "axes") - ax = axes - fig = ax.get_figure() - else: - fig, ax = plt.subplots() - - ax.set_title("Gaze heatmap") - ax.set_xlabel("X position") - ax.set_ylabel("Y position") - - if make_transparent: - # Make heatmap transparent - norm = mcolors.Normalize(vmin=0, vmax=np.nanmax(heatmap)) - alphas = norm(heatmap) - else: - alphas = 1.0 - - # Prepare the heatmap - vmin = np.nanmin(heatmap) if vmin is None else vmin - vmax = np.nanmax(heatmap) if vmax is None else vmax - extent = [0, width, height, 0] # origin is the top left of the screen - # Plot heatmap - im = ax.imshow( - heatmap, - aspect="equal", - interpolation="gaussian", - alpha=alphas, - cmap=cmap, - extent=extent, - origin="upper", - vmin=vmin, - vmax=vmax, - ) - - # Prepare the colorbar - cbar = fig.colorbar(im, ax=ax, label="Dwell time (seconds)") - # Prepare the colorbar transparency - if make_transparent: - cbar.set_alpha(1.0) - cbar.solids.set(alpha=np.linspace(0, np.max(alphas), 256)) - - plt_show(show) + fig = plot_heatmap_array(hist, width, height, cmap, vmin, vmax, axes) + if return_array: + return fig, hist return fig From 320fc39deb0489465954278d5cd06114763fdc44 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 09:25:31 -0400 Subject: [PATCH 04/33] ENH, DOC: refactor plot_gaze and add to API doc - took another pass at this function to simplify it. mainly, made - plot_heatmap_array private --- doc/visualization.rst | 15 +++ mne/viz/eyetracking/__init__.py | 2 +- mne/viz/eyetracking/heatmap.py | 203 ++++++++++++++------------------ 3 files changed, 107 insertions(+), 113 deletions(-) diff --git a/doc/visualization.rst b/doc/visualization.rst index 14f906d1693..a88d373c875 100644 --- a/doc/visualization.rst +++ b/doc/visualization.rst @@ -87,6 +87,21 @@ Visualization get_browser_backend use_browser_backend +Eyetracking +----------- + +.. currentmodule:: mne.viz.eyetracking + +:py:mod:`mne.viz.eyetracking`: + +.. automodule:: mne.viz.eyetracking + :no-members: + :no-inherited-members: +.. autosummary:: + :toctree: generated/ + + plot_gaze + UI Events --------- diff --git a/mne/viz/eyetracking/__init__.py b/mne/viz/eyetracking/__init__.py index 907c1d041e3..7de13fd8900 100644 --- a/mne/viz/eyetracking/__init__.py +++ b/mne/viz/eyetracking/__init__.py @@ -2,4 +2,4 @@ # # License: BSD-3-Clause -from .heatmap import plot_heatmap_array, plot_gaze +from .heatmap import plot_gaze diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 7808f0e8c36..12a69cd8e57 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -3,142 +3,60 @@ # License: BSD-3-Clause import numpy as np +from scipy.ndimage import gaussian_filter -from ..utils import plt_show -from ...utils import _validate_type, logger - - -def plot_heatmap_array( - data, - width, - height, - cmap=None, - alpha=None, - vmin=None, - vmax=None, - axes=None, - show=True, -): - """Plot a heatmap of eyetracking gaze data from a numpy array. - - Parameters - ---------- - data : numpy array - The heatmap data to plot. - width : int - The width dimension of the plot canvas. For example, if the eyegaze data units - are pixels, and the display screens resolution was 1920x1080, then the width - should be 1920. - height : int - The height dimension of the plot canvas. For example, if the eyegaze data units - are pixels, and the display screens resolution was 1920x1080, then the height - should be 1080. - cmap : matplotlib colormap | str | None - The matplotlib colormap to use. Defaults to None, which means the colormap will - default to matplotlib's default. - alpha : float | array-like | None - The alpha value(s) to use for the colormap. If ``None``, the alpha value is set - to 1. Default is ``None``. If an array-like object is passed, the shape must - match the shape of the data array. - vmin : float | None - The minimum value for the colormap. The unit is seconds, for the dwell time to - the pixel coordinate. If ``None``, the minimum value is set to the minimum value - of the heatmap. Default is ``None``. - vmax : float | None - The maximum value for the colormap. The unit is seconds, for the dwell time to - the pixel coordinate. If ``None``, the maximum value is set to the maximum value - of the heatmap. Default is ``None``. - axes : matplotlib.axes.Axes | None - The axes to plot on. If ``None``, a new figure and axes are created. - show : bool - Whether to show the plot. Default is ``True``. - """ - import matplotlib.pyplot as plt - - # Prepare axes - if axes is not None: - from matplotlib.axes import Axes - - _validate_type(axes, Axes, "axes") - ax = axes - fig = ax.get_figure() - else: - fig, ax = plt.subplots() - - ax.set_title("Gaze heatmap") - ax.set_xlabel("X position") - ax.set_ylabel("Y position") - - # Prepare the heatmap - alphas = 1 if alpha is None else alpha - vmin = np.nanmin(data) if vmin is None else vmin - vmax = np.nanmax(data) if vmax is None else vmax - extent = [0, width, height, 0] # origin is the top left of the screen - # Plot heatmap - im = ax.imshow( - data, - aspect="equal", - interpolation="none", - cmap=cmap, - alpha=alphas, - extent=extent, - origin="upper", - vmin=vmin, - vmax=vmax, - ) - # Prepare the colorbar - fig.colorbar(im, ax=ax, label="Dwell time (seconds)") - plt_show(show) - return fig +from ..utils import plt_show +from ...utils import _validate_type, logger, fill_doc +@fill_doc def plot_gaze( epochs, width, height, - bin_width, + bin_width=10, sigma=1, cmap=None, + alpha=None, vmin=None, vmax=None, axes=None, show=True, - return_array=False, ): """Plot a heatmap of eyetracking gaze data. Parameters ---------- epochs : mne.Epochs - The epochs object containing the gaze data. + The :class:`~mne.Epochs` object containing eyegaze channels. width : int The width dimension of the plot canvas. For example, if the eyegaze data units - are pixels, and the display screens resolution was 1920x1080, then the width + are pixels, and the participant screen resolution was 1920x1080, then the width should be 1920. height : int The height dimension of the plot canvas. For example, if the eyegaze data units - are pixels, and the display screens resolution was 1920x1080, then the height + are pixels, and the participant screen resolution was 1920x1080, then the height should be 1080. bin_width : int - The number of bins to use for the heatmap. Default is 25, which means the - heatmap will be a 25x25 grid. - sigma : float | None - The sigma value for the gaussian kernel used to smooth the heatmap. - If ``None``, no smoothing is applied. Default is 1. + The number of eyegaze units per square bin that are used to create the heatmap. + Default is 10, meaning that if the eyegaze units are pixels, each bin is 10x10 + pixels. See the appendix section of :ref:`tut-eyetrack-heatmap` for more + detail. + sigma : int | float + The amount of smoothing applied to the heatmap data. If ``None``, + no smoothing is applied. Default is 1. cmap : matplotlib colormap | str | None - The matplotlib colormap to use. Defaults to None, which means the - colormap will default to matplotlib's default. + The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning + the colormap will default to matplotlib's default. vmin : float | None The minimum value for the colormap. The unit is seconds, for dwell time - to the pixel coordinate. If ``None``, the minimum value is set to the + to the bin coordinate. If ``None``, the minimum value is set to the minimum value of the heatmap. Default is ``None``. vmax : float | None The maximum value for the colormap. The unit is seconds, for dwell time - to the pixel coordinate. If ``None``, the maximum value is set to the + to the bin coordinate. If ``None``, the maximum value is set to the maximum value of the heatmap. Default is ``None``. - make_transparent : bool - Whether to make the background transparent. Default is ``True``. axes : matplotlib.axes.Axes | None The axes to plot on. If ``None``, a new figure and axes are created. Default is ``None``. @@ -149,16 +67,17 @@ def plot_gaze( ------- fig : instance of matplotlib.figure.Figure The resulting figure object for the heatmap plot. + + Notes + ----- + .. versionadded:: 1.6 """ - from scipy.ndimage import gaussian_filter from mne import BaseEpochs _validate_type(epochs, BaseEpochs, "epochs") - # Find xpos and ypos channels: - # We could check the channeltype for eyegaze channels, which could be more robust if - # future readers use different channel names? However channel type will not - # differentiate between x-position and y-position. + # Find xpos and ypos channels. if future readers use different channel names than + # xpos/ypos, we will need a different way to identify these channels xpos_indices = np.where(np.char.startswith(epochs.ch_names, "xpos"))[0] ypos_indices = np.where(np.char.startswith(epochs.ch_names, "ypos"))[0] @@ -172,7 +91,7 @@ def plot_gaze( x_data = x_data.flatten() y_data = y_data.flatten() gaze_data = np.vstack((x_data, y_data)).T # shape (n_samples, 2) - # filter out gaze data that is outside screen bounds + # mask gaze data that is outside screen bounds mask = ( (gaze_data[:, 0] > 0) & (gaze_data[:, 1] > 0) @@ -193,11 +112,71 @@ def plot_gaze( hist = hist.T # transpose to match screen coordinates. i.e. width > height # Convert density from samples to seconds hist /= epochs.info["sfreq"] + # Smooth the heatmap if sigma: - # Smooth heatmap hist = gaussian_filter(hist, sigma=sigma) - fig = plot_heatmap_array(hist, width, height, cmap, vmin, vmax, axes) - if return_array: - return fig, hist + return _plot_heatmap_array( + hist, + width=width, + height=height, + cmap=cmap, + alpha=alpha, + vmin=vmin, + vmax=vmax, + axes=axes, + show=show, + ) + + +def _plot_heatmap_array( + data, + width, + height, + cmap=None, + alpha=None, + vmin=None, + vmax=None, + axes=None, + show=True, +): + """Plot a heatmap of eyetracking gaze data from a numpy array.""" + import matplotlib.pyplot as plt + + # Prepare axes + if axes is not None: + from matplotlib.axes import Axes + + _validate_type(axes, Axes, "axes") + ax = axes + fig = ax.get_figure() + else: + fig, ax = plt.subplots() + + ax.set_title("Gaze heatmap") + ax.set_xlabel("X position") + ax.set_ylabel("Y position") + + # Prepare the heatmap + alphas = 1 if alpha is None else alpha + vmin = np.nanmin(data) if vmin is None else vmin + vmax = np.nanmax(data) if vmax is None else vmax + extent = [0, width, height, 0] # origin is the top left of the screen + + # Plot heatmap + im = ax.imshow( + data, + aspect="equal", + cmap=cmap, + alpha=alphas, + extent=extent, + origin="upper", + vmin=vmin, + vmax=vmax, + ) + + # Prepare the colorbar + # stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph + fig.colorbar(im, ax=ax, label="Dwell time (seconds)", fraction=0.046, pad=0.04) + plt_show(show) return fig From 09ce77d8e2b154fb78848c8628586accd2d9e5fd Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 09:27:33 -0400 Subject: [PATCH 05/33] DOC: update eyelink dataset and its description - added a second eyelink dataset for a heatmap tutorial --- doc/overview/datasets_index.rst | 13 ++++++++----- mne/datasets/config.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/doc/overview/datasets_index.rst b/doc/overview/datasets_index.rst index 3946ef64be5..2b40ebe1d75 100644 --- a/doc/overview/datasets_index.rst +++ b/doc/overview/datasets_index.rst @@ -481,15 +481,18 @@ EYELINK ======= :func:`mne.datasets.eyelink.data_path` -A small example dataset from a pupillary light reflex experiment. Both EEG (EGI) and -eye-tracking (SR Research EyeLink; ASCII format) data were recorded and stored in -separate files. 1 participant fixated on the screen while short light flashes appeared. -Event onsets were recorded by a photodiode attached to the screen and were -sent to both the EEG and eye-tracking systems. +Two small example datasets of eye-tracking data from SR Research EyeLink. the "eeg-et" +dataset contains both EEG (EGI) and eye-tracking (ASCII format) data recorded from a +pupillary light reflex experiment, stored in separate files. 1 participant fixated +on the screen while short light flashes appeared. Event onsets were recorded by a +photodiode attached to the screen and were sent to both the EEG and eye-tracking +systems. The second dataset, in the "freeviewing" directory, contains only eye-tracking +data (ASCII format) from 1 participant who was free-viewing a natural scene. .. topic:: Examples * :ref:`tut-eyetrack` + * :ref:`tut-eyetrack-heatmap` References ========== diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 21f35a50c3f..f8e9713ef16 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -345,9 +345,9 @@ # eyelink dataset MNE_DATASETS["eyelink"] = dict( - archive_name="eeg-eyetrack_data.zip", - hash="md5:c4fc788fe01737e08e9086c90cab642d", - url=("https://osf.io/63fjm/download?version=1"), - folder_name="eyelink-example-data", + archive_name="MNE-eyelink-data.zip", + hash="md5:68a6323ef17d655f1a659c3290ee1c3f", + url=("https://osf.io/xsu4g/download?version=1"), + folder_name="MNE-eyelink-data", config_key="MNE_DATASETS_EYELINK_PATH", ) From 26888cd0b72143369126024d3899b6098d3ee886 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 09:30:02 -0400 Subject: [PATCH 06/33] DOC: add a new eyetracking tutorial - added a new tutorial for plotting eyetracking heatmaps - included a brief example of heatmap plotting in current tutorial --- .../preprocessing/90_eyetracking_data.py | 14 +- .../visualization/30_eyetracking_heatmap.py | 177 ++++++++++++++++++ 2 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tutorials/visualization/30_eyetracking_heatmap.py diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index a82a7147d24..4b1fce73c00 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -32,8 +32,8 @@ from mne.datasets.eyelink import data_path from mne.preprocessing.eyetracking import read_eyelink_calibration -et_fpath = data_path() / "sub-01_task-plr_eyetrack.asc" -eeg_fpath = data_path() / "sub-01_task-plr_eeg.mff" +et_fpath = data_path() / "eeg-et" / "sub-01_task-plr_eyetrack.asc" +eeg_fpath = data_path() / "eeg-et" / "sub-01_task-plr_eeg.mff" raw_et = mne.io.read_raw_eyelink(et_fpath, create_annotations=["blinks"]) raw_eeg = mne.io.read_raw_egi(eeg_fpath, preload=True, verbose="warning") @@ -197,6 +197,16 @@ epochs = mne.Epochs(raw_et, events=et_events, event_id=event_dict, tmin=-0.3, tmax=3) epochs[:8].plot(events=et_events, event_id=event_dict, order=picks_idx) +# %% +# For this experiment, the participant was instructed to fixate on a crosshair in the +# center of the screen. Let's plot the gaze position data to confirm that the +# participant primarily kept their gaze fixated at the center of the screen. + +mne.viz.eyetracking.plot_gaze(epochs, width=1920, height=1080, sigma=2) + +# %% +# .. seealso:: :ref:`tut-eyetrack-heatmap` + # %% # Finally, let's plot the evoked responses to the light flashes to get a sense of the # average pupillary light response, and the associated ERP in the EEG data. diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py new file mode 100644 index 00000000000..d129e403695 --- /dev/null +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +""" +.. _tut-eyetrack-heatmap: + +============================================= +Plotting eye-tracking heatmaps in MNE-Python +============================================= + +This tutorial covers plotting eye-tracking position data as a heatmap. + +.. seealso:: :ref:`tut-importing-eyetracking-data` + +""" + +# %% +# Data loading +# ------------ +# +# As usual we start by importing the modules we need and loading some +# :ref:`example data `: eye-tracking data recorded from SR research's +# ``'.asc'`` file format. We'll also define a helper function to plot image files. + + +import matplotlib.pyplot as plt + +import mne +from mne.viz.eyetracking import plot_gaze + + +# Define a function to plot stimuli photos +def plot_images(image_paths, ax, titles=None): + for i, image_path in enumerate(image_paths): + ax[i].imshow(plt.imread(image_path)) + if titles: + ax[i].set_title(titles[i]) + return fig + + +# define variables to pass to the plot_gaze function +px_width, px_height = 1920, 1080 + +task_fpath = mne.datasets.eyelink.data_path() / "freeviewing" +et_fpath = task_fpath / "sub-01_task-freeview_eyetrack.asc" +natural_stim_fpath = task_fpath / "stim" / "naturalistic.png" +scrambled_stim_fpath = task_fpath / "stim" / "scrambled.png" +image_paths = list([natural_stim_fpath, scrambled_stim_fpath]) + + +raw = mne.io.read_raw_eyelink(et_fpath) + +# %% +# Task background +# --------------- +# +# Participants watched videos while eye-tracking data was collected. The videos showed +# people dancing, or scrambled versions of those videos. Each video lasted about 20 +# seconds. An image of each video is shown below. + +fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) +plot_images(image_paths, ax, ["Natural", "Scrambled"]) + + +# %% +# Process and epoch the data +# -------------------------- +# +# First we will interpolate missing data during blinks and epoch the data. + +mne.preprocessing.eyetracking.interpolate_blinks(raw, interpolate_gaze=True) +raw.annotations.rename({"dvns": "natural", "dvss": "scrambled"}) # more intuitive +event_ids = {"natural": 1, "scrambled": 2} +events, event_dict = mne.events_from_annotations(raw, event_id=event_ids) + +epochs = mne.Epochs( + raw, events=events, event_id=event_dict, tmin=0, tmax=20, baseline=None +) + +# %% +# .. seealso:: :ref:`tut-eyetrack` +# + +# %% +# Plot a heatmap of the eye-tracking data +# --------------------------------------- +# +# To make a heatmap of the eye-tracking data, we can use the function +# :func:`~mne.viz.eyetracking.plot_gaze`. We will need to define the dimensions of our +# canvas; for this file, the eye position data are reported in pixels, so we'll use the +# screen resolution of the participant screen (1920x1080) as the width and height. We +# can also use the sigma parameter to smooth the plot. + +fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) +plot_gaze( + epochs["natural"], + width=px_width, + height=px_height, + sigma=5, + cmap="jet", + axes=ax[0], + show=False, +) +ax[0].set_title("Gaze Heatmap (Natural)") +plot_gaze( + epochs["scrambled"], + width=px_width, + height=px_height, + sigma=5, + cmap="jet", + axes=ax[1], + show=False, +) +ax[1].set_title("Gaze Heatmap (Scrambled)") +plt.show() + +# %% +# Overlaying plots with images +# ---------------------------- +# +# We can use matplotlib to plot the gaze heatmaps on top of the stimuli images. We'll +# customize a :class:`~matplotlib.colors.Colormap` to make some values of the heatmap +# (in this case, the color black) completely transparent. We'll then use the ``vmin`` +# parameter to force the heatmap to start at a value greater than the darkest value in +# our previous heatmap, which will make those values transparent. We'll also set the +# ``alpha`` parameter of :func:`~mne.viz.eyetracking.plot_gaze` to make even the +# visible colors of the heatmap semi-transparent so that we can see the image +# underneath. + +cmap = plt.get_cmap("jet") +cmap.set_under("k", alpha=0) # make the lowest values transparent +fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) + +plot_images(image_paths, ax) +plot_gaze( + epochs["natural"], + width=px_width, + height=px_height, + vmin=0.01, + alpha=0.8, + sigma=5, + cmap=cmap, + axes=ax[0], + show=False, +) +ax[0].set_title("Natural)") + +plot_gaze( + epochs["scrambled"], + width=px_width, + height=px_height, + sigma=5, + vmin=0.01, + alpha=0.8, + cmap=cmap, + axes=ax[1], + show=False, +) +ax[1].set_title("Scrambled") +plt.show() + + +# %% +# Appendix: Understanding the ``bin_width`` parameter +# --------------------------------------------------- +# +# The ``bin_width`` parameter of :func:`~mne.viz.eyetracking.plot_gaze` controls the +# resolution of the heatmap. The heatmap is created by binning the gaze data into +# square bins of a given width. The value of each bin is the sum of the dwell time +# (in seconds) of all gaze samples that fall within the bin. The heatmap is then +# smoothed using a Gaussian filter, which can be controlled via the ``sigma`` +# parameter. If we set ``sigma`` to 0, we can clearly see the binned data: + +# make each bin 120x120 pixels and don't smooth the data +plot_gaze(epochs["natural"], width=px_width, height=px_height, bin_width=120, sigma=0) + +# %% +# Making ``bin_width`` smaller results in a higher resolution heatmap: +plot_gaze(epochs["natural"], width=px_width, height=px_height, bin_width=20, sigma=0) From 2cfc3e1c885069a96aaae44fe433fbe3396b7e4c Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 09:38:34 -0400 Subject: [PATCH 07/33] TST: remove requires_pandas decorator from pytest --- mne/viz/eyetracking/tests/test_heatmap.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index 87b3d9df7d2..cb9f995e738 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -5,14 +5,13 @@ from mne.datasets.testing import data_path, requires_testing_data from mne.io import read_raw_eyelink from mne.preprocessing.eyetracking import interpolate_blinks -from mne.utils import requires_pandas from mne.viz.eyetracking import plot_gaze fname = data_path(download=False) / "eyetrack" / "test_eyelink.asc" +pd = pytest.importorskip("pandas") @requires_testing_data -@requires_pandas @pytest.mark.parametrize( "fname, axes", [(fname, None), (fname, True)], From 0e8985f2222dd9ee7c52bc05f9cde67c1bcbb988 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 09:40:46 -0400 Subject: [PATCH 08/33] DOC: fix typo --- tutorials/visualization/30_eyetracking_heatmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index d129e403695..13f63bc526f 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -141,7 +141,7 @@ def plot_images(image_paths, ax, titles=None): axes=ax[0], show=False, ) -ax[0].set_title("Natural)") +ax[0].set_title("Natural") plot_gaze( epochs["scrambled"], From 26054f0b566a65d0992d0993f29df4cfd141999c Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 10:12:22 -0400 Subject: [PATCH 09/33] FIX, DOC: add alpha parameter to API docstring - the alpha parameter was missing from the docstring which caused a numpydoc error --- mne/viz/eyetracking/heatmap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 12a69cd8e57..9c0d25718b5 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -49,6 +49,9 @@ def plot_gaze( cmap : matplotlib colormap | str | None The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning the colormap will default to matplotlib's default. + alpha : int | float | None + The transparency value of the heatmap. If ``None``, the alpha value is set to 1, + meaning the heatmap colors are fully opaque. Default is ``None``. vmin : float | None The minimum value for the colormap. The unit is seconds, for dwell time to the bin coordinate. If ``None``, the minimum value is set to the From aec89f8b74608804f774c3b7234dfd93da04834c Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Thu, 21 Sep 2023 16:50:48 -0400 Subject: [PATCH 10/33] DOC: in tutorial, dont baseline correct eyegaze channels - This is a tough one. In the "working with eye tracker data" tut, I DO want to baseline correct the pupil channel, but I DONT want to baseline correct eyegaze channels. I am not sure that it EVER makes sense to baseline correct eyegaze channels, given that they USUALLY represent a gaze coordinate, but I need to think about this more. - I may open a separate ticket about this matter --- tutorials/preprocessing/90_eyetracking_data.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index 4b1fce73c00..1b210734ec9 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -31,6 +31,7 @@ import mne from mne.datasets.eyelink import data_path from mne.preprocessing.eyetracking import read_eyelink_calibration +from mne.viz.eyetracking import plot_gaze et_fpath = data_path() / "eeg-et" / "sub-01_task-plr_eyetrack.asc" eeg_fpath = data_path() / "eeg-et" / "sub-01_task-plr_eeg.mff" @@ -123,7 +124,9 @@ # window 50 ms before and 200 ms after the blink, so that the noisy data surrounding # the blink is also interpolated. -mne.preprocessing.eyetracking.interpolate_blinks(raw_et, buffer=(0.05, 0.2)) +mne.preprocessing.eyetracking.interpolate_blinks( + raw_et, buffer=(0.05, 0.2), interpolate_gaze=True +) # %% # .. important:: By default, :func:`~mne.preprocessing.eyetracking.interpolate_blinks`, @@ -202,7 +205,9 @@ # center of the screen. Let's plot the gaze position data to confirm that the # participant primarily kept their gaze fixated at the center of the screen. -mne.viz.eyetracking.plot_gaze(epochs, width=1920, height=1080, sigma=2) +# extract new epochs without baseline correction, only for plotting purposes. +gaze_epochs = mne.Epochs(raw_et, events=et_events, tmin=-0.3, tmax=3, baseline=None) +plot_gaze(gaze_epochs, width=1920, height=1080, sigma=2) # %% # .. seealso:: :ref:`tut-eyetrack-heatmap` From 29eb623efca0e05cd11a1b8680ccef0c4679e41f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 28 Sep 2023 13:43:34 -0400 Subject: [PATCH 11/33] FIX: Need module --- mne/viz/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/viz/__init__.py b/mne/viz/__init__.py index 2ec36d48941..2a3ae16029b 100644 --- a/mne/viz/__init__.py +++ b/mne/viz/__init__.py @@ -4,7 +4,7 @@ __getattr__, __dir__, __all__ = lazy.attach( __name__, - submodules=["backends", "_scraper", "ui_events"], + submodules=["backends", "_scraper", "ui_events", "eyetracking"], submod_attrs={ "backends._abstract": ["Figure3D"], "backends.renderer": [ From 6962df96e9c01e556c050958d6281191103b7a9e Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Fri, 29 Sep 2023 10:42:16 -0400 Subject: [PATCH 12/33] FIX: Code suggestions from Eric Co-authored-by: Eric Larson - Use a smarter way to find gaze position channels, dont rely on the ch_name - No need to mask out values outside the width and height, histogram2d will do it for you - remove bin_width parameter in favor of relying on sigma param --- mne/viz/eyetracking/heatmap.py | 40 +++++-------------- .../visualization/30_eyetracking_heatmap.py | 34 +++------------- 2 files changed, 16 insertions(+), 58 deletions(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 9c0d25718b5..3419a3a1c16 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -15,8 +15,7 @@ def plot_gaze( epochs, width, height, - bin_width=10, - sigma=1, + sigma=25, cmap=None, alpha=None, vmin=None, @@ -38,11 +37,6 @@ def plot_gaze( The height dimension of the plot canvas. For example, if the eyegaze data units are pixels, and the participant screen resolution was 1920x1080, then the height should be 1080. - bin_width : int - The number of eyegaze units per square bin that are used to create the heatmap. - Default is 10, meaning that if the eyegaze units are pixels, each bin is 10x10 - pixels. See the appendix section of :ref:`tut-eyetrack-heatmap` for more - detail. sigma : int | float The amount of smoothing applied to the heatmap data. If ``None``, no smoothing is applied. Default is 1. @@ -76,40 +70,28 @@ def plot_gaze( .. versionadded:: 1.6 """ from mne import BaseEpochs + from mne._fiff.pick import _picks_to_idx _validate_type(epochs, BaseEpochs, "epochs") - # Find xpos and ypos channels. if future readers use different channel names than - # xpos/ypos, we will need a different way to identify these channels - xpos_indices = np.where(np.char.startswith(epochs.ch_names, "xpos"))[0] - ypos_indices = np.where(np.char.startswith(epochs.ch_names, "ypos"))[0] + pos_picks = _picks_to_idx(epochs.info, "eyegaze") + gaze_data = epochs.get_data(picks=pos_picks) + gaze_ch_loc = np.array([epochs.info["chs"][idx]["loc"] for idx in pos_picks]) + x_data = gaze_data[:, np.where(gaze_ch_loc[:, 4] == -1)[0], :] + y_data = gaze_data[:, np.where(gaze_ch_loc[:, 4] == 1)[0], :] - data = epochs.get_data() - x_data = data[:, xpos_indices, :] - y_data = data[:, ypos_indices, :] - if xpos_indices.size > 1: # binocular recording. Average across eyes + if x_data.shape[1] > 1: # binocular recording. Average across eyes logger.info("Detected binocular recording. Averaging positions across eyes.") x_data = np.nanmean(x_data, axis=1) # shape (n_epochs, n_samples) y_data = np.nanmean(y_data, axis=1) - x_data = x_data.flatten() - y_data = y_data.flatten() - gaze_data = np.vstack((x_data, y_data)).T # shape (n_samples, 2) - # mask gaze data that is outside screen bounds - mask = ( - (gaze_data[:, 0] > 0) - & (gaze_data[:, 1] > 0) - & (gaze_data[:, 0] < width) - & (gaze_data[:, 1] < height) - ) - canvas = gaze_data[mask].astype(float) + x_data, y_data = x_data.flatten(), y_data.flatten() + canvas = np.vstack((x_data, y_data)).T # shape (n_samples, 2) # Create 2D histogram - x_bins = np.linspace(0, width, width // bin_width) - y_bins = np.linspace(0, height, height // bin_width) hist, _, _ = np.histogram2d( canvas[:, 0], canvas[:, 1], - bins=(x_bins, y_bins), + bins=(width, height), range=[[0, width], [0, height]], ) hist = hist.T # transpose to match screen coordinates. i.e. width > height diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index 13f63bc526f..d7d8ab174f5 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -120,10 +120,7 @@ def plot_images(image_paths, ax, titles=None): # customize a :class:`~matplotlib.colors.Colormap` to make some values of the heatmap # (in this case, the color black) completely transparent. We'll then use the ``vmin`` # parameter to force the heatmap to start at a value greater than the darkest value in -# our previous heatmap, which will make those values transparent. We'll also set the -# ``alpha`` parameter of :func:`~mne.viz.eyetracking.plot_gaze` to make even the -# visible colors of the heatmap semi-transparent so that we can see the image -# underneath. +# our previous heatmap, which will make the darkest colors of the heatmap transparent. cmap = plt.get_cmap("jet") cmap.set_under("k", alpha=0) # make the lowest values transparent @@ -134,9 +131,8 @@ def plot_images(image_paths, ax, titles=None): epochs["natural"], width=px_width, height=px_height, - vmin=0.01, - alpha=0.8, - sigma=5, + vmin=0.0003, + sigma=50, cmap=cmap, axes=ax[0], show=False, @@ -147,31 +143,11 @@ def plot_images(image_paths, ax, titles=None): epochs["scrambled"], width=px_width, height=px_height, - sigma=5, - vmin=0.01, - alpha=0.8, + sigma=50, + vmin=0.0001, cmap=cmap, axes=ax[1], show=False, ) ax[1].set_title("Scrambled") plt.show() - - -# %% -# Appendix: Understanding the ``bin_width`` parameter -# --------------------------------------------------- -# -# The ``bin_width`` parameter of :func:`~mne.viz.eyetracking.plot_gaze` controls the -# resolution of the heatmap. The heatmap is created by binning the gaze data into -# square bins of a given width. The value of each bin is the sum of the dwell time -# (in seconds) of all gaze samples that fall within the bin. The heatmap is then -# smoothed using a Gaussian filter, which can be controlled via the ``sigma`` -# parameter. If we set ``sigma`` to 0, we can clearly see the binned data: - -# make each bin 120x120 pixels and don't smooth the data -plot_gaze(epochs["natural"], width=px_width, height=px_height, bin_width=120, sigma=0) - -# %% -# Making ``bin_width`` smaller results in a higher resolution heatmap: -plot_gaze(epochs["natural"], width=px_width, height=px_height, bin_width=20, sigma=0) From 20e3056fd33c4a523d0def9b0a3641d4a51a1a2c Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Fri, 29 Sep 2023 13:31:29 -0400 Subject: [PATCH 13/33] TST: add a test --- mne/viz/eyetracking/tests/test_heatmap.py | 43 ++++++++++++++--------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index cb9f995e738..cfa22328108 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -1,27 +1,38 @@ -import matplotlib.pyplot as plt import pytest -from mne import make_fixed_length_epochs -from mne.datasets.testing import data_path, requires_testing_data -from mne.io import read_raw_eyelink -from mne.preprocessing.eyetracking import interpolate_blinks -from mne.viz.eyetracking import plot_gaze +import matplotlib.pyplot as plt +import numpy as np -fname = data_path(download=False) / "eyetrack" / "test_eyelink.asc" -pd = pytest.importorskip("pandas") +import mne -@requires_testing_data @pytest.mark.parametrize( - "fname, axes", - [(fname, None), (fname, True)], + "axes", + [(None), (True)], ) -def test_plot_heatmap(fname, axes): +def test_plot_heatmap(axes): """Test plot_gaze.""" - raw = read_raw_eyelink(fname, find_overlaps=True) - interpolate_blinks(raw, interpolate_gaze=True, buffer=(0.05, 0.2)) - epochs = make_fixed_length_epochs(raw, duration=5) + # Create a toy epochs instance + info = info = mne.create_info( + ch_names=["xpos", "ypos"], sfreq=100, ch_types="eyegaze" + ) + # here we pretend that the subject was looking at the center of the screen + # we limit the gaze data between 860-1060px horizontally and 440-640px vertically + data = np.vstack([np.full((1, 100), 1920 / 2), np.full((1, 100), 1080 / 2)]) + epochs = mne.EpochsArray(data[None, ...], info) + epochs.info["chs"][0]["loc"][4] = -1 + epochs.info["chs"][1]["loc"][4] = 1 if axes: axes = plt.subplot() - plot_gaze(epochs, width=1920, height=1080, axes=axes) + fig = mne.viz.eyetracking.plot_gaze( + epochs, width=1920, height=1080, axes=axes, cmap="Greys" + ) + img = fig.axes[0].images[0].get_array() + # the pixels in the center of canvas + assert 960 in np.where(img)[1] + assert np.isclose(np.min(np.where(img)[1]), 860) + assert np.isclose(np.max(np.where(img)[1]), 1060) + assert 540 in np.where(img)[0] + assert np.isclose(np.min(np.where(img)[0]), 440) + assert np.isclose(np.max(np.where(img)[0]), 640) From 6bedba67de8089ef8f6e77bf16e5c04243657065 Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Mon, 2 Oct 2023 11:41:24 -0400 Subject: [PATCH 14/33] Apply suggestions from code review [ci skip] Co-authored-by: Eric Larson --- mne/viz/eyetracking/heatmap.py | 32 +++++++++++------------ mne/viz/eyetracking/tests/test_heatmap.py | 4 +++ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 3419a3a1c16..9382e5953f9 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -15,9 +15,10 @@ def plot_gaze( epochs, width, height, + *, sigma=25, cmap=None, - alpha=None, + alpha=1., vmin=None, vmax=None, axes=None, @@ -27,7 +28,7 @@ def plot_gaze( Parameters ---------- - epochs : mne.Epochs + epochs : instance of Epochs The :class:`~mne.Epochs` object containing eyegaze channels. width : int The width dimension of the plot canvas. For example, if the eyegaze data units @@ -37,15 +38,14 @@ def plot_gaze( The height dimension of the plot canvas. For example, if the eyegaze data units are pixels, and the participant screen resolution was 1920x1080, then the height should be 1080. - sigma : int | float - The amount of smoothing applied to the heatmap data. If ``None``, - no smoothing is applied. Default is 1. + sigma : float | None + The amount of Gaussian smoothing applied to the heatmap data (standard + deviation in pixels). If ``None``, no smoothing is applied. Default is 25. cmap : matplotlib colormap | str | None The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning the colormap will default to matplotlib's default. - alpha : int | float | None - The transparency value of the heatmap. If ``None``, the alpha value is set to 1, - meaning the heatmap colors are fully opaque. Default is ``None``. + alpha : float + The opacity of the heatmap (default is 1). vmin : float | None The minimum value for the colormap. The unit is seconds, for dwell time to the bin coordinate. If ``None``, the minimum value is set to the @@ -57,8 +57,7 @@ def plot_gaze( axes : matplotlib.axes.Axes | None The axes to plot on. If ``None``, a new figure and axes are created. Default is ``None``. - show : bool - Whether to show the plot. Default is ``True``. + %(show)s Returns ------- @@ -88,13 +87,13 @@ def plot_gaze( canvas = np.vstack((x_data, y_data)).T # shape (n_samples, 2) # Create 2D histogram + # Bin into image-like format hist, _, _ = np.histogram2d( - canvas[:, 0], canvas[:, 1], - bins=(width, height), - range=[[0, width], [0, height]], + canvas[:, 0], + bins=(height, width), + range=[[0, height], [0, width]], ) - hist = hist.T # transpose to match screen coordinates. i.e. width > height # Convert density from samples to seconds hist /= epochs.info["sfreq"] # Smooth the heatmap @@ -136,7 +135,7 @@ def _plot_heatmap_array( ax = axes fig = ax.get_figure() else: - fig, ax = plt.subplots() + fig, ax = plt.subplots(constrained_layout=True) ax.set_title("Gaze heatmap") ax.set_xlabel("X position") @@ -161,7 +160,6 @@ def _plot_heatmap_array( ) # Prepare the colorbar - # stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph - fig.colorbar(im, ax=ax, label="Dwell time (seconds)", fraction=0.046, pad=0.04) + fig.colorbar(im, ax=ax, label="Dwell time (seconds)") plt_show(show) return fig diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index cfa22328108..26e0635f246 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -1,3 +1,7 @@ +# Authors: Scott Huberty +# +# License: Simplified BSD + import pytest import matplotlib.pyplot as plt From d0a2caaa49d11e071845c1331460c7cd026700d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:42:17 +0000 Subject: [PATCH 15/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mne/viz/eyetracking/heatmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 9382e5953f9..37439e98820 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -18,7 +18,7 @@ def plot_gaze( *, sigma=25, cmap=None, - alpha=1., + alpha=1.0, vmin=None, vmax=None, axes=None, From c421d5063a5cd9fc1166573a24a87693318f8ffa Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 2 Oct 2023 17:13:28 -0400 Subject: [PATCH 16/33] FIX, DOC: Code suggestions from eric and mne.utils.doc addition - I added a new entry into the doc_dict in mne.utils.doc for the matplotlib cmap parameter, which I believe could prove to be useful elsewhere, for example in mne.viz.misc.plot_csd. FWIW The cmap_topo docstring that was already in the doc_dict says that if None is passed then "reds" will be used. But For any mne functions that have cmap=None as the default, if None is passed into the matplotlib function, the default will actually be whatever matplotlib's default is, which is usually viridis. --- mne/utils/docs.py | 8 +++++ mne/viz/eyetracking/heatmap.py | 32 +++++++------------ .../visualization/30_eyetracking_heatmap.py | 4 +-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 9367562c7a1..6fc20f1e950 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -771,6 +771,14 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): otherwise defaults to 'RdBu_r'. """ +docdict[ + "cmap_simple" +] = """ +cmap : matplotlib colormap | str | None + The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning + the colormap will default to matplotlib's default. +""" + docdict[ "cnorm" ] = """ diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 37439e98820..fe512c21101 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -7,7 +7,7 @@ from ..utils import plt_show -from ...utils import _validate_type, logger, fill_doc +from ...utils import _ensure_int, _validate_type, logger, fill_doc @fill_doc @@ -19,8 +19,7 @@ def plot_gaze( sigma=25, cmap=None, alpha=1.0, - vmin=None, - vmax=None, + vlim=(None, None), axes=None, show=True, ): @@ -41,22 +40,11 @@ def plot_gaze( sigma : float | None The amount of Gaussian smoothing applied to the heatmap data (standard deviation in pixels). If ``None``, no smoothing is applied. Default is 25. - cmap : matplotlib colormap | str | None - The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning - the colormap will default to matplotlib's default. + %(cmap_simple)s alpha : float The opacity of the heatmap (default is 1). - vmin : float | None - The minimum value for the colormap. The unit is seconds, for dwell time - to the bin coordinate. If ``None``, the minimum value is set to the - minimum value of the heatmap. Default is ``None``. - vmax : float | None - The maximum value for the colormap. The unit is seconds, for dwell time - to the bin coordinate. If ``None``, the maximum value is set to the - maximum value of the heatmap. Default is ``None``. - axes : matplotlib.axes.Axes | None - The axes to plot on. If ``None``, a new figure and axes are created. - Default is ``None``. + %(vlim_plot_topomap)s + %(axes_plot_topomap)s %(show)s Returns @@ -72,6 +60,10 @@ def plot_gaze( from mne._fiff.pick import _picks_to_idx _validate_type(epochs, BaseEpochs, "epochs") + _validate_type(alpha, "numeric", "alpha") + _validate_type(sigma, ("numeric", None), "sigma") + _ensure_int(width, "width") + _ensure_int(width, "height") pos_picks = _picks_to_idx(epochs.info, "eyegaze") gaze_data = epochs.get_data(picks=pos_picks) @@ -106,8 +98,8 @@ def plot_gaze( height=height, cmap=cmap, alpha=alpha, - vmin=vmin, - vmax=vmax, + vmin=vlim[0], + vmax=vlim[1], axes=axes, show=show, ) @@ -160,6 +152,6 @@ def _plot_heatmap_array( ) # Prepare the colorbar - fig.colorbar(im, ax=ax, label="Dwell time (seconds)") + fig.colorbar(im, ax=ax, shrink=0.6, label="Dwell time (seconds)") plt_show(show) return fig diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index d7d8ab174f5..11ab5b9eac0 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -94,7 +94,7 @@ def plot_images(image_paths, ax, titles=None): epochs["natural"], width=px_width, height=px_height, - sigma=5, + sigma=50, cmap="jet", axes=ax[0], show=False, @@ -104,7 +104,7 @@ def plot_images(image_paths, ax, titles=None): epochs["scrambled"], width=px_width, height=px_height, - sigma=5, + sigma=50, cmap="jet", axes=ax[1], show=False, From 379abf028e9fb0cadaabc0b57c4473c633a3a502 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 2 Oct 2023 17:47:22 -0400 Subject: [PATCH 17/33] FIX: doc_dict addition wasnt in alphabetical order --- mne/utils/docs.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 6fc20f1e950..c30b6447a53 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -744,6 +744,14 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): ``pos_lims``, as the surface plot must show the magnitude. """ +docdict[ + "cmap" +] = """ +cmap : matplotlib colormap | str | None + The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning + the colormap will default to matplotlib's default. +""" + docdict[ "cmap_topomap" ] = """ @@ -771,14 +779,6 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): otherwise defaults to 'RdBu_r'. """ -docdict[ - "cmap_simple" -] = """ -cmap : matplotlib colormap | str | None - The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning - the colormap will default to matplotlib's default. -""" - docdict[ "cnorm" ] = """ From 6531e53edbb5b209c004a02d3dff89cd4f9dceef Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 2 Oct 2023 17:47:43 -0400 Subject: [PATCH 18/33] DOC: be more memory efficient in eyetracking tutorial - memory profiling revealed that this tutorial was using around 700mb of memory. --- mne/viz/eyetracking/heatmap.py | 2 +- tutorials/preprocessing/90_eyetracking_data.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index fe512c21101..3e4c1959701 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -40,7 +40,7 @@ def plot_gaze( sigma : float | None The amount of Gaussian smoothing applied to the heatmap data (standard deviation in pixels). If ``None``, no smoothing is applied. Default is 25. - %(cmap_simple)s + %(cmap)s alpha : float The opacity of the heatmap (default is 1). %(vlim_plot_topomap)s diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index 1b210734ec9..7856dca5480 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -179,6 +179,7 @@ ) # Add EEG channels to the eye-tracking raw object raw_et.add_channels([raw_eeg], force_update_info=True) +del raw_eeg # free up some memory # Define a few channel groups of interest and plot the data frontal = ["E19", "E11", "E4", "E12", "E5"] @@ -197,7 +198,11 @@ # Now let's extract epochs around our flash events. We should see a clear pupil # constriction response to the flashes. -epochs = mne.Epochs(raw_et, events=et_events, event_id=event_dict, tmin=-0.3, tmax=3) +# Skip baseline correction for now. We will apply baseline correction later. +epochs = mne.Epochs( + raw_et, events=et_events, event_id=event_dict, tmin=-0.3, tmax=3, baseline=None +) +del raw_et # free up some memory epochs[:8].plot(events=et_events, event_id=event_dict, order=picks_idx) # %% @@ -205,9 +210,7 @@ # center of the screen. Let's plot the gaze position data to confirm that the # participant primarily kept their gaze fixated at the center of the screen. -# extract new epochs without baseline correction, only for plotting purposes. -gaze_epochs = mne.Epochs(raw_et, events=et_events, tmin=-0.3, tmax=3, baseline=None) -plot_gaze(gaze_epochs, width=1920, height=1080, sigma=2) +plot_gaze(epochs, width=1920, height=1080) # %% # .. seealso:: :ref:`tut-eyetrack-heatmap` @@ -216,4 +219,4 @@ # Finally, let's plot the evoked responses to the light flashes to get a sense of the # average pupillary light response, and the associated ERP in the EEG data. -epochs.average().plot(picks=occipital + pupil) +epochs.apple_baseline().average().plot(picks=occipital + pupil) From 3b17ec113b27b48b5f22344aaa25bb7edb661f3e Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 2 Oct 2023 18:15:54 -0400 Subject: [PATCH 19/33] FIX: obvious typo.... --- tutorials/preprocessing/90_eyetracking_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/preprocessing/90_eyetracking_data.py b/tutorials/preprocessing/90_eyetracking_data.py index 7856dca5480..ff35e3f8940 100644 --- a/tutorials/preprocessing/90_eyetracking_data.py +++ b/tutorials/preprocessing/90_eyetracking_data.py @@ -219,4 +219,4 @@ # Finally, let's plot the evoked responses to the light flashes to get a sense of the # average pupillary light response, and the associated ERP in the EEG data. -epochs.apple_baseline().average().plot(picks=occipital + pupil) +epochs.apply_baseline().average().plot(picks=occipital + pupil) From 6085bc53c6207548cbef71fcd4ff6fc15bf7e31a Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 2 Oct 2023 18:38:29 -0400 Subject: [PATCH 20/33] FIX, DOC: More doc fixes... --- tutorials/visualization/30_eyetracking_heatmap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index 11ab5b9eac0..6f88a7fb609 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -118,7 +118,7 @@ def plot_images(image_paths, ax, titles=None): # # We can use matplotlib to plot the gaze heatmaps on top of the stimuli images. We'll # customize a :class:`~matplotlib.colors.Colormap` to make some values of the heatmap -# (in this case, the color black) completely transparent. We'll then use the ``vmin`` +# (in this case, the color black) completely transparent. We'll then use the ``vlim`` # parameter to force the heatmap to start at a value greater than the darkest value in # our previous heatmap, which will make the darkest colors of the heatmap transparent. @@ -131,7 +131,7 @@ def plot_images(image_paths, ax, titles=None): epochs["natural"], width=px_width, height=px_height, - vmin=0.0003, + vlim=(0.0003, None), sigma=50, cmap=cmap, axes=ax[0], @@ -144,7 +144,7 @@ def plot_images(image_paths, ax, titles=None): width=px_width, height=px_height, sigma=50, - vmin=0.0001, + vlim=(0.0001, None), cmap=cmap, axes=ax[1], show=False, From 8305a1dca617e7ff90e7aef25e91881ef04bfc60 Mon Sep 17 00:00:00 2001 From: Scott Huberty <52462026+scott-huberty@users.noreply.github.com> Date: Tue, 3 Oct 2023 08:55:30 -0400 Subject: [PATCH 21/33] Apply suggestions from code review [ci skip] Co-authored-by: Daniel McCloy Co-authored-by: Mathieu Scheltienne --- mne/utils/docs.py | 4 ++-- mne/viz/eyetracking/heatmap.py | 6 +++--- mne/viz/eyetracking/tests/test_heatmap.py | 12 +++++------- tutorials/visualization/30_eyetracking_heatmap.py | 6 +++--- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/mne/utils/docs.py b/mne/utils/docs.py index c30b6447a53..8df680ac380 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -748,8 +748,8 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): "cmap" ] = """ cmap : matplotlib colormap | str | None - The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, meaning - the colormap will default to matplotlib's default. + The :class:`~matplotlib.colors.Colormap` to use. Defaults to ``None``, which + will use the matplotlib default colormap. """ docdict[ diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 3e4c1959701..7b8cdd6c96a 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -49,7 +49,7 @@ def plot_gaze( Returns ------- - fig : instance of matplotlib.figure.Figure + fig : instance of Figure The resulting figure object for the heatmap plot. Notes @@ -62,8 +62,8 @@ def plot_gaze( _validate_type(epochs, BaseEpochs, "epochs") _validate_type(alpha, "numeric", "alpha") _validate_type(sigma, ("numeric", None), "sigma") - _ensure_int(width, "width") - _ensure_int(width, "height") + width = _ensure_int(width, "width") + height = _ensure_int(width, "height") pos_picks = _picks_to_idx(epochs.info, "eyegaze") gaze_data = epochs.get_data(picks=pos_picks) diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index 26e0635f246..af33938d93b 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -10,19 +10,17 @@ import mne -@pytest.mark.parametrize( - "axes", - [(None), (True)], -) +@pytest.mark.parametrize("axes", [None, True]) def test_plot_heatmap(axes): """Test plot_gaze.""" # Create a toy epochs instance info = info = mne.create_info( ch_names=["xpos", "ypos"], sfreq=100, ch_types="eyegaze" ) - # here we pretend that the subject was looking at the center of the screen - # we limit the gaze data between 860-1060px horizontally and 440-640px vertically - data = np.vstack([np.full((1, 100), 1920 / 2), np.full((1, 100), 1080 / 2)]) + # simulate a steady fixation at the center of the screen + width, height = (1920, 1080) + shape = (1, 100) # x or y, time + data = np.vstack([np.full(shape, width / 2), np.full(shape, height / 2)]) epochs = mne.EpochsArray(data[None, ...], info) epochs.info["chs"][0]["loc"][4] = -1 epochs.info["chs"][1]["loc"][4] = 1 diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index 6f88a7fb609..81f3e285123 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -95,7 +95,7 @@ def plot_images(image_paths, ax, titles=None): width=px_width, height=px_height, sigma=50, - cmap="jet", + cmap="viridis", axes=ax[0], show=False, ) @@ -105,7 +105,7 @@ def plot_images(image_paths, ax, titles=None): width=px_width, height=px_height, sigma=50, - cmap="jet", + cmap="viridis", axes=ax[1], show=False, ) @@ -122,7 +122,7 @@ def plot_images(image_paths, ax, titles=None): # parameter to force the heatmap to start at a value greater than the darkest value in # our previous heatmap, which will make the darkest colors of the heatmap transparent. -cmap = plt.get_cmap("jet") +cmap = plt.get_cmap("viridis") cmap.set_under("k", alpha=0) # make the lowest values transparent fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) From baaa7bca463089d6db405e81759b18ff2b43b1bc Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 4 Oct 2023 16:25:17 -0400 Subject: [PATCH 22/33] FIX: fixed a mistake in _ensure_int parameter --- mne/viz/eyetracking/heatmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 7b8cdd6c96a..55d106cac71 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -63,7 +63,7 @@ def plot_gaze( _validate_type(alpha, "numeric", "alpha") _validate_type(sigma, ("numeric", None), "sigma") width = _ensure_int(width, "width") - height = _ensure_int(width, "height") + height = _ensure_int(height, "height") pos_picks = _picks_to_idx(epochs.info, "eyegaze") gaze_data = epochs.get_data(picks=pos_picks) From b5c6079411afc4cbcf5735db91acc2a7029a4449 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 4 Oct 2023 16:25:50 -0400 Subject: [PATCH 23/33] Simplify tutorial --- .../visualization/30_eyetracking_heatmap.py | 87 +++---------------- 1 file changed, 13 insertions(+), 74 deletions(-) diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/tutorials/visualization/30_eyetracking_heatmap.py index 81f3e285123..cf432a052c2 100644 --- a/tutorials/visualization/30_eyetracking_heatmap.py +++ b/tutorials/visualization/30_eyetracking_heatmap.py @@ -18,7 +18,7 @@ # # As usual we start by importing the modules we need and loading some # :ref:`example data `: eye-tracking data recorded from SR research's -# ``'.asc'`` file format. We'll also define a helper function to plot image files. +# ``'.asc'`` file format. import matplotlib.pyplot as plt @@ -27,39 +27,16 @@ from mne.viz.eyetracking import plot_gaze -# Define a function to plot stimuli photos -def plot_images(image_paths, ax, titles=None): - for i, image_path in enumerate(image_paths): - ax[i].imshow(plt.imread(image_path)) - if titles: - ax[i].set_title(titles[i]) - return fig - - # define variables to pass to the plot_gaze function px_width, px_height = 1920, 1080 +cmap = plt.get_cmap("viridis") task_fpath = mne.datasets.eyelink.data_path() / "freeviewing" et_fpath = task_fpath / "sub-01_task-freeview_eyetrack.asc" -natural_stim_fpath = task_fpath / "stim" / "naturalistic.png" -scrambled_stim_fpath = task_fpath / "stim" / "scrambled.png" -image_paths = list([natural_stim_fpath, scrambled_stim_fpath]) - +stim_fpath = task_fpath / "stim" / "naturalistic.png" raw = mne.io.read_raw_eyelink(et_fpath) -# %% -# Task background -# --------------- -# -# Participants watched videos while eye-tracking data was collected. The videos showed -# people dancing, or scrambled versions of those videos. Each video lasted about 20 -# seconds. An image of each video is shown below. - -fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) -plot_images(image_paths, ax, ["Natural", "Scrambled"]) - - # %% # Process and epoch the data # -------------------------- @@ -67,8 +44,8 @@ def plot_images(image_paths, ax, titles=None): # First we will interpolate missing data during blinks and epoch the data. mne.preprocessing.eyetracking.interpolate_blinks(raw, interpolate_gaze=True) -raw.annotations.rename({"dvns": "natural", "dvss": "scrambled"}) # more intuitive -event_ids = {"natural": 1, "scrambled": 2} +raw.annotations.rename({"dvns": "natural"}) # more intuitive +event_ids = {"natural": 1} events, event_dict = mne.events_from_annotations(raw, event_id=event_ids) epochs = mne.Epochs( @@ -89,44 +66,21 @@ def plot_images(image_paths, ax, titles=None): # screen resolution of the participant screen (1920x1080) as the width and height. We # can also use the sigma parameter to smooth the plot. -fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) -plot_gaze( - epochs["natural"], - width=px_width, - height=px_height, - sigma=50, - cmap="viridis", - axes=ax[0], - show=False, -) -ax[0].set_title("Gaze Heatmap (Natural)") -plot_gaze( - epochs["scrambled"], - width=px_width, - height=px_height, - sigma=50, - cmap="viridis", - axes=ax[1], - show=False, -) -ax[1].set_title("Gaze Heatmap (Scrambled)") -plt.show() +plot_gaze(epochs["natural"], width=px_width, height=px_height, cmap=cmap, sigma=50) # %% # Overlaying plots with images # ---------------------------- # -# We can use matplotlib to plot the gaze heatmaps on top of the stimuli images. We'll +# We can use matplotlib to plot gaze heatmaps on top of stimuli images. We'll # customize a :class:`~matplotlib.colors.Colormap` to make some values of the heatmap -# (in this case, the color black) completely transparent. We'll then use the ``vlim`` -# parameter to force the heatmap to start at a value greater than the darkest value in -# our previous heatmap, which will make the darkest colors of the heatmap transparent. +# completely transparent. We'll then use the ``vlim`` parameter to force the heatmap to +# start at a value greater than the darkest value in our previous heatmap, which will +# make the darkest colors of the heatmap transparent. -cmap = plt.get_cmap("viridis") cmap.set_under("k", alpha=0) # make the lowest values transparent -fig, ax = plt.subplots(1, 2, figsize=(10, 5), constrained_layout=True) - -plot_images(image_paths, ax) +ax = plt.subplot() +ax.imshow(plt.imread(stim_fpath)) plot_gaze( epochs["natural"], width=px_width, @@ -134,20 +88,5 @@ def plot_images(image_paths, ax, titles=None): vlim=(0.0003, None), sigma=50, cmap=cmap, - axes=ax[0], - show=False, -) -ax[0].set_title("Natural") - -plot_gaze( - epochs["scrambled"], - width=px_width, - height=px_height, - sigma=50, - vlim=(0.0001, None), - cmap=cmap, - axes=ax[1], - show=False, + axes=ax, ) -ax[1].set_title("Scrambled") -plt.show() From 093a0033a6065d9a2a30e08585757df0c7e83a51 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 4 Oct 2023 16:26:58 -0400 Subject: [PATCH 24/33] Move new tutorial to examples --- {tutorials => examples}/visualization/30_eyetracking_heatmap.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tutorials => examples}/visualization/30_eyetracking_heatmap.py (100%) diff --git a/tutorials/visualization/30_eyetracking_heatmap.py b/examples/visualization/30_eyetracking_heatmap.py similarity index 100% rename from tutorials/visualization/30_eyetracking_heatmap.py rename to examples/visualization/30_eyetracking_heatmap.py From 7c693db41cce8577e2cd619ffb6bec4527619df6 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 4 Oct 2023 16:27:47 -0400 Subject: [PATCH 25/33] DOC: Rename tutorial, remove 30_ that was prepended to filename --- .../{30_eyetracking_heatmap.py => eyetracking_plot_heatmap} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/visualization/{30_eyetracking_heatmap.py => eyetracking_plot_heatmap} (100%) diff --git a/examples/visualization/30_eyetracking_heatmap.py b/examples/visualization/eyetracking_plot_heatmap similarity index 100% rename from examples/visualization/30_eyetracking_heatmap.py rename to examples/visualization/eyetracking_plot_heatmap From bba5d7d7c297af261a43130fba3a187518553f56 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 4 Oct 2023 16:29:20 -0400 Subject: [PATCH 26/33] FIX: add py extension back to tutorial filename --- .../{eyetracking_plot_heatmap => eyetracking_plot_heatmap.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/visualization/{eyetracking_plot_heatmap => eyetracking_plot_heatmap.py} (100%) diff --git a/examples/visualization/eyetracking_plot_heatmap b/examples/visualization/eyetracking_plot_heatmap.py similarity index 100% rename from examples/visualization/eyetracking_plot_heatmap rename to examples/visualization/eyetracking_plot_heatmap.py From 927fdada76f7b4ac53d09edfa054cb9620709dc8 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 9 Oct 2023 14:54:25 -0700 Subject: [PATCH 27/33] STY: remove unnecessary transpose --- mne/viz/eyetracking/heatmap.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index 55d106cac71..e815cd1cd21 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -75,14 +75,13 @@ def plot_gaze( logger.info("Detected binocular recording. Averaging positions across eyes.") x_data = np.nanmean(x_data, axis=1) # shape (n_epochs, n_samples) y_data = np.nanmean(y_data, axis=1) - x_data, y_data = x_data.flatten(), y_data.flatten() - canvas = np.vstack((x_data, y_data)).T # shape (n_samples, 2) + canvas = np.vstack((x_data.flatten(), y_data.flatten())) # shape (2, n_samples) # Create 2D histogram # Bin into image-like format hist, _, _ = np.histogram2d( - canvas[:, 1], - canvas[:, 0], + canvas[1, :], + canvas[0, :], bins=(height, width), range=[[0, height], [0, width]], ) From 0c5b8f10cb43ce2e302673d8600d42d04c62298f Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Mon, 9 Oct 2023 14:55:18 -0700 Subject: [PATCH 28/33] TST: set sigma to None in test and check that fig data match input data --- mne/viz/eyetracking/tests/test_heatmap.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index af33938d93b..ca55d74e09b 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -28,13 +28,10 @@ def test_plot_heatmap(axes): if axes: axes = plt.subplot() fig = mne.viz.eyetracking.plot_gaze( - epochs, width=1920, height=1080, axes=axes, cmap="Greys" + epochs, width=width, height=height, axes=axes, cmap="Greys", sigma=None ) img = fig.axes[0].images[0].get_array() - # the pixels in the center of canvas - assert 960 in np.where(img)[1] - assert np.isclose(np.min(np.where(img)[1]), 860) - assert np.isclose(np.max(np.where(img)[1]), 1060) - assert 540 in np.where(img)[0] - assert np.isclose(np.min(np.where(img)[0]), 440) - assert np.isclose(np.max(np.where(img)[0]), 640) + # We simulated a 2D histogram where only values of 960 and 540 are present + # Check that the heatmap data only contains these values + np.testing.assert_array_almost_equal(np.where(img.T)[0], data[0].mean()) # 960 + np.testing.assert_array_almost_equal(np.where(img.T)[1], data[1].mean()) # 540 From 9896f2b58fda96a0ca5bb00c075459ce80bc494e Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Tue, 10 Oct 2023 16:01:47 -0700 Subject: [PATCH 29/33] DOC, STY: split Eyelink dataset description --- doc/documentation/datasets.rst | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/documentation/datasets.rst b/doc/documentation/datasets.rst index 2b40ebe1d75..348da772e90 100644 --- a/doc/documentation/datasets.rst +++ b/doc/documentation/datasets.rst @@ -481,17 +481,32 @@ EYELINK ======= :func:`mne.datasets.eyelink.data_path` -Two small example datasets of eye-tracking data from SR Research EyeLink. the "eeg-et" -dataset contains both EEG (EGI) and eye-tracking (ASCII format) data recorded from a +Two small example datasets of eye-tracking data from SR Research EyeLink. + +EEG-Eyetracking +^^^^^^^^^^^^^^^ +:func:`mne.datasets.eyelink.data_path`. Data exists at ``/eeg-et/``. + +Contains both EEG (EGI) and eye-tracking (ASCII format) data recorded from a pupillary light reflex experiment, stored in separate files. 1 participant fixated on the screen while short light flashes appeared. Event onsets were recorded by a photodiode attached to the screen and were sent to both the EEG and eye-tracking -systems. The second dataset, in the "freeviewing" directory, contains only eye-tracking -data (ASCII format) from 1 participant who was free-viewing a natural scene. +systems. .. topic:: Examples * :ref:`tut-eyetrack` + +Freeviewing +^^^^^^^^^^^ +:func:`mne.datasets.eyelink.data_path`. Data exists at ``/freeviewing/``. + +Contains eye-tracking data (ASCII format) from 1 participant who was free-viewing a +video of a natural scene. In some videos, the natural scene was pixelated such that +the people in the scene were unrecognizable. + +.. topic:: Examples + * :ref:`tut-eyetrack-heatmap` References From 32a3e64bafdd001ad14a2d02978f6ed26ea6c7c7 Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Tue, 10 Oct 2023 16:26:36 -0700 Subject: [PATCH 30/33] TST: Make test more direct Co-authored-by: Daniel McCloy --- mne/viz/eyetracking/tests/test_heatmap.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index ca55d74e09b..45401379d67 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -31,7 +31,6 @@ def test_plot_heatmap(axes): epochs, width=width, height=height, axes=axes, cmap="Greys", sigma=None ) img = fig.axes[0].images[0].get_array() - # We simulated a 2D histogram where only values of 960 and 540 are present - # Check that the heatmap data only contains these values - np.testing.assert_array_almost_equal(np.where(img.T)[0], data[0].mean()) # 960 - np.testing.assert_array_almost_equal(np.where(img.T)[1], data[1].mean()) # 540 + # We simulated a 2D histogram where only the central pixel (960, 540) was active + assert img.T[width // 2, height // 2] == 1 # central pixel is active + assert np.sum(img) == 1 # only the central pixel should be active From 5add060ac78de91f96898711f8eb94e404e53f5b Mon Sep 17 00:00:00 2001 From: Scott Huberty Date: Wed, 11 Oct 2023 09:57:00 -0700 Subject: [PATCH 31/33] DOC: minor documentation revisions suggested by Dan Co-authored-by: Daniel McCloy --- examples/visualization/eyetracking_plot_heatmap.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/examples/visualization/eyetracking_plot_heatmap.py b/examples/visualization/eyetracking_plot_heatmap.py index cf432a052c2..407070c1fa0 100644 --- a/examples/visualization/eyetracking_plot_heatmap.py +++ b/examples/visualization/eyetracking_plot_heatmap.py @@ -10,6 +10,8 @@ .. seealso:: :ref:`tut-importing-eyetracking-data` +.. seealso:: :ref:`tut-eyetrack` + """ # %% @@ -26,11 +28,6 @@ import mne from mne.viz.eyetracking import plot_gaze - -# define variables to pass to the plot_gaze function -px_width, px_height = 1920, 1080 -cmap = plt.get_cmap("viridis") - task_fpath = mne.datasets.eyelink.data_path() / "freeviewing" et_fpath = task_fpath / "sub-01_task-freeview_eyetrack.asc" stim_fpath = task_fpath / "stim" / "naturalistic.png" @@ -52,9 +49,6 @@ raw, events=events, event_id=event_dict, tmin=0, tmax=20, baseline=None ) -# %% -# .. seealso:: :ref:`tut-eyetrack` -# # %% # Plot a heatmap of the eye-tracking data @@ -66,6 +60,8 @@ # screen resolution of the participant screen (1920x1080) as the width and height. We # can also use the sigma parameter to smooth the plot. +px_width, px_height = 1920, 1080 +cmap = plt.get_cmap("viridis") plot_gaze(epochs["natural"], width=px_width, height=px_height, cmap=cmap, sigma=50) # %% From ea7e653204f0c3f5b810877b0f0ad411c448303f Mon Sep 17 00:00:00 2001 From: Daniel McCloy Date: Wed, 11 Oct 2023 13:21:57 -0500 Subject: [PATCH 32/33] Update examples/visualization/eyetracking_plot_heatmap.py --- examples/visualization/eyetracking_plot_heatmap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/visualization/eyetracking_plot_heatmap.py b/examples/visualization/eyetracking_plot_heatmap.py index 407070c1fa0..00c9fee6611 100644 --- a/examples/visualization/eyetracking_plot_heatmap.py +++ b/examples/visualization/eyetracking_plot_heatmap.py @@ -8,9 +8,10 @@ This tutorial covers plotting eye-tracking position data as a heatmap. -.. seealso:: :ref:`tut-importing-eyetracking-data` +.. seealso:: -.. seealso:: :ref:`tut-eyetrack` + :ref:`tut-importing-eyetracking-data` + :ref:`tut-eyetrack` """ From d89ef77148c0be1785162c736b115aecdca81b73 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 12 Oct 2023 13:35:39 -0400 Subject: [PATCH 33/33] FIX: Order --- mne/viz/eyetracking/heatmap.py | 3 +-- mne/viz/eyetracking/tests/test_heatmap.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mne/viz/eyetracking/heatmap.py b/mne/viz/eyetracking/heatmap.py index e815cd1cd21..d3ff4756d8d 100644 --- a/mne/viz/eyetracking/heatmap.py +++ b/mne/viz/eyetracking/heatmap.py @@ -5,9 +5,8 @@ import numpy as np from scipy.ndimage import gaussian_filter - +from ...utils import _ensure_int, _validate_type, fill_doc, logger from ..utils import plt_show -from ...utils import _ensure_int, _validate_type, logger, fill_doc @fill_doc diff --git a/mne/viz/eyetracking/tests/test_heatmap.py b/mne/viz/eyetracking/tests/test_heatmap.py index 45401379d67..99103c552b1 100644 --- a/mne/viz/eyetracking/tests/test_heatmap.py +++ b/mne/viz/eyetracking/tests/test_heatmap.py @@ -2,10 +2,9 @@ # # License: Simplified BSD -import pytest - import matplotlib.pyplot as plt import numpy as np +import pytest import mne