From 3b242a3c6600458cbb4265ae25019fc1e94548ba Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 16:01:47 +0000 Subject: [PATCH 1/7] Convert format on read --- mne/annotations.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mne/annotations.py b/mne/annotations.py index 629ee7b20cb..12a8a0f2990 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -1264,7 +1264,13 @@ def _read_annotations_csv(fname): "onsets in seconds." ) except ValueError: - pass + # remove nanoseconds for ISO8601 (microsecond) compliance + timestamp = pd.Timestamp(orig_time) + timespec = "microseconds" + if timestamp == pd.Timestamp(_handle_meas_date(0)).astimezone(None): + timespec = "auto" # use default timespec for `orig_time=None` + orig_time = timestamp.isoformat(sep=" ", timespec=timespec) + onset_dt = pd.to_datetime(df["onset"]) onset = (onset_dt - onset_dt[0]).dt.total_seconds() duration = df["duration"].values.astype(float) From ae5addf4a26049d64ecd48f68d57014e73462678 Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 16:03:12 +0000 Subject: [PATCH 2/7] Convert before df --- mne/utils/dataframe.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mne/utils/dataframe.py b/mne/utils/dataframe.py index 0511433001d..68302787c90 100644 --- a/mne/utils/dataframe.py +++ b/mne/utils/dataframe.py @@ -46,7 +46,9 @@ def _convert_times(times, time_format, meas_date=None, first_time=0): elif time_format == "timedelta": times = to_timedelta(times, unit="s") elif time_format == "datetime": - times = to_timedelta(times + first_time, unit="s") + meas_date + times = (to_timedelta(times + first_time, unit="s") + meas_date).astype( + "datetime64[us]" + ) # make ISO8601 (microsecond) compatible return times From 2015ceb28631da84f090014113b4a2e7b17c360e Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 18:04:49 +0000 Subject: [PATCH 3/7] Update tests --- mne/tests/test_annotations.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index 4d0db170e2a..c02d37bc92a 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -1041,6 +1041,24 @@ def test_broken_csv(tmp_path): read_annotations(fname) +def test_nanosecond_csv(tmp_path): + """Test .csv with nanosecond timestamps for onsets read correctly.""" + pytest.importorskip("pandas") + import pandas as pd + + onset = ( + pd.Timestamp(_ORIG_TIME) + .astimezone(None) + .isoformat(sep=" ", timespec="nanoseconds") + ) + content = f"onset,duration,description\n{onset},1.0,AA" + fname = tmp_path / "annotations_broken.csv" + with open(fname, "w") as f: + f.write(content) + annot = read_annotations(fname) + assert annot.orig_time == _ORIG_TIME + + # Test for IO with .txt files @@ -1462,6 +1480,8 @@ def test_repr(): def test_annotation_to_data_frame(time_format): """Test annotation class to data frame conversion.""" pytest.importorskip("pandas") + import pandas as pd + onset = np.arange(1, 10) durations = np.full_like(onset, [4, 5, 6, 4, 5, 6, 4, 5, 6]) description = ["yy"] * onset.shape[0] @@ -1481,6 +1501,12 @@ def test_annotation_to_data_frame(time_format): assert want == got assert df.groupby("description").count().onset["yy"] == 9 + # Check nanoseconds omitted from onset times + if time_format == "datetime": + a.onset += 1e-7 # >6 decimals to trigger nanosecond component + df = a.to_data_frame(time_format=time_format) + assert pd.Timestamp(df.onset[0]).nanosecond == 0 + def test_annotation_ch_names(): """Test annotation ch_names updating and pruning.""" From 70200a9b704dff3c019bed52e0699f29779e1c94 Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 18:23:26 +0000 Subject: [PATCH 4/7] Update docs and add warning --- mne/annotations.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mne/annotations.py b/mne/annotations.py index 12a8a0f2990..87386ddd6e1 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -142,7 +142,8 @@ class Annotations: the annotations with raw data if their acquisition is started at the same time. If it is a string, it should conform to the ISO8601 format. More precisely to this '%%Y-%%m-%%d %%H:%%M:%%S.%%f' particular case of - the ISO8601 format where the delimiter between date and time is ' '. + the ISO8601 format where the delimiter between date and time is ' ' and at most + microsecond precision (nanoseconds are not supported). %(ch_names_annot)s .. versionadded:: 0.23 @@ -276,6 +277,12 @@ class Annotations: def __init__(self, onset, duration, description, orig_time=None, ch_names=None): self._orig_time = _handle_meas_date(orig_time) + if isinstance(orig_time, str) and self._orig_time is None: + warnings.warn( + "The format of the `orig_time` string is not recognised. It must " + "conform to the ISO8601 format with at most microsecond precision and " + "where the delimiter between date and time is ' '." + ) self.onset, self.duration, self.description, self.ch_names = _check_o_d_s_c( onset, duration, description, ch_names ) From a9558c41ab6817ee3c5ac96294ad6db38a255d52 Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 18:24:32 +0000 Subject: [PATCH 5/7] Update docs and add warning --- mne/annotations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mne/annotations.py b/mne/annotations.py index 87386ddd6e1..5606ff26fad 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -281,7 +281,8 @@ def __init__(self, onset, duration, description, orig_time=None, ch_names=None): warnings.warn( "The format of the `orig_time` string is not recognised. It must " "conform to the ISO8601 format with at most microsecond precision and " - "where the delimiter between date and time is ' '." + "where the delimiter between date and time is ' '.", + RuntimeWarning, ) self.onset, self.duration, self.description, self.ch_names = _check_o_d_s_c( onset, duration, description, ch_names From c6d075d32d61acd0ac468d03d1f5d8a34df98236 Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 18:31:47 +0000 Subject: [PATCH 6/7] Add changelog entry --- doc/changes/devel/13109.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/devel/13109.bugfix.rst diff --git a/doc/changes/devel/13109.bugfix.rst b/doc/changes/devel/13109.bugfix.rst new file mode 100644 index 00000000000..b200b90cd9c --- /dev/null +++ b/doc/changes/devel/13109.bugfix.rst @@ -0,0 +1 @@ +Fix reading annotations with :func:`mne.read_annotations` from .csv files containing nanoseconds in times, and make times saved in .csv files by :meth:`mne.Annotations.save` and returned from :meth:`mne.Annotations.to_data_frame` ISO8601 compliant, by `Thomas Binns`_. \ No newline at end of file From 924bebffb82745812cdb987a6f05fda8d82dc5c0 Mon Sep 17 00:00:00 2001 From: "Thomas S. Binns" Date: Mon, 10 Feb 2025 20:07:01 +0000 Subject: [PATCH 7/7] Update warning --- mne/annotations.py | 19 +++++++++++++------ mne/tests/test_annotations.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/mne/annotations.py b/mne/annotations.py index 5606ff26fad..845a8b7228f 100644 --- a/mne/annotations.py +++ b/mne/annotations.py @@ -278,12 +278,19 @@ class Annotations: def __init__(self, onset, duration, description, orig_time=None, ch_names=None): self._orig_time = _handle_meas_date(orig_time) if isinstance(orig_time, str) and self._orig_time is None: - warnings.warn( - "The format of the `orig_time` string is not recognised. It must " - "conform to the ISO8601 format with at most microsecond precision and " - "where the delimiter between date and time is ' '.", - RuntimeWarning, - ) + try: # only warn if `orig_time` is not the default '1970-01-01 00:00:00' + if _handle_meas_date(0) == datetime.strptime( + orig_time, "%Y-%m-%d %H:%M:%S" + ).replace(tzinfo=timezone.utc): + pass + except ValueError: # error if incorrect datetime format AND not the default + warn( + "The format of the `orig_time` string is not recognised. It " + "must conform to the ISO8601 format with at most microsecond " + "precision and where the delimiter between date and time is " + "' '.", + RuntimeWarning, + ) self.onset, self.duration, self.description, self.ch_names = _check_o_d_s_c( onset, duration, description, ch_names ) diff --git a/mne/tests/test_annotations.py b/mne/tests/test_annotations.py index c02d37bc92a..55f97360916 100644 --- a/mne/tests/test_annotations.py +++ b/mne/tests/test_annotations.py @@ -76,6 +76,9 @@ def windows_like_datetime(monkeypatch): def test_basics(): """Test annotation class.""" + pytest.importorskip("pandas") + import pandas as pd + raw = read_raw_fif(fif_fname) assert raw.annotations is not None assert len(raw.annotations.onset) == 0 @@ -95,6 +98,17 @@ def test_basics(): assert isinstance(annot.orig_time, datetime) assert annot.orig_time.tzinfo is timezone.utc + # Test bad format `orig_time` str -> `None` raises warning + with pytest.warns( + RuntimeWarning, match="The format of the `orig_time` string is not recognised." + ): + bad_orig_time = ( + pd.Timestamp(_ORIG_TIME) + .astimezone(None) + .isoformat(sep=" ", timespec="nanoseconds") + ) + Annotations(onset, duration, description, bad_orig_time) + pytest.raises(ValueError, Annotations, onset, duration, description[:9]) pytest.raises(ValueError, Annotations, [onset, 1], duration, description) pytest.raises(ValueError, Annotations, onset, [duration, 1], description)