From c42bb5b4518a1500e26d87cbd3b5b9e80bc63646 Mon Sep 17 00:00:00 2001 From: Christian Boehm Date: Mon, 3 Aug 2026 22:09:40 +0200 Subject: [PATCH 1/2] Enable reading scalar attributes stored as shape-(1,) arrays. This was probably never intended as PyASDF's own writer would use a scalar attribute in the first place. However, external writers might use H5LTset_attribute_*(..., size=1) which creates a size-1 array instead of an H5S_SCALAR. This still works fine with numpy <2.4.0, but from 2.4.0 onwards, it raises a type error rather than silently squeezing to a float: TypeError: only 0-dimensional arrays can be converted to Python scalars. --- pyasdf/asdf_data_set.py | 24 ++++++--- pyasdf/tests/test_asdf_data_set.py | 86 ++++++++++++++++++++++++++++++ pyasdf/utils.py | 33 ++++++++++-- 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/pyasdf/asdf_data_set.py b/pyasdf/asdf_data_set.py index 9411c4f..9257ef4 100644 --- a/pyasdf/asdf_data_set.py +++ b/pyasdf/asdf_data_set.py @@ -68,6 +68,7 @@ ProvenanceAccessor, split_qualified_name, _read_string_array, + attr_to_scalar, FilteredWaveformAccessor, label2string, labelstring2list, @@ -811,11 +812,13 @@ def _get_idx_and_size_estimate(self, waveform_name, starttime, endtime): idx_start = 0 idx_end = data.shape[0] - dt = 1.0 / data.attrs["sampling_rate"] + dt = 1.0 / attr_to_scalar(data.attrs["sampling_rate"], "sampling_rate") # Starttime is a timestamp in nanoseconds. # Get time of first and time of last sample. - data_starttime = obspy.UTCDateTime(float(data.attrs["starttime"]) / 1.0e9) + data_starttime = obspy.UTCDateTime( + float(attr_to_scalar(data.attrs["starttime"], "starttime")) / 1.0e9 + ) data_endtime = data_starttime + float((idx_end - 1) * dt) # Modify the data indices to restrict the data if necessary. @@ -879,11 +882,13 @@ def _get_waveform(self, waveform_name, starttime=None, endtime=None): else: idx_start = 0 idx_end = data.shape[0] - data_starttime = obspy.UTCDateTime(float(attrs["starttime"]) / 1.0e9) + data_starttime = obspy.UTCDateTime( + float(attr_to_scalar(attrs["starttime"], "starttime")) / 1.0e9 + ) tr = obspy.Trace(data=data[idx_start:idx_end]) tr.stats.starttime = data_starttime - tr.stats.sampling_rate = attrs["sampling_rate"] + tr.stats.sampling_rate = attr_to_scalar(attrs["sampling_rate"], "sampling_rate") tr.stats.network = network tr.stats.station = station tr.stats.location = location @@ -1075,10 +1080,11 @@ def _get_dataset_within_tolerance(station_group, trace): if not ds_name.startswith(i): continue ds = station_group._WaveformAccessor__hdf5_group[ds_name] - t = ( - ds.attrs["starttime"] / 1e9 - + ds.shape[0] * 1.0 / ds.attrs["sampling_rate"] + ds_starttime = attr_to_scalar(ds.attrs["starttime"], "starttime") + ds_sampling_rate = attr_to_scalar( + ds.attrs["sampling_rate"], "sampling_rate" ) + t = ds_starttime / 1e9 + ds.shape[0] * 1.0 / ds_sampling_rate del ds if min_t <= t <= max_t: return ds_name @@ -1123,7 +1129,9 @@ def _get_dataset_within_tolerance(station_group, trace): sta=trace.stats.station, loc=trace.stats.location, cha=trace.stats.channel, - start=obspy.UTCDateTime(ds.attrs["starttime"] / 1e9), + start=obspy.UTCDateTime( + attr_to_scalar(ds.attrs["starttime"], "starttime") / 1e9 + ), end=trace.stats.endtime, tag=tag, ) diff --git a/pyasdf/tests/test_asdf_data_set.py b/pyasdf/tests/test_asdf_data_set.py index f608622..69ab5fb 100644 --- a/pyasdf/tests/test_asdf_data_set.py +++ b/pyasdf/tests/test_asdf_data_set.py @@ -191,6 +191,92 @@ def test_equality_checks(example_data_set): assert data_set_1 != data_set_2 +def _rewrite_scalar_attrs_as_arrays(filename, size=1): + """ + Rewrite the scalar waveform attributes of a file as arrays of the given + size. Returns the number of data sets that have been modified. + """ + count = 0 + with h5py.File(filename, "a") as f: + for station in f["Waveforms"].values(): + for name, ds in station.items(): + if name == "StationXML": + continue + ds.attrs["starttime"] = np.array( + [ds.attrs["starttime"]] * size, dtype=np.int64 + ) + ds.attrs["sampling_rate"] = np.array( + [ds.attrs["sampling_rate"]] * size, dtype=np.float64 + ) + count += 1 + return count + + +def test_reading_files_with_size_1_array_attributes(example_data_set): + """ + pyasdf stores 'starttime' and 'sampling_rate' as true scalars but other + ASDF producers sometimes store them as size-1 arrays. NumPy >= 2.4 raises + a TypeError when converting an array with ndim > 0 to a Python scalar, so + make sure these are squeezed down while reading. + """ + filename_1 = example_data_set.filename + filename_2 = os.path.join(example_data_set.tmpdir, "array_attributes.h5") + shutil.copyfile(filename_1, filename_2) + + assert _rewrite_scalar_attrs_as_arrays(filename_2) == 6 + + with ASDFDataSet(filename_1) as ds_1, ASDFDataSet(filename_2) as ds_2: + # Plain read via the dot accessors. + st_1 = ds_1.waveforms.AE_113A.raw_recording + st_2 = ds_2.waveforms.AE_113A.raw_recording + assert st_1 == st_2 + for tr in st_2: + assert isinstance(tr.stats.sampling_rate, float) + assert tr.stats.starttime == UTCDateTime(2013, 5, 24, 5, 40) + + # Partial read - this goes through a different code path that + # computes the indices from the attributes. + starttime = UTCDateTime(2013, 5, 24, 5, 45) + endtime = UTCDateTime(2013, 5, 24, 5, 50) + kwargs = dict( + network="AE", + station="113A", + location="", + channel="BHZ", + tag="raw_recording", + starttime=starttime, + endtime=endtime, + ) + assert ds_1.get_waveforms(**kwargs) == ds_2.get_waveforms(**kwargs) + + # The query interface also reads the attributes. + assert [ + _i._station_name + for _i in ds_2.ifilter( + ds_2.q.starttime <= starttime, ds_2.q.sampling_rate == 40.0 + ) + ] == ["AE.113A", "TA.POKR"] + assert list(ds_2.ifilter(ds_2.q.starttime >= endtime)) == [] + + +def test_reading_files_with_larger_array_attributes_raises(example_data_set): + """ + Attributes that are arrays with more than one element are ambiguous and + are thus rejected. + """ + filename = os.path.join(example_data_set.tmpdir, "array_attributes.h5") + shutil.copyfile(example_data_set.filename, filename) + _rewrite_scalar_attrs_as_arrays(filename, size=2) + + with ASDFDataSet(filename) as ds: + with pytest.raises(ASDFValueError) as err: + ds.waveforms.AE_113A.raw_recording + + assert err.value.args[0] == ( + "Attribute 'starttime' must be a scalar but has shape (2,)." + ) + + def test_adding_same_event_twice_raises(tmpdir): """ Adding the same event twice raises. diff --git a/pyasdf/utils.py b/pyasdf/utils.py index 3868567..8153b8d 100644 --- a/pyasdf/utils.py +++ b/pyasdf/utils.py @@ -91,6 +91,25 @@ def _read_string_array(data): return data[()].tobytes().strip() +def attr_to_scalar(value, name=None): + """ + Coerce an HDF5 attribute to a Python/NumPy scalar. + + pyasdf writes scalar attributes as scalars but other ASDF producers + sometimes store them as size-1 arrays. NumPy >= 2.4 raises a TypeError + when converting an array with ``ndim > 0`` to a scalar (deprecated in + NumPy 1.25), so squeeze those down before they are used in arithmetic. + """ + if isinstance(value, np.ndarray) and value.ndim > 0: + if value.size != 1: + raise ASDFValueError( + "Attribute '%s' must be a scalar but has shape %s." + % (name or "unknown", value.shape) + ) + return value.reshape(-1)[0] + return value + + class SimpleBuffer: """ Object that can be used as a cache. @@ -667,7 +686,12 @@ def filter_waveforms(self, queries): attrs = group[wf].attrs if queries["sampling_rate"]: - if queries["sampling_rate"](attrs["sampling_rate"]) is False: + if ( + queries["sampling_rate"]( + attr_to_scalar(attrs["sampling_rate"], "sampling_rate") + ) + is False + ): continue if queries["npts"]: @@ -675,9 +699,12 @@ def filter_waveforms(self, queries): continue if queries["starttime"] or queries["endtime"]: - starttime = obspy.UTCDateTime(float(attrs["starttime"]) / 1.0e9) + starttime = obspy.UTCDateTime( + float(attr_to_scalar(attrs["starttime"], "starttime")) + / 1.0e9 + ) endtime = starttime + (len(group[wf]) - 1) * 1.0 / float( - attrs["sampling_rate"] + attr_to_scalar(attrs["sampling_rate"], "sampling_rate") ) if queries["starttime"]: From c90ef0bb46ee4a8f3e1c3db7cb4c92fb9b4a0132 Mon Sep 17 00:00:00 2001 From: Christian Boehm Date: Tue, 4 Aug 2026 12:48:54 +0200 Subject: [PATCH 2/2] Run actions with 3.14 and filter warning. --- .github/workflows/pyasdf.yml | 2 +- pytest.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pyasdf.yml b/.github/workflows/pyasdf.yml index 153c23d..b1ad3dd 100644 --- a/.github/workflows/pyasdf.yml +++ b/.github/workflows/pyasdf.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 diff --git a/pytest.ini b/pytest.ini index 5db120c..5414e8f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,4 @@ filterwarnings = ignore:.*pkg_resources is deprecated as an API.*:DeprecationWarning ignore:.*Deprecated call to `pkg_resources.declare_namespace.*:DeprecationWarning ignore:.*The pkg_resources package is slated for removal.*:UserWarning + ignore:.*SelectableGroups dict interface is deprecated\. Use select*:DeprecationWarning