aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Scholten2025-10-28 12:57:55 +1000
committerSam Scholten2025-10-28 12:57:55 +1000
commit1c33fe6e84dd2dc732f2c56064494388bdee7bb2 (patch)
tree59413676fd1c3fcf847c5f5863fa3211fa4ebc15
parenta6770b50fcbc49bbbdd25ed4b7511fdd1498e864 (diff)
downloadtransivent-1c33fe6e84dd2dc732f2c56064494388bdee7bb2.tar.gz
transivent-1c33fe6e84dd2dc732f2c56064494388bdee7bb2.zip
Fix time array monotonicity for very large arrays
- Generate time arrays with float64 precision before converting to float32 - This prevents precision loss that causes non-monotonic time arrays - Affects datasets with >10 million samples (e.g., 200M points at 20 MHz) - Changed empty time array warning to raise RuntimeError for better error handling - Applied fix to both rd() and rd_chunked() functions
-rw-r--r--src/transivent/io.py19
1 files changed, 10 insertions, 9 deletions
diff --git a/src/transivent/io.py b/src/transivent/io.py
index 5eb46f6..b272283 100644
--- a/src/transivent/io.py
+++ b/src/transivent/io.py
@@ -335,16 +335,16 @@ def rd(
# x = x.astype(np.float32) # NB: data is already in physical units (V)
# Use np.linspace for more robust time array generation
+ # Generate with float64 precision to handle very large arrays, then convert to float32
num_points = len(x)
- if num_points > 0:
- t = np.linspace(0, (num_points - 1) * si, num_points, dtype=np.float32)
- else:
- t = np.array([], dtype=np.float32)
- warnings.warn(
- f"Generated an empty time array for file {rel_fp}. "
- f"Length of signal: {len(x)}, sampling interval: {si}. "
- "This might indicate an issue with input data or sampling interval."
+ if num_points == 0:
+ raise RuntimeError(
+ f"No data points found in file {rel_fp}. "
+ f"This indicates an empty or corrupted binary file."
)
+
+ t = np.linspace(0, (num_points - 1) * si, num_points, dtype=np.float64)
+ t = t.astype(np.float32)
return t, x
@@ -445,8 +445,9 @@ def rd_chunked(
start_time,
start_time + (num_points - 1) * si,
num_points,
- dtype=np.float32,
+ dtype=np.float64,
)
+ t_chunk = t_chunk.astype(np.float32)
yield t_chunk, x_chunk.astype(np.float32)