diff options
| author | Sam Scholten | 2025-10-28 12:57:55 +1000 |
|---|---|---|
| committer | Sam Scholten | 2025-10-28 12:57:55 +1000 |
| commit | 1c33fe6e84dd2dc732f2c56064494388bdee7bb2 (patch) | |
| tree | 59413676fd1c3fcf847c5f5863fa3211fa4ebc15 | |
| parent | a6770b50fcbc49bbbdd25ed4b7511fdd1498e864 (diff) | |
| download | transivent-1c33fe6e84dd2dc732f2c56064494388bdee7bb2.tar.gz transivent-1c33fe6e84dd2dc732f2c56064494388bdee7bb2.zip | |
Fix time array monotonicity for very large arrays
- Generate time arrays with float64 precision before converting to float32
- This prevents precision loss that causes non-monotonic time arrays
- Affects datasets with >10 million samples (e.g., 200M points at 20 MHz)
- Changed empty time array warning to raise RuntimeError for better error handling
- Applied fix to both rd() and rd_chunked() functions
| -rw-r--r-- | src/transivent/io.py | 19 |
1 files changed, 10 insertions, 9 deletions
diff --git a/src/transivent/io.py b/src/transivent/io.py index 5eb46f6..b272283 100644 --- a/src/transivent/io.py +++ b/src/transivent/io.py @@ -335,16 +335,16 @@ def rd( # x = x.astype(np.float32) # NB: data is already in physical units (V) # Use np.linspace for more robust time array generation + # Generate with float64 precision to handle very large arrays, then convert to float32 num_points = len(x) - if num_points > 0: - t = np.linspace(0, (num_points - 1) * si, num_points, dtype=np.float32) - else: - t = np.array([], dtype=np.float32) - warnings.warn( - f"Generated an empty time array for file {rel_fp}. " - f"Length of signal: {len(x)}, sampling interval: {si}. " - "This might indicate an issue with input data or sampling interval." + if num_points == 0: + raise RuntimeError( + f"No data points found in file {rel_fp}. " + f"This indicates an empty or corrupted binary file." ) + + t = np.linspace(0, (num_points - 1) * si, num_points, dtype=np.float64) + t = t.astype(np.float32) return t, x @@ -445,8 +445,9 @@ def rd_chunked( start_time, start_time + (num_points - 1) * si, num_points, - dtype=np.float32, + dtype=np.float64, ) + t_chunk = t_chunk.astype(np.float32) yield t_chunk, x_chunk.astype(np.float32) |
