Memory error when extracting single channel raw data

Hello,

So I am trying to extract a raw single channel from a biocam recording to use as an electrical stimulus onset TTL pulse timer for removing stimulation artefacts. However when I try to extract it, the script trying to create an array which is the full recording size (>90Gb) and it obviously crashes. This happens whether I try and get the single channel from either the full recording or whether I split off the channel with rec.channel_slice(). The function works find if I work in small time chunks but I would really prefer to get the full trace together. See the code below:

rec = si.BiocamRecordingExtractor('Test_stim_rawData_00.brw')
print(Rec)

BiocamRecordingExtractor: 4096 channels - 19.8kHz - 1 segments - 11,852,265 samples
600.00s (10.00 minutes) - uint16 dtype - 90.43 GiB
file_path: D:\Data\MEA_recording\CloseNIT\20240812\Test_stim_rawData_00.brw

numSamps = rec.get_total_samples()
triggerRec = rec.channel_slice(channel_ids=['1'])
print(triggerRec)
triggerTrace = triggerRec.get_traces(segment_index=0, channel_ids=['1'], start_frame=0, end_frame=numSamps)

ChannelSliceRecording: 1 channels - 19.8kHz - 1 segments - 11,852,265 samples
600.00s (10.00 minutes) - uint16 dtype - 22.61 MiB

MemoryError Traceback (most recent call last)
Cell In[47], line 6
4 triggerRec = rec.channel_slice(channel_ids=['1'])
5 print(triggerRec)
----> 6 triggerTrace = triggerRec.get_traces(segment_index=0, channel_ids=['1'], start_frame=0, end_frame=numSamps)

File D:\Code\python\spikeinterface\src\spikeinterface\core\baserecording.py:342, in BaseRecording.get_traces(self, segment_index, start_frame, end_frame, channel_ids, order, return_scaled, cast_unsigned)
340 num_samples = rs.get_num_samples()
341 end_frame = int(min(end_frame, num_samples)) if end_frame is not None else num_samples
--> 342 traces = rs.get_traces(start_frame=start_frame, end_frame=end_frame, channel_indices=channel_indices)
343 if order is not None:
344 assert order in ["C", "F"]

File D:\Code\python\spikeinterface\src\spikeinterface\core\channelslice.py:99, in ChannelSliceRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
92 def get_traces(
93 self,
94 start_frame: int | None = None,
95 end_frame: int | None = None,
96 channel_indices: list | None = None,
97 ) -> np.ndarray:
98 parent_indices = self._parent_channel_indices[channel_indices]
---> 99 traces = self._parent_recording_segment.get_traces(start_frame, end_frame, parent_indices)
100 return traces

File D:\Code\python\spikeinterface\src\spikeinterface\extractors\neoextractors\neobaseextractor.py:351, in NeoRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
345 def get_traces(
346 self,
347 start_frame: Union[int, None] = None,
348 end_frame: Union[int, None] = None,
349 channel_indices: Union[List, None] = None,
350 ) -> np.ndarray:
--> 351 raw_traces = self.neo_reader.get_analogsignal_chunk(
352 block_index=self.block_index,
353 seg_index=self.segment_index,
354 i_start=start_frame,
355 i_stop=end_frame,
356 stream_index=self.stream_index,
357 channel_indexes=channel_indices,
358 )
359 if self.inverted_gain:
360 raw_traces = -raw_traces

File C:\ProgramData\anaconda3\envs\si_env2\Lib\site-packages\neo\rawio\baserawio.py:805, in BaseRawIO.get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes, channel_names, channel_ids, prefer_slice)
802 if np.all(np.diff(channel_indexes) == 1):
803 channel_indexes = slice(channel_indexes[0], channel_indexes[-1] + 1)
--> 805 raw_chunk = self._get_analogsignal_chunk(block_index, seg_index, i_start, i_stop, stream_index, channel_indexes)
807 return raw_chunk

File C:\ProgramData\anaconda3\envs\si_env2\Lib\site-packages\neo\rawio\biocamrawio.py:127, in BiocamRawIO._get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes)
125 if channel_indexes is None:
126 channel_indexes = slice(None)
--> 127 data = self._read_function(self._filehandle, i_start, i_stop, self._num_channels)
128 return data[:, channel_indexes]

File C:\ProgramData\anaconda3\envs\si_env2\Lib\site-packages\neo\rawio\biocamrawio.py:253, in readHDF5t_brw4(rf, t0, t1, nch)
251 for key in rf:
252 if key[:5] == "Well_":
--> 253 return rf[key]["Raw"][nch * t0 : nch * t1].reshape((t1 - t0, nch), order="C")

File h5py\_objects.pyx:54, in h5py._objects.with_phil.wrapper()

File h5py\_objects.pyx:55, in h5py._objects.with_phil.wrapper()

File C:\ProgramData\anaconda3\envs\si_env2\Lib\site-packages\h5py_hl\dataset.py:758, in Dataset.getitem(self, args, new_dtype)
756 if self._fast_read_ok and (new_dtype is None):
757 try:
--> 758 return self._fast_reader.read(args)
759 except TypeError:
760 pass # Fall back to Python read pathway below

File h5py\_selector.pyx:368, in h5py._selector.Reader.read()

File h5py\_selector.pyx:342, in h5py._selector.Reader.make_array()

MemoryError: Unable to allocate 90.4 GiB for an array with shape (48546877440,) and data type uint16

Any help would be greatly appreciated!

Start with src/spikeinterface/extractors/neoextractors/neobaseextractor.py and the Biocam path shown in neo/rawio/biocamrawio.py, especially readHDF5t_brw4. Run the provided single-channel get_traces reproduction against the 90.43 GiB recording and inspect how the full time range is read. Done means the full single-channel trace can be extracted without attempting a 90 GiB allocation.