SpikeInterface / SpikeInterface/spikeinterface
Question about PAIRED_KAMPFF datasets accuracy on various sorters
Nobody has claimed this yet.
- Dominant language
- Python
- Stars
- 847
- Forks
- 280
- Avg merge
- 3d 9h
- Merged PRs (30d)
- 29
Description
Dear,
I am trying to use different sorters for the dataset PAIRED_KAMPFF. However, all 5 sorters I used give a lot more spikes than expected. The code is shown here, with the raster plot from the most accurate sorter (spykingcircus 2), which is also only 20% accuracy. I also plot the raster plots of ground_truth and timeseries. Could you please help a little bit about where to start debugging? Thank you very much!
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import spikeinterface.extractors as se
from spikeinterface.preprocessing import bandpass_filter
from probeinterface import Probe
import spikeinterface.comparison as sc
##############################file loading###########################
file_folder = "a"
file_name = "sub-paired-kampff_ses-paired-kampff-2014-11-25-Pair-3-0_ecephys.nwb"
print(file_name)
recording,sorting_GT = se.read_nwb(file_path=os.path.join( file_folder, file_name), load_recording=True,load_sorting=True)
recording = recording.frame_slice(start_frame=0,end_frame = 60*recording.get_sampling_frequency())
sorting_GT = sorting_GT.frame_slice(start_frame=0,end_frame = 60*recording.get_sampling_frequency())
recording = bandpass_filter(recording)
probe_folder = "a"
probe_name = "probe_sub-paired-kampff_ses-paired-kampff-2014-11-25-Pair-3-0_ecephys.csv"
probe_df = pd.read_csv(os.path.join(probe_folder,probe_name))
xycoords = np.vstack( (probe_df['x'], probe_df['y'])).T #shape = 32,2
probe = Probe(ndim=2, si_units='um')
probe.set_contacts(positions=xycoords, shapes='circle', shape_params={'radius': 5})
channel_indices = np.arange(32)
probe.set_device_channel_indices(channel_indices)
recording.set_probe(probe)
recording.set_channel_locations(probe.contact_positions)
#############################sorting#######################################
from pathlib import Path
import spikeinterface.sorters as ss
import sys
from spikeinterface.preprocessing import common_reference, zscore
base_folder = Path("./test")
original_stdout = sys.stdout # Save a reference to the original standard output
sorter_params = dict(n_jobs=-1, chunk_duration="1s", progress_bar=True)
ss.IronClustSorter.set_ironclust_path('/imec/other/macaw/chen14/spikeinterface/Official_Tutorial_SI_0.96_Oct22/ironclust')
ss.Kilosort3Sorter.set_kilosort3_path('/imec/other/macaw/chen14/spikeinterface/Official_Tutorial_SI_0.96_Oct22/Kilosort')
cmp_method = ("raw_count", "by_unit", "pooled_with_average")
def sorter_loop_nofilt(srtr,sorter,recording,sorting_GT,set_name):
acc_list1 = []
acc_list2 = []
well_detected = []
filename = 'test.txt'
#recording = recording.frame_slice(start_frame=0*fs, end_frame=60*fs)
path = base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name )
if srtr == 'IC':
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, filter = False, verbose=False,**sorter_params)
elif srtr == 'KS':
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, freq_min = 40,verbose=True,**sorter_params)
elif srtr == 'pyKS' :
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), fshigh = 40,verbose=True,**sorter_params)
elif srtr == 'HS':
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), filter = False, verbose=False)
elif srtr == 'TDC':
recording = common_reference(recording)
recording = zscore(recording, dtype='float32')
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, apply_preprocessing = False, verbose=False)
elif srtr == 'TDC0':
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name ), freq_min = 40, freq_max = None, verbose=False)
elif srtr == 'SC':
recording = common_reference(recording)
sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder / ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, apply_preprocessing = False, verbose=False)
file_path = base_folder/("sorting_results/results_nofilt"+srtr+set_name)
se.NpzSortingExtractor.write_sorting(sorting_sorter, file_path)
comp1 = sc.compare_sorter_to_ground_truth(sorting_GT, sorting_sorter,match_score = 0.0,exhaustive_gt=False)
tmp_accuracy1 = comp1.get_performance(method = cmp_method[2])['accuracy']
acc_list1.append(tmp_accuracy1)
comp2 = sc.compare_sorter_to_ground_truth(sorting_GT, sorting_sorter,chance_score = 0.1,match_mode = 'best', exhaustive_gt=False)
tmp_accuracy2 = comp2.get_performance(method = cmp_method[2])['accuracy']
acc_list2.append(tmp_accuracy2)
tmp_well_units = [len(comp2.get_well_detected_units())]
well_detected.append(tmp_well_units)
print('no filter, set = ',set_name,', Hungrian 0.0 accuracy = ', tmp_accuracy1, ',best match 0.1 accuracy = ', tmp_accuracy2)
print('no filter, set = ',set_name,', best match 0.1 summary(well/bad/fp/overmerged/redundant) = ', tmp_well_units)
cmd = "rm -rf mearec_sorters_test/results_nofilt"+srtr+set_name
os.system(cmd)
print('-----------'+srtr+', finish--------')
with open(filename, "a") as f:
sys.stdout = f # Change the standard output to the file we created.
print('------------------Results for '+ sorter+'---------------')
print('no filter, set = ',file_name,',Hungrian 0.0 accuracy = ', acc_list1,',best match 0.1 accuracy = ', acc_list2 )
print('no filter, set = ',file_name,', best match 0.1 summary(well/bad/fp/overmerged/redundant) = ', well_detected)
sys.stdout = original_stdout
breakpoint()
return
#sorter_loop_nofilt('HS' ,'herdingspikes' ,recording,sorting_GT,'kampff') #too many false positives ac= 8%
sorter_loop_nofilt('IC' ,'ironclust' ,recording,sorting_GT,'kampff')
#sorter_loop_nofilt('SC' ,'spykingcircus2',recording,sorting_GT,'kampff') #acc = 20%
#sorter_loop_nofilt('TDC0','tridesclous' ,recording,sorting_GT,'kampff') #acc = 2%, too many false positives
sorter_loop_nofilt('KS' ,'kilosort3' ,recording,sorting_GT,'kampff') #acc=12.6%
The plot is here:
Contributor guide
No contributing guide indexed for this repository
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Reproduce the PAIRED_KAMPFF case from the shown script, starting with read_nwb, frame_slice, and bandpass_filter, then inspect each run_sorter call and its preprocessing parameters. Compare the outputs from compare_sorter_to_ground_truth with the plotted ground truth and sorting results; done means identifying and documenting the source of the excess spikes or confirming the relevant sorter configuration.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- matplotlib, numpy, pandas, python
- Domain
- data, testing-qa
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 25/100