SpikeInterface / SpikeInterface/spikeinterface

Question about PAIRED_KAMPFF datasets accuracy on various sorters

Open
#3,703 11 comments 0 reactions 0 assignees View on GitHub

Nobody has claimed this yet.

question
Dominant language
Python
Stars
847
Forks
280
Avg merge
3d 9h
Merged PRs (30d)
29

Description

Dear,
I am trying to use different sorters for the dataset PAIRED_KAMPFF. However, all 5 sorters I used give a lot more spikes than expected. The code is shown here, with the raster plot from the most accurate sorter (spykingcircus 2), which is also only 20% accuracy. I also plot the raster plots of ground_truth and timeseries. Could you please help a little bit about where to start debugging? Thank you very much!

import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import spikeinterface.extractors as se 
from spikeinterface.preprocessing import bandpass_filter
from probeinterface import Probe
import spikeinterface.comparison as sc
##############################file loading###########################
file_folder =  "a"
file_name = "sub-paired-kampff_ses-paired-kampff-2014-11-25-Pair-3-0_ecephys.nwb"

print(file_name)
recording,sorting_GT = se.read_nwb(file_path=os.path.join( file_folder, file_name), load_recording=True,load_sorting=True)
recording = recording.frame_slice(start_frame=0,end_frame = 60*recording.get_sampling_frequency())  
sorting_GT = sorting_GT.frame_slice(start_frame=0,end_frame = 60*recording.get_sampling_frequency())  
recording = bandpass_filter(recording)

probe_folder = "a"
probe_name = "probe_sub-paired-kampff_ses-paired-kampff-2014-11-25-Pair-3-0_ecephys.csv"

probe_df = pd.read_csv(os.path.join(probe_folder,probe_name))
xycoords = np.vstack( (probe_df['x'], probe_df['y'])).T #shape = 32,2
probe = Probe(ndim=2, si_units='um')
probe.set_contacts(positions=xycoords, shapes='circle', shape_params={'radius': 5})
channel_indices = np.arange(32)
probe.set_device_channel_indices(channel_indices)
recording.set_probe(probe)
recording.set_channel_locations(probe.contact_positions)


#############################sorting#######################################
from pathlib import Path
import spikeinterface.sorters as ss
import sys
from spikeinterface.preprocessing import common_reference, zscore
base_folder = Path("./test")
original_stdout = sys.stdout # Save a reference to the original standard output
sorter_params = dict(n_jobs=-1, chunk_duration="1s", progress_bar=True)
ss.IronClustSorter.set_ironclust_path('/imec/other/macaw/chen14/spikeinterface/Official_Tutorial_SI_0.96_Oct22/ironclust')
ss.Kilosort3Sorter.set_kilosort3_path('/imec/other/macaw/chen14/spikeinterface/Official_Tutorial_SI_0.96_Oct22/Kilosort')
cmp_method = ("raw_count", "by_unit", "pooled_with_average")


def sorter_loop_nofilt(srtr,sorter,recording,sorting_GT,set_name): 
    acc_list1 = []
    acc_list2 = []
    well_detected = []
    filename = 'test.txt'
    #recording = recording.frame_slice(start_frame=0*fs, end_frame=60*fs)
    path = base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name )
    if srtr == 'IC': 
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, filter = False, verbose=False,**sorter_params)
    elif srtr == 'KS': 
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True, freq_min = 40,verbose=True,**sorter_params)
    elif srtr == 'pyKS' :
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name), fshigh = 40,verbose=True,**sorter_params)
    elif srtr == 'HS':
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name),  filter = False, verbose=False)
    elif srtr == 'TDC': 
        recording = common_reference(recording)
        recording = zscore(recording, dtype='float32')
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True,  apply_preprocessing = False, verbose=False)
    elif srtr == 'TDC0':
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name ),  freq_min = 40, freq_max = None, verbose=False)

    elif srtr == 'SC': 
        recording = common_reference(recording)
        sorting_sorter = ss.run_sorter(sorter, recording, output_folder=base_folder /  ("14449576_sorters_test/results_nofilt"+srtr+set_name), delete_output_folder = True,  apply_preprocessing = False, verbose=False)
    file_path = base_folder/("sorting_results/results_nofilt"+srtr+set_name)
    se.NpzSortingExtractor.write_sorting(sorting_sorter, file_path)
    comp1 = sc.compare_sorter_to_ground_truth(sorting_GT, sorting_sorter,match_score = 0.0,exhaustive_gt=False)
    tmp_accuracy1 = comp1.get_performance(method = cmp_method[2])['accuracy']
    acc_list1.append(tmp_accuracy1)
    comp2 = sc.compare_sorter_to_ground_truth(sorting_GT, sorting_sorter,chance_score = 0.1,match_mode = 'best', exhaustive_gt=False)
    tmp_accuracy2 = comp2.get_performance(method = cmp_method[2])['accuracy']
    acc_list2.append(tmp_accuracy2)
    tmp_well_units = [len(comp2.get_well_detected_units())]
    well_detected.append(tmp_well_units)
    print('no filter, set = ',set_name,', Hungrian 0.0 accuracy = ', tmp_accuracy1, ',best match 0.1 accuracy = ', tmp_accuracy2)
    print('no filter, set = ',set_name,', best match 0.1 summary(well/bad/fp/overmerged/redundant) = ', tmp_well_units)
    cmd = "rm -rf mearec_sorters_test/results_nofilt"+srtr+set_name
    os.system(cmd)
    print('-----------'+srtr+', finish--------')

    with open(filename, "a") as f:    
        sys.stdout = f # Change the standard output to the file we created.   
        print('------------------Results for '+ sorter+'---------------')  

        print('no filter, set = ',file_name,',Hungrian 0.0 accuracy = ', acc_list1,',best match 0.1 accuracy = ', acc_list2 )

        print('no filter, set = ',file_name,', best match 0.1 summary(well/bad/fp/overmerged/redundant) = ', well_detected)
    sys.stdout = original_stdout
    breakpoint()
    return


#sorter_loop_nofilt('HS'  ,'herdingspikes' ,recording,sorting_GT,'kampff') #too many false positives ac= 8%
sorter_loop_nofilt('IC'  ,'ironclust'     ,recording,sorting_GT,'kampff') 
#sorter_loop_nofilt('SC'  ,'spykingcircus2',recording,sorting_GT,'kampff') #acc = 20%
#sorter_loop_nofilt('TDC0','tridesclous'   ,recording,sorting_GT,'kampff') #acc = 2%, too many false positives
sorter_loop_nofilt('KS'  ,'kilosort3'     ,recording,sorting_GT,'kampff') #acc=12.6%

The plot is here:

Image

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

Reproduce the PAIRED_KAMPFF case from the shown script, starting with read_nwb, frame_slice, and bandpass_filter, then inspect each run_sorter call and its preprocessing parameters. Compare the outputs from compare_sorter_to_ground_truth with the plotted ground truth and sorting results; done means identifying and documenting the source of the excess spikes or confirming the relevant sorter configuration.

Written by the indexing model from the issue text.

Assessment

Tech stack
matplotlib, numpy, pandas, python
Domain
data, testing-qa
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.