1_process_bpod_output.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EJThompson 02/12/24\n",
    "\n",
    "## import functions from .py utils file: \n",
    "from Utilities.preprocessing import *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# set paths and params "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# list of Animal IDs (can be a single animal or multiple)\n",
    "Animal_ID = ['EJT100']\n",
    "\n",
    "# path to where data lives \n",
    "InputPath = (r'D:\\BPOD_data_example\\\\')\n",
    "# path to where processed data should go\n",
    "OutputPath = (r'D:\\Behavioural_data_output\\\\')\n",
    "\n",
    "CameraPath = (r'Z:\\data\\\\EJT_FlyCap_SeqTracking\\\\')\n",
    "\n",
    "# replace and redo previoulsy processed data = True, else = False\n",
    "Replace = True\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MAIN:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<b>Processing data for: EJT100</b>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EJT100_Sequence_Automated_20210503_152617.mat\n",
      "EJT100_Sequence_Automated_20210504_130853.mat\n",
      "EJT100_Sequence_Automated_20210505_150127.mat\n",
      "EJT100_Sequence_Automated_20210506_130707.mat\n",
      "EJT100_Sequence_Automated_20210507_124355.mat\n",
      "EJT100_Sequence_Automated_20210510_163634.mat\n",
      "EJT100_Sequence_Automated_20210511_130108.mat\n",
      "EJT100_Sequence_Automated_20210512_142005.mat\n",
      "EJT100_Sequence_Automated_20210513_132308.mat\n",
      "EJT100_Sequence_Automated_20210514_135821.mat\n",
      "EJT100_Sequence_Automated_20210517_140010.mat\n",
      "EJT100_Sequence_Automated_20210518_145854.mat\n",
      "EJT100_Sequence_Automated_20210520_113831.mat\n",
      "EJT100_Sequence_Automated_20210521_130841.mat\n",
      "EJT100_Sequence_Automated_20210524_113227.mat\n",
      "EJT100_Sequence_Automated_20210525_110423.mat\n",
      "EJT100_Sequence_Automated_20210526_121530.mat\n",
      "EJT100_Sequence_Automated_20210527_125621.mat\n",
      "EJT100_Sequence_Automated_20210528_130246.mat\n",
      "EJT100_Sequence_Automated_20210531_141633.mat\n",
      "EJT100_Sequence_Automated_20210601_115359.mat\n",
      "EJT100_Sequence_Automated_20210602_112555.mat\n",
      "EJT100_Sequence_Automated_20210604_141852.mat\n",
      "EJT100_Sequence_Automated_20210605_151407.mat\n",
      "EJT100_Sequence_Automated_20210607_104140.mat\n",
      "EJT100_Sequence_Automated_20210608_133449.mat\n",
      "EJT100_Sequence_Automated_20210609_103018.mat\n",
      "EJT100_Sequence_Automated_20210610_094933.mat\n",
      "EJT100_Sequence_Automated_20210611_094326.mat\n",
      "EJT100_Sequence_Automated_20210614_085710.mat\n",
      "EJT100_Sequence_Automated_20210615_093008.mat\n",
      "EJT100_Sequence_Automated_20210616_093103.mat\n",
      "EJT100_Sequence_Automated_20210617_101225.mat\n",
      "EJT100_Sequence_Automated_20210618_110451.mat\n",
      "EJT100_Sequence_Automated_20210621_092700.mat\n",
      "EJT100_Sequence_Automated_20210622_124158.mat\n",
      "EJT100_Sequence_Automated_20210628_121130.mat\n",
      "EJT100_Sequence_Automated_20210629_094458.mat\n",
      "EJT100_Sequence_Automated_20210630_092336.mat\n",
      "EJT100_Sequence_Automated_20210701_101524.mat\n",
      "EJT100_Sequence_Automated_20210702_100216.mat\n",
      "EJT100_Sequence_Automated_20210705_094405.mat\n",
      "EJT100_Sequence_Automated_20210708_153131.mat\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 43/43 [00:12<00:00,  3.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Already Processed so skipped: \n",
      "Processed: 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,\n",
      "finished\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# reloads imports (utils) if something changes\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "#### MAIN ####\n",
    "for Animal_index, CurrentAnimal in enumerate(Animal_ID):\n",
    "    display(HTML(f\"<b>Processing data for: {CurrentAnimal}</b>\"))\n",
    "    InputPathCurrent = (InputPath + CurrentAnimal + r'\\Sequence_Automated\\Session Data\\\\')\n",
    "    \n",
    "    #Load Behavioural data in its current .mat format:\n",
    "    Behav_data, Sessions,Path,FileDates = Import_Bpod_DataFiles(InputPathCurrent)\n",
    "    \n",
    "    Processed = ('')\n",
    "    Skipped = ('')\n",
    " \n",
    "\n",
    "    for Session in tqdm(range(0, Sessions)):\n",
    "\n",
    "        # make savepath \n",
    "        filedate = FileDates[Session] + '_' + str(Behav_data[Session]['__header__'])[-25:-22]\n",
    "        if Session < 10:\n",
    "            Save_path = (OutputPath + CurrentAnimal + r'\\Preprocessed' + '\\\\0' + str(Session) + '_' + filedate)\n",
    "        else:\n",
    "            Save_path = (OutputPath + CurrentAnimal + r'\\Preprocessed' + '\\\\' + str(Session) + '_' + filedate)\n",
    "\n",
    "        # see if dir exists already then use bool logic to detemrine if it needs to be processed or not\n",
    "        Process = check_and_create_directory(Save_path, Replace)\n",
    "\n",
    "        if  Process == True:\n",
    "            \n",
    "            #Convert to python friendly format:\n",
    "            convert_nested_structs(Behav_data[Session])\n",
    "            # Extract GUI info\n",
    "            Trial_settings = todict(Behav_data[Session]['SessionData']['TrialSettings'][0])\n",
    "            FinalRewardAmount = []\n",
    "            for item in Behav_data[Session]['SessionData']['SessionVariables']['TLevel']:\n",
    "                TLevel = item\n",
    "                FinalRewardAmount = FinalRewardAmount + [Behav_data[Session]['SessionData']['SessionVariables']['TrainingLevels'][TLevel-1][4]]\n",
    "\n",
    "            # save out training levels on their own\n",
    "            filename = 'Preprocessed_TrainingLevels' \n",
    "            with open(Save_path + '\\\\'+ filename, 'wb') as fp:\n",
    "                pickle.dump(Behav_data[Session]['SessionData']['SessionVariables']['TLevel'], fp)\n",
    "\n",
    "            # save out led intensites and reward amounts on their own:\n",
    "            LED_Intensities = pd.DataFrame({'Port2':Behav_data[Session]['SessionData']['SessionVariables']['LEDIntensitys']['port2'],\n",
    "                                    'Port3':Behav_data[Session]['SessionData']['SessionVariables']['LEDIntensitys']['port3'],\n",
    "                                    'Port4':Behav_data[Session]['SessionData']['SessionVariables']['LEDIntensitys']['port4'],\n",
    "                                    'Port5':Behav_data[Session]['SessionData']['SessionVariables']['LEDIntensitys']['port5']})\n",
    "            LED_Intensities.to_csv(Save_path + '/Preprocessed_LED_Intensities.csv')\n",
    "            RewardAmounts = pd.DataFrame({'Port1':Behav_data[Session]['SessionData']['SessionVariables']['RewardAmount']['port1'],\n",
    "                                            'Port2':Behav_data[Session]['SessionData']['SessionVariables']['RewardAmount']['port2'],\n",
    "                                            'Port3':Behav_data[Session]['SessionData']['SessionVariables']['RewardAmount']['port3'],\n",
    "                                            'Port4':Behav_data[Session]['SessionData']['SessionVariables']['RewardAmount']['port4']})\n",
    "            RewardAmounts.to_csv(Save_path + '/Preprocessed_RewardAmounts.csv')\n",
    "\n",
    "\n",
    "\n",
    "            #Extract PortIn times for each port and check for errors (inside this function):\n",
    "            All_PortIn_Times,All_PortOut_Times,All_Port_references = extract_poke_times(Behav_data[Session])\n",
    "\n",
    "            \n",
    "            #remove nans (times when part [in or out poke] of the event was dropped for some reason by bpod)\n",
    "            All_PortIn_Times_fixed,All_PortOut_Times_fixed ,All_Port_references_fixed = remove_dropped_in_events(All_PortIn_Times,All_PortOut_Times,All_Port_references)\n",
    "            \n",
    "            # Resort these in time:\n",
    "            All_PortIn_Times_sorted,All_PortOut_Times_sorted,All_Port_references_sorted = time_sort(All_PortIn_Times,All_PortOut_Times,All_Port_references)\n",
    "            \n",
    "            #fix stupid error that suddently appeared\n",
    "            if len(All_Port_references_sorted) == 1:\n",
    "                All_Port_references_sorted = All_Port_references_sorted[0]\n",
    "                All_PortIn_Times_sorted = All_PortIn_Times_sorted[0]\n",
    "                All_PortOut_Times_sorted = All_PortOut_Times_sorted[0]\n",
    "            #extract reward times:\n",
    "            Reward_ts = extract_reward_times(Behav_data[Session])\n",
    "\n",
    "            # find reward inds and align rewarded ts to poke events:\n",
    "            Rewarded_event_inds = find_reward_inds(All_PortIn_Times_sorted,All_Port_references_sorted,Reward_ts)\n",
    "            Reward_ts = np.asarray(Reward_ts)\n",
    "            Reward_ts = Reward_ts[np.logical_not(np.isnan(Reward_ts))]\n",
    "            Reward_ts = list(Reward_ts)\n",
    "            Reward_ts_aligned = align_trigger_to_index(Reward_ts,Rewarded_event_inds,All_Port_references_sorted)\n",
    "\n",
    "            #extract trial start time stamps\n",
    "            Trial_start_ts = extract_trial_timestamps(Behav_data[Session])\n",
    "\n",
    "            #extract trial end times:\n",
    "            Trial_end_ts = extract_trial_end_times(Behav_data[Session])\n",
    "\n",
    "            #determine trial IDs\n",
    "            trial_id = determine_trial_id(All_PortIn_Times_sorted,Trial_end_ts)\n",
    "\n",
    "            # align trial start and end times to poke events\n",
    "            trialstart_index = find_trialstart_index(trial_id)\n",
    "            trial_start_ts_aligned = align_trial_start_end_timestamps(trial_id,trialstart_index,Trial_start_ts)\n",
    "            trial_end_ts_aligned = align_trial_start_end_timestamps(trial_id,trialstart_index,Trial_end_ts)\n",
    "\n",
    "            # optostim data:\n",
    "            if Trial_settings['GUI']['OptoStim'] == 1: \n",
    "                ## save out opto settings as a dataframe\n",
    "                Opto_StimPoke = Trial_settings['GUI']['StimPoke']\n",
    "                Opto_PulsePower = Trial_settings['GUI']['PulsePower']\n",
    "                Opto_OptoChance = Trial_settings['GUI']['OptoChance']\n",
    "                Opto_PulseDuration = Trial_settings['GUI']['PulseDuration']\n",
    "                Opto_PulseInterval = Trial_settings['GUI']['PulseInterval']\n",
    "                Opto_TrainDuration = Trial_settings['GUI']['TrainDuration']\n",
    "                #create opto dataframe\n",
    "                Opto_settings = pd.DataFrame(\n",
    "                {'StimPoke' : [Opto_StimPoke],\n",
    "                 'PulsePower':[Opto_PulsePower],\n",
    "                 'OptoChance':[Opto_OptoChance],\n",
    "                 'PulseDuration' : [Opto_PulseDuration],\n",
    "                 'PulseInterval':[Opto_PulseInterval],\n",
    "                 'TrainDuration':[Opto_TrainDuration]})\n",
    "                #save\n",
    "                Opto_settings.to_csv(Save_path + '/Opto_settings.csv')\n",
    "\n",
    "                #pull out optotrials from data\n",
    "                optotrials = Behav_data[Session]['SessionData']['SessionVariables']['OptoStim']\n",
    "                #align these to dataframe:\n",
    "                executed_optotrials = optotrials[0:trial_id[-1]]\n",
    "                optotrials_aligned = align_opto_trials_to_dataframe(trial_id,executed_optotrials)\n",
    "            else:\n",
    "                #no optostim so fill this column with NaNs\n",
    "                optotrials_aligned = ['NaN'] * len(trial_id)\n",
    "\n",
    "            #determine LED and reward states for each trial and align them to trials:\n",
    "            IntermediateRewards = []\n",
    "            LED_intensities = []\n",
    "            for item in Behav_data[Session]['SessionData']['SessionVariables']['TLevel']:\n",
    "                TLevel = item\n",
    "                IntermediateRewards = IntermediateRewards + [list(Behav_data[Session]['SessionData']['SessionVariables']['TrainingLevels'][TLevel-1][0:4])]\n",
    "                LED_intensities = LED_intensities + [list(Behav_data[Session]['SessionData']['SessionVariables']['TrainingLevels'][TLevel-1][6:10])]\n",
    "            aligned_LED_intensities = align_trial_start_end_timestamps(trial_id,trialstart_index,LED_intensities)\n",
    "            aligned_IntermediateRewards = align_trial_start_end_timestamps(trial_id,trialstart_index,IntermediateRewards)\n",
    "\n",
    "            #Search for timestamps for given animal and session \n",
    "            TimeStampsExist, TimeStampPath = FindTimestamps(filedate,CameraPath,CurrentAnimal)\n",
    "\n",
    "            if TimeStampsExist:\n",
    "                print('Timestamps Found')\n",
    "                #Load camera timestamps:\n",
    "                Camera_ts_raw = load_camera_timestamps(TimeStampPath)\n",
    "                #Convert to seconds and uncycle:\n",
    "                Camera_ts = convert_uncycle_Timestamps(Camera_ts_raw)\n",
    "                #check for dropped frames:\n",
    "                check_timestamps(Camera_ts, Frame_rate = 40)\n",
    "                # Find triggers:\n",
    "                Camera_trig_states = find_trigger_states(Camera_ts_raw)\n",
    "                #check if triggers are working:\n",
    "                result = np.max(Camera_trig_states) == np.min(Camera_trig_states)\n",
    "                if not result:\n",
    "                    # make camera dataframe:\n",
    "                    Camera_dataframe = pd.DataFrame(\n",
    "                        {'Time Stamps': Camera_ts,\n",
    "                        'Trigger State': Camera_trig_states,\n",
    "                        'DataPath': ([TimeStampPath] * len(Camera_ts))})\n",
    "\n",
    "                    #Save Data\n",
    "                    Camera_dataframe.to_csv(Save_path + '/PreProcessed_CameraData.csv')\n",
    "\n",
    "                    #find camera inds that align with trigger start and stop events (trial start and first poke):\n",
    "                    Trial_start_camera_inds,Poke1_camera_inds = Find_TrialStart_and_Poke1_camera_inds(Camera_trig_states)\n",
    "                    #align behavioural data (trial starts) with camera timestamps \n",
    "                    Trial_start_Camera_Ts_aligned = align_trial_start_end_timestamps(trial_id,trialstart_index,Camera_ts[Trial_start_camera_inds])\n",
    "                    \n",
    "                    \n",
    "\n",
    "                    #align behavioural data (trial ends) with camera timestamps \n",
    "                    Trial_end_Camera_Ts_aligned = generate_aligned_trial_end_camera_ts(Trial_start_camera_inds,trial_id,trialstart_index,Camera_ts)\n",
    "                    #align behavioural data (first poke in port1) with camera timestamps \n",
    "                    First_poke_Camera_Ts_aligned = align_firstpoke_camera_timestamps(trial_id,trialstart_index,Camera_ts[Poke1_camera_inds],All_Port_references_sorted)\n",
    "                \n",
    "                else:\n",
    "                    print('Triggers Broken')\n",
    "                    TimeStampsExist = False\n",
    "                    Trial_start_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "                    Trial_end_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "                    First_poke_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "            else:\n",
    "                Trial_start_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "                Trial_end_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "                First_poke_Camera_Ts_aligned = ['NaN'] * len(trial_id)\n",
    "\n",
    "            ## align Training level for each trial:\n",
    "            Training_Levels = align_opto_trials_to_dataframe(trial_id,Behav_data[Session]['SessionData']['SessionVariables']['TLevel'])\n",
    "\n",
    "            # make portin dataframe:\n",
    "            PortIn_df = pd.DataFrame(\n",
    "                {'Trial_id' : trial_id,\n",
    "                 'Trial_Start' : trial_start_ts_aligned,\n",
    "                 'Port': All_Port_references_sorted,\n",
    "                 'PokeIn_Time': All_PortIn_Times_sorted,\n",
    "                 'PokeOut_Time': All_PortOut_Times_sorted,\n",
    "                 'Reward_Times': Reward_ts_aligned,\n",
    "                 'Trial_End' : trial_end_ts_aligned,\n",
    "                 'Trial_Start_Camera_Time':Trial_start_Camera_Ts_aligned,\n",
    "                 'Trial_End_Camera_Time':Trial_end_Camera_Ts_aligned,\n",
    "                 'First_poke_camera_timestamp': First_poke_Camera_Ts_aligned,\n",
    "                 'Port 2,3,4,5 LED intensities': aligned_LED_intensities,\n",
    "                 'Port 1,2,3,4 RewardAmount':aligned_IntermediateRewards,\n",
    "                 'OptoCondition':optotrials_aligned,\n",
    "                'TrainingLevel': Training_Levels})\n",
    "            \n",
    "            #Save Data\n",
    "            PortIn_df.to_csv(Save_path +'/PreProcessed_RawPokeData.csv')\n",
    "\n",
    "            #PART2##########################Transitions:\n",
    "\n",
    "            #Determine Transition times and types for all events \n",
    "            ##old way:\n",
    "#             Transition_times, Transition_types,transition_reference_time = Determine_Transition_Times_and_Types(All_PortIn_Times_sorted , All_Port_references_sorted)\n",
    "            out_in_Transition_times, in_in_Transition_times, Transition_types, out_in_transition_reference,in_in_transition_reference = Determine_Transition_Times_and_Types(All_PortIn_Times_sorted ,All_PortOut_Times_sorted, All_Port_references_sorted)\n",
    "            \n",
    "            \n",
    "            #Split transtion types into first and last ports: \n",
    "            start_port_ids = Start_End_port_id(Transition_types,0)\n",
    "            end_port_ids = Start_End_port_id(Transition_types,1)\n",
    "            #Align start and end port times\n",
    "            End_Port_in_time = All_PortIn_Times_sorted[1::]\n",
    "            Start_Port_in_time = All_PortIn_Times_sorted[0:-1]\n",
    "            End_Port_out_time = All_PortOut_Times_sorted[1::]\n",
    "            Start_Port_out_time = All_PortOut_Times_sorted[0:-1]\n",
    "            #Find Port repeat events (double pokes)\n",
    "            Non_Port_repeat = determine_RepeatPort_events(start_port_ids,end_port_ids)\n",
    "            #Determine which transitions are good: less than 1.5s\n",
    "            out_in_Filtered_transitions = filter_transitons_by_latency(out_in_Transition_times, 2)\n",
    "            in_in_Filtered_transitions = filter_transitons_by_latency(in_in_Transition_times, 2)\n",
    "\n",
    "            if TimeStampsExist:\n",
    "                # Align camera ts to each transition event \n",
    "                ## calculate from each time stamp\n",
    "                first_port_camera_ts = port_events_in_camera_time(trial_start_ts_aligned,Start_Port_in_time,Trial_start_Camera_Ts_aligned)\n",
    "                second_port_camera_ts = first_port_camera_ts + in_in_Transition_times\n",
    "            else:\n",
    "                first_port_camera_ts = ['NaN'] * len(Transition_types)\n",
    "                second_port_camera_ts = ['NaN'] * len(Transition_types)\n",
    "\n",
    "            # make dataframe:\n",
    "            Transition_df = pd.DataFrame(\n",
    "                {'Trial_id' : trial_id[0:-1],\n",
    "                'Transition_type' : Transition_types,\n",
    "                 'Start_Port':start_port_ids, \n",
    "                 'End_Port':end_port_ids,\n",
    "                 'Start_Poke_in_time': Start_Port_in_time,\n",
    "                 'Start_Poke_out_time': Start_Port_out_time,\n",
    "                 'End_Poke_in_time': End_Port_in_time,\n",
    "                 'End_Poke_out_time': End_Port_out_time,\n",
    "                 'out_in_Latency' : out_in_Transition_times,\n",
    "                 'in_in_Latency': in_in_Transition_times,\n",
    "                 'First_port_in_camera_time':first_port_camera_ts,\n",
    "                 'Second_port_in_camera_time':second_port_camera_ts,\n",
    "                 'Repeat_Filter':Non_Port_repeat,\n",
    "                 '2s_Time_Filter_out_in': out_in_Filtered_transitions,\n",
    "                 '2s_Time_Filter_in_in': in_in_Filtered_transitions,\n",
    "                 'OptoCondition':optotrials_aligned[0:-1],\n",
    "                 'TrainingLevel': Training_Levels[0:-1],\n",
    "                 'Port 2,3,4,5 LED intensities': aligned_LED_intensities[0:-1],\n",
    "                 'Port 1,2,3,4 RewardAmount':aligned_IntermediateRewards[0:-1]})\n",
    "            #Save Data\n",
    "            Transition_df.to_csv(Save_path + '/PreProcessed_TransitionData.csv')\n",
    "\n",
    "            #PART3#########################Sequences:\n",
    "\n",
    "            # Define useful port/seq related information\n",
    "            port1 = Behav_data[Session]['SessionData']['TrialSequence'][0][0]\n",
    "            port2 = Behav_data[Session]['SessionData']['TrialSequence'][0][1]\n",
    "            port3 = Behav_data[Session]['SessionData']['TrialSequence'][0][2]\n",
    "            port4 = Behav_data[Session]['SessionData']['TrialSequence'][0][3]\n",
    "            port5 = Behav_data[Session]['SessionData']['TrialSequence'][0][4]\n",
    "            sequence1 = int(str(port1) + str(port2))\n",
    "            sequence2 = int(str(port2) + str(port3))\n",
    "            sequence3 = int(str(port3) + str(port4))\n",
    "            sequence4 = int(str(port4) + str(port5))\n",
    "\n",
    "            #Filter transitons into sequences for each port by finding port and then transitions that happen after that port which are in a string that satisfies the transition filter time\n",
    "            # there is now a lower filter - ie. any sequences that are shorter than 0.05 are discounted:\n",
    "            ## these are from in to in as we dont care about transiton times, just poke in times for eahc port \n",
    "            Port1_Time_Filtered_seq_ids,Port1_Time_Filtered_seq_times,Port1_Reference_times = CreateSequences_TimeandPort(Transition_types,in_in_Transition_times,port1,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "            Port2_Time_Filtered_seq_ids,Port2_Time_Filtered_seq_times,Port2_Reference_times = CreateSequences_TimeandPort(Transition_types,in_in_Transition_times,port2,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "            Port3_Time_Filtered_seq_ids,Port3_Time_Filtered_seq_times,Port3_Reference_times = CreateSequences_TimeandPort(Transition_types,in_in_Transition_times,port3,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "            Port4_Time_Filtered_seq_ids,Port4_Time_Filtered_seq_times,Port4_Reference_times = CreateSequences_TimeandPort(Transition_types,in_in_Transition_times,port4,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "            Port5_Time_Filtered_seq_ids,Port5_Time_Filtered_seq_times,Port5_Reference_times = CreateSequences_TimeandPort(Transition_types,in_in_Transition_times,port5,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "\n",
    "            #Filter transitons into sequences that are within the transition filter time (not filtered to start at first poke):\n",
    "            Time_Filtered_seq_ids,Time_Filtered_seq_times,Reference_times = CreateSequences_Time(Transition_types,out_in_Transition_times,port1,in_in_transition_reference,Transition_filter_time = 2.0)\n",
    "\n",
    "            # make dataframes:\n",
    "            Sequence_df_timefiltered_port1aligned = pd.DataFrame(\n",
    "                {'Sequence_ids' : Port1_Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Port1_Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Port1_Reference_times})\n",
    "            Sequence_df_timefiltered_port2aligned = pd.DataFrame({\n",
    "                 'Sequence_ids' : Port2_Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Port2_Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Port2_Reference_times})\n",
    "            Sequence_df_timefiltered_port3aligned = pd.DataFrame({    \n",
    "                 'Sequence_ids' : Port3_Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Port3_Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Port3_Reference_times})\n",
    "            Sequence_df_timefiltered_port4aligned = pd.DataFrame({     \n",
    "                 'Sequence_ids' : Port4_Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Port4_Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Port4_Reference_times})\n",
    "            Sequence_df_timefiltered_port5aligned = pd.DataFrame({\n",
    "                 'Sequence_ids' : Port5_Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Port5_Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Port5_Reference_times})\n",
    "\n",
    "            Sequence_timefilteredonly_df = pd.DataFrame(\n",
    "                {'Sequence_ids' : Time_Filtered_seq_ids,\n",
    "                 'Sequence_times':Time_Filtered_seq_times,\n",
    "                 'SessionTime_Reference':Reference_times})\n",
    "\n",
    "            #Save Data\n",
    "            Sequence_df_timefiltered_port1aligned.to_csv(Save_path + '/PreProcessed_Sequence_df_timefiltered_port1aligned.csv')\n",
    "            Sequence_df_timefiltered_port2aligned.to_csv(Save_path + '/PreProcessed_Sequence_df_timefiltered_port2aligned.csv')\n",
    "            Sequence_df_timefiltered_port3aligned.to_csv(Save_path + '/PreProcessed_Sequence_df_timefiltered_port3aligned.csv')\n",
    "            Sequence_df_timefiltered_port4aligned.to_csv(Save_path + '/PreProcessed_Sequence_df_timefiltered_port4aligned.csv')\n",
    "            Sequence_df_timefiltered_port5aligned.to_csv(Save_path + '/PreProcessed_Sequence_df_timefiltered_port5aligned.csv')\n",
    "\n",
    "            Sequence_timefilteredonly_df.to_csv(Save_path + '/PreProcessed_Sequence_timefilteredonly_df.csv')\n",
    "\n",
    "            #Make final session information dataframe\n",
    "            TrainingLevels = list(Trial_settings['GUIMeta']['TrainingLevel']['String'])\n",
    "            SessionLevel = Trial_settings['GUI']['TrainingLevel']\n",
    "            no_rewarded_events = number_of_rewarded_events(Reward_ts_aligned)\n",
    "            \n",
    "            #experiment or training session:\n",
    "            if Trial_settings['GUI']['ExperimentType'] == 2:\n",
    "                Experiment = 1\n",
    "            else:\n",
    "                Experiment = 0\n",
    "\n",
    "            Session_information = pd.DataFrame(\n",
    "                {'Port1' : [port1],\n",
    "                 'Port2':[port2],\n",
    "                 'Port3':[port3],\n",
    "                 'Port4':[port4],\n",
    "                 'Port5':[port5],\n",
    "                'Transition1':sequence1,\n",
    "                'Transition2':sequence2,\n",
    "                'Transition3':sequence3,\n",
    "                'Transition4':sequence4,\n",
    "                'n_Trials': trial_id[-1],\n",
    "                'n_rewards': no_rewarded_events,\n",
    "                'FinalRewardAmount': [FinalRewardAmount],\n",
    "                'SessionLevel':[TLevel],\n",
    "                'Experiment':Experiment,\n",
    "                'CameraData': TimeStampsExist,})\n",
    "\n",
    "            #Save Data\n",
    "            Session_information.to_csv(Save_path + '/PreProcessed_SessionInfo.csv')\n",
    "            Processed = (Processed + str(Session) +',')\n",
    "        else:\n",
    "            Skipped = (Skipped + str(Session) +',')\n",
    "    if len(Skipped) == 0:\n",
    "        Skipped == 'None'\n",
    "    if len(Processed) == 0:\n",
    "        Processed == 'None'\n",
    "    print('Already Processed so skipped: ' + Skipped)\n",
    "    print('Processed: ' + Processed)\n",
    "\n",
    "print('finished')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}