From 493006087d36c8a344e117c22a87420836a659d7 Mon Sep 17 00:00:00 2001 From: Marco Galardini Date: Tue, 16 Jul 2019 09:30:21 -0400 Subject: [PATCH 1/3] New client for new server/arduino configuration Still missing: - add a semaphore to avoid recieving multiple broadcasts - especially when pausing for long periods of time --- experiment/template/custom_script.py | 121 +++--- experiment/template/eVOLVER.py | 569 ++++++++++++++++++++++++++ experiment/template/eVOLVER_module.py | 382 ----------------- 3 files changed, 644 insertions(+), 428 deletions(-) create mode 100644 experiment/template/eVOLVER.py delete mode 100644 experiment/template/eVOLVER_module.py diff --git a/experiment/template/custom_script.py b/experiment/template/custom_script.py index b40a2755..71b7a2b7 100644 --- a/experiment/template/custom_script.py +++ b/experiment/template/custom_script.py @@ -1,19 +1,23 @@ -import eVOLVER_module +#!/usr/bin/env python3 + import numpy as np +import logging import os.path import time +# logger setup +logger = logging.getLogger(__name__) + ##### USER DEFINED GENERAL SETTINGS ##### #set new name for each experiment, otherwise files will be overwritten EXP_NAME = 'test_expt' -GUI_NAME = 'eVOLVER Experiment' EVOLVER_IP = '192.168.1.2' EVOLVER_PORT = 8081 ##### Identify pump calibration files, define initial values for temperature, stirring, volume, power settings -TEMP_INITIAL = [30] * 16 #degrees C, makes 16-value list +TEMP_INITIAL = [30] * 16 #degrees C, makes 16-value list #Alternatively enter 16-value list to set different values #TEMP_INITIAL = [30,30,30,30,32,32,32,32,34,34,34,34,36,36,36,36] @@ -24,12 +28,13 @@ VOLUME = 25 #mL, determined by vial cap straw length OD_POWER = 2125 #must match value used for OD calibration PUMP_CAL_FILE = 'pump_cal.txt' #tab delimited, mL/s with 16 influx pumps on first row, etc. -OPERATION_MODE = 'chemostat' #use to choose between 'turbidostat' and 'chemostat' functions +OPERATION_MODE = 'turbidostat' #use to choose between 'turbidostat' and 'chemostat' functions +# if using a different mode, name your function as the OPERATION_MODE variable ##### END OF USER DEFINED GENERAL SETTINGS ##### - -def turbidostat (OD_data, temp_data, vials, elapsed_time): +def turbidostat(eVOLVER, input_data, vials, elapsed_time): + OD_data = input_data['transformed']['od_90'] ##### USER DEFINED VARIABLES ##### @@ -39,7 +44,7 @@ def turbidostat (OD_data, temp_data, vials, elapsed_time): lower_thresh = [0.2] * len(vials) #to set all vials to the same value, creates 16-value list upper_thresh = [0.4] * len(vials) #to set all vials to the same value, creates 16-value list - #Alternatively, use 16 value list to set different thresholds, use 9999 for vials not being used + #Alternatively, use 16 value list to set different thresholds, use 9999 for vials not being used #lower_thresh = [0.2, 0.2, 0.3, 0.3, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999] #upper_thresh = [0.4, 0.4, 0.4, 0.4, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999, 9999] @@ -51,37 +56,38 @@ def turbidostat (OD_data, temp_data, vials, elapsed_time): #Tunable settings for overflow protection, pump scheduling etc. Unlikely to change between expts time_out = 5 #(sec) additional amount of time to run efflux pump - pump_wait = 3 # (min) minimum amount of time to wait between pump events + pump_wait = 3 # (min) minimum amount of time to wait between pump events ##### End of Turbidostat Settings ##### - control = np.power(2,range(0,32)) #vial addresses save_path = os.path.dirname(os.path.realpath(__file__)) #save path - flow_rate = eVOLVER_module.get_flow_rate() #read from calibration file + flow_rate = eVOLVER.get_flow_rate() #read from calibration file ##### Turbidostat Control Code Below ##### + # fluidic message: initialized so that no change is sent + MESSAGE = ['--'] * 48 for x in turbidostat_vials: #main loop through each vial # Update turbidostat configuration files for each vial # initialize OD and find OD path file_name = "vial{0}_ODset.txt".format(x) - ODset_path = os.path.join(save_path,EXP_NAME,'ODset',file_name) + ODset_path = os.path.join(save_path, EXP_NAME, 'ODset', file_name) data = np.genfromtxt(ODset_path, delimiter=',') ODset = data[len(data)-1][1] ODsettime = data[len(data)-1][0] num_curves=len(data)/2; file_name = "vial{0}_OD.txt".format(x) - OD_path = os.path.join(save_path,EXP_NAME,'OD',file_name) + OD_path = os.path.join(save_path, EXP_NAME, 'OD', file_name) data = np.genfromtxt(OD_path, delimiter=',') average_OD = 0 # Determine whether turbidostat dilutions are needed enough_ODdata = (len(data) > 7) #logical, checks to see if enough data points (couple minutes) for sliding window - collecting_more_curves = (num_curves <= (stop_after_n_curves+2)) #logical, checks to see if enough growth curves have happened + collecting_more_curves = (num_curves <= (stop_after_n_curves + 2)) #logical, checks to see if enough growth curves have happened if enough_ODdata: # Take median to avoid outlier @@ -92,16 +98,17 @@ def turbidostat (OD_data, temp_data, vials, elapsed_time): #if recently exceeded upper threshold, note end of growth curve in ODset, allow dilutions to occur and growthrate to be measured if (average_OD > upper_thresh[x]) and (ODset != lower_thresh[x]): - text_file = open(ODset_path,"a+") - text_file.write("{0},{1}\n".format(elapsed_time, lower_thresh[x])) + text_file = open(ODset_path, "a+") + text_file.write("{0},{1}\n".format(elapsed_time, + lower_thresh[x])) text_file.close() ODset = lower_thresh[x] # calculate growth rate - eVOLVER_module.calc_growth_rate(x, ODsettime, elapsed_time) + eVOLVER.calc_growth_rate(x, ODsettime, elapsed_time) #if have approx. reached lower threshold, note start of growth curve in ODset - if (average_OD < (lower_thresh[x]+(upper_thresh[x] - lower_thresh[x])/3)) and (ODset != upper_thresh[x]): - text_file = open(ODset_path,"a+") + if (average_OD < (lower_thresh[x] + (upper_thresh[x] - lower_thresh[x]) / 3)) and (ODset != upper_thresh[x]): + text_file = open(ODset_path, "a+") text_file.write("{0},{1}\n".format(elapsed_time, upper_thresh[x])) text_file.close() ODset = upper_thresh[x] @@ -116,21 +123,38 @@ def turbidostat (OD_data, temp_data, vials, elapsed_time): time_in = round(time_in, 2) - save_path = os.path.dirname(os.path.realpath(__file__)) file_name = "vial{0}_pump_log.txt".format(x) - file_path = os.path.join(save_path,EXP_NAME,'pump_log',file_name) + file_path = os.path.join(save_path, EXP_NAME, + 'pump_log', file_name) data = np.genfromtxt(file_path, delimiter=',') last_pump = data[len(data)-1][0] if ((elapsed_time - last_pump)*60) >= pump_wait: # if sufficient time since last pump, send command to Arduino - MESSAGE = {'pumps_binary':"{0:b}".format(control[x]), 'pump_time': time_in, 'efflux_pump_time': time_out, 'delay_interval': 0, 'times_to_repeat': 0, 'run_efflux': 1} - eVOLVER_module.fluid_command(MESSAGE, x, elapsed_time, pump_wait *60, time_in, 'y') + logger.info('turbidostat dilution for vial %d' % x) + # influx pump + MESSAGE[x] = str(time_in) + # efflux pump + MESSAGE[x + 16] = str(time_in + time_out) + + file_name = "vial{0}_pump_log.txt".format(x) + file_path = os.path.join(save_path, EXP_NAME, 'pump_log', file_name) + + text_file = open(file_path, "a+") + text_file.write("{0},{1}\n".format(elapsed_time, time_in)) + text_file.close() + else: + logger.debug('not enough OD measurements for vial %d' % x) + + # send fluidic command only if we are actually turning on any of the pumps + if MESSAGE != ['--'] * 48: + eVOLVER.fluid_command(MESSAGE) # your_FB_function_here() #good spot to call feedback functions for dynamic temperature, stirring, etc for ind. vials # your_function_here() #good spot to call non-feedback functions for dynamic temperature, stirring, etc. # end of turbidostat() fxn -def chemostat (OD_data, temp_data, vials, elapsed_time): +def chemostat(eVOLVER, input_data, vials, elapsed_time): + OD_data = input_data['transformed']['od_90'] ##### USER DEFINED VARIABLES ##### start_OD = 0 # ~OD600, set to 0 to start chemostate dilutions at any positive OD @@ -139,10 +163,10 @@ def chemostat (OD_data, temp_data, vials, elapsed_time): chemostat_vials = vials #vials is all 16, can set to different range (ex. [0,1,2,3]) to only trigger tstat on those vials - rate_config = [0.5] * len(vials) #to set all vials to the same value, creates 16-value list + rate_config = [0.5] * 16 #to set all vials to the same value, creates 16-value list #UNITS of 1/hr, NOT mL/hr, rate = flowrate/volume, so dilution rate ~ growth rate, set to 0 for unused vials - #Alternatively, use 16 value list to set different rates, use 0 for vials not being used + #Alternatively, use 16 value list to set different rates, use 0 for vials not being used #rate_config = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.1,1.2,1.3,1.4,1.5,1.6] ##### END OF USER DEFINED VARIABLES ##### @@ -155,32 +179,21 @@ def chemostat (OD_data, temp_data, vials, elapsed_time): ##### End of Chemostat Settings ##### - control = np.power(2,range(0,32)) #vial addresses - save_path = os.path.dirname(os.path.realpath(__file__)) #save path - flow_rate = eVOLVER_module.get_flow_rate() #read from calibration file + flow_rate = eVOLVER.get_flow_rate() #read from calibration file period_config = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] #initialize array bolus_in_s = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] #initialize array - + ##### Chemostat Control Code Below ##### for x in chemostat_vials: #main loop through each vial - #calculate time needed to pump bolus for each pump - bolus_in_s[x] = bolus/flow_rate[x] - - # calculate the period (i.e. frequency of dilution events) based on user specified growth rate and bolus size - if rate_config[x] > 0: - period_config[x] = (3600*bolus)/((rate_config[x])*VOLUME) #scale dilution rate by bolus size and volume - else: # if no dilutions needed, then just loops with no dilutions - period_config[x] = 0 - # Update chemostat configuration files for each vial #initialize OD and find OD path file_name = "vial{0}_OD.txt".format(x) - OD_path = os.path.join(save_path,EXP_NAME,'OD',file_name) + OD_path = os.path.join(save_path, EXP_NAME, 'OD', file_name) data = np.genfromtxt(OD_path, delimiter=',') average_OD = 0 enough_ODdata = (len(data) > 7) #logical, checks to see if enough data points (couple minutes) for sliding window @@ -189,13 +202,14 @@ def chemostat (OD_data, temp_data, vials, elapsed_time): #calculate median OD od_values_from_file = [] - for n in range(1,7): + for n in range(1, 7): od_values_from_file.append(data[len(data)-n][1]) average_OD = float(np.median(od_values_from_file)) # set chemostat config path and pull current state from file file_name = "vial{0}_chemoconfig.txt".format(x) - chemoconfig_path = os.path.join(save_path,EXP_NAME,'chemo_config',file_name) + chemoconfig_path = os.path.join(save_path, EXP_NAME, + 'chemo_config', file_name) chemo_config = np.genfromtxt(chemoconfig_path, delimiter=',') last_chemoset = chemo_config[len(chemo_config)-1][0] #should t=0 initially, changes each time a new command is written to file last_chemophase = chemo_config[len(chemo_config)-1][1] #should be zero initially, changes each time a new command is written to file @@ -204,20 +218,35 @@ def chemostat (OD_data, temp_data, vials, elapsed_time): # once start time has passed and culture hits start OD, if no command has been written, write new chemostat command to file if ((elapsed_time > start_time) & (average_OD > start_OD)): + #calculate time needed to pump bolus for each pump + bolus_in_s[x] = bolus/flow_rate[x] + + # calculate the period (i.e. frequency of dilution events) based on user specified growth rate and bolus size + if rate_config[x] > 0: + period_config[x] = (3600*bolus)/((rate_config[x])*VOLUME) #scale dilution rate by bolus size and volume + else: # if no dilutions needed, then just loops with no dilutions + period_config[x] = 0 + if (last_chemorate != period_config[x]): print('Chemostat updated in vial {0}'.format(x)) + logger.info('chemostat initiated for vial %d, period %.2f' + % (x, period_config[x])) # writes command to chemo_config file, for storage - text_file = open(chemoconfig_path,"a+") - text_file.write("{0},{1},{2}\n".format(elapsed_time,(last_chemophase+1),period_config[x])) #note that this changes chemophase + text_file = open(chemoconfig_path, "a+") + text_file.write("{0},{1},{2}\n".format(elapsed_time, + (last_chemophase+1), + period_config[x])) #note that this changes chemophase text_file.close() + else: + logger.debug('not enough OD measurements for vial %d' % x) # your_FB_function_here() #good spot to call feedback functions for dynamic temperature, stirring, etc for ind. vials - # your_function_here() #good spot to call non-feedback functions for dynamic temperature, stirring, etc. + # your_function_here() #good spot to call non-feedback functions for dynamic temperature, stirring, etc. - eVOLVER_module.update_chemo(vials, bolus_in_s) #uses values stored in chemo_config files + eVOLVER.update_chemo(input_data, chemostat_vials, bolus_in_s, period_config) #compares computed chemostat config to the remote one # end of chemostat() fxn # def your_function_here(): # good spot to define modular functions for dynamics or feedback if __name__ == '__main__': - print('Please run main_eVOLVER.py instead') \ No newline at end of file + print('Please run eVOLVER.py instead') diff --git a/experiment/template/eVOLVER.py b/experiment/template/eVOLVER.py new file mode 100644 index 00000000..dee1d36c --- /dev/null +++ b/experiment/template/eVOLVER.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import pickle +import shutil +import logging +import argparse +import numpy as np +from scipy import stats +from socketIO_client import SocketIO, BaseNamespace + +import custom_script +from custom_script import EXP_NAME, OD_POWER, PUMP_CAL_FILE +from custom_script import EVOLVER_IP, EVOLVER_PORT, OPERATION_MODE +from custom_script import STIR_INITIAL, TEMP_INITIAL + +# Should not be changed +# vials to be considered/excluded should be handled +# inside the custom functions +VIALS = [x for x in range(16)] + +SAVE_PATH = os.path.dirname(os.path.realpath(__file__)) +EXP_DIR = os.path.join(SAVE_PATH, EXP_NAME) + +logger = logging.getLogger('eVOLVER') + +EVOLVER_NS = None + +class EvolverNamespace(BaseNamespace): + start_time = None + use_blank = False + OD_initial = None + + def on_connect(self, *args): + print("Connected to eVOLVER as client") + logger.info('connected to eVOLVER as client') + + def on_disconnect(self, *args): + print("Disconected from eVOLVER as client") + logger.info('disconnected to eVOLVER as client') + + def on_reconnect(self, *args): + print("Reconnected to eVOLVER as client") + logger.info("reconnected to eVOLVER as client") + + def on_broadcast(self, data): + logger.debug('broadcast received') + elapsed_time = round((time.time() - self.start_time) / 3600, 4) + logger.debug('elapsed time: %.4f hours' % elapsed_time) + print("{0}: {1} Hours".format(EXP_NAME, elapsed_time)) + # are the calibrations in yet? + if not self.check_for_calibrations(): + logger.warning('calibration files still missing, skipping custom ' + 'functions') + return + + # apply calibrations + # update temperatures if needed + data = self.transform_data(data, VIALS) + if data is None: + logger.error('could not tranform raw data, skipping user-' + 'defined functions') + return + + # should we "blank" the OD? + if self.use_blank and self.OD_initial is None: + logger.info('setting initial OD reading') + self.OD_initial = data['transformed']['od_90'] + elif self.OD_initial is None: + self.OD_initial = np.zeros(len(VIALS)) + data['transformed']['od_90'] = (data['transformed']['od_90'] - + self.OD_initial) + # save data + self.save_data(data['transformed']['od_90'], elapsed_time, + VIALS, 'OD') + self.save_data(data['transformed']['temp'], elapsed_time, + VIALS, 'temp') + # run custom functions + self.custom_functions(data, VIALS, elapsed_time) + # save variables + self.save_variables(self.start_time, self.OD_initial) + + def on_calibrationod(self, data): + file_path = os.path.join(EXP_DIR, 'od_cal.txt') + with open(file_path, 'w') as f: + f.write(data) + logger.debug("OD calibration: %s" % data) + + def on_calibrationtemp(self, data): + file_path = os.path.join(EXP_DIR, 'temp_calibration.txt') + with open(file_path , 'w') as f: + f.write(data) + logger.debug("temperature calibration: %s" % data) + + def request_od_calibration(self): + logger.debug('requesting OD calibrations') + self.emit('getcalibrationod', + {}, namespace = '/dpu-evolver') + + def request_temp_calibration(self): + logger.debug('requesting temperature calibrations') + self.emit('getcalibrationtemp', + {}, namespace = '/dpu-evolver') + + def transform_data(self, data, vials): + odcal_path = os.path.join(EXP_DIR, 'od_cal.txt') + od_cal = np.genfromtxt(odcal_path, delimiter=',') + + tempcal_path = os.path.join(EXP_DIR, 'temp_calibration.txt') + temp_cal = np.genfromtxt(tempcal_path, delimiter=',') + + od_data = data['data'].get('od_90', None) + temp_data = data['data'].get('temp', None) + set_temp_data = data['config'].get('temp', {}).get('value', None) + + if od_data is None or temp_data is None or set_temp_data is None: + print('Incomplete data recieved, Error with measurement') + logger.error('Incomplete data received, error with measurements') + return None + if 'NaN' in od_data or 'NaN' in temp_data or 'NaN' in set_temp_data: + print('NaN recieved, Error with measurement') + logger.error('NaN received, error with measurements') + return None + + od_data = np.array([float(x) for x in od_data]) + temp_data = np.array([float(x) for x in temp_data]) + set_temp_data = np.array([float(x) for x in set_temp_data]) + + temps = [] + for x in vials: + file_name = "vial{0}_tempconfig.txt".format(x) + file_path = os.path.join(EXP_DIR, 'temp_config', file_name) + temp_set_data = np.genfromtxt(file_path, delimiter=',') + temp_set = temp_set_data[len(temp_set_data)-1][1] + temps.append(temp_set) + try: + if (od_cal.shape[0] == 4): + #convert raw photodiode data into ODdata using calibration curve + od_data[x] = np.real(od_cal[2,x] - + ((np.log10((od_cal[1,x] - + od_cal[0,x]) / + (float(od_data[x]) - + od_cal[0,x])-1)) / + od_cal[3,x])) + if not np.isfinite(od_data[x]): + od_data[x] = 'NaN' + logger.debug('OD from vial %d: %s' % (x, od_data[x])) + else: + logger.debug('OD from vial %d: %.3f' % (x, od_data[x])) + else: + logger.error('OD calibration not formatted correctly') + od_data[x] = 'NaN' + except ValueError: + print("OD Read Error") + logger.error('OD read error for vial %d, setting to NaN' % x) + od_data[x] = 'NaN' + try: + temp_data[x] = (float(temp_data[x]) * + temp_cal[0][x]) + temp_cal[1][x] + logger.debug('temperature from vial %d: %.3f' % (x, temp_data[x])) + except ValueError: + print("Temp Read Error") + logger.error('temperature read error for vial %d, setting to NaN' + % x) + temp_data[x] = 'NaN' + try: + set_temp_data[x] = (float(set_temp_data[x]) * + temp_cal[0][x]) + temp_cal[1][x] + logger.debug('set_temperature from vial %d: %.3f' % (x, + set_temp_data[x])) + except ValueError: + print("Set Temp Read Error") + logger.error('set temperature read error for vial %d, setting to NaN' + % x) + set_temp_data[x] = 'NaN' + + temps = np.array(temps) + # update temperatures only if difference with expected + # value is above 0.2 degrees celsius + delta_t = np.abs(set_temp_data - temps).max() + if delta_t > 0.2: + logger.info('updating temperatures (max. deltaT is %.2f)' % + delta_t) + raw_temperatures = [str(int((temps[x] - temp_cal[1][x]) / + temp_cal[0][x])) + for x in vials] + self.update_temperature(raw_temperatures) + else: + # config from server agrees with local config + # report if actual temperature doesn't match + delta_t = np.abs(temps - temp_data).max() + if delta_t > 0.2: + logger.info('actual temperature doesn\'t match configuration ' + '(yet? max deltaT is %.2f)' % delta_t) + logger.debug('temperature config: %s' % temps) + logger.debug('actual temperatures: %s' % temp_data) + + # add a new field in the data dictionary + data['transformed'] = {} + data['transformed']['od_90'] = od_data + data['transformed']['temp'] = temp_data + return data + + def update_stir_rate(self, stir_rates): + data = {'param': 'stir', 'value': stir_rates, + 'immediate': False, 'recurring': True} + logger.debug('stir rate command: %s' % data) + self.emit('command', data, namespace = '/dpu-evolver') + + def update_temperature(self, temperatures): + data = {'param': 'temp', 'value': temperatures, + 'immediate': False, 'recurring': True} + logger.debug('temperature command: %s' % data) + self.emit('command', data, namespace = '/dpu-evolver') + + def fluid_command(self, MESSAGE): + logger.debug('fluid command: %s' % MESSAGE) + command = {'param': 'pump', 'value': MESSAGE, + 'recurring': False ,'immediate': True} + self.emit('command', command, namespace='/dpu-evolver') + + def update_chemo(self, data, vials, bolus_in_s, period_config): + current_pump = data['config']['pump']['value'] + + MESSAGE = {'fields_expected_incoming': 49, + 'fields_expected_outgoing': 49, + 'recurring': True, + 'immediate': False, + 'value': ['--'] * 48, + 'param': 'pump'} + + for x in vials: + # stop pumps if period is zero + if period_config[x] == 0: + # influx + MESSAGE['value'][x] = '0|0' + # efflux + MESSAGE['value'][x + 16] = '0|0' + else: + # influx + MESSAGE['value'][x] = '%.2f|%d' % (bolus_in_s[x], period_config[x]) + # efflux + MESSAGE['value'][x + 16] = '%.2f|%d' % (bolus_in_s[x] * 2, + period_config[x]) + + if MESSAGE['value'] != current_pump: + logger.info('updating chemostat: %s' % MESSAGE) + self.emit('command', MESSAGE, namespace = '/dpu-evolver') + + def stop_all_pumps(self, ): + data = {'param': 'pump', + 'value': ['0'] * 48, + 'recurring': False, + 'immediate': True} + logger.info('stopping all pumps') + self.emit('command', data, namespace = '/dpu-evolver') + + def _create_file(self, vial, param, directory=None, defaults=None): + if defaults is None: + defaults = [] + if directory is None: + directory = param + file_name = "vial{0}_{1}.txt".format(vial, param) + file_path = os.path.join(EXP_DIR, directory, file_name) + text_file = open(file_path, "w") + for default in defaults: + text_file.write(default + '\n') + text_file.close() + + def initialize_exp(self, vials, always_yes=False): + logger.debug('initializing experiment') + + if os.path.exists(EXP_DIR): + logger.info('found an existing experiment') + exp_continue = None + if always_yes: + exp_continue = 'y' + else: + while exp_continue not in ['y', 'n']: + exp_continue = input('Continue from existing experiment? (y/n): ') + else: + exp_continue = 'n' + + if exp_continue == 'n': + if os.path.exists(EXP_DIR): + exp_overwrite = None + if always_yes: + exp_overwrite = 'y' + else: + while exp_overwrite not in ['y', 'n']: + exp_overwrite = input('Directory aleady exists. ' + 'Overwrite with new experiment? (y/n): ') + logger.info('data directory already exists') + if exp_overwrite == 'y': + logger.info('deleting existing data directory') + shutil.rmtree(EXP_DIR) + else: + print('Change experiment name in custom_script.py ' + 'and then restart...') + logger.warning('not deleting existing data directory, exiting') + sys.exit(1) + + start_time = time.time() + + self.request_od_calibration() + self.request_temp_calibration() + + logger.debug('creating data directories') + os.makedirs(os.path.join(EXP_DIR, 'OD')) + os.makedirs(os.path.join(EXP_DIR, 'temp')) + os.makedirs(os.path.join(EXP_DIR, 'temp_config')) + os.makedirs(os.path.join(EXP_DIR, 'pump_log')) + os.makedirs(os.path.join(EXP_DIR, 'ODset')) + os.makedirs(os.path.join(EXP_DIR, 'growthrate')) + os.makedirs(os.path.join(EXP_DIR, 'chemo_config')) + for x in vials: + exp_str = "Experiment: {0} vial {1}, {2}".format(EXP_NAME, + x, + time.strftime("%c")) + # make OD file + self._create_file(x, 'OD', defaults=[exp_str]) + # make temperature data file + self._create_file(x, 'temp') + # make temperature configuration file + self._create_file(x, 'tempconfig', + defaults=[exp_str, + "0,{0}".format(TEMP_INITIAL[x])]) + # make pump log file + self._create_file(x, 'pump_log', + defaults=[exp_str, + "0,0"]) + # make ODset file + self._create_file(x, 'ODset', + defaults=[exp_str, + "0,0"]) + # make growth rate file + self._create_file(x, 'gr', + defaults=[exp_str, + "0,0"], + directory='growthrate') + # make chemostat file + self._create_file(x, 'chemoconfig', + defaults=["0,0,0", + "0,0,0"], + directory='chemo_config') + + self.update_stir_rate(STIR_INITIAL) + + if always_yes: + exp_blank = 'y' + else: + exp_blank = input('Calibrate vials to blank? (y/n): ') + if exp_blank == 'y': + # will do it with first broadcast + self.use_blank = True + logger.info('will use initial OD measurement as blank') + else: + self.use_blank = False + self.OD_initial = np.zeros(len(vials)) + else: + # load existing experiment + pickle_name = "{0}.pickle".format(EXP_NAME) + pickle_path = os.path.join(EXP_DIR, pickle_name) + logger.info('loading previous experiment data: %s' % pickle_path) + with open(pickle_path, 'rb') as f: + loaded_var = pickle.load(f) + x = loaded_var + start_time = x[0] + self.OD_initial = x[1] + + # copy current custom script to txt file + backup_filename = '{0}_{1}.txt'.format(EXP_NAME, + time.strftime('%y%m%d_%H%M')) + shutil.copy('custom_script.py', os.path.join(EXP_DIR, + backup_filename)) + logger.info('saved a copy of current custom_script.py as %s' % + backup_filename) + + return start_time + + def check_for_calibrations(self): + result = True + odcal_path = os.path.join(EXP_DIR, 'od_cal.txt') + tempcal_path = os.path.join(EXP_DIR, 'temp_calibration.txt') + if not os.path.exists(odcal_path): + # log and request again + logger.warning('OD calibrations not received yet, requesting again') + self.request_od_calibration() + result = False + if not os.path.exists(tempcal_path): + # log and request again + logger.warning('temperature calibrations not received yet, ' + 'requesting again') + self.request_temp_calibration() + result = False + return result + + def save_data(self, data, elapsed_time, vials, parameter): + for x in vials: + file_name = "vial{0}_{1}.txt".format(x, parameter) + file_path = os.path.join(EXP_DIR, parameter, file_name) + text_file = open(file_path, "a+") + text_file.write("{0},{1}\n".format(elapsed_time, data[x])) + text_file.close() + + def save_variables(self, start_time, OD_initial): + # save variables needed for restarting experiment later + save_path = os.path.dirname(os.path.realpath(__file__)) + pickle_name = "{0}.pickle".format(EXP_NAME) + pickle_path = os.path.join(EXP_DIR, pickle_name) + logger.debug('saving all variables: %s' % pickle_path) + with open(pickle_path, 'wb') as f: + pickle.dump([start_time, OD_initial], f) + + def get_flow_rate(self): + file_path = os.path.join(SAVE_PATH, PUMP_CAL_FILE) + flow_calibration = np.loadtxt(file_path, delimiter="\t") + if len(flow_calibration) == 16: + flow_rate = flow_calibration + else: + # Currently just implementing influx flow rate + flow_rate = flow_calibration[0,:] + return flow_rate + + def calc_growth_rate(self, vial, gr_start, elapsed_time): + ODfile_name = "vial{0}_OD.txt".format(vial) + # Grab Data and make setpoint + OD_path = os.path.join(EXP_DIR, 'OD', ODfile_name) + OD_data = np.genfromtxt(OD_path, delimiter=',') + raw_time = OD_data[:, 0] + raw_OD = OD_data[:, 1] + raw_time = raw_time[np.isfinite(raw_OD)] + raw_OD = raw_OD[np.isfinite(raw_OD)] + + # Trim points prior to gr_start + trim_time = raw_time[np.nonzero(np.where(raw_time > gr_start, 1, 0))] + trim_OD = raw_OD[np.nonzero(np.where(raw_time > gr_start, 1, 0))] + + # Take natural log, calculate slope + log_OD = np.log(trim_OD) + slope, intercept, r_value, p_value, std_err = stats.linregress( + trim_time[np.isfinite(log_OD)], + log_OD[np.isfinite(log_OD)]) + logger.debug('growth rate for vial %s: %.2f' % (vial, slope)) + + # Save slope to file + file_name = "vial{0}_gr.txt".format(vial) + gr_path = os.path.join(EXP_DIR, 'growthrate', file_name) + text_file = open(gr_path, "a+") + text_file.write("{0},{1}\n".format(elapsed_time, slope)) + text_file.close() + + def custom_functions(self, data, vials, elapsed_time): + # load user script from custom_script.py + if OPERATION_MODE == 'turbidostat': + custom_script.turbidostat(self, data, vials, elapsed_time) + elif OPERATION_MODE == 'chemostat': + custom_script.chemostat(self, data, vials, elapsed_time) + else: + # try to load the user function + # if failing report to user + logger.info('user-defined operation mode %s' % OPERATION_MODE) + try: + func = getattr(custom_script, OPERATION_MODE) + func(self, data, vials, elapsed_time) + except AttributeError: + logger.error('could not find function %s in custom_script.py' % + OPERATION_MODE) + print('Could not find function %s in custom_script.py ' + '- Skipping user defined functions'% + OPERATION_MODE) + + def stop_exp(self): + self.stop_all_pumps() + +def get_options(): + description = 'Run an eVOLVER experiment from the command line' + parser = argparse.ArgumentParser(description=description) + + parser.add_argument('--always-yes', action='store_true', + default=False, + help='Answer yes to all questions ' + '(i.e. continues from existing experiment, ' + 'overwrites existing data and blanks OD ' + 'measurements)') + parser.add_argument('--log-name', + default=os.path.join(EXP_DIR, 'evolver.log'), + help='Log file name directory (default: %(default)s)') + + log_nolog = parser.add_mutually_exclusive_group() + log_nolog.add_argument('--verbose', action='count', + default=0, + help='Increase logging verbosity level to DEBUG ' + '(default: INFO)') + log_nolog.add_argument('--quiet', action='store_true', + default=False, + help='Disable logging to file entirely') + + return parser.parse_args() + +if __name__ == '__main__': + options = get_options() + + #changes terminal tab title in OSX + print('\x1B]0;eVOLVER EXPERIMENT: PRESS Ctrl-C TO PAUSE\x07') + + # silence logging until experiment is initialized + logging.level = logging.CRITICAL + 10 + + socketIO = SocketIO(EVOLVER_IP, EVOLVER_PORT) + EVOLVER_NS = socketIO.define(EvolverNamespace, '/dpu-evolver') + + # start by stopping any existing chemostat + EVOLVER_NS.stop_all_pumps() + # + EVOLVER_NS.start_time = EVOLVER_NS.initialize_exp(VIALS, + options.always_yes) + + # logging setup + if options.quiet: + logging.basicConfig(level=logging.CRITICAL + 10) + else: + if options.verbose == 0: + level = logging.INFO + elif options.verbose >= 1: + level = logging.DEBUG + logging.basicConfig(format='%(asctime)s - %(name)s - [%(levelname)s] ' + '- %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + filename=options.log_name, + level=level) + + while True: + try: + # infinite loop + socketIO.wait() + except KeyboardInterrupt: + try: + print('Ctrl-C detected, pausing experiment') + logger.warning('interrupt received, pausing experiment') + EVOLVER_NS.stop_exp() + while True: + key = input('Experiment paused. Press enter key to restart ' + ' or hit Ctrl-C again to terminate experiment') + logger.warning('resuming experiment') + # no need to have something like "restart_chemo" here + # with the new server logic + break + except KeyboardInterrupt: + print('Second Ctrl-C detected, shutting down') + logger.warning('second interrupt received, terminating ' + 'experiment') + EVOLVER_NS.stop_exp() + print('Experiment stopped, goodbye!') + logger.warning('experiment stopped, goodbye!') + break + except Exception as e: + logger.critical('exception %s stopped the experiment' % str(e)) + print('error "%s" stopped the experiment' % str(e)) + EVOLVER_NS.stop_exp() + print('Experiment stopped, goodbye!') + logger.warning('experiment stopped, goodbye!') + break + + # stop experiment one last time + # covers corner case where user presses Ctrl-C twice quickly + EVOLVER_NS.stop_exp() diff --git a/experiment/template/eVOLVER_module.py b/experiment/template/eVOLVER_module.py deleted file mode 100644 index 0a9e37bf..00000000 --- a/experiment/template/eVOLVER_module.py +++ /dev/null @@ -1,382 +0,0 @@ -import socket -import os.path -import shutil -import time -import pickle -import numpy as np -import numpy.matlib -import matplotlib -import scipy.signal -matplotlib.use('Agg') -import matplotlib.pyplot as plt -from socketIO_client import SocketIO, BaseNamespace -from threading import Thread -import asyncio -import custom_script - -plt.ioff() - -dpu_evolver_ns = None -received_data = {} -od_cal = None -temp_cal = None - -save_path = os.path.dirname(os.path.realpath(__file__)) -shared_name = None -shared_ip = None -shared_port = None - -# TODO: use proper async/await -wait_for_data = True -control = np.power(2,range(0,32)) -current_chemo = [0] * 16 -current_temps = [0] * 16 -connected = False - -class EvolverNamespace(BaseNamespace): - def on_connect(self, *args): - global connected, current_temps - print("Connected to eVOLVER as client") - connected = True - - def on_disconnect(self, *args): - global connected - print("Disconected from eVOLVER as client") - connected = False - - def on_reconnect(self, *args): - global connected, stop_waiting, current_temps - current_temps = [0] * 16 - print("Reconnected to eVOLVER as client") - connected = True - stop_waiting = True - - def on_dataresponse(self, data): - #print(data) - global received_data, wait_for_data - received_data = data - wait_for_data = False - - def on_calibrationod(self, data): - global od_cal, save_path, shared_name - file_path = os.path.join(save_path,shared_name,'od_cal.txt') - with open(file_path, 'w') as f: - f.write(data) - od_cal = True - - def on_calibrationtemp(self, data): - global temp_cal, save_path, shared_name - file_path = os.path.join(save_path,shared_name,'temp_calibration.txt') - with open(file_path , 'w') as f: - f.write(data) - temp_cal = True - -def read_data(vials): - global wait_for_data, received_data, current_temps, connected, stop_waiting, shared_ip, shared_port - - save_path = os.path.dirname(os.path.realpath(__file__)) - - odcal_path = os.path.join(save_path,custom_script.EXP_NAME,'od_cal.txt') - od_cal = np.genfromtxt(odcal_path, delimiter=',') - - tempcal_path = os.path.join(save_path,custom_script.EXP_NAME,'temp_calibration.txt') - temp_cal = np.genfromtxt(tempcal_path, delimiter=',') - - wait_for_data = True - stop_waiting = False - dpu_evolver_ns.emit('data', {'config':{'od':[custom_script.OD_POWER] * 16, 'temp':['NaN'] * 16}}, namespace='/dpu-evolver') - start_time = time.time() - # print('Fetching data from eVOLVER') - while(wait_for_data): - if not connected or stop_waiting or (time.time() - start_time > 30): - wait_for_data = False - print('Issue with eVOLVER communication - skipping data acquisition') - return None, None - pass - - - od_data = received_data['od'] - temp_data = received_data['temp'] - if 'NaN' in od_data or 'NaN' in temp_data: - print('NaN recieved, Error with measurement') - return None, None - temps = [] - for x in vials: - file_name = "vial{0}_tempconfig.txt".format(x) - file_path = os.path.join(save_path,custom_script.EXP_NAME,'temp_config',file_name) - temp_set_data = np.genfromtxt(file_path, delimiter=',') - temp_set = temp_set_data[len(temp_set_data)-1][1] - temp_set = int((temp_set - temp_cal[1][x])/temp_cal[0][x]) #convert raw thermistor data into temp using calibration fit - temps.append(temp_set) - - try: - if (od_cal.shape[0] == 4): #convert raw photodiode data into ODdata using calibration curve - od_data[x] = np.real(od_cal[2,x] - ((np.log10((od_cal[1,x]-od_cal[0,x])/(float(od_data[x]) - od_cal[0,x])-1))/od_cal[3,x])) - except ValueError: - print("OD Read Error") - od_data[x] = 'NaN' - try: - temp_data[x] = (float(temp_data[x]) * temp_cal[0][x]) + temp_cal[1][x] - except ValueError: - print("Temp Read Error") - temp_data[x] = 'NaN' - if not temps == current_temps: - MESSAGE = list(temps) - command = {'param':'temp', 'message':MESSAGE} - dpu_evolver_ns.emit('command', command, namespace='/dpu-evolver') - current_temps = temps - - return od_data, temp_data - -def fluid_command(MESSAGE, vial, elapsed_time, pump_wait, time_on, file_write): - command = {'param':'pump', 'message':MESSAGE} - dpu_evolver_ns.emit('command', command, namespace='/dpu-evolver') - - save_path = os.path.dirname(os.path.realpath(__file__)) - - file_name = "vial{0}_pump_log.txt".format(vial) - file_path = os.path.join(save_path,custom_script.EXP_NAME,'pump_log',file_name) - - if file_write == 'y': - text_file = open(file_path,"a+") - text_file.write("{0},{1}\n".format(elapsed_time, time_on)) - text_file.close() - -def update_chemo(vials, bolus_in_s): - global current_chemo - - save_path = os.path.dirname(os.path.realpath(__file__)) - MESSAGE = {} - for x in vials: - file_name = "vial{0}_chemoconfig.txt".format(x) - file_path = os.path.join(save_path,custom_script.EXP_NAME,'chemo_config',file_name) - - data = np.genfromtxt(file_path, delimiter=',') - chemo_set = data[len(data)-1][2] - if not chemo_set == current_chemo[x]: - current_chemo[x] = chemo_set - if chemo_set == 0: - time_on = 0 - time_off = 0 - t_rep = 0 - else: - time_on = bolus_in_s[x] - time_off = (bolus_in_s[x]*2) - t_rep = -1 - MESSAGE = {'pumps_binary':"{0:b}".format(control[x]), 'pump_time': time_on, 'efflux_pump_time': time_off, 'delay_interval': chemo_set, 'times_to_repeat': t_rep, 'run_efflux': 1} - command = {'param': 'pump', 'message': MESSAGE} - dpu_evolver_ns.emit('command', command, namespace = '/dpu-evolver') - -def stir_rate (MESSAGE): - command = {'param':'stir', 'message':MESSAGE} - dpu_evolver_ns.emit('command', command, namespace='/dpu-evolver') - -def get_flow_rate(): - file_path = os.path.join(save_path,custom_script.PUMP_CAL_FILE) - flow_calibration = np.loadtxt(file_path, delimiter="\t") - if len(flow_calibration)==16: - flow_rate=flow_calibration - else: - flow_rate=flow_calibration[0,:] #Currently just implementing influx flow rate - return flow_rate - -def calc_growth_rate(vial, gr_start, elapsed_time): - save_path = os.path.dirname(os.path.realpath(__file__)) - ODfile_name = "vial{0}_OD.txt".format(vial) - # Grab Data and make setpoint - OD_path = os.path.join(save_path,custom_script.EXP_NAME,'OD',ODfile_name) - OD_data = np.genfromtxt(OD_path, delimiter=',') - raw_time=OD_data[:,0] - raw_OD=OD_data[:,1] - raw_time=raw_time[np.isfinite(raw_OD)] - raw_OD=raw_OD[np.isfinite(raw_OD)] - - # Trim points prior to gr_start - trim_time=raw_time[np.nonzero(np.where(raw_time>gr_start,1,0))] - trim_OD=raw_OD[np.nonzero(np.where(raw_time>gr_start,1,0))] - - # Take natural log, calculate slope - log_OD = np.log(trim_OD) - slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(trim_time[np.isfinite(log_OD)],log_OD[np.isfinite(log_OD)]) - - # Save slope to file - file_name = "vial{0}_gr.txt".format(vial) - gr_path = os.path.join(save_path,custom_script.EXP_NAME,'growthrate',file_name) - text_file = open(gr_path,"a+") - text_file.write("{0},{1}\n".format(elapsed_time,slope)) - text_file.close() - -def parse_data(data, elapsed_time, vials, parameter): - save_path = os.path.dirname(os.path.realpath(__file__)) - if data == 'empty': - print("%s Data Empty! Skipping data log...".format(parameter)) - else: - for x in vials: - file_name = "vial{0}_{1}.txt".format(x, parameter) - file_path = os.path.join(save_path,custom_script.EXP_NAME,parameter,file_name) - text_file = open(file_path,"a+") - text_file.write("{0},{1}\n".format(elapsed_time, data[x])) - text_file.close() - -def start_background_loop(loop): - asyncio.set_event_loop(loop) - loop.run_forever() - -def run(): - global dpu_evolver_ns - socketIO = SocketIO(custom_script.EVOLVER_IP, custom_script.EVOLVER_PORT) - dpu_evolver_ns = socketIO.define(EvolverNamespace, '/dpu-evolver') - socketIO.wait() - -def initialize_exp(vials): - global od_cal, temp_cal, shared_name, shared_ip, shared_port - shared_name = custom_script.EXP_NAME - shared_ip = custom_script.EVOLVER_IP - shared_port = custom_script.EVOLVER_PORT - new_loop = asyncio.new_event_loop() - t = Thread(target = start_background_loop, args = (new_loop,)) - t.daemon = True - t.start() - new_loop.call_soon_threadsafe(run) - - if dpu_evolver_ns is None: - print("Waiting for evolver connection...") - - save_path = os.path.dirname(os.path.realpath(__file__)) - dir_path = os.path.join(save_path,custom_script.EXP_NAME) - - - - while dpu_evolver_ns is None: - pass - - if os.path.exists(dir_path): - exp_continue = input('Continue from existing experiment? (y/n): ') - else: - exp_continue = 'n' - - if exp_continue == 'n': - - start_time = time.time() - if os.path.exists(dir_path): - exp_overwrite = input('Directory aleady exists. Overwrite with new experiment? (y/n): ') - if exp_overwrite == 'y': - shutil.rmtree(dir_path) - else: - print('Change experiment name in custom_script.py and then restart...') - exit() #exit - - os.makedirs(os.path.join(dir_path,'OD')) - os.makedirs(os.path.join(dir_path,'temp')) - os.makedirs(os.path.join(dir_path,'temp_config')) - if custom_script.OPERATION_MODE == 'turbidostat': - os.makedirs(os.path.join(dir_path,'pump_log')) - os.makedirs(os.path.join(dir_path,'ODset')) - os.makedirs(os.path.join(dir_path,'growthrate')) - if custom_script.OPERATION_MODE == 'chemostat': - os.makedirs(os.path.join(dir_path,'chemo_config')) - - dpu_evolver_ns.emit('getcalibrationod', {}, namespace = '/dpu-evolver') - while od_cal is None: - pass - - dpu_evolver_ns.emit('getcalibrationtemp', {}, namespace = '/dpu-evolver') - while temp_cal is None: - pass - - for x in vials: - file_name = "vial{0}_OD.txt".format(x) # make OD file - file_path = os.path.join(dir_path,'OD',file_name) - text_file = open(file_path,"w") - text_file.write("Experiment: {0} vial {1}, {2}\n".format(custom_script.EXP_NAME, x, time.strftime("%c"))) - text_file.close() - - file_name = "vial{0}_temp.txt".format(x) # make temperature data file - file_path = os.path.join(dir_path,'temp',file_name) - text_file = open(file_path,"w").close() - - file_name = "vial{0}_tempconfig.txt".format(x) # make temperature configuration file - file_path = os.path.join(dir_path,'temp_config',file_name) - text_file = open(file_path,"w") - text_file.write("Experiment: {0} vial {1}, {2}\n".format(custom_script.EXP_NAME, x, time.strftime("%c"))) - text_file.write("0,{0}\n".format(custom_script.TEMP_INITIAL[x])) #initialize based on custom_script.choose_setup() - text_file.close() - - if custom_script.OPERATION_MODE == 'turbidostat': - file_name = "vial{0}_pump_log.txt".format(x) # make pump log file - file_path = os.path.join(dir_path,'pump_log',file_name) - text_file = open(file_path,"w") - text_file.write("Experiment: {0} vial {1}, {2}\n".format(custom_script.EXP_NAME, x, time.strftime("%c"))) - text_file.write("0,0\n") - text_file.close() - - file_name = "vial{0}_ODset.txt".format(x) # make ODset file - file_path = os.path.join(dir_path,'ODset',file_name) - text_file = open(file_path,"w") - text_file.write("Experiment: {0} vial {1}, {2}\n".format(custom_script.EXP_NAME, x, time.strftime("%c"))) - text_file.write("0,0\n") - text_file.close() - - file_name = "vial{0}_gr.txt".format(x) # make growth rate file - file_path = os.path.join(dir_path,'growthrate',file_name) - text_file = open(file_path,"w") - text_file.write("Experiment: {0} vial {1}, {2}\n".format(custom_script.EXP_NAME, x, time.strftime("%c"))) - text_file.write("0,0\n") #initialize to 0 - text_file.close() - - if custom_script.OPERATION_MODE == 'chemostat': - file_name = "vial{0}_chemoconfig.txt".format(x) #make chemostat file - file_path = os.path.join(dir_path,'chemo_config',file_name) - text_file = open(file_path,"w") - text_file.write("0,0,0\n") #header - text_file.write("0,0,0\n") #initialize to 0 - text_file.close() - - stir_rate(custom_script.STIR_INITIAL) - OD_read = None - temp_read = None - print('Getting initial values from eVOLVER...') - while OD_read is None and temp_read is None: - OD_read, temp_read = read_data(vials) - exp_blank = input('Calibrate vials to blank? (y/n): ') - if exp_blank == 'y': - OD_initial = OD_read # take an OD measurement, subtract OD_initial from all future measurements, note that this is not stored in data files - else: - OD_initial = np.zeros(len(vials)) # just use zeros, may lead to offsets in data if vial/medai is slightly diff, but can correct in post-processing unless needed for feedback - - else: #load existing experiment - pickle_name = "{0}.pickle".format(custom_script.EXP_NAME) - pickle_path = os.path.join(save_path,custom_script.EXP_NAME,pickle_name) - with open(pickle_path, 'rb') as f: - loaded_var = pickle.load(f) - x = loaded_var - start_time = x[0] - OD_initial = x[1] - - # Restart chemostat pumps - current_chemo = [0] * 16 - - # copy current custom script to txt file - backup_filename = '{0}_{1}.txt'.format(custom_script.EXP_NAME,(time.strftime('%y%m%d_%H%M'))) - shutil.copy('custom_script.py',os.path.join(save_path,custom_script.EXP_NAME,backup_filename)) - return start_time, OD_initial - -def stop_all_pumps(): - command = {'param':'pump', 'message':'stop'} - dpu_evolver_ns.emit('command', command, namespace='/dpu-evolver') - print('All Pumps Stopped!') - -def save_var(start_time, OD_initial): - # save variables needed for restarting experiment later - save_path = os.path.dirname(os.path.realpath(__file__)) - pickle_name = "{0}.pickle".format(custom_script.EXP_NAME) - pickle_path = os.path.join(save_path,custom_script.EXP_NAME,pickle_name) - with open(pickle_path, 'wb') as f: - pickle.dump([start_time, OD_initial], f) - -def restart_chemo(): - global current_chemo - current_chemo = [0] * 16 - -if __name__ == '__main__': - print('Please run main_eVOLVER.py instead') From 8a1af619eb93dd0c79f9f4ae0423463ad0a23ed6 Mon Sep 17 00:00:00 2001 From: Zachary Heins Date: Wed, 17 Jul 2019 10:10:01 -0400 Subject: [PATCH 2/3] Save raw data and removed main_eVOLVER.py --- experiment/template/eVOLVER.py | 8 +++ experiment/template/main_eVOLVER.py | 92 ----------------------------- 2 files changed, 8 insertions(+), 92 deletions(-) delete mode 100644 experiment/template/main_eVOLVER.py diff --git a/experiment/template/eVOLVER.py b/experiment/template/eVOLVER.py index dee1d36c..0c819dd1 100644 --- a/experiment/template/eVOLVER.py +++ b/experiment/template/eVOLVER.py @@ -77,6 +77,10 @@ def on_broadcast(self, data): VIALS, 'OD') self.save_data(data['transformed']['temp'], elapsed_time, VIALS, 'temp') + self.save_data(data['od_90'], elapsed_time, + VIALS, 'OD90_raw') + self.save_data(data['od_135'], elapsed_time, + VIALS, 'OD135_raw') # run custom functions self.custom_functions(data, VIALS, elapsed_time) # save variables @@ -309,6 +313,8 @@ def initialize_exp(self, vials, always_yes=False): logger.debug('creating data directories') os.makedirs(os.path.join(EXP_DIR, 'OD')) + os.makedirs(os.path.join(EXP_DIR, 'OD90_raw')) + os.makedirs(os.path.join(EXP_DIR, 'OD135_raw')) os.makedirs(os.path.join(EXP_DIR, 'temp')) os.makedirs(os.path.join(EXP_DIR, 'temp_config')) os.makedirs(os.path.join(EXP_DIR, 'pump_log')) @@ -321,6 +327,8 @@ def initialize_exp(self, vials, always_yes=False): time.strftime("%c")) # make OD file self._create_file(x, 'OD', defaults=[exp_str]) + self._create_file(x, 'OD90_raw', defaults=[exp_str]) + self._create_file(x, 'OD135_raw', defaults=[exp_str]) # make temperature data file self._create_file(x, 'temp') # make temperature configuration file diff --git a/experiment/template/main_eVOLVER.py b/experiment/template/main_eVOLVER.py deleted file mode 100644 index 9f0ec586..00000000 --- a/experiment/template/main_eVOLVER.py +++ /dev/null @@ -1,92 +0,0 @@ -from tkinter import * -from tkinter.ttk import * -import eVOLVER_module -import time -import pickle -import os.path -import custom_script -import smtplib - -#### Where the GUI is called and widgets are placed -class make_GUI: - - def __init__(self, master): - master.wm_title(custom_script.GUI_NAME) - note = Notebook(master) - home = Frame(note) - note.add(home, text = "Home") - - self.quitButton = Button(home, text="Pause Measuring / Stop Pumps", command=stop_exp) - self.quitButton.pack(side=BOTTOM) - - self.printButton = Button(home, text="Start / Measure now", command=start_exp) - self.printButton.pack(side=BOTTOM) - - note.pack(fill=BOTH, expand=YES) - - -## Task done by pressing printButton -def start_exp(): - stop_exp() - eVOLVER_module.restart_chemo() - update_eVOLVER() - -def stop_exp(): - global run_exp - try: - run_exp - except NameError: - print("Name Error") - else: - print("Experiment Stopped!") - root.after_cancel(run_exp) - #stop all pumps, including chemostat commands - eVOLVER_module.stop_all_pumps() - -#### Updates Temperature and OD values (repeated every 10 seconds) -def update_eVOLVER(): - global OD_data, temp_data - ##Read and record OD - elapsed_time = round((time.time() - start_time)/3600,4) - print("Time: {0} Hours".format(elapsed_time)) - OD_data, temp_data = eVOLVER_module.read_data(vials) - skip = False - if OD_data is not None and temp_data is not None: - if OD_data == 'empty': - print("Data Empty! Skipping data log...") - else: - for x in vials: - OD_data[x] = OD_data[x] - OD_initial[x] - eVOLVER_module.parse_data(OD_data, elapsed_time, vials, 'OD') - eVOLVER_module.parse_data(temp_data, elapsed_time, vials, 'temp') - - #Make decision - custom_functions(elapsed_time) - - #Save Variables - global run_exp - eVOLVER_module.save_var(start_time, OD_initial) - run_exp = root.after(1000,update_eVOLVER) - - -#### Custom Script -def custom_functions(elapsed_time): - global OD_data, temp_data - if OD_data == 'empty': - print("UDP Empty, did not execute program!") - else: - ###load script from another python file - if custom_script.OPERATION_MODE == 'turbidostat': - custom_script.turbidostat(OD_data, temp_data, vials, elapsed_time) - if custom_script.OPERATION_MODE == 'chemostat': - custom_script.chemostat(OD_data, temp_data, vials, elapsed_time) - - -#### Runs if this is main script -if __name__ == '__main__': - vials = range(0,16) - start_time, OD_initial = eVOLVER_module.initialize_exp(vials) - root=Tk() - make_GUI(root) - update_eVOLVER() - root.mainloop() From 9ad260c3edde91c3791249a68d5d504679f72c09 Mon Sep 17 00:00:00 2001 From: Marco Galardini Date: Mon, 22 Jul 2019 13:57:08 -0400 Subject: [PATCH 3/3] Handle raw data missing and other corner cases --- experiment/template/custom_script.py | 2 +- experiment/template/eVOLVER.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/experiment/template/custom_script.py b/experiment/template/custom_script.py index 71b7a2b7..6add94dc 100644 --- a/experiment/template/custom_script.py +++ b/experiment/template/custom_script.py @@ -207,7 +207,7 @@ def chemostat(eVOLVER, input_data, vials, elapsed_time): average_OD = float(np.median(od_values_from_file)) # set chemostat config path and pull current state from file - file_name = "vial{0}_chemoconfig.txt".format(x) + file_name = "vial{0}_chemo_config.txt".format(x) chemoconfig_path = os.path.join(save_path, EXP_NAME, 'chemo_config', file_name) chemo_config = np.genfromtxt(chemoconfig_path, delimiter=',') diff --git a/experiment/template/eVOLVER.py b/experiment/template/eVOLVER.py index 0c819dd1..b9eea355 100644 --- a/experiment/template/eVOLVER.py +++ b/experiment/template/eVOLVER.py @@ -77,10 +77,10 @@ def on_broadcast(self, data): VIALS, 'OD') self.save_data(data['transformed']['temp'], elapsed_time, VIALS, 'temp') - self.save_data(data['od_90'], elapsed_time, + self.save_data(data['data'].get('od_90', []), elapsed_time, VIALS, 'OD90_raw') - self.save_data(data['od_135'], elapsed_time, - VIALS, 'OD135_raw') + self.save_data(data['data'].get('od_135', []), elapsed_time, + VIALS, 'OD135_raw') # run custom functions self.custom_functions(data, VIALS, elapsed_time) # save variables @@ -134,7 +134,7 @@ def transform_data(self, data, vials): temps = [] for x in vials: - file_name = "vial{0}_tempconfig.txt".format(x) + file_name = "vial{0}_temp_config.txt".format(x) file_path = os.path.join(EXP_DIR, 'temp_config', file_name) temp_set_data = np.genfromtxt(file_path, delimiter=',') temp_set = temp_set_data[len(temp_set_data)-1][1] @@ -332,7 +332,7 @@ def initialize_exp(self, vials, always_yes=False): # make temperature data file self._create_file(x, 'temp') # make temperature configuration file - self._create_file(x, 'tempconfig', + self._create_file(x, 'temp_config', defaults=[exp_str, "0,{0}".format(TEMP_INITIAL[x])]) # make pump log file @@ -349,7 +349,7 @@ def initialize_exp(self, vials, always_yes=False): "0,0"], directory='growthrate') # make chemostat file - self._create_file(x, 'chemoconfig', + self._create_file(x, 'chemo_config', defaults=["0,0,0", "0,0,0"], directory='chemo_config') @@ -406,6 +406,8 @@ def check_for_calibrations(self): return result def save_data(self, data, elapsed_time, vials, parameter): + if len(data) == 0: + return for x in vials: file_name = "vial{0}_{1}.txt".format(x, parameter) file_path = os.path.join(EXP_DIR, parameter, file_name)