diff --git a/scan_precision.py b/scan_precision.py index 652ab00..f083306 100644 --- a/scan_precision.py +++ b/scan_precision.py @@ -8,7 +8,7 @@ import json from train import trainCNN -from util import plotGraphErr +from utils import plotGraphErr def plotScan(x,outs,name,odir,xtitle="n bits"): outs = pd.concat(outs) @@ -26,7 +26,7 @@ def BitScan(options, args): updates = [{'nBits_input':{'total': b, 'integer': 2}} for b in bits] outputs = [trainCNN(options,args,u) for u in updates] plotScan(bits,outputs,"test_input_bits",options.odir,xtitle="total input bits") - + if False: # test weights bits = [i+1 for i in range(8)] diff --git a/tests/test_qkeras.py b/tests/test_qkeras.py new file mode 100644 index 0000000..67d1333 --- /dev/null +++ b/tests/test_qkeras.py @@ -0,0 +1,155 @@ +import tensorflow as tf +import tensorflow.keras as kr +from tensorflow.keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D, Flatten, \ + Conv2DTranspose, Reshape, Activation +from tensorflow.keras.models import Model +from tensorflow.keras import backend as K +import qkeras as qkr +from qkeras import QDense, QConv2D, QActivation +#from qkeras.qlayers import QConv2D,QActivation,QDense +import numpy as np +import json + +def main(): + test_inputs() + test_dense() + exit(0) + +def test_inputs(): + data = getData() + + nBits=8 + nBitsInt=4 + qbits_param_input = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=0) + + # simple model only quantizes + inputs = Input(shape=(4, 4, 3)) + x = inputs + x = Flatten(name="flatten")(x) + x = QActivation(qbits_param_input, name='q_decoder_output')(x) + model = Model(inputs, x, name='encoder') + model.summary() + + model.compile(loss='mse', optimizer='adam') + + val_input, train_input = split(data, 0.5) + train_output = np.ones(240).reshape(5,48) # garbage outputs for training + val_output = np.ones(240).reshape(5,48) # garbage outputs for validation + + es = kr.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3) + history = model.fit(train_input, train_output, + epochs=1, + batch_size=500, + shuffle=True, + validation_data=(val_input,val_output), + callbacks=[es]) + + val_output = model.predict(val_input) + + print('\nTEST INPUTS') + print('\nRaw validation output: \n',val_output) + print('\nMultiplied by 2^(decimal bits): \n',val_output*(2**(nBits-nBitsInt))) + return + + +def test_dense(): + data = getData() + + nBits=8 + nBitsInt=4 + qbits_param_input = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=0) + qbits_param = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=1) + + # simple model only quantizes + inputs = Input(shape=(4, 4, 3)) + x = inputs + x = Flatten(name="flatten")(x) + x = QActivation(qbits_param_input, name='q_decoder_output')(x) + encodedLayer = QDense(10, activation='relu', name='encoded_vector', + kernel_quantizer=qbits_param, bias_quantizer=qbits_param)(x) + model = Model(inputs, encodedLayer, name='encoder') + model.summary() + + model.compile(loss='mse', optimizer='adam') + + val_input, train_input = split(data, 0.5) + train_output = np.ones(50).reshape(5,10) # garbage outputs for training + val_output = np.ones(50).reshape(5,10) # garbage outputs for validation + + es = kr.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3) + history = model.fit(train_input, train_output, + epochs=1, + batch_size=500, + shuffle=True, + validation_data=(val_input,val_output), + callbacks=[es]) + + val_output = model.predict(val_input) + + print('\nTEST DENSE') + print('\nRaw validation output: \n',val_output) + print('\nMultiplied by 2^(decimal bits): \n Results should be integers * weight precision... \n',val_output*(2**(nBits-nBitsInt))) + return + + + +def getData(): + # read 10 events of 48 inputs + d = np.array( + [[73,6,13,41,28,21,37,48,13,46,51,10,29,19,19,53,0,0,4,12,0,45,20,79,26,68,23,61,35,14,14,36,9,52,15,21,0,0,0,0,0,0,0,0,0,0,0,0], + [41,59,3,0,32,24,12,6,18,140,24,53,34,44,127,15,0,0,0,0,0,0,0,0,0,0,0,7,0,0,3,13,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], + [25,11,24,56,10,33,9,33,70,35,29,0,3,22,5,53,0,0,0,118,0,0,0,21,0,0,0,25,0,0,0,44,32,68,21,11,8,16,47,0,47,34,14,0,82,0,0,0], + [0,0,0,0,0,0,0,9,22,26,25,42,5,105,23,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,58,103,29,0,86,23,0,0,62,8,0,0,75,7,0,0], + [0,0,38,61,0,10,27,36,26,6,43,53,42,24,7,13,37,59,29,78,0,0,0,0,0,0,0,0,0,0,0,0,15,14,20,55,26,27,55,97,73,31,66,112,9,25,28,42], + [0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,60,23,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,10,3,0,2,44,29,10,4,7,9,53,67,10,39,124], + [0,0,0,23,0,0,0,52,0,0,0,50,0,0,7,11,79,45,18,30,34,627,23,20,58,44,28,0,76,13,0,0,64,7,31,21,24,14,40,20,47,70,85,25,41,73,10,8], + [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,29,62,14,0,50,17,10,0,111,42,0,0,8,6,0,0,0,0,0,0,0,0,9,19,10,21,37,76,19,63,58,37], + [160,8,15,104,0,0,0,0,0,0,0,0,0,0,0,0,6,10,29,13,14,21,73,38,45,23,95,139,8,43,13,35,0,0,127,40,0,24,22,40,26,62,79,23,167,80,61,32], + [29,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,33,0,0,49,47,52,5,44,26,37,59,9,10,85,22,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,2],], + dtype=np.float32) + # normalize + for i in range(len(d)): + d[i] = d[i]*1./d[i].sum() + + #sort inputs + arrange443 = np.array([0,16, 32, + 1,17, 33, + 2,18, 34, + 3,19, 35, + 4,20, 36, + 5,21, 37, + 6,22, 38, + 7,23, 39, + 8,24, 40, + 9,25, 41, + 10,26, 42, + 11,27, 43, + 12,28, 44, + 13,29, 45, + 14,30, 46, + 15,31, 47]) + d = d[:, arrange443] + d = d.reshape(len(d),4,4,3) + return d + +def split(shaped_data, validation_frac): + N = round(len(shaped_data)*validation_frac) + + #randomly select 25% entries + index = np.random.choice(shaped_data.shape[0], N, replace=False) + #select the indices of the other 75% + full_index = np.array(range(0,len(shaped_data))) + train_index = np.logical_not(np.in1d(full_index,index)) + + val_input = shaped_data[index] + train_input = shaped_data[train_index] + + print(train_input.shape) + print(val_input.shape) + + return val_input,train_input + + + +if __name__ == "__main__": + main() diff --git a/train.py b/train.py index 742aa56..e95ef5d 100644 --- a/train.py +++ b/train.py @@ -34,7 +34,7 @@ def normalize(data,rescaleInputToMax=False): data[i] = 1.*data[i]/data[i].sum() return data,np.array(norm) -def plotHist(vals,name,odir='.',xtitle="",ytitle="",nbins=40, +def plotHist(vals,name,odir='.',xtitle="",ytitle="",nbins=40, stats=True, logy=False, leg=None): plt.figure(figsize=(6,4)) if leg: @@ -58,7 +58,7 @@ def split(shaped_data, validation_frac=0.2): N = round(len(shaped_data)*validation_frac) #randomly select 25% entries - val_index = np.random.choice(shaped_data.shape[0], N, replace=False) + val_index = np.random.choice(shaped_data.shape[0], N, replace=False) #select the indices of the other 75% full_index = np.array(range(0,len(shaped_data))) train_index = np.logical_not(np.in1d(full_index,val_index)) @@ -143,7 +143,7 @@ def ssd(x,y): [-1.5697594, -16.0113], [-3.6627693, -14.802895], [-5.7557793, -13.594498], [-7.848793, -12.386101]]) hexMetric = ot.dist(hexCoords, hexCoords, 'euclidean') MAXDIST = 16.08806614 -def emd(_x, _y, threshold=-1): +def emd(_x, _y, threshold=-1): if (np.sum(_x)==0 or np.sum(_y)==0): return MAXDIST x = np.array(_x, dtype=np.float64) y = np.array(_y, dtype=np.float64) @@ -242,11 +242,11 @@ def visDisplays(index,input_Q,decoded_Q,encoded_Q=np.array([]),name='model_X'): plt.tight_layout() plt.savefig("%s_examples.png"%name) plt.close() - -def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False): - + +def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False): + plotHist(vals,name,options.odir,xtitle=longMetric[mname]) - + plt.figure(figsize=(6,4)) plt.hist([input_Q.flatten(),decoded_Q.flatten()],20,label=['input','output']) plt.yscale('log') @@ -257,7 +257,7 @@ def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False): input_Q_abs = np.array([input_Q[i] * maxQ[i] for i in range(0,len(input_Q))]) decoded_Q_abs = np.array([decoded_Q[i]*maxQ[i] for i in range(0,len(decoded_Q))]) - + nonzeroQs = np.count_nonzero(input_Q_abs.reshape(len(input_Q_abs),48),axis=1) occbins = [0,5,10,20,48] fig, axes = plt.subplots(1,len(occbins)-1, figsize=(16, 4)) @@ -276,12 +276,23 @@ def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False): plt.close() return cross_corr_arr,ssd_arr,emd_arr - -def GetBitsString(In, Accum, Weight): + +def GetBitsString(In, Accum, Weight, Encoded, Dense=False, Conv=False): s="" s += "Input{}b{}i".format(In['total'],In['integer']) s += "_Accum{}b{}i".format(Accum['total'],Accum['integer']) - s += "_Weight{}b{}i".format(Weight['total'],Weight['integer']) + if Dense: + s += "_Dense{}b{}i".format(Dense['total'], Dense['integer']) + if Conv: + s += "_Conv{}b{}i".format(Conv['total'], Conv['integer']) + else: + s += "_Conv{}b{}i".format(Weight['total'], Weight['integer']) + elif Conv: + s += "_Dense{}b{}i".format(Weight['total'], Weight['integer']) + s += "_Conv{}b{}i".format(Conv['total'], Conv['integer']) + else: + s += "_Weight{}b{}i".format(Weight['total'],Weight['integer']) + s += "_Encod{}b{}i".format(Encoded['total'], Encoded['integer']) return s def trainCNN(options, args, pam_updates=None): @@ -291,10 +302,10 @@ def trainCNN(options, args, pam_updates=None): print("Is GPU available? ", tf.test.is_gpu_available()) # default precisions for quantized training - nBits_input = {'total': 16, 'integer': 6} - nBits_accum = {'total': 16, 'integer': 6} - nBits_weight = {'total': 16, 'integer': 6} - nBits_encod = {'total': 16, 'integer': 6} + nBits_input = {'total': 32, 'integer': 4} + nBits_accum = {'total': 32, 'integer': 4} + nBits_weight = {'total': 32, 'integer': 4} + nBits_encod = {'total': 32, 'integer': 4} # model-dependent -- use common weights unless overridden conv_qbits = nBits_weight dense_qbits = nBits_weight @@ -306,8 +317,9 @@ def trainCNN(options, args, pam_updates=None): df_arr = [] for infile in os.listdir(options.inputFile): infile = os.path.join(options.inputFile,infile) - df_arr.append(pd.read_csv(infile, dtype=np.float64, header=0)) + df_arr.append(pd.read_csv(infile, dtype=np.float64, header=0, usecols=[*range(1, 49)])) data = pd.concat(df_arr) + data = data.loc[(data.sum(axis=1) != 0)] #drop rows where occupancy = 0 print(data.shape) data.describe() else: @@ -353,12 +365,21 @@ def trainCNN(options, args, pam_updates=None): 15,31, 47]) models = [ - {'name': '4x4_norm_d10', 'ws': '', - 'pams': {'shape': (4, 4, 3), + #{'name': '4x4_norm_d10', 'ws': '', + # 'pams': {'shape': (4, 4, 3), + # 'channels_first': False, + # 'arrange': arrange443, + # 'encoded_dim': 10, + # 'loss': 'weightedMSE'}}, + {'name': '4x4_norm_v7', 'ws': '', + 'pams': {'shape': (4, 4, 3), 'channels_first': False, - 'arrange': arrange443, - 'encoded_dim': 10, - 'loss': 'weightedMSE'}}, + 'arrange': arrange443, + 'loss': 'weightedMSE', + 'CNN_layer_nodes': [4, 4, 4], + 'CNN_kernel_size': [5, 5, 3], + 'CNN_pool': [False, False, False], }}, + ] #{'name':'denseCNN', 'ws':'denseCNN.hdf5', 'pams':{'shape':(1,8,8) } }, @@ -484,7 +505,7 @@ def trainCNN(options, args, pam_updates=None): 'dMean':'difference in energy-weighted mean', 'zero_frac':'zero fraction',} summary_entries=['name','en_pams','tot_pams'] - for algname in algnames: + for algname in algnames: for mname in metrics: name = mname+"_"+algname summary_entries.append(mname+"_"+algname) @@ -497,9 +518,10 @@ def trainCNN(options, args, pam_updates=None): for model in models: model_name = model['name'] if options.quantize: - bit_str = GetBitsString(m['pams']['nBits_input'], - m['pams']['nBits_accum'], - m['pams']['nBits_weight']) + bit_str = GetBitsString(model['pams']['nBits_input'], model['pams']['nBits_accum'], + model['pams']['nBits_weight'], model['pams']['nBits_encod'], + (model['pams']['nBits_dense'] if 'nBits_dense' in model['pams'] else False), + (model['pams']['nBits_conv'] if 'nBits_conv' in model['pams'] else False)) model_name += "_" + bit_str if not os.path.exists(model_name): os.mkdir(model_name) os.chdir(model_name) @@ -514,7 +536,7 @@ def trainCNN(options, args, pam_updates=None): val_input, train_input, val_ind = split(shaped_data) m_autoCNN , m_autoCNNen = m.get_models() val_max = maxdata[val_ind] - + if model['ws']=='': if options.quickTrain: train_input = train_input[:5000] history = train(m_autoCNN,m_autoCNNen,train_input,val_input,name=model_name,n_epochs = options.epochs) @@ -525,7 +547,7 @@ def trainCNN(options, args, pam_updates=None): 'name':model_name, 'en_pams' : m_autoCNNen.count_params(), 'tot_pams': m_autoCNN.count_params(),} - + input_Q,cnn_deQ ,cnn_enQ = m.predict(val_input) ## csv files for RTL verification @@ -546,7 +568,7 @@ def trainCNN(options, args, pam_updates=None): 'thr_hi': thr_hi_Q, } - # to generate event displays + # to generate event displays Nevents = 8 index = np.random.choice(input_Q.shape[0], Nevents, replace=False) @@ -569,7 +591,7 @@ def trainCNN(options, args, pam_updates=None): vals = np.sort(vals) model[name] = np.round(np.mean(vals), 3) model[name+'_err'] = np.round(np.std(vals), 3) - summary_dict[name] = model[name] + summary_dict[name] = model[name] summary_dict[name+'_err'] = model[name+'_err'] if(not options.skipPlot) and (not('zero_frac' in mname)): plotHist(vals,"hist_"+name,xtitle=longMetric[mname]) @@ -577,15 +599,15 @@ def trainCNN(options, args, pam_updates=None): lo_index = (np.where(vals0: - hi_index = np.random.choice(hi_index, min(Nevents,len(hi_index)), replace=False) + hi_index = np.random.choice(hi_index, min(Nevents,len(hi_index)), replace=False) visDisplays(hi_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname) if len(lo_index)>0: - lo_index = np.random.choice(lo_index, min(Nevents,len(lo_index)), replace=False) - visDisplays(lo_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname) + lo_index = np.random.choice(lo_index, min(Nevents,len(lo_index)), replace=False) + visDisplays(lo_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname) print('summary_dict',summary_dict) summary = summary.append(summary_dict, ignore_index=True) - + with open(model_name+"_pams.json",'w') as f: f.write(json.dumps(m.get_pams(),indent=4))