Fixes data loading for nELinks datasets

scan_precision.py

@@ -8,7 +8,7 @@
 import json
 
 from train import trainCNN
-from util import plotGraphErr
+from utils import plotGraphErr
 
 def plotScan(x,outs,name,odir,xtitle="n bits"):
     outs = pd.concat(outs)
@@ -26,7 +26,7 @@ def BitScan(options, args):
         updates = [{'nBits_input':{'total': b, 'integer': 2}} for b in bits]
         outputs = [trainCNN(options,args,u) for u in updates]
         plotScan(bits,outputs,"test_input_bits",options.odir,xtitle="total input bits")
-    
+
     if False:
         # test weights
         bits = [i+1 for i in range(8)]

tests/test_qkeras.py

@@ -0,0 +1,155 @@
+import tensorflow as tf
+import tensorflow.keras as kr
+from tensorflow.keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D, Flatten, \
+    Conv2DTranspose, Reshape, Activation
+from tensorflow.keras.models import Model
+from tensorflow.keras import backend as K
+import qkeras as qkr
+from qkeras import QDense, QConv2D, QActivation
+#from qkeras.qlayers import QConv2D,QActivation,QDense
+import numpy as np
+import json
+
+def main():
+    test_inputs()
+    test_dense()
+    exit(0)
+
+def test_inputs():
+    data = getData()
+
+    nBits=8
+    nBitsInt=4
+    qbits_param_input = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=0)
+
+    # simple model only quantizes
+    inputs = Input(shape=(4, 4, 3))
+    x = inputs
+    x = Flatten(name="flatten")(x)
+    x = QActivation(qbits_param_input, name='q_decoder_output')(x)
+    model = Model(inputs, x, name='encoder')
+    model.summary()
+
+    model.compile(loss='mse', optimizer='adam')
+
+    val_input, train_input = split(data, 0.5)
+    train_output = np.ones(240).reshape(5,48) # garbage outputs for training
+    val_output   = np.ones(240).reshape(5,48) # garbage outputs for validation
+
+    es = kr.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
+    history = model.fit(train_input, train_output,
+                        epochs=1,
+                        batch_size=500,
+                        shuffle=True,
+                        validation_data=(val_input,val_output),
+                        callbacks=[es])
+
+    val_output = model.predict(val_input)
+
+    print('\nTEST INPUTS')
+    print('\nRaw validation output: \n',val_output)
+    print('\nMultiplied by 2^(decimal bits): \n',val_output*(2**(nBits-nBitsInt)))    
+    return
+
+
+def test_dense():
+    data = getData()
+
+    nBits=8
+    nBitsInt=4
+    qbits_param_input = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=0)
+    qbits_param = qkr.quantized_bits(bits=nBits,integer=nBitsInt,keep_negative=1)
+
+    # simple model only quantizes
+    inputs = Input(shape=(4, 4, 3))
+    x = inputs
+    x = Flatten(name="flatten")(x)
+    x = QActivation(qbits_param_input, name='q_decoder_output')(x)
+    encodedLayer = QDense(10, activation='relu', name='encoded_vector',
+                          kernel_quantizer=qbits_param, bias_quantizer=qbits_param)(x)
+    model = Model(inputs, encodedLayer, name='encoder')
+    model.summary()
+
+    model.compile(loss='mse', optimizer='adam')
+
+    val_input, train_input = split(data, 0.5)
+    train_output = np.ones(50).reshape(5,10) # garbage outputs for training
+    val_output   = np.ones(50).reshape(5,10) # garbage outputs for validation
+
+    es = kr.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
+    history = model.fit(train_input, train_output,
+                        epochs=1,
+                        batch_size=500,
+                        shuffle=True,
+                        validation_data=(val_input,val_output),
+                        callbacks=[es])
+
+    val_output = model.predict(val_input)
+
+    print('\nTEST DENSE')
+    print('\nRaw validation output: \n',val_output)
+    print('\nMultiplied by 2^(decimal bits): \n  Results should be integers * weight precision... \n',val_output*(2**(nBits-nBitsInt)))
+    return
+
+
+
+def getData():
+    # read 10 events of 48 inputs
+    d = np.array(
+        [[73,6,13,41,28,21,37,48,13,46,51,10,29,19,19,53,0,0,4,12,0,45,20,79,26,68,23,61,35,14,14,36,9,52,15,21,0,0,0,0,0,0,0,0,0,0,0,0],
+         [41,59,3,0,32,24,12,6,18,140,24,53,34,44,127,15,0,0,0,0,0,0,0,0,0,0,0,7,0,0,3,13,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
+         [25,11,24,56,10,33,9,33,70,35,29,0,3,22,5,53,0,0,0,118,0,0,0,21,0,0,0,25,0,0,0,44,32,68,21,11,8,16,47,0,47,34,14,0,82,0,0,0],
+         [0,0,0,0,0,0,0,9,22,26,25,42,5,105,23,56,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,58,103,29,0,86,23,0,0,62,8,0,0,75,7,0,0],
+         [0,0,38,61,0,10,27,36,26,6,43,53,42,24,7,13,37,59,29,78,0,0,0,0,0,0,0,0,0,0,0,0,15,14,20,55,26,27,55,97,73,31,66,112,9,25,28,42],
+         [0,0,0,0,0,0,0,0,0,0,0,14,0,0,0,60,23,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,10,3,0,2,44,29,10,4,7,9,53,67,10,39,124],
+         [0,0,0,23,0,0,0,52,0,0,0,50,0,0,7,11,79,45,18,30,34,627,23,20,58,44,28,0,76,13,0,0,64,7,31,21,24,14,40,20,47,70,85,25,41,73,10,8],
+         [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,29,62,14,0,50,17,10,0,111,42,0,0,8,6,0,0,0,0,0,0,0,0,9,19,10,21,37,76,19,63,58,37],
+         [160,8,15,104,0,0,0,0,0,0,0,0,0,0,0,0,6,10,29,13,14,21,73,38,45,23,95,139,8,43,13,35,0,0,127,40,0,24,22,40,26,62,79,23,167,80,61,32],
+         [29,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,33,0,0,49,47,52,5,44,26,37,59,9,10,85,22,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,2],],
+        dtype=np.float32)
+    # normalize
+    for i in range(len(d)): 
+        d[i] = d[i]*1./d[i].sum()
+
+    #sort inputs
+    arrange443 = np.array([0,16, 32,
+                           1,17, 33,
+                           2,18, 34,
+                           3,19, 35,
+                           4,20, 36,
+                           5,21, 37,
+                           6,22, 38,
+                           7,23, 39,
+                           8,24, 40,
+                           9,25, 41,
+                           10,26, 42,
+                           11,27, 43,
+                           12,28, 44,
+                           13,29, 45,
+                           14,30, 46,
+                           15,31, 47])
+    d = d[:, arrange443]    
+    d = d.reshape(len(d),4,4,3)
+    return d
+
+def split(shaped_data, validation_frac):
+  N = round(len(shaped_data)*validation_frac)
+
+  #randomly select 25% entries
+  index = np.random.choice(shaped_data.shape[0], N, replace=False)  
+  #select the indices of the other 75%
+  full_index = np.array(range(0,len(shaped_data)))
+  train_index = np.logical_not(np.in1d(full_index,index))
+
+  val_input = shaped_data[index]
+  train_input = shaped_data[train_index]
+
+  print(train_input.shape)
+  print(val_input.shape)
+
+  return val_input,train_input
+
+
+
+if __name__ == "__main__":
+    main()

train.py

@@ -34,7 +34,7 @@ def normalize(data,rescaleInputToMax=False):
             data[i] = 1.*data[i]/data[i].sum()
     return data,np.array(norm)
 
-def plotHist(vals,name,odir='.',xtitle="",ytitle="",nbins=40, 
+def plotHist(vals,name,odir='.',xtitle="",ytitle="",nbins=40,
              stats=True, logy=False, leg=None):
     plt.figure(figsize=(6,4))
     if leg:
@@ -58,7 +58,7 @@ def split(shaped_data, validation_frac=0.2):
     N = round(len(shaped_data)*validation_frac)
 
     #randomly select 25% entries
-    val_index = np.random.choice(shaped_data.shape[0], N, replace=False)  
+    val_index = np.random.choice(shaped_data.shape[0], N, replace=False)
     #select the indices of the other 75%
     full_index = np.array(range(0,len(shaped_data)))
     train_index = np.logical_not(np.in1d(full_index,val_index))
@@ -143,7 +143,7 @@ def ssd(x,y):
     [-1.5697594, -16.0113], [-3.6627693, -14.802895], [-5.7557793, -13.594498], [-7.848793, -12.386101]])
 hexMetric = ot.dist(hexCoords, hexCoords, 'euclidean')
 MAXDIST = 16.08806614
-def emd(_x, _y, threshold=-1):    
+def emd(_x, _y, threshold=-1):
     if (np.sum(_x)==0 or np.sum(_y)==0): return MAXDIST
     x = np.array(_x, dtype=np.float64)
     y = np.array(_y, dtype=np.float64)
@@ -242,11 +242,11 @@ def visDisplays(index,input_Q,decoded_Q,encoded_Q=np.array([]),name='model_X'):
     plt.tight_layout()
     plt.savefig("%s_examples.png"%name)
     plt.close()
-    
-def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False): 
-    
+
+def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False):
+
     plotHist(vals,name,options.odir,xtitle=longMetric[mname])
-  
+
     plt.figure(figsize=(6,4))
     plt.hist([input_Q.flatten(),decoded_Q.flatten()],20,label=['input','output'])
     plt.yscale('log')
@@ -257,7 +257,7 @@ def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False):
 
     input_Q_abs   = np.array([input_Q[i] * maxQ[i] for i in range(0,len(input_Q))])
     decoded_Q_abs = np.array([decoded_Q[i]*maxQ[i] for i in range(0,len(decoded_Q))])
-  
+
     nonzeroQs = np.count_nonzero(input_Q_abs.reshape(len(input_Q_abs),48),axis=1)
     occbins = [0,5,10,20,48]
     fig, axes = plt.subplots(1,len(occbins)-1, figsize=(16, 4))
@@ -276,12 +276,23 @@ def visMetric(input_Q,decoded_Q,metric,name,odir,skipPlot=False):
         plt.close()
 
     return cross_corr_arr,ssd_arr,emd_arr
-  
-def GetBitsString(In, Accum, Weight):
+
+def GetBitsString(In, Accum, Weight, Encoded, Dense=False, Conv=False):
     s=""
     s += "Input{}b{}i".format(In['total'],In['integer'])
     s += "_Accum{}b{}i".format(Accum['total'],Accum['integer'])
-    s += "_Weight{}b{}i".format(Weight['total'],Weight['integer'])
+    if Dense:
+        s += "_Dense{}b{}i".format(Dense['total'], Dense['integer'])
+        if Conv:
+            s += "_Conv{}b{}i".format(Conv['total'], Conv['integer'])
+        else:
+            s += "_Conv{}b{}i".format(Weight['total'], Weight['integer'])
+    elif Conv:
+        s += "_Dense{}b{}i".format(Weight['total'], Weight['integer'])
+        s += "_Conv{}b{}i".format(Conv['total'], Conv['integer'])
+    else:
+        s += "_Weight{}b{}i".format(Weight['total'],Weight['integer'])
+    s += "_Encod{}b{}i".format(Encoded['total'], Encoded['integer'])
     return s
 
 def trainCNN(options, args, pam_updates=None):
@@ -291,10 +302,10 @@ def trainCNN(options, args, pam_updates=None):
     print("Is GPU available? ", tf.test.is_gpu_available())
 
     # default precisions for quantized training
-    nBits_input  = {'total': 16, 'integer': 6}
-    nBits_accum  = {'total': 16, 'integer': 6}
-    nBits_weight = {'total': 16, 'integer': 6}
-    nBits_encod  = {'total': 16, 'integer': 6}
+    nBits_input  = {'total': 32, 'integer': 4}
+    nBits_accum  = {'total': 32, 'integer': 4}
+    nBits_weight = {'total': 32, 'integer': 4}
+    nBits_encod  = {'total': 32, 'integer': 4}
     # model-dependent -- use common weights unless overridden
     conv_qbits = nBits_weight
     dense_qbits = nBits_weight
@@ -306,8 +317,9 @@ def trainCNN(options, args, pam_updates=None):
         df_arr = []
         for infile in os.listdir(options.inputFile):
             infile = os.path.join(options.inputFile,infile)
-            df_arr.append(pd.read_csv(infile, dtype=np.float64, header=0))
+            df_arr.append(pd.read_csv(infile, dtype=np.float64, header=0, usecols=[*range(1, 49)]))
         data = pd.concat(df_arr)
+        data = data.loc[(data.sum(axis=1) != 0)] #drop rows where occupancy = 0
         print(data.shape)
         data.describe()
     else:
@@ -353,12 +365,21 @@ def trainCNN(options, args, pam_updates=None):
                            15,31, 47])
 
     models = [
-        {'name': '4x4_norm_d10', 'ws': '',
-         'pams': {'shape': (4, 4, 3), 
+        #{'name': '4x4_norm_d10', 'ws': '',
+        # 'pams': {'shape': (4, 4, 3),
+        #          'channels_first': False,
+        #          'arrange': arrange443,
+        #          'encoded_dim': 10,
+        #          'loss': 'weightedMSE'}},
+        {'name': '4x4_norm_v7', 'ws': '',
+         'pams': {'shape': (4, 4, 3),
                   'channels_first': False,
-                  'arrange': arrange443, 
-                  'encoded_dim': 10, 
-                  'loss': 'weightedMSE'}},
+                  'arrange': arrange443,
+                  'loss': 'weightedMSE',
+                  'CNN_layer_nodes': [4, 4, 4],
+                  'CNN_kernel_size': [5, 5, 3],
+                  'CNN_pool': [False, False, False], }},
+
     ]
 
     #{'name':'denseCNN',  'ws':'denseCNN.hdf5', 'pams':{'shape':(1,8,8) } },
@@ -484,7 +505,7 @@ def trainCNN(options, args, pam_updates=None):
                   'dMean':'difference in energy-weighted mean',
                   'zero_frac':'zero fraction',}
     summary_entries=['name','en_pams','tot_pams']
-    for algname in algnames: 
+    for algname in algnames:
         for mname in metrics:
             name = mname+"_"+algname
             summary_entries.append(mname+"_"+algname)
@@ -497,9 +518,10 @@ def trainCNN(options, args, pam_updates=None):
     for model in models:
         model_name = model['name']
         if options.quantize: 
-            bit_str = GetBitsString(m['pams']['nBits_input'],
-                                    m['pams']['nBits_accum'],
-                                    m['pams']['nBits_weight'])
+            bit_str = GetBitsString(model['pams']['nBits_input'], model['pams']['nBits_accum'],
+                                    model['pams']['nBits_weight'], model['pams']['nBits_encod'],
+                                    (model['pams']['nBits_dense'] if 'nBits_dense'  in model['pams'] else False),
+                                    (model['pams']['nBits_conv'] if 'nBits_conv' in model['pams'] else False))
             model_name += "_" + bit_str
         if not os.path.exists(model_name): os.mkdir(model_name)
         os.chdir(model_name)
@@ -514,7 +536,7 @@ def trainCNN(options, args, pam_updates=None):
         val_input, train_input, val_ind = split(shaped_data)
         m_autoCNN , m_autoCNNen         = m.get_models()
         val_max = maxdata[val_ind]
-        
+
         if model['ws']=='':
             if options.quickTrain: train_input = train_input[:5000]
             history = train(m_autoCNN,m_autoCNNen,train_input,val_input,name=model_name,n_epochs = options.epochs)
@@ -525,7 +547,7 @@ def trainCNN(options, args, pam_updates=None):
             'name':model_name,
             'en_pams' : m_autoCNNen.count_params(),
             'tot_pams': m_autoCNN.count_params(),}
-                
+
         input_Q,cnn_deQ ,cnn_enQ   = m.predict(val_input)
 
         ## csv files for RTL verification
@@ -546,7 +568,7 @@ def trainCNN(options, args, pam_updates=None):
                     'thr_hi': thr_hi_Q,
                 }
 
-        # to generate event displays 
+        # to generate event displays
         Nevents = 8
         index = np.random.choice(input_Q.shape[0], Nevents, replace=False)
 
@@ -569,23 +591,23 @@ def trainCNN(options, args, pam_updates=None):
                 vals = np.sort(vals)
                 model[name]        = np.round(np.mean(vals), 3)
                 model[name+'_err'] = np.round(np.std(vals), 3)
-                summary_dict[name]        = model[name]       
+                summary_dict[name]        = model[name]
                 summary_dict[name+'_err'] = model[name+'_err']
                 if(not options.skipPlot) and (not('zero_frac' in mname)):
                     plotHist(vals,"hist_"+name,xtitle=longMetric[mname])
                     hi_index = (np.where(vals>np.quantile(vals,0.9)))[0]
                     lo_index = (np.where(vals<np.quantile(vals,0.2)))[0]
                     # visualize(input_Q,cnn_deQ,cnn_enQ,index,name=model_name)
                     if len(hi_index)>0:
-                        hi_index = np.random.choice(hi_index, min(Nevents,len(hi_index)), replace=False)  
+                        hi_index = np.random.choice(hi_index, min(Nevents,len(hi_index)), replace=False)
                         visDisplays(hi_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname)
                     if len(lo_index)>0:
-                        lo_index = np.random.choice(lo_index, min(Nevents,len(lo_index)), replace=False)  
-                        visDisplays(lo_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname)                        
+                        lo_index = np.random.choice(lo_index, min(Nevents,len(lo_index)), replace=False)
+                        visDisplays(lo_index, input_Q, alg_out, (cnn_enQ if algname=='ae' else np.array([])), name=algname)
 
         print('summary_dict',summary_dict)
         summary = summary.append(summary_dict, ignore_index=True)
-        
+
         with open(model_name+"_pams.json",'w') as f:
             f.write(json.dumps(m.get_pams(),indent=4))