add deeper domain cnn; refactor hyperband using load_data function

arguments.py

@@ -32,10 +32,13 @@ parser.add_argument("--models", action="store", dest="model_paths", nargs="+",
                     default=[])
 
 parser.add_argument("--type", action="store", dest="model_type",
-                    default="final") # inter, final, staggered
+                    default="final")
 
-parser.add_argument("--depth", action="store", dest="model_depth",
+parser.add_argument("--embd_type", action="store", dest="embedding_type",
-                    default="flat1")  # small, medium
+                    default="small")
+
+# parser.add_argument("--depth", action="store", dest="model_depth",
+#                     default="flat1")
 
 parser.add_argument("--model_output", action="store", dest="model_output",
                     default="both")

main.py

@@ -58,6 +58,7 @@ if args.gpu:
 # default parameter
 PARAMS = {
     "type": args.model_type,
+    "embedding_type": args.embedding_type,
     # "depth": args.model_depth,
     "batch_size": args.batch_size,
     "window_size": args.window,
@@ -83,6 +84,7 @@ def get_param_dist(dist_size="small"):
         return {
             # static params
             "type": [args.model_type],
+            "embedding_type": [args.embedding_type],
             # "depth": [args.model_depth],
             "model_output": [args.model_output],
             "batch_size": [args.batch_size],
@@ -103,6 +105,7 @@ def get_param_dist(dist_size="small"):
         return {
             # static params
             "type": [args.model_type],
+            "embedding_type": [args.embedding_type],
             # "depth": [args.model_depth],
             "model_output": [args.model_output],
             "batch_size": [args.batch_size],
@@ -131,7 +134,7 @@ def shuffle_training_data(domain, flow, client, server):
 
 
 def main_paul_best():
-    pauls_best_params = best_config = {
+    pauls_best_params = {
         "type": "paul",
         "batch_size": 64,
         "window_size": 10,
@@ -155,18 +158,8 @@ def main_paul_best():
 
 
 def main_hyperband(data, domain_length, window_size, model_type, result_file, max_iter, dist_size="small"):
-    param_dist = get_param_dist(dist_size)
-    
     logger.info("create training dataset")
-    domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(data, domain_length,
+    domain_tr, flow_tr, client_tr, server_tr = load_data(data, domain_length, window_size, model_type, shuffled=True)
-                                                                                                window)
-    server_tr = np.max(server_windows_tr, axis=1)
-    
-    if model_type in ("inter", "staggered"):
-        server_tr = np.expand_dims(server_windows_tr, 2)
-    
-    domain_tr, flow_tr, client_tr, server_tr = shuffle_training_data(domain_tr, flow_tr, client_tr, server_tr)
-
     return run_hyperband(dist_size, domain_tr, flow_tr, client_tr, server_tr, max_iter, result_file)
 
 
@@ -186,13 +179,15 @@ def train(parameters, features, labels):
     pass
 
 
-def load_data(data, domain_length, window_size, model_type):
+def load_data(data, domain_length, window_size, model_type, shuffled=False):
     # data preparation
     domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(data, domain_length,
                                                                                                 window_size)
     server_tr = np.max(server_windows_tr, axis=1)
     if model_type in ("inter", "staggered"):
         server_tr = np.expand_dims(server_windows_tr, 2)
+    if shuffled:
+        domain_tr, flow_tr, client_tr, server_tr = shuffle_training_data(domain_tr, flow_tr, client_tr, server_tr)
     return domain_tr, flow_tr, client_tr, server_tr
 
 

models/__init__.py

@@ -26,6 +26,7 @@ def get_models_by_params(params: dict):
     K.clear_session()
     # decomposing param section
     # mainly embedding model
+    embedding_type = params.get("embedding_type", "small")
     network_type = params.get("type")
     # network_depth = params.get("depth")
     embedding_size = params.get("embedding")
@@ -42,8 +43,14 @@ def get_models_by_params(params: dict):
     dense_dim = params.get("dense_main")
     model_output = params.get("model_output", "both")
 
-    domain_cnn = networks.get_domain_embedding_model(embedding_size, domain_length, filter_embedding, kernel_embedding,
+    if embedding_type == "small":
-                                                     hidden_embedding, 0.5)
+        domain_cnn = networks.get_domain_embedding_model(embedding_size, domain_length, filter_embedding,
+                                                         kernel_embedding, hidden_embedding, 0.5)
+    elif embedding_type == "deep":
+        domain_cnn = networks.get_domain_embedding_model2(embedding_size, domain_length, filter_embedding,
+                                                          kernel_embedding, hidden_embedding, 0.5)
+    else:
+        raise ValueError("embedding type not found")
     
     if network_type == "final":
         model = networks.get_final_model(0.25, flow_features, window_size, domain_length,
@@ -65,7 +72,7 @@ def get_models_by_params(params: dict):
         conv_client = model.get_layer("conv_client").trainable_weights
         l1 = [0.001 * K.sum(K.abs(x - y)) for (x, y) in zip(conv_server, conv_client)]
         model.add_loss(l1)
-    
+
         dense_server = model.get_layer("dense_server").trainable_weights
         dense_client = model.get_layer("dense_client").trainable_weights
         l2 = [0.001 * K.sum(K.abs(x - y)) for (x, y) in zip(dense_server, dense_client)]

models/networks.py

@@ -2,7 +2,7 @@ from collections import namedtuple
 
 import keras
 from keras.engine import Input, Model as KerasModel
-from keras.layers import Conv1D, Dense, Dropout, Embedding, GlobalMaxPooling1D, TimeDistributed
+from keras.layers import Conv1D, Dense, Dropout, Embedding, GlobalAveragePooling1D, GlobalMaxPooling1D, TimeDistributed
 
 import dataset
 
@@ -22,6 +22,24 @@ def get_domain_embedding_model(embedding_size, input_length, filter_size, kernel
     return KerasModel(x, y)
 
 
+def get_domain_embedding_model2(embedding_size, input_length, filter_size, kernel_size, hidden_dims,
+                                drop_out=0.5) -> KerasModel:
+    x = y = Input(shape=(input_length,))
+    y = Embedding(input_dim=dataset.get_vocab_size(), output_dim=embedding_size)(y)
+    y = Conv1D(filter_size,
+               kernel_size,
+               activation='relu')(y)
+    y = Conv1D(filter_size,
+               kernel_size,
+               activation='relu')(y)
+    y = Conv1D(filter_size,
+               kernel_size,
+               activation='relu')(y)
+    y = GlobalAveragePooling1D()(y)
+    y = Dense(hidden_dims, activation="relu")(y)
+    return KerasModel(x, y)
+
+
 def get_final_model(cnnDropout, flow_features, window_size, domain_length, cnn_dims, kernel_size,
                     dense_dim, cnn) -> Model:
     ipt_domains = Input(shape=(window_size, domain_length), name="ipt_domains")