rene/ma_cisco_malware
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add staggered model training for intermediate sever prediction; refactor model return values

Browse Source
This commit is contained in:
René Knaebel
2017-09-07 14:24:55 +02:00
parent 2080444fb7
commit 5bd8e41711
6 changed files with 92 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
models/__init__.py
View File

@@ -34,13 +34,13 @@ def get_models_by_params(params: dict):
embedding_model = networks.get_embedding(embedding_size, input_length, filter_embedding, kernel_embedding,
hidden_embedding, dropout)
predict_model = networks.get_model(dropout, flow_features, domain_features, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
old_model = networks.get_model(dropout, flow_features, domain_features, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
new_model = networks.get_new_model(dropout, flow_features, domain_features, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
return embedding_model, predict_model, new_model
return embedding_model, old_model, new_model
def get_metrics():

43
models/pauls_networks.py
View File

@@ -1,9 +1,13 @@
import keras
from keras.engine import Input, Model
from keras.engine import Input, Model as KerasModel
from keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense, Dropout, Activation, TimeDistributed
import dataset
from collections import namedtuple
Model = namedtuple("Model", ["in_domains", "in_flows", "out_client", "out_server"])
best_config = {
"type": "paul",
"batch_size": 64,
@@ -26,7 +30,7 @@ best_config = {
}
def get_embedding(embedding_size, input_length, filter_size, kernel_size, hidden_dims, drop_out=0.5):
def get_embedding(embedding_size, input_length, filter_size, kernel_size, hidden_dims, drop_out=0.5) -> KerasModel:
x = y = Input(shape=(input_length,))
y = Embedding(input_dim=dataset.get_vocab_size(), output_dim=embedding_size)(y)
y = Conv1D(filter_size, kernel_size, activation='relu')(y)
@@ -34,11 +38,11 @@ def get_embedding(embedding_size, input_length, filter_size, kernel_size, hidden
y = Dropout(drop_out)(y)
y = Dense(hidden_dims)(y)
y = Activation('relu')(y)
return Model(x, y)
return KerasModel(x, y)
def get_model(cnnDropout, flow_features, domain_features, window_size, domain_length, cnn_dims, kernel_size,
dense_dim, cnn, model_output="both"):
dense_dim, cnn, model_output="both") -> Model:
ipt_domains = Input(shape=(window_size, domain_length), name="ipt_domains")
encoded = TimeDistributed(cnn)(ipt_domains)
ipt_flows = Input(shape=(window_size, flow_features), name="ipt_flows")
@@ -52,40 +56,31 @@ def get_model(cnnDropout, flow_features, domain_features, window_size, domain_le
y = GlobalMaxPooling1D()(y)
y = Dropout(cnnDropout)(y)
y = Dense(dense_dim, activation='relu')(y)
y1 = Dense(1, activation='sigmoid', name="client")(y)
y2 = Dense(1, activation='sigmoid', name="server")(y)
out_client = Dense(1, activation='sigmoid', name="client")(y)
out_server = Dense(1, activation='sigmoid', name="server")(y)
if model_output == "both":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1, y2))
elif model_output == "client":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1,))
elif model_output == "server":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y2,))
return Model(ipt_domains, ipt_flows, out_client, out_server)
def get_new_model(dropout, flow_features, domain_features, window_size, domain_length, cnn_dims, kernel_size,
dense_dim, cnn, model_output="both"):
dense_dim, cnn, model_output="both") -> Model:
ipt_domains = Input(shape=(window_size, domain_length), name="ipt_domains")
ipt_flows = Input(shape=(window_size, flow_features), name="ipt_flows")
encoded = TimeDistributed(cnn)(ipt_domains)
merged = keras.layers.concatenate([encoded, ipt_flows], -1)
y = Dense(dense_dim, activation="relu")(merged)
y2 = Dense(1, activation="sigmoid", name="server")(y)
y = Dense(dense_dim, activation="relu", name="dense_server")(merged)
out_server = Dense(1, activation="sigmoid", name="server")(y)
merged = keras.layers.concatenate([merged, y], -1)
# CNN processing a small slides of flow windows
y = Conv1D(cnn_dims,
kernel_size,
activation='relu',
input_shape=(window_size, domain_features + flow_features))(y)
input_shape=(window_size, domain_features + flow_features))(merged)
# remove temporal dimension by global max pooling
y = GlobalMaxPooling1D()(y)
y = Dropout(dropout)(y)
y = Dense(dense_dim, activation='relu')(y)
y = Dense(dense_dim, activation='relu', name="dense_client")(y)
y1 = Dense(1, activation='sigmoid', name="client")(y)
out_client = Dense(1, activation='sigmoid', name="client")(y)
if model_output == "both":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1, y2))
elif model_output == "client":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1,))
elif model_output == "server":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y2,))
return Model(ipt_domains, ipt_flows, out_client, out_server)

31
models/renes_networks.py
View File

@@ -1,10 +1,14 @@
import keras
from keras.engine import Input, Model
from keras.engine import Input, Model as KerasModel
from keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense, Dropout, TimeDistributed, MaxPool1D, \
GlobalAveragePooling1D
import dataset
from collections import namedtuple
Model = namedtuple("Model", ["in_domains", "in_flows", "out_client", "out_server"])
def get_embedding(embedding_size, input_length, filter_size, kernel_size, hidden_dims, drop_out=0.5):
x = y = Input(shape=(input_length,))
@@ -14,7 +18,7 @@ def get_embedding(embedding_size, input_length, filter_size, kernel_size, hidden
y = Conv1D(filter_size, kernel_size=3, activation='relu')(y)
y = GlobalAveragePooling1D()(y)
y = Dense(hidden_dims, activation="relu")(y)
return Model(x, y)
return KerasModel(x, y)
def get_model(cnnDropout, flow_features, domain_features, window_size, domain_length, cnn_dims, kernel_size,
@@ -35,15 +39,10 @@ def get_model(cnnDropout, flow_features, domain_features, window_size, domain_le
y = Dropout(cnnDropout)(y)
y = Dense(dense_dim, activation='relu')(y)
y = Dense(dense_dim // 2, activation='relu')(y)
y1 = Dense(1, activation='sigmoid', name="client")(y)
y2 = Dense(1, activation='sigmoid', name="server")(y)
out_client = Dense(1, activation='sigmoid', name="client")(y)
out_server = Dense(1, activation='sigmoid', name="server")(y)
if model_output == "both":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1, y2))
elif model_output == "client":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1,))
elif model_output == "server":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y2,))
return Model(ipt_domains, ipt_flows, out_client, out_server)
def get_new_model(dropout, flow_features, domain_features, window_size, domain_length, cnn_dims, kernel_size,
@@ -53,7 +52,7 @@ def get_new_model(dropout, flow_features, domain_features, window_size, domain_l
encoded = TimeDistributed(cnn)(ipt_domains)
merged = keras.layers.concatenate([encoded, ipt_flows], -1)
y = Dense(dense_dim, activation="relu")(merged)
y2 = Dense(1, activation="sigmoid", name="server")(y)
out_server = Dense(1, activation="sigmoid", name="server")(y)
# CNN processing a small slides of flow windows
y = Conv1D(filters=cnn_dims, kernel_size=kernel_size, activation='relu', padding="same",
input_shape=(window_size, domain_features + flow_features))(y)
@@ -66,11 +65,7 @@ def get_new_model(dropout, flow_features, domain_features, window_size, domain_l
y = Dropout(dropout)(y)
y = Dense(dense_dim, activation='relu')(y)
y1 = Dense(1, activation='sigmoid', name="client")(y)
out_client = Dense(1, activation='sigmoid', name="client")(y)
return Model(ipt_domains, ipt_flows, out_client, out_server)
if model_output == "both":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1, y2))
elif model_output == "client":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y1,))
elif model_output == "server":
return Model(inputs=[ipt_domains, ipt_flows], outputs=(y2,))