removed old files from paul
This commit is contained in:
parent
d19036a611
commit
deac7f9e58
@ -1,846 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
import sys
|
|
||||||
sys.path.append('..')
|
|
||||||
sys.path.append('/mnt/projekte/pmlcluster/home/prasse/projects/ciscoSVN/cisco/trunk/code/')
|
|
||||||
import os
|
|
||||||
import numpy as np
|
|
||||||
import joblib
|
|
||||||
from keras.preprocessing.sequence import pad_sequences
|
|
||||||
from keras.utils import np_utils
|
|
||||||
from keras.models import Sequential
|
|
||||||
from keras.layers import Dense
|
|
||||||
from keras.layers import LSTM
|
|
||||||
from keras.layers import Dropout
|
|
||||||
import csv
|
|
||||||
import pandas as pd
|
|
||||||
import random
|
|
||||||
from keras.models import model_from_json
|
|
||||||
import time
|
|
||||||
import re
|
|
||||||
# import mongoDBConnector as mongoDBConnector
|
|
||||||
import stackedNeuralModels as stackedNeuralModels
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
|
|
||||||
def getCiscoDomainLabel(curDomain,curSIP,hostSet,sipSet,sldSet):
|
|
||||||
# check server-ip
|
|
||||||
if curSIP in sipSet:
|
|
||||||
return 1.0
|
|
||||||
# check second level domain
|
|
||||||
splitDomain = curDomain.split('.')
|
|
||||||
if len(splitDomain) >= 2:
|
|
||||||
curSLD = splitDomain[-2] + '.' + splitDomain[-1]
|
|
||||||
else:
|
|
||||||
curSLD = curDomain
|
|
||||||
if curSLD in sldSet:
|
|
||||||
return 1.0
|
|
||||||
|
|
||||||
# check domain
|
|
||||||
if curDomain in hostSet:
|
|
||||||
return 1.0
|
|
||||||
else:
|
|
||||||
if curSLD in hostSet:
|
|
||||||
return 1.0
|
|
||||||
else:
|
|
||||||
return 0.0
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
def getSubSample(useDir,numUser,threshold=3,
|
|
||||||
windowSize=10,minFlowsPerUser=10,
|
|
||||||
maxLen=40,flagUseCiscoFeatures=False,
|
|
||||||
urlSIPDIct=dict(),characterDict=dict(),
|
|
||||||
maxLengthInSeconds=-1,
|
|
||||||
timesNeg=-1,
|
|
||||||
mongoHost='',mongoPort=0,dbName='',
|
|
||||||
collectionName='',metaCollectionName=''):
|
|
||||||
curDFs = mongoDBConnector.sampleDataFromDir(mongoHost=mongoHost,mongoPort=mongoPort,dbName=dbName,
|
|
||||||
useDir=useDir,collectionName=collectionName,
|
|
||||||
metaCollectionName=metaCollectionName,
|
|
||||||
numUser=numUser,minFlowsPerUser=minFlowsPerUser)
|
|
||||||
|
|
||||||
domainLists = []
|
|
||||||
dfLists = []
|
|
||||||
for i in tqdm(np.arange(len(curDFs)), miniters=10):
|
|
||||||
(domainListsTmp,dfListsTmp) = stackedNeuralModels.getChunksFromUserDataFrame(curDFs[i],
|
|
||||||
windowSize=windowSize,overlapping=False,maxLengthInSeconds=maxLengthInSeconds)
|
|
||||||
domainLists += domainListsTmp
|
|
||||||
dfLists += dfListsTmp
|
|
||||||
|
|
||||||
(testData,testLabel,testHits,testNames) = stackedNeuralModels.createTrainData(
|
|
||||||
domainLists=domainLists,dfLists=dfLists,charachterDict=characterDict,
|
|
||||||
maxLen=maxLen,threshold = threshold,
|
|
||||||
flagUseCiscoFeatures=flagUseCiscoFeatures,urlSIPDIct=urlSIPDIct,
|
|
||||||
windowSize=windowSize)
|
|
||||||
|
|
||||||
useIDs = np.where(np.array(testLabel) == 1.0)[0]
|
|
||||||
useIDs = np.concatenate([useIDs, np.where(np.array(testLabel) == 0.0)[0]])
|
|
||||||
|
|
||||||
if timesNeg != -1:
|
|
||||||
posIDs = np.where(np.array(testLabel)[useIDs] == 1.0)[0]
|
|
||||||
negIDs = np.where(np.array(testLabel)[useIDs] == 0.0)[0]
|
|
||||||
if len(negIDs) > len(posIDs) * timesNeg:
|
|
||||||
negIDs = np.random.permutation(negIDs)
|
|
||||||
negIDs = negIDs[0:len(posIDs) * timesNeg]
|
|
||||||
negIDs = useIDs[negIDs]
|
|
||||||
posIDs = useIDs[posIDs]
|
|
||||||
useIDs = np.concatenate([negIDs,posIDs])
|
|
||||||
testLabel = testLabel[useIDs]
|
|
||||||
testHits = testHits[useIDs]
|
|
||||||
testNames = testNames[useIDs]
|
|
||||||
for i in range(len(testData)):
|
|
||||||
testData[i] = testData[i][useIDs]
|
|
||||||
return (testData,testLabel,testHits,testNames)
|
|
||||||
|
|
||||||
|
|
||||||
def getSubSampleAllPositiveUsers(useDir,threshold=3,
|
|
||||||
windowSize=10,minFlowsPerUser=10,
|
|
||||||
maxLen=40,flagUseCiscoFeatures=False,
|
|
||||||
urlSIPDIct=dict(),characterDict=dict(),
|
|
||||||
maxLengthInSeconds=-1,
|
|
||||||
numNegUser=10000,
|
|
||||||
mongoHost='',mongoPort=0,dbName='',
|
|
||||||
collectionName='',metaCollectionName=''):
|
|
||||||
|
|
||||||
curDFs = mongoDBConnector.sampleAllPositiveUserFromDir(mongoHost=mongoHost,mongoPort=mongoPort,dbName=dbName,
|
|
||||||
useDir=useDir,collectionName=collectionName,
|
|
||||||
metaCollectionName=metaCollectionName,
|
|
||||||
numNegUser=numNegUser,minFlowsPerUser=minFlowsPerUser)
|
|
||||||
domainLists = []
|
|
||||||
dfLists = []
|
|
||||||
for i in tqdm(np.arange(len(curDFs)), miniters=10):
|
|
||||||
(domainListsTmp,dfListsTmp) = stackedNeuralModels.getChunksFromUserDataFrame(curDFs[i],
|
|
||||||
windowSize=windowSize,overlapping=False,maxLengthInSeconds=maxLengthInSeconds)
|
|
||||||
domainLists += domainListsTmp
|
|
||||||
dfLists += dfListsTmp
|
|
||||||
|
|
||||||
(testData,testLabel,testHits,testNames) = stackedNeuralModels.createTrainData(
|
|
||||||
domainLists=domainLists,dfLists=dfLists,charachterDict=characterDict,
|
|
||||||
maxLen=maxLen,threshold = threshold,
|
|
||||||
flagUseCiscoFeatures=flagUseCiscoFeatures,urlSIPDIct=urlSIPDIct,
|
|
||||||
windowSize=windowSize)
|
|
||||||
|
|
||||||
useIDs = np.where(np.array(testLabel) == 1.0)[0]
|
|
||||||
useIDs = np.concatenate([useIDs, np.where(np.array(testLabel) == 0.0)[0]])
|
|
||||||
|
|
||||||
testLabel = testLabel[useIDs]
|
|
||||||
testHits = testHits[useIDs]
|
|
||||||
testNames = testNames[useIDs]
|
|
||||||
for i in range(len(testData)):
|
|
||||||
testData[i] = testData[i][useIDs]
|
|
||||||
return (testData,testLabel,testHits,testNames)
|
|
||||||
|
|
||||||
def sequenceGenerator(useDir,numUser,threshold=3,
|
|
||||||
windowSize=10,minFlowsPerUser=10,
|
|
||||||
maxLen=40,flagUseCiscoFeatures=False,
|
|
||||||
urlSIPDIct=dict(),characterDict=dict(),
|
|
||||||
maxLengthInSeconds=-1,
|
|
||||||
timesNeg=-1,
|
|
||||||
mongoHost='',mongoPort=0,dbName='',
|
|
||||||
collectionName='',metaCollectionName=''):
|
|
||||||
while 1:
|
|
||||||
(testData,testLabel,testHits,testNames) = getSubSample(useDir,numUser,threshold=threshold,
|
|
||||||
windowSize=windowSize,minFlowsPerUser=minFlowsPerUser,
|
|
||||||
maxLen=maxLen,flagUseCiscoFeatures=flagUseCiscoFeatures,
|
|
||||||
urlSIPDIct=urlSIPDIct,characterDict=characterDict,
|
|
||||||
maxLengthInSeconds=maxLengthInSeconds,
|
|
||||||
timesNeg=timesNeg,
|
|
||||||
mongoHost=mongoHost,mongoPort=mongoPort,dbName=dbName,
|
|
||||||
collectionName=collectionName,metaCollectionName=metaCollectionName)
|
|
||||||
testLabel = np_utils.to_categorical(testLabel, 2)
|
|
||||||
#print(testData.shape)
|
|
||||||
yield (testData, testLabel)
|
|
||||||
|
|
||||||
|
|
||||||
def sequenceGeneratorTest(data,label):
|
|
||||||
while 1:
|
|
||||||
yield (data, label)
|
|
||||||
|
|
||||||
# three different modes
|
|
||||||
# if mode == 'correct' -> dont permute or touch the ordering
|
|
||||||
# if mode == 'permutate' -> permute the ordering
|
|
||||||
# if mode == 'sort' -> sort the flows by sent bytes
|
|
||||||
def dataGenerator(trainData,trainLabel,numTimesPos,mode='correct'):
|
|
||||||
return True
|
|
||||||
|
|
||||||
def getMalwareClassDict(path):
|
|
||||||
outDict = dict()
|
|
||||||
for line in file(path):
|
|
||||||
lineSplit = line.strip().split('\t')
|
|
||||||
if len(lineSplit) == 3:
|
|
||||||
outDict[lineSplit[0]] = (lineSplit[1],lineSplit[2])
|
|
||||||
return outDict
|
|
||||||
|
|
||||||
def applyLower(inStr):
|
|
||||||
try:
|
|
||||||
return inStr.lower()
|
|
||||||
except:
|
|
||||||
return inStr
|
|
||||||
|
|
||||||
def logTransformData(inputMatrix):
|
|
||||||
# delete timestamps
|
|
||||||
try:
|
|
||||||
return np.log1p(np.array(inputMatrix,dtype='float64'))
|
|
||||||
except:
|
|
||||||
return inputMatrix
|
|
||||||
|
|
||||||
def getTrainMatrixLabelFromDataFrame(dataFrame,parameter=dict(),\
|
|
||||||
hostDict=dict(),sipDict=dict(),vtDF = dict(),
|
|
||||||
flagReturnDomains=False):
|
|
||||||
if len(dataFrame) == 0:
|
|
||||||
return ([],-1)
|
|
||||||
if 'flowFeatures' in parameter:
|
|
||||||
flowFeatures = parameter['flowFeatures']
|
|
||||||
else:
|
|
||||||
flowFeatures = ['duration','bytes_down','bytes_up']
|
|
||||||
# extract flow features
|
|
||||||
data = dataFrame[flowFeatures].values
|
|
||||||
# get time-gap feature
|
|
||||||
timeStamps = np.array(dataFrame['timeStamp'].values,dtype='float32')
|
|
||||||
timeStampsPre = np.zeros([len(timeStamps),])
|
|
||||||
timeStampsPre[1:] = timeStamps[0:len(timeStamps)-1]
|
|
||||||
diffTimeStamps = timeStamps - timeStampsPre
|
|
||||||
diffTimeStamps[0] = 0.0
|
|
||||||
negIDs = np.where(diffTimeStamps < 0.0)[0]
|
|
||||||
diffTimeStamps[negIDs] = 0.0
|
|
||||||
diffTimeStamps = np.reshape(diffTimeStamps,[len(diffTimeStamps),1])
|
|
||||||
data = np.hstack([data,diffTimeStamps])
|
|
||||||
# log transform
|
|
||||||
data = logTransformData(data)
|
|
||||||
if 'urlFeature' in dataFrame:
|
|
||||||
urlFeatures = np.zeros([len(dataFrame),len(dataFrame.iloc[0]['urlFeature'])])
|
|
||||||
for i in range(len(dataFrame)):
|
|
||||||
urlFeatures[i,:] = dataFrame.iloc[i]['urlFeature']
|
|
||||||
data = np.hstack([data,urlFeatures])
|
|
||||||
# cisco feature
|
|
||||||
numCiscoFeature = 30
|
|
||||||
ciscoFeatures = np.zeros([data.shape[0],2*numCiscoFeature])
|
|
||||||
if len(hostDict) > 0:
|
|
||||||
counter = 0
|
|
||||||
for i in range(len(dataFrame)):
|
|
||||||
row = dataFrame.iloc[i]
|
|
||||||
curHost = extractHost(row['domain'])
|
|
||||||
if curHost in hostDict:
|
|
||||||
ciscoFeatures[counter,0:numCiscoFeature] = hostDict[curHost]
|
|
||||||
if len(sipDict) > 0:
|
|
||||||
counter = 0
|
|
||||||
for i in range(len(dataFrame)):
|
|
||||||
row = dataFrame.iloc[i]
|
|
||||||
curSIP = row['server_ip']
|
|
||||||
if curSIP in sipDict:
|
|
||||||
ciscoFeatures[counter,numCiscoFeature:] = sipDict[curSIP]
|
|
||||||
data = np.hstack([data,ciscoFeatures])
|
|
||||||
if len(vtDF) != 0:
|
|
||||||
vtHashSet = set(vtDF['hash'])
|
|
||||||
hitNums = []
|
|
||||||
hashes = dataFrame['anyConnect_hash']
|
|
||||||
for curHash in hashes:
|
|
||||||
#print(vtDF.keys())
|
|
||||||
try:
|
|
||||||
if curHash.lower() in vtHashSet:
|
|
||||||
curID = np.where(vtDF['hash'] == curHash.lower())[0]
|
|
||||||
if len(curID) >= 1:
|
|
||||||
curID = curID[0]
|
|
||||||
hitNums.append(float(vtDF.iloc[curID]['hits']))
|
|
||||||
else:
|
|
||||||
hitNums.append(-1.0)
|
|
||||||
else:
|
|
||||||
hitNums.append(-1.0)
|
|
||||||
except:
|
|
||||||
hitNums.append(-1.0)
|
|
||||||
maxHits = np.max(hitNums)
|
|
||||||
else:
|
|
||||||
if 'hits' in dataFrame:
|
|
||||||
maxHits = np.max(dataFrame['hits'])
|
|
||||||
else:
|
|
||||||
maxHits = -1
|
|
||||||
label = np.max(dataFrame['label'])
|
|
||||||
if flagReturnDomains:
|
|
||||||
return (data,label,maxHits,dataFrame['domain'])
|
|
||||||
else:
|
|
||||||
return (data,label,maxHits)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def getDomainChunksByUser(data,useUserName,blockSize):
|
|
||||||
outData = []
|
|
||||||
outLabel = []
|
|
||||||
useDataAll = data[data['user_hash'] == useUserName]
|
|
||||||
userIDs = np.arange(len(useDataAll))
|
|
||||||
#print('number of found flows for user: ' + str(len(userIDs)))
|
|
||||||
numBlocks = int(np.ceil(float(len(userIDs)) / float(blockSize)))
|
|
||||||
for blockID in range(numBlocks):
|
|
||||||
curIDs = userIDs[(blockID * blockSize):((blockID+1)*blockSize)]
|
|
||||||
#print(curIDs)
|
|
||||||
useData = useDataAll.iloc[curIDs]
|
|
||||||
curDomains = useData['domain']
|
|
||||||
curLabel = np.max(useData['label'])
|
|
||||||
outData.append(curDomains)
|
|
||||||
outLabel.append(curLabel)
|
|
||||||
return (outData,outLabel)
|
|
||||||
|
|
||||||
|
|
||||||
def getChunksByUser(data,useUserName,blockSize,parameter=dict(),\
|
|
||||||
hostDict=dict(),sipDict=dict(), vtDF = dict, flagOnlyOneUser = False,
|
|
||||||
flagReturnDomains=False):
|
|
||||||
outData = []
|
|
||||||
outLabel = []
|
|
||||||
outHits = []
|
|
||||||
outDomains = []
|
|
||||||
if flagOnlyOneUser:
|
|
||||||
useDataAll = data
|
|
||||||
else:
|
|
||||||
useDataAll = data[data['user_hash'] == useUserName]
|
|
||||||
userIDs = np.arange(len(useDataAll))
|
|
||||||
#print('number of found flows for user: ' + str(len(userIDs)))
|
|
||||||
numBlocks = int(np.ceil(float(len(userIDs)) / float(blockSize)))
|
|
||||||
for blockID in range(numBlocks):
|
|
||||||
curIDs = userIDs[(blockID * blockSize):((blockID+1)*blockSize)]
|
|
||||||
#print(curIDs)
|
|
||||||
useData = useDataAll.iloc[curIDs]
|
|
||||||
if flagReturnDomains:
|
|
||||||
(curTrainData,curLabel,curMaxHits,curDomains) = getTrainMatrixLabelFromDataFrame(useData,\
|
|
||||||
parameter,hostDict,sipDict,vtDF=vtDF,flagReturnDomains=flagReturnDomains)
|
|
||||||
else:
|
|
||||||
(curTrainData,curLabel,curMaxHits) = getTrainMatrixLabelFromDataFrame(useData,\
|
|
||||||
parameter,hostDict,sipDict,vtDF=vtDF,flagReturnDomains=flagReturnDomains)
|
|
||||||
outData.append(curTrainData)
|
|
||||||
outLabel.append(curLabel)
|
|
||||||
outHits.append(curMaxHits)
|
|
||||||
if flagReturnDomains:
|
|
||||||
outDomains.append(curDomains)
|
|
||||||
if flagReturnDomains:
|
|
||||||
return (outData,outLabel,outHits,outDomains)
|
|
||||||
else:
|
|
||||||
return (outData,outLabel,outHits)
|
|
||||||
|
|
||||||
|
|
||||||
def getLSTMModel(blockSize=10,input_dim=103,lstmUnits=10,denseSize=128):
|
|
||||||
nb_classes = 2
|
|
||||||
model = Sequential()
|
|
||||||
model.add(LSTM(lstmUnits, input_dim=input_dim, input_length=blockSize))
|
|
||||||
model.add(Dense(denseSize, activation='relu'))
|
|
||||||
model.add(Dropout(0.5))
|
|
||||||
model.add(Dense(nb_classes, activation='softmax'))
|
|
||||||
model.compile(loss='binary_crossentropy',
|
|
||||||
optimizer='adam', metrics=['accuracy'])
|
|
||||||
# number of params:
|
|
||||||
# params = 4 * ((size_of_input + 1) * size_of_output + size_of_output^2)
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def getCiscoURLFeatureForRow(row):
|
|
||||||
sortKeys = list(row.keys())
|
|
||||||
sortKeys.sort()
|
|
||||||
featureVec = np.zeros([len(sortKeys)-1,])
|
|
||||||
counter = 0
|
|
||||||
for keyName in sortKeys:
|
|
||||||
if 'key' in keyName:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
featureVec[counter] = float(row[keyName])
|
|
||||||
except:
|
|
||||||
featureVec[counter] = 0.0
|
|
||||||
counter += 1
|
|
||||||
featureVec[np.where(np.isnan(featureVec))[0]] = 0.0
|
|
||||||
return featureVec
|
|
||||||
|
|
||||||
|
|
||||||
def getCiscoFeatureDictForHost(headerPath,dataPath):
|
|
||||||
# get header
|
|
||||||
header = []
|
|
||||||
for line in file(headerPath):
|
|
||||||
header.append(line.strip())
|
|
||||||
|
|
||||||
header = ['key'] + header
|
|
||||||
|
|
||||||
fobj = open(dataPath,'r')
|
|
||||||
csvReader = csv.DictReader(fobj,fieldnames = header,delimiter='\t')
|
|
||||||
hostDict = dict()
|
|
||||||
counter = 0
|
|
||||||
for row in csvReader:
|
|
||||||
featureVec = getCiscoURLFeatureForRow(row)
|
|
||||||
curHost = row['key']
|
|
||||||
curHost = extractHost(curHost)
|
|
||||||
hostDict[curHost] = featureVec
|
|
||||||
counter += 1
|
|
||||||
if counter % 10000 == 0:
|
|
||||||
print(str(counter) + ' host features collected')
|
|
||||||
return hostDict
|
|
||||||
|
|
||||||
def getCiscoFeatureDictForSIP(headerPath,dataPath):
|
|
||||||
# get header
|
|
||||||
header = []
|
|
||||||
for line in file(headerPath):
|
|
||||||
header.append(line.strip())
|
|
||||||
|
|
||||||
header = ['key'] + header
|
|
||||||
|
|
||||||
fobj = open(dataPath,'r')
|
|
||||||
csvReader = csv.DictReader(fobj,fieldnames = header,delimiter='\t')
|
|
||||||
hostDict = dict()
|
|
||||||
counter = 0
|
|
||||||
for row in csvReader:
|
|
||||||
featureVec = getCiscoURLFeatureForRow(row)
|
|
||||||
curHost = row['key']
|
|
||||||
hostDict[curHost] = featureVec
|
|
||||||
counter += 1
|
|
||||||
if counter % 10000 == 0:
|
|
||||||
print(str(counter) + ' sip features collected')
|
|
||||||
return hostDict
|
|
||||||
|
|
||||||
def getCiscoFeatureDictForSLD(headerPath,dataPath):
|
|
||||||
# get header
|
|
||||||
header = []
|
|
||||||
for line in file(headerPath):
|
|
||||||
header.append(line.strip())
|
|
||||||
|
|
||||||
header = ['key'] + header
|
|
||||||
|
|
||||||
fobj = open(dataPath,'r')
|
|
||||||
csvReader = csv.DictReader(fobj,fieldnames = header,delimiter='\t')
|
|
||||||
hostDict = dict()
|
|
||||||
counter = 0
|
|
||||||
for row in csvReader:
|
|
||||||
featureVec = getCiscoURLFeatureForRow(row)
|
|
||||||
curHost = row['key']
|
|
||||||
hostDict[curHost] = featureVec
|
|
||||||
counter += 1
|
|
||||||
if counter % 10000 == 0:
|
|
||||||
print(str(counter) + ' sld features collected')
|
|
||||||
return hostDict
|
|
||||||
|
|
||||||
|
|
||||||
def extractHost(domain):
|
|
||||||
curHostSplit = domain.split('.')
|
|
||||||
try:
|
|
||||||
curHost = curHostSplit[-2] + '.' + curHostSplit[-1]
|
|
||||||
return curHost
|
|
||||||
except:
|
|
||||||
return domain
|
|
||||||
|
|
||||||
|
|
||||||
def loadDataSetFromJoblib(trainDirs,minFlowsPerUser = 10,numTimesPos = 20):
|
|
||||||
for dirID in range(len(trainDirs)):
|
|
||||||
curDir = trainDirs[dirID]
|
|
||||||
curFiles = os.listdir(curDir)
|
|
||||||
dayJoblibCounter = 0
|
|
||||||
for curFile in curFiles:
|
|
||||||
curFile = curDir + curFile
|
|
||||||
if curFile.endswith('.joblib'):
|
|
||||||
curData = joblib.load(curFile)
|
|
||||||
if dayJoblibCounter == 0:
|
|
||||||
dayData = curData
|
|
||||||
else:
|
|
||||||
dayData = dayData.append(curData,ignore_index=True)
|
|
||||||
dayJoblibCounter += 1
|
|
||||||
print('processed file number: ' + str(dayJoblibCounter) + ' (dir ' + str(curDir) +')')
|
|
||||||
# use flows with min minFlowsPerUser flows
|
|
||||||
if minFlowsPerUser != -1:
|
|
||||||
grouped = dayData.groupby('user_hash')
|
|
||||||
useUsers = set()
|
|
||||||
for grouping in grouped:
|
|
||||||
numFlowsCurUser = len(grouping[1])
|
|
||||||
userLabel = np.max(grouping[1]['label'])
|
|
||||||
if numFlowsCurUser >= minFlowsPerUser and userLabel != -1.0:
|
|
||||||
useUsers.add(grouping[0])
|
|
||||||
# get ids
|
|
||||||
userIDs = dayData.loc[dayData['user_hash'].isin(useUsers)].index.values
|
|
||||||
dayData = dayData.iloc[userIDs]
|
|
||||||
dayData = dayData.reset_index(drop=True)
|
|
||||||
if numTimesPos != -1:
|
|
||||||
grouped = dayData.groupby('user_hash')
|
|
||||||
curUserLabel = []
|
|
||||||
curUserNames = []
|
|
||||||
for grouping in grouped:
|
|
||||||
numFlowsCurUser = len(grouping[1])
|
|
||||||
userLabel = np.max(grouping[1]['label'])
|
|
||||||
curUserLabel.append(userLabel)
|
|
||||||
curUserNames.append(grouping[1]['user_hash'].values[0])
|
|
||||||
posIDs = np.where(np.array(curUserLabel) == 1.0)[0]
|
|
||||||
negIDs = np.where(np.array(curUserLabel) == 0.0)[0]
|
|
||||||
maxNegLabel = len(posIDs) * numTimesPos
|
|
||||||
if len(negIDs) > maxNegLabel:
|
|
||||||
np.random.seed(1)
|
|
||||||
np.random.shuffle(negIDs)
|
|
||||||
negIDs = negIDs[0:maxNegLabel]
|
|
||||||
useIDs = np.concatenate([posIDs,negIDs])
|
|
||||||
useUsers = np.array(curUserNames)[useIDs]
|
|
||||||
useUsers = set(useUsers)
|
|
||||||
# get ids
|
|
||||||
userIDs = dayData.loc[dayData['user_hash'].isin(useUsers)].index.values
|
|
||||||
dayData = dayData.iloc[userIDs]
|
|
||||||
dayData = dayData.reset_index(drop=True)
|
|
||||||
if dirID == 0:
|
|
||||||
allData = dayData
|
|
||||||
else:
|
|
||||||
allData = allData.append(dayData,ignore_index=True)
|
|
||||||
return allData
|
|
||||||
|
|
||||||
def tokenizeDomain(domain,n=3):
|
|
||||||
domain = domain.replace('https://','')
|
|
||||||
domain = domain.replace('www.','')
|
|
||||||
domain = domain.replace('/','')
|
|
||||||
# reverse domain
|
|
||||||
domain = domain[::-1]
|
|
||||||
outList = []
|
|
||||||
splitCriterion = n
|
|
||||||
# overlapping n-grams
|
|
||||||
outList = [domain[i:i+splitCriterion] for i in range(0, len(domain), 1)]
|
|
||||||
return outList
|
|
||||||
|
|
||||||
|
|
||||||
def getDomainsInWindowData(allData,numNeg=-1,blockSize=10):
|
|
||||||
uniqueTrainUser = np.unique(allData['user_hash'])
|
|
||||||
userLabel = []
|
|
||||||
for curTrainUser in uniqueTrainUser:
|
|
||||||
userIDs = allData.loc[allData['user_hash'] == curTrainUser].index.values
|
|
||||||
curLabel = np.max(allData.iloc[userIDs]['label'])
|
|
||||||
userLabel.append(curLabel)
|
|
||||||
negIDs = np.where(np.array(userLabel) == 0.0)[0]
|
|
||||||
userLabel = np.array(userLabel)
|
|
||||||
posUser = np.where(userLabel == 1.0)[0]
|
|
||||||
negUser = np.where(userLabel == 0.0)[0]
|
|
||||||
|
|
||||||
if numNeg != -1:
|
|
||||||
if len(negUser) > numNeg:
|
|
||||||
np.random.shuffle(negUser)
|
|
||||||
negUser = negIDs[0:numNeg]
|
|
||||||
|
|
||||||
useUser = posUser
|
|
||||||
useUser = np.concatenate([posUser,negUser])
|
|
||||||
counter = 0
|
|
||||||
trainDomains = []
|
|
||||||
trainBlockLabel = []
|
|
||||||
trainNames = []
|
|
||||||
for uID in range(len(useUser)):
|
|
||||||
curTrainUser = uniqueTrainUser[useUser[uID]]
|
|
||||||
(curUserData,curUserLabel) = getDomainChunksByUser(allData,curTrainUser,blockSize)
|
|
||||||
for i in range(len(curUserLabel)):
|
|
||||||
trainNames.append(curTrainUser)
|
|
||||||
trainDomains += curUserData
|
|
||||||
trainBlockLabel += curUserLabel
|
|
||||||
print('processed ' + str(counter) + ' users of ' + str(len(useUser)))
|
|
||||||
counter+= 1
|
|
||||||
return (trainDomains,trainBlockLabel,trainNames)
|
|
||||||
|
|
||||||
def getPaddedData(allData,numNeg=-1,blockSize=10,parameterDict=dict(),\
|
|
||||||
hostDict=dict(),sipDict = dict(),vtLabelPath=''):
|
|
||||||
if vtLabelPath != '':
|
|
||||||
vtDF = pd.read_csv(vtLabelPath,sep='\t')
|
|
||||||
else:
|
|
||||||
vtDF = dict()
|
|
||||||
uniqueTrainUser = np.unique(allData['user_hash'])
|
|
||||||
userLabel = []
|
|
||||||
for curTrainUser in uniqueTrainUser:
|
|
||||||
userIDs = allData.loc[allData['user_hash'] == curTrainUser].index.values
|
|
||||||
curLabel = np.max(allData.iloc[userIDs]['label'])
|
|
||||||
userLabel.append(curLabel)
|
|
||||||
negIDs = np.where(np.array(userLabel) == 0.0)[0]
|
|
||||||
userLabel = np.array(userLabel)
|
|
||||||
posUser = np.where(userLabel == 1.0)[0]
|
|
||||||
negUser = np.where(userLabel == 0.0)[0]
|
|
||||||
|
|
||||||
if numNeg != -1:
|
|
||||||
if len(negUser) > numNeg:
|
|
||||||
np.random.shuffle(negUser)
|
|
||||||
negUser = negIDs[0:numNeg]
|
|
||||||
|
|
||||||
useUser = posUser
|
|
||||||
useUser = np.concatenate([posUser,negUser])
|
|
||||||
counter = 0
|
|
||||||
trainBlocks = []
|
|
||||||
trainBlockLabel = []
|
|
||||||
trainNames = []
|
|
||||||
trainBlockHits = []
|
|
||||||
for uID in range(len(useUser)):
|
|
||||||
curTrainUser = uniqueTrainUser[useUser[uID]]
|
|
||||||
(curUserData,curUserLabel,curHits) = getChunksByUser(allData,curTrainUser,blockSize,\
|
|
||||||
parameter=parameterDict,hostDict=hostDict,sipDict=sipDict,vtDF = vtDF)
|
|
||||||
for i in range(len(curUserLabel)):
|
|
||||||
trainNames.append(curTrainUser)
|
|
||||||
trainBlocks += curUserData
|
|
||||||
trainBlockLabel += curUserLabel
|
|
||||||
trainBlockHits += curHits
|
|
||||||
print('processed ' + str(counter) + ' users of ' + str(len(useUser)))
|
|
||||||
counter+= 1
|
|
||||||
paddedData = pad_sequences(trainBlocks, maxlen=blockSize,dtype='float32')
|
|
||||||
#paddedData = paddedData[:,:,featureTypeDict[useFeatureType]]
|
|
||||||
return (paddedData,trainBlockLabel,trainNames,trainBlockHits)
|
|
||||||
|
|
||||||
|
|
||||||
def createTrainDataFromJoblibsPerUser(joblibPaths,minFlowsPerUser = 10,blockSize=10,
|
|
||||||
hostDict=dict(),sipDict=dict(),
|
|
||||||
vtLabelPath='',maxFlowsPerUser = 50000):
|
|
||||||
trainBlockLabel = []
|
|
||||||
trainNames = []
|
|
||||||
trainBlockHits = []
|
|
||||||
parameterDict = dict()
|
|
||||||
numBlocksToInitialize = 10000
|
|
||||||
paddedData = np.zeros([numBlocksToInitialize,blockSize,globalNumFeatures])
|
|
||||||
overallCounter = 0
|
|
||||||
startTime = time.time()
|
|
||||||
for uID in range(len(joblibPaths)):
|
|
||||||
curSavePath = joblibPaths[uID]
|
|
||||||
curData = joblib.load(curSavePath)['dataFrame']
|
|
||||||
if len(curData) < minFlowsPerUser:
|
|
||||||
continue
|
|
||||||
#curUserName = np.unique(curData['user_hash'])[0]
|
|
||||||
(curUserData,curUserLabel,curHits) = getChunksByUser(curData,'',blockSize,\
|
|
||||||
parameter=parameterDict,hostDict=hostDict,sipDict=sipDict,vtDF=dict(),flagOnlyOneUser = True)
|
|
||||||
curPaddedData = pad_sequences(curUserData, maxlen=blockSize,dtype='float32')
|
|
||||||
if (curPaddedData.shape[0] > maxFlowsPerUser):
|
|
||||||
curPaddedData = curPaddedData[0:maxFlowsPerUser]
|
|
||||||
curUserLabel = list(np.array(curUserLabel)[0:maxFlowsPerUser])
|
|
||||||
curHits = list(np.array(curHits)[0:maxFlowsPerUser])
|
|
||||||
for i in range(len(curPaddedData)):
|
|
||||||
trainNames.append(curSavePath)
|
|
||||||
trainBlockLabel += curUserLabel
|
|
||||||
trainBlockHits += curHits
|
|
||||||
#curPaddedData = curPaddedData[:,:,featureTypeDict[useFeatureType]]
|
|
||||||
numCurInstances = curPaddedData.shape[0]
|
|
||||||
while overallCounter+numCurInstances > paddedData.shape[0]:
|
|
||||||
paddedData = np.vstack([paddedData,np.zeros([numBlocksToInitialize,blockSize,globalNumFeatures])])
|
|
||||||
paddedData[overallCounter:overallCounter+numCurInstances,:] = curPaddedData
|
|
||||||
overallCounter += numCurInstances
|
|
||||||
if uID % 1000 == 0:
|
|
||||||
elapsedTime = time.time() - startTime
|
|
||||||
startTime = time.time()
|
|
||||||
print(str(uID+1) + ' user processed [' + str(elapsedTime) + ']')
|
|
||||||
paddedData = paddedData[0:overallCounter]
|
|
||||||
return (paddedData,trainBlockLabel,trainNames,trainBlockHits)
|
|
||||||
|
|
||||||
def loadDataSetFromJoblibPerUser(trainDirs,minFlowsPerUser = 10,numNegPerDay = 50000,
|
|
||||||
blockSize = 10,hostDict=dict(),sipDict=dict(),
|
|
||||||
seed =1,flagSkipNoLabelUser=False,
|
|
||||||
vtLabelPath='',maxFlowsPerUser = 50000,
|
|
||||||
flagReturnDomains=False):
|
|
||||||
if vtLabelPath != '':
|
|
||||||
vtDF = pd.read_csv(vtLabelPath,sep='\t')
|
|
||||||
else:
|
|
||||||
vtDF = dict()
|
|
||||||
trainBlockLabel = []
|
|
||||||
trainNames = []
|
|
||||||
trainBlockHits = []
|
|
||||||
trainBlockDomains = []
|
|
||||||
parameterDict = dict()
|
|
||||||
numBlocksToInitialize = 10000
|
|
||||||
paddedData = np.zeros([numBlocksToInitialize,blockSize,globalNumFeatures])
|
|
||||||
overallCounter = 0
|
|
||||||
for curDirID in range(len(trainDirs)):
|
|
||||||
curDir = trainDirs[curDirID]
|
|
||||||
curLabelFile = curDir + 'data_label.joblib'
|
|
||||||
labelData = joblib.load(curLabelFile)
|
|
||||||
posIDs = np.where(np.array(labelData['label']) == 1.0)[0]
|
|
||||||
negIDs = np.where(np.array(labelData['label']) == 0.0)[0]
|
|
||||||
random.seed(seed)
|
|
||||||
random.shuffle(negIDs)
|
|
||||||
useIDs = np.concatenate([posIDs,negIDs])
|
|
||||||
counter = 0
|
|
||||||
negCounter = 0
|
|
||||||
startTime = time.time()
|
|
||||||
for uID in range(len(useIDs)):
|
|
||||||
curID = useIDs[uID]
|
|
||||||
curUserName = labelData['usernames'][curID]
|
|
||||||
curSavePath = curDir + str(curUserName) + '.joblib'
|
|
||||||
curData = joblib.load(curSavePath)['dataFrame']
|
|
||||||
if flagSkipNoLabelUser:
|
|
||||||
curUserLabel = np.max(curData['label'])
|
|
||||||
if curUserLabel == -1.0:
|
|
||||||
continue
|
|
||||||
if len(curData) < minFlowsPerUser:
|
|
||||||
continue
|
|
||||||
if numNegPerDay != -1:
|
|
||||||
if negCounter > numNegPerDay:
|
|
||||||
break
|
|
||||||
if flagReturnDomains:
|
|
||||||
(curUserData,curUserLabel,curHits,curDomains) = getChunksByUser(curData,curUserName,blockSize,\
|
|
||||||
parameter=parameterDict,hostDict=hostDict,sipDict=sipDict,vtDF=vtDF,\
|
|
||||||
flagReturnDomains=flagReturnDomains)
|
|
||||||
else:
|
|
||||||
(curUserData,curUserLabel,curHits) = getChunksByUser(curData,curUserName,blockSize,\
|
|
||||||
parameter=parameterDict,hostDict=hostDict,sipDict=sipDict,vtDF=vtDF,\
|
|
||||||
flagReturnDomains=flagReturnDomains)
|
|
||||||
curPaddedData = pad_sequences(curUserData, maxlen=blockSize,dtype='float32')
|
|
||||||
if (curPaddedData.shape[0] > maxFlowsPerUser):
|
|
||||||
curPaddedData = curPaddedData[0:maxFlowsPerUser]
|
|
||||||
curUserLabel = list(np.array(curUserLabel)[0:maxFlowsPerUser])
|
|
||||||
curHits = list(np.array(curHits)[0:maxFlowsPerUser])
|
|
||||||
if 'curDomains' in locals():
|
|
||||||
curDomains = list(np.array(curDomains)[0:maxFlowsPerUser])
|
|
||||||
for i in range(len(curPaddedData)):
|
|
||||||
trainNames.append(curUserName)
|
|
||||||
trainBlockLabel += curUserLabel
|
|
||||||
trainBlockHits += curHits
|
|
||||||
trainBlockDomains += curDomains
|
|
||||||
#curPaddedData = curPaddedData[:,:,featureTypeDict[useFeatureType]]
|
|
||||||
numCurInstances = curPaddedData.shape[0]
|
|
||||||
while overallCounter+numCurInstances > paddedData.shape[0]:
|
|
||||||
paddedData = np.vstack([paddedData,np.zeros([numBlocksToInitialize,blockSize,globalNumFeatures])])
|
|
||||||
paddedData[overallCounter:overallCounter+numCurInstances,:] = curPaddedData
|
|
||||||
overallCounter += numCurInstances
|
|
||||||
#print('num of instances: ' + str(numCurInstances))
|
|
||||||
if (counter+1) % 1000 == 0:
|
|
||||||
elapsedTime = time.time() - startTime
|
|
||||||
print('processed ' + str(counter+1) + ' users of ' +\
|
|
||||||
str(len(useIDs)) + ' with ' + str(len(curData['label'])) +\
|
|
||||||
' flows [dir ' + str(curDirID+1) + ' of ' +\
|
|
||||||
str(len(trainDirs)) + '] in ' + str(elapsedTime) + ' sec')
|
|
||||||
startTime = time.time()
|
|
||||||
if np.max(np.array(curUserLabel)) == 0.0:
|
|
||||||
negCounter += 1
|
|
||||||
counter+= 1
|
|
||||||
paddedData = paddedData[0:overallCounter]
|
|
||||||
if flagReturnDomains:
|
|
||||||
return (paddedData,trainBlockLabel,trainNames,trainBlockHits,trainBlockDomains)
|
|
||||||
else:
|
|
||||||
return (paddedData,trainBlockLabel,trainNames,trainBlockHits)
|
|
||||||
|
|
||||||
def loadRawDataSetFromJoblibPerUser(trainDirs,numNegPerDay = 2000, seed = 1):
|
|
||||||
dataFrameList = []
|
|
||||||
overallCounter = 0
|
|
||||||
from tqdm import tqdm
|
|
||||||
for curDirID in tqdm(np.arange(len(trainDirs)), miniters=1):
|
|
||||||
curDir = trainDirs[curDirID]
|
|
||||||
curLabelFile = curDir + 'data_label.joblib'
|
|
||||||
labelData = joblib.load(curLabelFile)
|
|
||||||
posIDs = np.where(np.array(labelData['label']) == 1.0)[0]
|
|
||||||
negIDs = np.where(np.array(labelData['label']) == 0.0)[0]
|
|
||||||
random.seed(seed)
|
|
||||||
random.shuffle(negIDs)
|
|
||||||
if len(negIDs) >= numNegPerDay:
|
|
||||||
negIDs = negIDs[0:numNegPerDay]
|
|
||||||
useIDs = np.concatenate([posIDs,negIDs])
|
|
||||||
for uID in range(len(useIDs)):
|
|
||||||
curID = useIDs[uID]
|
|
||||||
curUserName = labelData['usernames'][curID]
|
|
||||||
curSavePath = curDir + str(curUserName) + '.joblib'
|
|
||||||
curData = joblib.load(curSavePath)['dataFrame']
|
|
||||||
dataFrameList.append(curData)
|
|
||||||
overallCounter += 1
|
|
||||||
return dataFrameList
|
|
||||||
|
|
||||||
|
|
||||||
def checkDomainForSecondLevelDomain(inDomain,sldDomainDict):
|
|
||||||
if not 'str' in str(type(inDomain)):
|
|
||||||
return False
|
|
||||||
splitDomain = inDomain.split('.')
|
|
||||||
if len(splitDomain) <= 2:
|
|
||||||
return False
|
|
||||||
sldDomain = splitDomain[-2] + '.' + splitDomain[-1]
|
|
||||||
if sldDomain in sldDomainDict:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
'''
|
|
||||||
out = False
|
|
||||||
for sldDomain in sldDomainDict:
|
|
||||||
if inDomain.endswith(sldDomain):
|
|
||||||
out = True
|
|
||||||
break
|
|
||||||
return out
|
|
||||||
'''
|
|
||||||
|
|
||||||
def save_model(model,jsonPath,h5Path):
|
|
||||||
# saving model
|
|
||||||
json_model = model.to_json()
|
|
||||||
open(jsonPath, 'w').write(json_model)
|
|
||||||
# saving weights
|
|
||||||
model.save_weights(h5Path, overwrite=True)
|
|
||||||
|
|
||||||
def load_model(jsonPath,h5Path):
|
|
||||||
# loading model
|
|
||||||
model = model_from_json(open(jsonPath).read())
|
|
||||||
model.load_weights(h5Path)
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
def getResultsFromSavedJoblibFile(joblibFiles,threshold=3):
|
|
||||||
testUserScores = []
|
|
||||||
testUserLabel = []
|
|
||||||
testLabel = []
|
|
||||||
testScores = []
|
|
||||||
testNames = []
|
|
||||||
for joblibPath in joblibFiles:
|
|
||||||
print('process: ' + joblibPath)
|
|
||||||
tmpJoblib = joblib.load(joblibPath)
|
|
||||||
if 'testBlockScores' in tmpJoblib.keys():
|
|
||||||
curTestBlockScores = tmpJoblib['testBlockScores']
|
|
||||||
for i in range(len(curTestBlockScores)):
|
|
||||||
if i == 0:
|
|
||||||
curTestScores = curTestBlockScores[i]
|
|
||||||
else:
|
|
||||||
curTestScores = np.concatenate([curTestScores,curTestBlockScores[i]])
|
|
||||||
curTestHits = tmpJoblib['blockHits']
|
|
||||||
curTestHits = np.array(curTestHits)
|
|
||||||
curTestScores = np.array(curTestScores)
|
|
||||||
curTestLabel = np.ones([len(curTestScores),]) * -1.0
|
|
||||||
curTestLabel[np.where(curTestHits == 0)[0]] = 0.0
|
|
||||||
curTestLabel[np.where(curTestHits >= threshold)[0]] = 1.0
|
|
||||||
curTestNames = tmpJoblib['testNames']
|
|
||||||
else:
|
|
||||||
curTestHits = tmpJoblib['testHits']
|
|
||||||
curTestScores = tmpJoblib['testScores']
|
|
||||||
curTestLabel = tmpJoblib['testLabel']
|
|
||||||
curTestNames = tmpJoblib['testNames']
|
|
||||||
|
|
||||||
useIDs = np.where(curTestHits >= threshold)[0]
|
|
||||||
useIDs = np.concatenate([useIDs,np.where(curTestHits == 0.0)[0]])
|
|
||||||
# old code
|
|
||||||
#useIDs = np.where(tmpJoblib['testLabel'] == 1.0)[0]
|
|
||||||
#useIDs = np.concatenate([useIDs,np.where(tmpJoblib['testLabel'] == 0.0)[0]])
|
|
||||||
curTestScoresT = curTestScores[useIDs]
|
|
||||||
curTestLabelT = curTestLabel[useIDs]
|
|
||||||
if len(testScores) == 0:
|
|
||||||
testScores = curTestScoresT
|
|
||||||
testLabel = curTestLabelT
|
|
||||||
else:
|
|
||||||
testScores = np.concatenate([testScores,curTestScoresT])
|
|
||||||
testLabel = np.concatenate([testLabel,curTestLabelT])
|
|
||||||
|
|
||||||
if 'testBlockScores' in tmpJoblib.keys():
|
|
||||||
tmpScores = np.array(tmpJoblib['testScores'])
|
|
||||||
tmpHits = np.array(tmpJoblib['testHits'])
|
|
||||||
tmpLabel = np.ones([len(tmpHits),])*-1
|
|
||||||
tmpLabel[np.where(tmpHits == 0.0)[0]] = 0.0
|
|
||||||
tmpLabel[np.where(tmpHits >= threshold)[0]] = 1.0
|
|
||||||
useIDs = np.where(tmpLabel == 1.0)[0]
|
|
||||||
useIDs = np.concatenate([useIDs,np.where(tmpLabel == 0.0)[0]])
|
|
||||||
testUserLabel += list(np.array(tmpLabel)[useIDs])
|
|
||||||
testUserScores += list(np.array(tmpScores)[useIDs])
|
|
||||||
else:
|
|
||||||
# get user label
|
|
||||||
uniqueTestNames = list(np.unique(curTestNames))
|
|
||||||
for testName in uniqueTestNames:
|
|
||||||
curIDs = np.where(curTestNames == testName)[0]
|
|
||||||
curMaxHits = np.max(curTestHits[curIDs])
|
|
||||||
if curMaxHits > 0 and curMaxHits < threshold:
|
|
||||||
continue
|
|
||||||
if curMaxHits >= threshold:
|
|
||||||
testUserLabel.append(1.0)
|
|
||||||
else:
|
|
||||||
testUserLabel.append(0.0)
|
|
||||||
curScore = np.max(curTestScores[curIDs])
|
|
||||||
testUserScores.append(curScore)
|
|
||||||
testNames.append(testName)
|
|
||||||
testUserScores = np.array(testUserScores)
|
|
||||||
testUserLabel = np.array(testUserLabel)
|
|
||||||
testNames = np.array(testNames)
|
|
||||||
return (testUserScores,testUserLabel,testLabel,testScores,testNames)
|
|
||||||
|
|
||||||
def checkIfIP(host):
|
|
||||||
ipMask = '^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
|
|
||||||
if re.search(ipMask, host) is not None:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
# GLOBAL VALUES
|
|
||||||
numCiscoFeatures = 30
|
|
||||||
featureTypeDict = {'neural':np.arange(4,104,1),\
|
|
||||||
'packet':np.array([0,1,2,3]),\
|
|
||||||
'neural+packet':np.arange(0,104,1),\
|
|
||||||
'neural+packet+cisco':np.arange(0,104+(2*numCiscoFeatures),1),\
|
|
||||||
'cisco':np.arange(104,104+(2*numCiscoFeatures),1)}
|
|
||||||
|
|
||||||
globalNumFeatures = len(featureTypeDict['neural+packet+cisco'])
|
|
||||||
@ -1,215 +0,0 @@
|
|||||||
# -*- coding: utf-8 -*-
|
|
||||||
import csv
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
from keras.layers import Dense, Activation, Embedding, Dropout, Conv1D, GlobalMaxPooling1D, Lambda
|
|
||||||
from keras.layers import Input
|
|
||||||
from keras.models import Model
|
|
||||||
from keras.models import Sequential
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
|
|
||||||
def getCiscoFeatures(curDataLine, urlSIPDict):
|
|
||||||
numCiscoFeatures = 30
|
|
||||||
try:
|
|
||||||
ciscoFeatures = urlSIPDict[str(curDataLine['domain']) + str(curDataLine['server_ip'])]
|
|
||||||
# print('cisco features: ' + str(ciscoFeatures))
|
|
||||||
# log transform
|
|
||||||
ciscoFeatures = np.log1p(ciscoFeatures, dtype='float32')
|
|
||||||
# print('log transformed: ' + str(ciscoFeatures))
|
|
||||||
return ciscoFeatures.ravel()
|
|
||||||
except:
|
|
||||||
return np.zeros([numCiscoFeatures, ]).ravel()
|
|
||||||
|
|
||||||
|
|
||||||
def getCNNWithoutLastLayer(vocabSize, embeddingSize, input_length, filters, kernel_size,
|
|
||||||
hidden_dims, drop_out):
|
|
||||||
model = Sequential()
|
|
||||||
model.add(Embedding(input_dim=vocabSize, output_dim=embeddingSize,
|
|
||||||
input_length=input_length))
|
|
||||||
|
|
||||||
model.add(Conv1D(filters,
|
|
||||||
kernel_size,
|
|
||||||
activation='relu'))
|
|
||||||
|
|
||||||
# we use max pooling:
|
|
||||||
model.add(GlobalMaxPooling1D())
|
|
||||||
|
|
||||||
# We add a vanilla hidden layer:
|
|
||||||
model.add(Dense(hidden_dims))
|
|
||||||
model.add(Dropout(drop_out))
|
|
||||||
model.add(Activation('relu'))
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
def getCNNWitoutLastLayerFunctional(vocabSize, embeddingSize, input_length, filters, kernel_size,
|
|
||||||
hidden_dims, drop_out):
|
|
||||||
a = Input(shape=(input_length,))
|
|
||||||
embedding = Embedding(input_dim=vocabSize, output_dim=embeddingSize)(a)
|
|
||||||
conv1 = Conv1D(filters, kernel_size, activation='relu')(embedding)
|
|
||||||
glob = GlobalMaxPooling1D()(conv1)
|
|
||||||
dense = Dense(hidden_dims)(glob)
|
|
||||||
drop = Dropout(drop_out)(dense)
|
|
||||||
model = Activation('relu')(drop)
|
|
||||||
model = Model(a, model)
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
def getFlowFeatureLayer(numFeatures):
|
|
||||||
model = Sequential()
|
|
||||||
# slpModel.add(Dense(1, input_shape=(1,)))
|
|
||||||
model.add(Lambda(lambda x: x + 0.0, input_shape=(numFeatures,)))
|
|
||||||
return model
|
|
||||||
|
|
||||||
|
|
||||||
def createCNNDataSet(domains, label, characterDict, maxLen=40):
|
|
||||||
# process domains in reverse order
|
|
||||||
outFeature = np.zeros([len(domains), maxLen])
|
|
||||||
outLabel = np.zeros([len(domains), ])
|
|
||||||
for i in range(len(domains)):
|
|
||||||
domain = domains[i]
|
|
||||||
curLabel = label[i]
|
|
||||||
curFeature = np.zeros([maxLen, ])
|
|
||||||
# print(domain + ' ' + str(len(domain)))
|
|
||||||
for j in range(np.min([len(domain), maxLen])):
|
|
||||||
# print(j)
|
|
||||||
curCharacter = domain[-j]
|
|
||||||
if curCharacter in characterDict:
|
|
||||||
curFeature[j] = characterDict[curCharacter]
|
|
||||||
outFeature[i] = curFeature
|
|
||||||
outLabel[i] = curLabel
|
|
||||||
return (outFeature, outLabel)
|
|
||||||
|
|
||||||
|
|
||||||
def getFeatureVecForDomain(domain, characterDict, maxLen=40):
|
|
||||||
curFeature = np.zeros([maxLen, ])
|
|
||||||
for j in range(np.min([len(domain), maxLen])):
|
|
||||||
# print(j)
|
|
||||||
curCharacter = domain[-j]
|
|
||||||
if curCharacter in characterDict:
|
|
||||||
curFeature[j] = characterDict[curCharacter]
|
|
||||||
return curFeature
|
|
||||||
|
|
||||||
|
|
||||||
def getFlowFeatures(curDataLine):
|
|
||||||
useKeys = ['duration', 'bytes_down', 'bytes_up']
|
|
||||||
curFeature = np.zeros([len(useKeys), ])
|
|
||||||
for i in range(len(useKeys)):
|
|
||||||
curKey = useKeys[i]
|
|
||||||
try:
|
|
||||||
curFeature[i] = np.log1p(curDataLine[curKey], dtype='float32')
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
return curFeature
|
|
||||||
|
|
||||||
|
|
||||||
def getChunksFromUserDataFrame(dataFrame, windowSize=10, overlapping=False,
|
|
||||||
maxLengthInSeconds=300):
|
|
||||||
# print('maxLength: ' + str(maxLengthInSeconds))
|
|
||||||
maxMilliSeconds = maxLengthInSeconds * 1000
|
|
||||||
outDomainLists = []
|
|
||||||
outDFFrames = []
|
|
||||||
if overlapping == False:
|
|
||||||
numBlocks = int(np.ceil(float(len(dataFrame)) / float(windowSize)))
|
|
||||||
userIDs = np.arange(len(dataFrame))
|
|
||||||
for blockID in np.arange(numBlocks):
|
|
||||||
curIDs = userIDs[(blockID * windowSize):((blockID + 1) * windowSize)]
|
|
||||||
# print(curIDs)
|
|
||||||
useData = dataFrame.iloc[curIDs]
|
|
||||||
curDomains = useData['domain']
|
|
||||||
if maxLengthInSeconds != -1:
|
|
||||||
curMinMilliSeconds = np.min(useData['timeStamp']) + maxMilliSeconds
|
|
||||||
underTimeOutIDs = np.where(np.array(useData['timeStamp']) <= curMinMilliSeconds)
|
|
||||||
if len(underTimeOutIDs) != len(curIDs):
|
|
||||||
curIDs = curIDs[underTimeOutIDs]
|
|
||||||
useData = dataFrame.iloc[curIDs]
|
|
||||||
curDomains = useData['domain']
|
|
||||||
outDomainLists.append(list(curDomains))
|
|
||||||
outDFFrames.append(useData)
|
|
||||||
else:
|
|
||||||
numBlocks = len(dataFrame) + 1 - windowSize
|
|
||||||
userIDs = np.arange(len(dataFrame))
|
|
||||||
for blockID in np.arange(numBlocks):
|
|
||||||
curIDs = userIDs[blockID:blockID + windowSize]
|
|
||||||
# print(curIDs)
|
|
||||||
useData = dataFrame.iloc[curIDs]
|
|
||||||
curDomains = useData['domain']
|
|
||||||
if maxLengthInSeconds != -1:
|
|
||||||
curMinMilliSeconds = np.min(useData['timeStamp']) + maxMilliSeconds
|
|
||||||
underTimeOutIDs = np.where(np.array(useData['timeStamp']) <= curMinMilliSeconds)
|
|
||||||
if len(underTimeOutIDs) != len(curIDs):
|
|
||||||
curIDs = curIDs[underTimeOutIDs]
|
|
||||||
useData = dataFrame.iloc[curIDs]
|
|
||||||
curDomains = useData['domain']
|
|
||||||
outDomainLists.append(list(curDomains))
|
|
||||||
outDFFrames.append(useData)
|
|
||||||
return (outDomainLists, outDFFrames)
|
|
||||||
|
|
||||||
|
|
||||||
def createTrainData(domainLists, dfLists, charachterDict, maxLen, threshold=3,
|
|
||||||
flagUseCiscoFeatures=False, urlSIPDIct=dict,
|
|
||||||
windowSize=10):
|
|
||||||
if 'hits' in dfLists[0].keys():
|
|
||||||
hitName = 'hits'
|
|
||||||
elif 'virusTotalHits' in dfLists[0].keys():
|
|
||||||
hitName = 'virusTotalHits'
|
|
||||||
numFlowFeatures = 3
|
|
||||||
numCiscoFeatures = 30
|
|
||||||
numFeatures = numFlowFeatures
|
|
||||||
if flagUseCiscoFeatures:
|
|
||||||
numFeatures += numCiscoFeatures
|
|
||||||
outputFeatures = []
|
|
||||||
label = []
|
|
||||||
hits = []
|
|
||||||
trainNames = []
|
|
||||||
for i in range(windowSize):
|
|
||||||
outputFeatures.append(np.zeros([len(domainLists), maxLen]))
|
|
||||||
outputFeatures.append(np.zeros([len(domainLists), numFeatures]))
|
|
||||||
|
|
||||||
for i in tqdm(np.arange(len(domainLists)), miniters=10):
|
|
||||||
curCounter = 0
|
|
||||||
# print('len domainList: ' + str(len(domainLists[i])))
|
|
||||||
# print('len df: ' + str(len(dfLists[i])))
|
|
||||||
for j in range(np.min([windowSize, len(domainLists[i])])):
|
|
||||||
outputFeatures[curCounter][i, :] = getFeatureVecForDomain(domainLists[i][j], charachterDict, maxLen)
|
|
||||||
curCounter += 1
|
|
||||||
if flagUseCiscoFeatures:
|
|
||||||
outputFeatures[curCounter][i, 0:numFlowFeatures] = getFlowFeatures(dfLists[i].iloc[j])
|
|
||||||
outputFeatures[curCounter][i, numFlowFeatures:] = getCiscoFeatures(dfLists[i].iloc[j], urlSIPDIct)
|
|
||||||
else:
|
|
||||||
outputFeatures[curCounter][i, :] = getFlowFeatures(dfLists[i].iloc[j])
|
|
||||||
curCounter += 1
|
|
||||||
curLabel = 0.0
|
|
||||||
if np.max(dfLists[i][hitName]) >= threshold:
|
|
||||||
curLabel = 1.0
|
|
||||||
elif np.max(dfLists[i][hitName]) == -1:
|
|
||||||
curLabel = -1.0
|
|
||||||
elif np.max(dfLists[i][hitName]) > 0 and np.max(dfLists[i][hitName]) < threshold:
|
|
||||||
curLabel = -2.0
|
|
||||||
label.append(curLabel)
|
|
||||||
hits.append(np.max(dfLists[i][hitName]))
|
|
||||||
trainNames.append(np.unique(dfLists[i]['user_hash']))
|
|
||||||
return (outputFeatures, np.array(label), np.array(hits), np.array(trainNames))
|
|
||||||
|
|
||||||
|
|
||||||
def transformStringListToNumpyArray(listString):
|
|
||||||
listString = listString.replace('[', '').replace(']', '')
|
|
||||||
return np.array(listString.split(','), dtype='float32')
|
|
||||||
|
|
||||||
|
|
||||||
def getCiscoFeatureDict(csvPathList):
|
|
||||||
outDict = dict()
|
|
||||||
for path in tqdm(csvPathList, miniters=1):
|
|
||||||
fobj = open(path, 'r')
|
|
||||||
csvReader = csv.DictReader(fobj, delimiter=',')
|
|
||||||
for row in csvReader:
|
|
||||||
urlSIPString = row['Domain'] + row['ServerIP']
|
|
||||||
ciscoFeatures = row['CiscoFeature']
|
|
||||||
outDict[urlSIPString] = transformStringListToNumpyArray(ciscoFeatures)
|
|
||||||
# if len(outDict) % 10000 == 0:
|
|
||||||
# print('numbers in dict: ' + str(len(outDict)))
|
|
||||||
return outDict
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
pass
|
|
||||||
Loading…
Reference in New Issue
Block a user