diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b70d71 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +venv/ +.idea/ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ed0a8ae..d6186fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ -matplotlib==3.2.0 -tensorflow_datasets==3.2.1 -tensorflow==2.1.0 -opencv_python==4.5.1.48 -numpy==1.18.2 -tensorflow_examples==e6e7388f90c6450edc9eda7bf9d293a79cbef6cc_ -pandas==1.2.3 -PyYAML==5.4.1 +matplotlib +tensorflow_datasets +tensorflow==2.2.0 +opencv_python +numpy +pandas +PyYAML diff --git a/src/config.py b/src/config.py index b5595c3..36f617a 100644 --- a/src/config.py +++ b/src/config.py @@ -10,7 +10,7 @@ def config(confFile): print("Make sure the correct path is passed to the inject call.") sys.exit() if(confFile.endswith(".yaml")): - fiConf = yaml.load(fiConfs) + fiConf = yaml.safe_load(fiConfs) else: print("Unsupported file format:", confFile) sys.exit() diff --git a/src/graph_node.py b/src/graph_node.py new file mode 100644 index 0000000..a06c595 --- /dev/null +++ b/src/graph_node.py @@ -0,0 +1,17 @@ +class GraphNode: + def __init__(self, name): + self.name = name + self.input_layers = [] + self.output_layers = [] + + def add_input_layer(self, layer): + self.input_layers.append(layer) + + def add_output_layer(self, layer): + self.output_layers.append(layer) + + def get_input_layers(self): + return self.input_layers + + def get_output_layers(self): + return self.output_layers diff --git a/src/tensorfi2.py b/src/tensorfi2.py index d6508a5..cbdc624 100644 --- a/src/tensorfi2.py +++ b/src/tensorfi2.py @@ -1,15 +1,17 @@ #!/usr/bin/python -import os, logging - -import tensorflow as tf +import logging +import math +import random from struct import pack, unpack import numpy as np -from tensorflow.keras import Model, layers +import tensorflow as tf from tensorflow.keras import backend as K -import random, math + from src import config +from src.utility import compute_fault_injected_prediction, get_fault_injection_configs + def bitflip(f, pos): @@ -23,11 +25,11 @@ def bitflip(f, pos): f = unpack('f', f_) return f[0] + class inject(): def __init__( self, model, confFile, log_level="ERROR", **kwargs ): - # Logging setup logging.basicConfig() logging.getLogger().setLevel(log_level) @@ -36,10 +38,14 @@ def __init__( # Retrieve config params fiConf = config.config(confFile) self.Model = model # No more passing or using a session variable in TF v2 + """Model graph and super nodes are model related. So, if multiple fault injection on same model is required, + then we can compute them priorly and pass them as parameter to this function to make the process faster """ + self.model_graph, self.super_nodes = get_fault_injection_configs(model) # Call the corresponding FI function fiFunc = getattr(self, fiConf["Target"]) - fiFunc(model, fiConf, **kwargs) + self.final_label = fiFunc(model, fiConf, **kwargs) + def layer_states(self, model, fiConf, **kwargs): @@ -180,7 +186,9 @@ def layer_outputs(self, model, fiConf, **kwargs): x_test = kwargs["x_test"] # Choose a random layer for injection - randnum = random.randint(0, len(model.layers) - 2) + randnum = random.randint(0, len(model.layers) - 3) + 1 + + # injection_layer_index = 30 fiLayer = model.layers[randnum] @@ -189,45 +197,50 @@ def layer_outputs(self, model, fiConf, **kwargs): fiLayerOutputs = get_output([x_test]) # Unstack elements into a single dimension - elem_shape = fiLayerOutputs[0].shape - fiLayerOutputs[0] = fiLayerOutputs[0].flatten() - num = fiLayerOutputs[0].shape[0] - - if(fiFault == "zeros"): - fiSz = (fiSz * num) / 100 - fiSz = math.floor(fiSz) - - # Choose the indices for FI - ind = random.sample(range(num), fiSz) - - # Inject the specified fault into the randomly chosen values - if(fiFault == "zeros"): - for item in ind: - fiLayerOutputs[0][item] = 0. - elif(fiFault == "random"): - for item in ind: - fiLayerOutputs[0][item] = np.random.random() - elif(fiFault == "bitflips"): - for item in ind: - val = fiLayerOutputs[0][item] - if(fiConf["Bit"] == "N"): - pos = random.randint(0, 31) - else: - pos = int(fiConf["Bit"]) - val_ = bitflip(val, pos) - fiLayerOutputs[0][item] = val_ + original_output_list = fiLayerOutputs[0] + batch_size = len(original_output_list) + faulty_output_list = None + for i in range(batch_size): + target_image = original_output_list[i] + elem_shape = target_image.shape + target_image = target_image.flatten() + num = target_image.shape[0] + if (fiFault == "zeros"): + fiSz = (fiSz * num) / 100 + fiSz = math.floor(fiSz) - # Reshape into original dimensions and get the final prediction - fiLayerOutputs[0] = fiLayerOutputs[0].reshape(elem_shape) - get_pred = K.function([model.layers[randnum + 1].input], [model.layers[-1].output]) - pred = get_pred([fiLayerOutputs]) + # Choose the indices for FI + ind = random.sample(range(num), fiSz) - # Uncomment below line and comment next two lines for ImageNet models - # return pred + # Inject the specified fault into the randomly chosen values + if (fiFault == "zeros"): + for item in ind: + target_image[item] = 0. + elif (fiFault == "random"): + for item in ind: + target_image[item] = np.random.random() + elif (fiFault == "bitflips"): + for item in ind: + val = target_image[item] + if (fiConf["Bit"] == "N"): + pos = random.randint(0, 31) + else: + pos = int(fiConf["Bit"]) + val_ = bitflip(val, pos) + target_image[item] = val_ + + # Reshape into original dimensions and get the final prediction + target_image = target_image.reshape(elem_shape) + target_image = np.expand_dims(target_image, axis=0) + if faulty_output_list is None: + faulty_output_list = target_image + else: + faulty_output_list = np.concatenate((faulty_output_list, target_image), axis=0) + + fiLayerOutputs[0] = faulty_output_list + pred = compute_fault_injected_prediction(self.model_graph, self.super_nodes, model.layers, randnum, fiLayerOutputs, x_test) labels = np.argmax(pred, axis=-1) return labels[0] - - logging.info("Completed injections... exiting") elif(fiConf["Mode"] == "multiple"): diff --git a/src/utility.py b/src/utility.py new file mode 100644 index 0000000..fd95433 --- /dev/null +++ b/src/utility.py @@ -0,0 +1,241 @@ +from tensorflow.keras import backend as K + +from src.graph_node import GraphNode + + +def compute_layer_output(layer_dependency_graph, model_layers, original_input, layer_output_dict, layer): + """ + This function computes the layer output. It searches for the longest consecutive model portion + to get the prediction. It recursively computes input if any layer has multiple inputs. It then stores + the layer output in the output dictionary. + + :param layer_dependency_graph: layer dependency graph of model + :param model_layers: list of layers of the model + :param original_input: original input tensor used for prediction + :param layer_output_dict: a dictionary which stores outputs of different layer for memorization + :param layer: the layer whose output needs to be calculated + :return: + """ + current_layer = layer + input_tensor_list = [] + while True: + inputs_of_current_layer = layer_dependency_graph[current_layer].get_input_layers() + if len(inputs_of_current_layer) == 1: + if inputs_of_current_layer[0] in layer_output_dict: + input_tensor_list.append(layer_output_dict[inputs_of_current_layer[0]]) + start_layer = current_layer + break + else: + current_layer = inputs_of_current_layer[0] + else: + for node in inputs_of_current_layer: + if node not in layer_output_dict: + compute_layer_output(layer_dependency_graph, model_layers, original_input, layer_output_dict, node) + input_tensor_list.append(layer_output_dict[node]) + start_layer = current_layer + break + get_pred = K.function([model_layers[start_layer].input], [model_layers[layer].output]) + layer_output_dict[layer] = get_pred(input_tensor_list) + + +def compute_fault_injected_prediction(layer_dependency_graph, super_nodes, model_layers, + injection_layer_index, injected_output, original_input): + """ + Calculate the model's final prediction using the fault-injected layer outputs and original input data. + It stores the layer output results in a dictionary to eliminate recomputation. It initially computes + the prediction of the immediate previous super layer and stores it in the dictionary because all the + branchingstarts from the previous super layer. Then it calculates the inputs of the next super layer + recursively with memorization to get the final prediction. + + + :param layer_dependency_graph: layer dependency graph of model + :param super_nodes: list of all super nodes + :param model_layers: list of layers of the model + :param injection_layer_index: the index where fault is injected + :param injected_output: fault injected output + :param original_input: original input tensor used for prediction + :return: faulty model predicted output + """ + layer_output_dict = {injection_layer_index: injected_output} + if injection_layer_index not in super_nodes: + previous_super_layer = get_previous_super_layer(layer_dependency_graph, super_nodes, injection_layer_index) + pred_function = K.function([model_layers[0].input], [model_layers[previous_super_layer].output]) + layer_output_dict[previous_super_layer] = pred_function(original_input) + + next_super_layer = get_next_super_layer(layer_dependency_graph, super_nodes, injection_layer_index) + next_super_layer_inputs = layer_dependency_graph[next_super_layer].get_input_layers() + input_tensor_list = [] + for layer in next_super_layer_inputs: + if layer not in layer_output_dict: + compute_layer_output(layer_dependency_graph, model_layers, original_input, layer_output_dict, layer) + input_tensor_list.append(layer_output_dict[layer]) + get_pred = K.function([model_layers[next_super_layer].input], [model_layers[-1].output]) + return get_pred(input_tensor_list) + + +def build_dependency_graph(model_layers, layer_name_to_index): + """ + This function builds a dependency graph using the model layers. For each layer + it stores the unique name of the output of that layer, the layer indices on which + it depends and the layer indices which depend on this layer. + + :param model_layers: List of layers of the model + :param layer_name_to_index: a dictionary that contains mapping of all layer output names to their index + :return: a dependency graph + """ + dependency_graph = [] + index = 0 + for layer in model_layers: + graph_element = GraphNode(layer.output.name) + if index != 0: + if type(layer.input) is list: + for layer_name in layer.input: + layer_index = layer_name_to_index[layer_name.name] + graph_element.add_input_layer(layer_index) + dependency_graph[layer_index].add_output_layer(index) + else: + layer_index = layer_name_to_index[layer.input.name] + graph_element.add_input_layer(layer_index) + dependency_graph[layer_index].add_output_layer(index) + dependency_graph.append(graph_element) + index += 1 + return dependency_graph + + +def map_layer_output_name_with_index(model_layers): + """ + Each layer has a unique output name. This function maps this unique name with + the layer index so that we can easily access each layer with index instead of name. + + :param model_layers: List of layers of the model + :return: a dictionary that contains mapping of all layer names to their index + """ + output_name_to_index = {} + total_layer_count = len(model_layers) + for i in range(total_layer_count): + output_name_to_index[model_layers[i].output.name] = i + return output_name_to_index + + +def get_super_nodes(layer_dependency_graph, start_node, end_node, super_node_dict): + """ + Returns super nodes list from start_node position to end_node position of layer_dependency_graph. + + :param layer_dependency_graph: dependency graph of the model + :param start_node: starting layer index from where super node searching starts + :param end_node: ending layer index of super node searching + :param super_node_dict: a temporary dictionary which keeps track of already searched portion for super nodes + :return: + """ + super_nodes = [start_node] + while start_node < end_node: + graph_node = layer_dependency_graph[start_node] + if len(graph_node.get_output_layers()) == 1: + start_node = graph_node.get_output_layers()[0] + super_nodes.append(start_node) + else: + # Recursively collect super nodes of each branch and combine them to get the overall supernode list + partial_super_nodes_list = [] + for node in graph_node.get_output_layers(): + if node not in super_node_dict: + partial_super_nodes = get_super_nodes(layer_dependency_graph, node, end_node, super_node_dict) + super_node_dict[node] = partial_super_nodes + partial_super_nodes_list.append(super_node_dict[node]) + super_nodes.extend(get_common_elements(partial_super_nodes_list)) + break + return super_nodes + + +def get_common_elements(element_list): + """ + :param element_list: list of list where each internal list contains values + :return: a sorted list of elements which are common in all the internal lists + """ + common_element_list = set(element_list[0]) + index = 1 + while index < len(element_list): + common_element_list = common_element_list.intersection(element_list[index]) + index += 1 + return sorted(list(common_element_list)) + + +def get_next_super_layer(layer_dependency_graph, super_nodes, current_layer): + """ + Return the immediate next super layer of current layer. + + :param layer_dependency_graph: dependency graph of the model + :param super_nodes: list of all super nodes + :param current_layer: the layer whose next super layer need to compute + :return: immediate next super layer + """ + current_layer = layer_dependency_graph[current_layer].get_output_layers()[0] + while True: + if current_layer in super_nodes: + return current_layer + current_layer = layer_dependency_graph[current_layer].get_output_layers()[0] + + +def get_previous_super_layer(layer_dependency_graph, super_nodes, current_layer): + """ + Return the immediate previous super layer of current layer. + + :param layer_dependency_graph: dependency graph of the model + :param super_nodes: list of all super nodes + :param current_layer: the layer whose previous super layer need to compute + :return: immediate previous super layer + """ + current_layer = layer_dependency_graph[current_layer].get_input_layers()[0] + while True: + if current_layer in super_nodes: + return current_layer + current_layer = layer_dependency_graph[current_layer].get_input_layers()[0] + + +def draw_graph(layer_dependency_graph, super_nodes, name): + """ + This method is used to draw the directed dependency graph. In the graph, supernodes are filled with red color. It + is helpful to check whether the drawn dependency graph and supernodes are correct or not. It also eases the + debugging process. + + + :param layer_dependency_graph: graph generated from model indicating input and output layers. + :param super_nodes: The nodes which are central to the model + :param name: graph is being saved by this name + :return: saves a pdf of the dependency graph + """ + import pygraphviz as pgv + + graph = pgv.AGraph(directed=True) + graph.node_attr['style'] = 'filled' + graph.node_attr['shape'] = 'circle' + graph.node_attr['fixedsize'] = 'true' + graph.node_attr['fontcolor'] = '#000000' + layer_len = len(layer_dependency_graph) + + for i in range(layer_len - 1): + model_elem = layer_dependency_graph[i] + for node in model_elem.get_output_layers(): + graph.add_edge(i, node) + for node in super_nodes: + n = graph.get_node(node) + n.attr['fillcolor'] = "#FF0000" + graph.draw(name + '.pdf', prog="circo") + + +def get_fault_injection_configs(model, graph_drawing=False): + """ + For each model, we need to develop a dependency graph of inputs and outputs of all the + layers. We also compute the super nodes list. Dependency graph and super nodes list are + model specific, so we can compute them before starting fault injection. + + :param model: keras model on which faults will be injected + :param graph_drawing: determines whether a pdf version of dependency graph will be saved or not. Default=False + :return: dependency graph and super node list + """ + layer_name_to_index = map_layer_output_name_with_index(model.layers) + dependency_graph = build_dependency_graph(model.layers, layer_name_to_index) + super_nodes = get_super_nodes(dependency_graph, 0, len(model.layers) - 1, {}) + + if graph_drawing: + draw_graph(dependency_graph, super_nodes, model.name) + return dependency_graph, super_nodes diff --git a/test_non_linear_fi.py b/test_non_linear_fi.py new file mode 100644 index 0000000..554b27d --- /dev/null +++ b/test_non_linear_fi.py @@ -0,0 +1,106 @@ +import os +import sys + +import numpy as np +from tensorflow.keras.applications import vgg16, vgg19, resnet, xception, nasnet, mobilenet, mobilenet_v2, \ + inception_resnet_v2, inception_v3, densenet +from tensorflow.keras.preprocessing.image import load_img, img_to_array + +from src import tensorfi2 as tfi + +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' + + +def get_model_from_name(model_name): + if model_name == "ResNet50": + return resnet.ResNet50() + elif model_name == "ResNet101": + return resnet.ResNet101() + elif model_name == "ResNet152": + return resnet.ResNet152() + elif model_name == "VGG16": + return vgg16.VGG16() + elif model_name == "VGG19": + return vgg19.VGG19() + elif model_name == "Xception": + return xception.Xception() + elif model_name == "NASNetMobile": + return nasnet.NASNetMobile() + elif model_name == "NASNetLarge": + return nasnet.NASNetLarge() + elif model_name == "MobileNet": + return mobilenet.MobileNet() + elif model_name == "MobileNetV2": + return mobilenet_v2.MobileNetV2() + elif model_name == "InceptionResNetV2": + return inception_resnet_v2.InceptionResNetV2() + elif model_name == "InceptionV3": + return inception_v3.InceptionV3() + elif model_name == "DenseNet121": + return densenet.DenseNet121() + elif model_name == "DenseNet169": + return densenet.DenseNet169() + elif model_name == "DenseNet201": + return densenet.DenseNet201() + + +def get_preprocessed_input_by_model_name(model_name, x_val): + if model_name == "ResNet50" or model_name == "ResNet101" or model_name == "ResNet152": + return resnet.preprocess_input(x_val) + elif model_name == "VGG16": + return vgg16.preprocess_input(x_val) + elif model_name == "VGG19": + return vgg19.preprocess_input(x_val) + elif model_name == "Xception": + return xception.preprocess_input(x_val) + elif model_name == "NASNetMobile" or model_name == "NASNetLarge": + return nasnet.preprocess_input(x_val) + elif model_name == "MobileNet": + return mobilenet.preprocess_input(x_val) + elif model_name == "MobileNetV2": + return mobilenet_v2.preprocess_input(x_val) + elif model_name == "InceptionResNetV2": + return inception_resnet_v2.preprocess_input(x_val) + elif model_name == "InceptionV3": + return inception_v3.preprocess_input(x_val) + elif model_name == "DenseNet121" or model_name == "DenseNet169" or model_name == "DenseNet201": + return densenet.preprocess_input(x_val) + + +def main(): + model_name = sys.argv[1] + model = get_model_from_name(model_name) + conf_file = sys.argv[2] + total_injection = int(sys.argv[3]) + input_dim = int(sys.argv[4]) + + # Golder run + path = 'ILSVRC2012_val_00000001.JPEG' + image = load_img(path, target_size=(input_dim, input_dim)) + image = img_to_array(image) + image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2])) + image = get_preprocessed_input_by_model_name(model_name, image) + out = model.predict(image).argmax(axis=-1)[0] + print("Fault free prediction " + str(out)) + + # Inject fault to single image + print("Injecting faults to single image") + for i in range(total_injection): + res = tfi.inject(model=model, x_test=image, confFile=conf_file) + print(res.final_label) + + # Inject fault to batch images + print("Injecting faults to batch images") + for i in range(total_injection): + image = load_img(path, target_size=(input_dim, input_dim)) + image = img_to_array(image) + image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2])) + batch_image = np.concatenate((image, image), axis=0) + batch_image = get_preprocessed_input_by_model_name(model_name, batch_image) + res = tfi.inject(model=model, x_test=batch_image, confFile=conf_file) + print(res.final_label) + print("Fault injection done") + + +if __name__ == '__main__': + main()