Mutation based test generation

example/hello_world.py

@@ -2,7 +2,7 @@
 import houston
 import bugzoo
 import json
-from houston.generator.rand import RandomMissionGenerator
+from houston.generator import *
 from houston.generator.resources import ResourceLimits
 from houston.mission import Mission
 from houston.runner import MissionRunnerPool
@@ -94,6 +94,23 @@ def generate_and_run(sut, initial, environment, number_of_missions):
     print("DONE")
 
 
+### Generate and run missions with mutation operator
+def generate_and_run_mutation(sut, initial_state, environment, initial_mission, number_of_missions):
+    mission_generator = MutationBasedMissionGenerator(sut, initial, environment, initial_mission, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)])
+    resource_limits = ResourceLimits(number_of_missions*5, 1000, number_of_missions)
+    mission_generator.generate_and_run(100, resource_limits, with_coverage=True)
+    print("DONE")
+    with open("example/missions-mutation.json", "w") as f:
+        mission_descriptions = list(map(Mission.to_json, mission_generator.history))
+        print(str(mission_descriptions))
+        json.dump(mission_descriptions, f)
+        f.write("\n")
+        mission_descriptions = list(map(Mission.to_json, mission_generator.most_fit_missions))
+        print(str(mission_descriptions))
+        json.dump(mission_descriptions, f)
+
+
+
 ### Generate and run missions with fault localization
 def generate_and_run_with_fl(sut, initial, environment, number_of_missions):
     mission_generator = RandomMissionGenerator(sut, initial, environment, max_num_actions=3, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)])
@@ -108,19 +125,19 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions):
 
 
 if __name__=="__main__":
-    #sut = houston.ardu.ArduRover('afrl:overflow')
-    sut = houston.ardu.ArduCopter('afrl:overflow')
+    sut = houston.ardu.ArduRover('afrl:overflow')
+    #sut = houston.ardu.ArduCopter('afrl:overflow')
 
     # mission description
     actions = [
         houston.action.Action("arm", {'arm': True}),
-        houston.action.Action("takeoff", {'altitude': 3.0}),
+        #houston.action.Action("takeoff", {'altitude': 3.0}),
         houston.action.Action("goto", {
             'latitude' : -35.361354,
             'longitude': 149.165218,
             'altitude' : 5.0
         }),
-        houston.action.Action("setmode", {'mode': 'LAND'}),
+        #houston.action.Action("setmode", {'mode': 'LAND'}),
         houston.action.Action("arm", {'arm': False})
     ]
     environment = houston.state.Environment({})
@@ -141,7 +158,8 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions):
 
     #run_single_mission_with_coverage(sandbox, mission)
     #generate(sut, initial, environment, 100, 10)
-    run_all_missions(sut, "example/missions.json", False)
+    #run_all_missions(sut, "example/missions.json", False)
+    generate_and_run_mutation(sut, initial, environment, mission, 3)
     #generate_and_run_with_fl(sut, initial, environment, 5)
     #run_single_mission_with_coverage(sandbox, mission)
 

houston/generator/__init__.py

@@ -0,0 +1,2 @@
+from houston.generator.rand import RandomMissionGenerator
+from houston.generator.mutation import MutationBasedMissionGenerator

houston/generator/base.py

@@ -244,8 +244,7 @@ def generate_and_run(self, seed, resource_limits, with_coverage=False):
                                                    self.threads,
                                                    stream,
                                                    self.record_outcome,
-                                                   with_coverage
-                                                  )
+                                                   with_coverage)
             self.__resource_usage = ResourceUsage()
             self.__start_time = timeit.default_timer()
             self.tick()

houston/generator/mutation.py

@@ -0,0 +1,179 @@
+from houston.generator.base import MissionGenerator
+from houston.mission import Mission
+
+
+class MutationBasedMissionGenerator(MissionGenerator):
+    def __init__(self,
+                 system,
+                 initial_state,
+                 env,
+                 initial_mission,
+                 threads = 1,
+                 action_generators = [],
+                 max_num_actions = 10):
+        super(MutationBasedMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions)
+        self.__initial_state = initial_state
+        self.__env = env
+        self.__initial_mission = initial_mission
+        self.__in_progress_missions = {}
+        self.__most_fit_missions = [] #TODO this can be a dictionary instead of a list
+
+
+    @property
+    def initial_state(self):
+        """
+        The initial state used by all missions produced by this generator.
+        """
+        return self.__initial_state
+
+
+    @property
+    def initial_mission(self):
+        """
+        Returns the initial mission to start the mutation from.
+        """
+        return self.__initial_mission
+
+
+    @property
+    def env(self):
+        """
+        The environment used by all missions produced by this generator.
+        """
+        return self.__env
+
+
+    @property
+    def most_fit_missions(self):
+        """
+        Return the most fit missions generated.
+        """
+        return self.__most_fit_missions
+
+
+    def _generate_action(self, schema):
+        generator = self.action_generator(schema)
+        if generator is None:
+            return schema.generate(self.rng)
+        return generator.generate_action_without_state(self.system, self.__env, self.rng)
+
+
+    def _generate_random_mission(self):
+        schemas = list(self.system.schemas.values())
+        actions = []
+        for _ in range(self.rng.randint(1, self.max_num_actions)):
+            schema = self.rng.choice(schemas)
+            actions.append(self._generate_action(schema))
+        return Mission(self.__env, self.__initial_state, actions)
+
+
+    def _get_fitness(self, mission):
+        """
+        Returns a float number as the fitness. Higher is better.
+        """
+        #TODO Come up with a reasonable fitness metric
+
+        outcome = self.outcomes[mission]
+        coverage = self.coverage[mission]
+        initial_coverage = self.coverage[self.__initial_mission]
+
+        fitness = 1.0
+
+        if not outcome.passed:
+            fitness *= 10.0
+
+        similar_coverage = coverage.intersection(initial_coverage)
+        fitness += (len(similar_coverage)/len(coverage))*15
+
+        fitness -= len(mission.actions)*3
+
+        return fitness
+
+
+    def _add_action_operator(self, mission):
+        actions = mission.actions
+        if len(actions) >= self.max_num_actions:
+            return None
+        schema = self.rng.choice(list(self.system.schemas.values()))
+        actions.insert(self.rng.randint(0, len(actions)-1), self._generate_action(schema))
+        return Mission(self.__env, self.__initial_state, actions)
+
+
+    def _delete_action_operator(self, mission):
+        actions = mission.actions
+        if len(actions) <= 1:
+            return None
+        actions.pop(self.rng.randint(0, len(actions)-1))
+        return Mission(self.__env, self.__initial_state, actions)
+
+
+    def _edit_action_operator(self, mission):
+        actions = mission.actions
+        index = self.rng.randint(0, len(actions)-1)
+        new_action = self._generate_action(self.system.schemas[actions[index].schema_name])
+        if new_action.values == actions[index].values:
+            return None
+        actions[index] = new_action
+
+        return Mission(self.__env, self.__initial_state, actions)
+
+
+    def _mutate_mission(self, mission):
+        mutation_operators = [self._add_action_operator, self._delete_action_operator,
+                              self._edit_action_operator]
+        generated_mission = None
+        while not generated_mission:
+            operator = self.rng.choice(mutation_operators)
+            print("Operator: {}".format(str(operator)))
+            generated_mission = operator(mission)
+        return generated_mission
+
+
+    def generate_mission(self):
+        if not self.__most_fit_missions:
+            self.__in_progress_missions[self.__initial_mission] = {'parent': None}
+            return self.__initial_mission
+
+        parent = self.rng.choice(self.__most_fit_missions)
+        mission = self._mutate_mission(parent)
+        self.__in_progress_missions[mission] = {'parent': parent}
+        print("Mutated: {}\nfrom: {}".format(mission.to_json(), parent.to_json()))
+        return mission
+
+
+    def record_outcome(self, mission, outcome, coverage):
+        """
+        Records the outcome of a given mission. The mission is logged to the
+        history, and its outcome is stored in the outcome dictionary. If the
+        mission failed, the mission is also added to the set of failed
+        missions.
+        """
+        self.history.append(mission)
+        self.outcomes[mission] = outcome
+        self.coverage[mission] = coverage
+
+
+        if outcome.failed:
+            self.failures.add(mission)
+
+        if not mission in self.__in_progress_missions:
+            print("Something went wrong! mission is not in progress")
+
+        fitness = self._get_fitness(mission)
+        parent = self.__in_progress_missions[mission]['parent']
+        if not parent:
+            # initial mission
+            self.most_fit_missions.append(mission)
+            self.__in_progress_missions.pop(mission)
+            return
+
+        parent_fitness = self._get_fitness(parent)
+        print("My fitness: {}, parent fitness: {}".format(fitness, parent_fitness))
+        if fitness >= parent_fitness or self.rng.random() <= 0.05:
+            if len(self.most_fit_missions) >= self.resource_limits.num_missions_selected:
+                self.most_fit_missions.remove(parent)
+            self.most_fit_missions.append(mission)
+        self.__in_progress_missions.pop(mission)
+
+
+

houston/generator/resources.py

@@ -28,12 +28,14 @@ class ResourceLimits(object):
     @staticmethod
     def from_json(jsn):
         return ResourceLimits(num_missions = jsn['num_missions'],
-                              running_time = jsn['running_time'])
+                              running_time = jsn['running_time'],
+                              num_missions_selected = jsn['num_missions_selected'])
 
 
-    def __init__(self, num_missions = None, running_time = None):
+    def __init__(self, num_missions = None, running_time = None, num_missions_selected = None):
         self.__num_missions = num_missions
         self.__running_time = running_time
+        self.__num_missions_selected = num_missions_selected
 
 
     def reached(self, usage):
@@ -48,6 +50,14 @@ def reached(self, usage):
         return False
 
 
+    @property
+    def num_missions_selected(self):
+        """
+        Number of missions that should be selected from all generated missions.
+        """
+        return self.__num_missions_selected
+
+
     @property
     def num_missions(self):
         """
@@ -89,5 +99,6 @@ def to_json(self):
         """
         return {
             'num_missions': self.__num_missions,
-            'running_time': self.__running_time
+            'running_time': self.__running_time,
+            'num_missions_selected': self.__num_missions_selected
         }