diff --git a/example/hello_world.py b/example/hello_world.py index fb6733a8..f975491a 100755 --- a/example/hello_world.py +++ b/example/hello_world.py @@ -2,7 +2,7 @@ import houston import bugzoo import json -from houston.generator.rand import RandomMissionGenerator +from houston.generator import * from houston.generator.resources import ResourceLimits from houston.mission import Mission from houston.runner import MissionRunnerPool @@ -94,6 +94,23 @@ def generate_and_run(sut, initial, environment, number_of_missions): print("DONE") +### Generate and run missions with mutation operator +def generate_and_run_mutation(sut, initial_state, environment, initial_mission, number_of_missions): + mission_generator = MutationBasedMissionGenerator(sut, initial, environment, initial_mission, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) + resource_limits = ResourceLimits(number_of_missions*5, 1000, number_of_missions) + mission_generator.generate_and_run(100, resource_limits, with_coverage=True) + print("DONE") + with open("example/missions-mutation.json", "w") as f: + mission_descriptions = list(map(Mission.to_json, mission_generator.history)) + print(str(mission_descriptions)) + json.dump(mission_descriptions, f) + f.write("\n") + mission_descriptions = list(map(Mission.to_json, mission_generator.most_fit_missions)) + print(str(mission_descriptions)) + json.dump(mission_descriptions, f) + + + ### Generate and run missions with fault localization def generate_and_run_with_fl(sut, initial, environment, number_of_missions): mission_generator = RandomMissionGenerator(sut, initial, environment, max_num_actions=3, action_generators=[CircleBasedGotoGenerator((-35.3632607, 149.1652351), 2.0)]) @@ -108,19 +125,19 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions): if __name__=="__main__": - #sut = houston.ardu.ArduRover('afrl:overflow') - sut = houston.ardu.ArduCopter('afrl:overflow') + sut = houston.ardu.ArduRover('afrl:overflow') + #sut = houston.ardu.ArduCopter('afrl:overflow') # mission description actions = [ houston.action.Action("arm", {'arm': True}), - houston.action.Action("takeoff", {'altitude': 3.0}), + #houston.action.Action("takeoff", {'altitude': 3.0}), houston.action.Action("goto", { 'latitude' : -35.361354, 'longitude': 149.165218, 'altitude' : 5.0 }), - houston.action.Action("setmode", {'mode': 'LAND'}), + #houston.action.Action("setmode", {'mode': 'LAND'}), houston.action.Action("arm", {'arm': False}) ] environment = houston.state.Environment({}) @@ -141,7 +158,8 @@ def generate_and_run_with_fl(sut, initial, environment, number_of_missions): #run_single_mission_with_coverage(sandbox, mission) #generate(sut, initial, environment, 100, 10) - run_all_missions(sut, "example/missions.json", False) + #run_all_missions(sut, "example/missions.json", False) + generate_and_run_mutation(sut, initial, environment, mission, 3) #generate_and_run_with_fl(sut, initial, environment, 5) #run_single_mission_with_coverage(sandbox, mission) diff --git a/houston/generator/__init__.py b/houston/generator/__init__.py index e69de29b..1e828ddc 100644 --- a/houston/generator/__init__.py +++ b/houston/generator/__init__.py @@ -0,0 +1,2 @@ +from houston.generator.rand import RandomMissionGenerator +from houston.generator.mutation import MutationBasedMissionGenerator diff --git a/houston/generator/base.py b/houston/generator/base.py index 856f278f..825ed524 100644 --- a/houston/generator/base.py +++ b/houston/generator/base.py @@ -244,8 +244,7 @@ def generate_and_run(self, seed, resource_limits, with_coverage=False): self.threads, stream, self.record_outcome, - with_coverage - ) + with_coverage) self.__resource_usage = ResourceUsage() self.__start_time = timeit.default_timer() self.tick() diff --git a/houston/generator/mutation.py b/houston/generator/mutation.py new file mode 100644 index 00000000..cdd137e7 --- /dev/null +++ b/houston/generator/mutation.py @@ -0,0 +1,179 @@ +from houston.generator.base import MissionGenerator +from houston.mission import Mission + + +class MutationBasedMissionGenerator(MissionGenerator): + def __init__(self, + system, + initial_state, + env, + initial_mission, + threads = 1, + action_generators = [], + max_num_actions = 10): + super(MutationBasedMissionGenerator, self).__init__(system, threads, action_generators, max_num_actions) + self.__initial_state = initial_state + self.__env = env + self.__initial_mission = initial_mission + self.__in_progress_missions = {} + self.__most_fit_missions = [] #TODO this can be a dictionary instead of a list + + + @property + def initial_state(self): + """ + The initial state used by all missions produced by this generator. + """ + return self.__initial_state + + + @property + def initial_mission(self): + """ + Returns the initial mission to start the mutation from. + """ + return self.__initial_mission + + + @property + def env(self): + """ + The environment used by all missions produced by this generator. + """ + return self.__env + + + @property + def most_fit_missions(self): + """ + Return the most fit missions generated. + """ + return self.__most_fit_missions + + + def _generate_action(self, schema): + generator = self.action_generator(schema) + if generator is None: + return schema.generate(self.rng) + return generator.generate_action_without_state(self.system, self.__env, self.rng) + + + def _generate_random_mission(self): + schemas = list(self.system.schemas.values()) + actions = [] + for _ in range(self.rng.randint(1, self.max_num_actions)): + schema = self.rng.choice(schemas) + actions.append(self._generate_action(schema)) + return Mission(self.__env, self.__initial_state, actions) + + + def _get_fitness(self, mission): + """ + Returns a float number as the fitness. Higher is better. + """ + #TODO Come up with a reasonable fitness metric + + outcome = self.outcomes[mission] + coverage = self.coverage[mission] + initial_coverage = self.coverage[self.__initial_mission] + + fitness = 1.0 + + if not outcome.passed: + fitness *= 10.0 + + similar_coverage = coverage.intersection(initial_coverage) + fitness += (len(similar_coverage)/len(coverage))*15 + + fitness -= len(mission.actions)*3 + + return fitness + + + def _add_action_operator(self, mission): + actions = mission.actions + if len(actions) >= self.max_num_actions: + return None + schema = self.rng.choice(list(self.system.schemas.values())) + actions.insert(self.rng.randint(0, len(actions)-1), self._generate_action(schema)) + return Mission(self.__env, self.__initial_state, actions) + + + def _delete_action_operator(self, mission): + actions = mission.actions + if len(actions) <= 1: + return None + actions.pop(self.rng.randint(0, len(actions)-1)) + return Mission(self.__env, self.__initial_state, actions) + + + def _edit_action_operator(self, mission): + actions = mission.actions + index = self.rng.randint(0, len(actions)-1) + new_action = self._generate_action(self.system.schemas[actions[index].schema_name]) + if new_action.values == actions[index].values: + return None + actions[index] = new_action + + return Mission(self.__env, self.__initial_state, actions) + + + def _mutate_mission(self, mission): + mutation_operators = [self._add_action_operator, self._delete_action_operator, + self._edit_action_operator] + generated_mission = None + while not generated_mission: + operator = self.rng.choice(mutation_operators) + print("Operator: {}".format(str(operator))) + generated_mission = operator(mission) + return generated_mission + + + def generate_mission(self): + if not self.__most_fit_missions: + self.__in_progress_missions[self.__initial_mission] = {'parent': None} + return self.__initial_mission + + parent = self.rng.choice(self.__most_fit_missions) + mission = self._mutate_mission(parent) + self.__in_progress_missions[mission] = {'parent': parent} + print("Mutated: {}\nfrom: {}".format(mission.to_json(), parent.to_json())) + return mission + + + def record_outcome(self, mission, outcome, coverage): + """ + Records the outcome of a given mission. The mission is logged to the + history, and its outcome is stored in the outcome dictionary. If the + mission failed, the mission is also added to the set of failed + missions. + """ + self.history.append(mission) + self.outcomes[mission] = outcome + self.coverage[mission] = coverage + + + if outcome.failed: + self.failures.add(mission) + + if not mission in self.__in_progress_missions: + print("Something went wrong! mission is not in progress") + + fitness = self._get_fitness(mission) + parent = self.__in_progress_missions[mission]['parent'] + if not parent: + # initial mission + self.most_fit_missions.append(mission) + self.__in_progress_missions.pop(mission) + return + + parent_fitness = self._get_fitness(parent) + print("My fitness: {}, parent fitness: {}".format(fitness, parent_fitness)) + if fitness >= parent_fitness or self.rng.random() <= 0.05: + if len(self.most_fit_missions) >= self.resource_limits.num_missions_selected: + self.most_fit_missions.remove(parent) + self.most_fit_missions.append(mission) + self.__in_progress_missions.pop(mission) + + + diff --git a/houston/generator/resources.py b/houston/generator/resources.py index 547b6793..01b2c413 100644 --- a/houston/generator/resources.py +++ b/houston/generator/resources.py @@ -28,12 +28,14 @@ class ResourceLimits(object): @staticmethod def from_json(jsn): return ResourceLimits(num_missions = jsn['num_missions'], - running_time = jsn['running_time']) + running_time = jsn['running_time'], + num_missions_selected = jsn['num_missions_selected']) - def __init__(self, num_missions = None, running_time = None): + def __init__(self, num_missions = None, running_time = None, num_missions_selected = None): self.__num_missions = num_missions self.__running_time = running_time + self.__num_missions_selected = num_missions_selected def reached(self, usage): @@ -48,6 +50,14 @@ def reached(self, usage): return False + @property + def num_missions_selected(self): + """ + Number of missions that should be selected from all generated missions. + """ + return self.__num_missions_selected + + @property def num_missions(self): """ @@ -89,5 +99,6 @@ def to_json(self): """ return { 'num_missions': self.__num_missions, - 'running_time': self.__running_time + 'running_time': self.__running_time, + 'num_missions_selected': self.__num_missions_selected }