From 20b87d3ab996bae9afd8b3308965621e124c5218 Mon Sep 17 00:00:00 2001 From: Ravi Jagannadhan Date: Fri, 16 Mar 2018 12:29:24 -0700 Subject: [PATCH 1/4] adding profiler hooks and collection to uploader --- conductor/lib/uploader.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/conductor/lib/uploader.py b/conductor/lib/uploader.py index cc9b1044..905e867d 100644 --- a/conductor/lib/uploader.py +++ b/conductor/lib/uploader.py @@ -5,10 +5,14 @@ import os import Queue import sys +import tempfile import thread from threading import Thread +import time import traceback +import yappi # yappi is a profiler specializing in multi-threading + from conductor import CONFIG from conductor.lib import api_client, common, worker, client_db, loggeria @@ -587,6 +591,27 @@ def handle_upload_response(self, project, upload_files, upload_id=None, md5_only def main(self, run_one_loop=False): logger.info('Uploader Started. Checking for uploads...') + # for profiling, grab the environment variable's value + # to determine where to put the profiling data + + if "CONDUCTOR_PROFILE_DATA_PATH" not in os.environ: + profile_path = tempfile.gettempdir() + else: + profile_path = os.environ["CONDUCTOR_PROFILE_DATA_PATH"] + + if not profile_path: + profile_path = tempfile.gettempdir() + + timestr = time.strftime("%Y%m%d-%H%M%S") + func_filename = '{}_func_profile.dmp'.format(timestr) + thread_filename = '{}_thread_profile.txt'.format(timestr) + + self.profile_func_filename = os.path.join(profile_path, func_filename) + self.profile_thread_filename = os.path.join(profile_path, thread_filename) + + # start profiling + yappi.start() + while not common.SIGINT_EXIT: try: # TODO: we should pass args as url params, not http data @@ -629,6 +654,20 @@ def main(self, run_one_loop=False): except KeyboardInterrupt: logger.info("ctrl-c exit") + + # stop profiling here + yappi.stop() + + # There are two collections we want + # the function stats and the thread stats + func_stats = yappi.get_func_stats() + pstats_func = yappi.convert2pstats(func_stats) + pstats_func.dump_stats(self.profile_func_filename) + + thread_stats = yappi.get_thread_stats() + thread_stats_file = open(self.profile_thread_filename, 'w') + thread_stats.print_all(thread_stats_file) + thread_stats_file.close() break except: logger.exception('Caught exception:\n') From a5d4a778682399cdd78de699025988aeedf7824b Mon Sep 17 00:00:00 2001 From: Ravi Jagannadhan Date: Fri, 16 Mar 2018 12:40:42 -0700 Subject: [PATCH 2/4] Removing duplicate import --- conductor/lib/uploader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conductor/lib/uploader.py b/conductor/lib/uploader.py index 905e867d..fc7e9c99 100644 --- a/conductor/lib/uploader.py +++ b/conductor/lib/uploader.py @@ -1,5 +1,4 @@ import datetime -import time import json import logging import os From 3b993a33b2c8b96e238b011ae2fe22f11abf2feb Mon Sep 17 00:00:00 2001 From: Ravi Jagannadhan Date: Sun, 25 Mar 2018 10:50:52 -0700 Subject: [PATCH 3/4] Adding SIGTERM handler --- conductor/lib/uploader.py | 51 +++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/conductor/lib/uploader.py b/conductor/lib/uploader.py index fc7e9c99..6fdc6cf8 100644 --- a/conductor/lib/uploader.py +++ b/conductor/lib/uploader.py @@ -3,6 +3,7 @@ import logging import os import Queue +import signal import sys import tempfile import thread @@ -586,10 +587,7 @@ def handle_upload_response(self, project, upload_files, upload_id=None, md5_only except: return traceback.format_exc() - - def main(self, run_one_loop=False): - logger.info('Uploader Started. Checking for uploads...') - + def profiling_start(self): # for profiling, grab the environment variable's value # to determine where to put the profiling data @@ -609,7 +607,39 @@ def main(self, run_one_loop=False): self.profile_thread_filename = os.path.join(profile_path, thread_filename) # start profiling + logger.info('starting profiling') yappi.start() + return True + + def profiling_stop(self): + logger.info('stopping profiling') + yappi.stop() + + # There are two collections we want + # the function stats and the thread stats + func_stats = yappi.get_func_stats() + pstats_func = yappi.convert2pstats(func_stats) + pstats_func.dump_stats(self.profile_func_filename) + + thread_stats = yappi.get_thread_stats() + thread_stats_file = open(self.profile_thread_filename, 'w') + thread_stats.print_all(thread_stats_file) + thread_stats_file.close() + + def sigterm_handler(self, signal, frame): + logger.info("Caught SIGTERM, exiting...") + self.profiling_stop() + sys.exit(0) + + def main(self, run_one_loop=False): + logger.info('Uploader Started. Checking for uploads...') + + # in some cases, this uploader will be terminated with a SIGTERM + # in which case, we need to exit gracefully via a handler + signal.signal(signal.SIGTERM, self.sigterm_handler) + + # begin profiling + self.profiling_start() while not common.SIGINT_EXIT: try: @@ -655,18 +685,7 @@ def main(self, run_one_loop=False): logger.info("ctrl-c exit") # stop profiling here - yappi.stop() - - # There are two collections we want - # the function stats and the thread stats - func_stats = yappi.get_func_stats() - pstats_func = yappi.convert2pstats(func_stats) - pstats_func.dump_stats(self.profile_func_filename) - - thread_stats = yappi.get_thread_stats() - thread_stats_file = open(self.profile_thread_filename, 'w') - thread_stats.print_all(thread_stats_file) - thread_stats_file.close() + self.profiling_stop() break except: logger.exception('Caught exception:\n') From 3388f6dad69e57d871849b2cb68bd5585305c63f Mon Sep 17 00:00:00 2001 From: Ravi Jagannadhan Date: Thu, 12 Apr 2018 11:16:24 -0700 Subject: [PATCH 4/4] Adding new MtoA Package ID for 3.0.0 --- conductor/resources/resources.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/conductor/resources/resources.yml b/conductor/resources/resources.yml index 3e75fab1..cbd75423 100644 --- a/conductor/resources/resources.yml +++ b/conductor/resources/resources.yml @@ -336,6 +336,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -409,6 +410,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -481,6 +483,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -554,6 +557,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -627,6 +631,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -700,6 +705,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -773,6 +779,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": 376c8d7c44c638545ee6f587a9503baa miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -846,6 +853,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": eafd6b0c0e336266fd63d74b378e8bd8 miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -919,6 +927,7 @@ package_ids: "2.1.0": ee7336e8a1799fc479617d5774bd43c7 "2.1.0.1": a0eddec3d5716765c8a11086abf88624 "2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf + "3.0.0": eafd6b0c0e336266fd63d74b378e8bd8 miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -975,6 +984,7 @@ package_ids: "2.1.0": ad6d64e2abb7877bcac1838e84d1a91d "2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f "2.1.0.2": bc992ba2e26a9820ade73266a6f300d8 + "3.0.0": eafd6b0c0e336266fd63d74b378e8bd8 renderman-maya: "21.3": c691fab2d2770adc01d614e16b8d6e64 "21.4": ebf7f44b9e0b119b29c2768c1092c4c4 @@ -1028,6 +1038,7 @@ package_ids: "2.1.0": ad6d64e2abb7877bcac1838e84d1a91d "2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f "2.1.0.2": bc992ba2e26a9820ade73266a6f300d8 + "3.0.0": eafd6b0c0e336266fd63d74b378e8bd8 miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -1083,6 +1094,7 @@ package_ids: "2.1.0": ad6d64e2abb7877bcac1838e84d1a91d "2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f "2.1.0.2": bc992ba2e26a9820ade73266a6f300d8 + "3.0.0": eafd6b0c0e336266fd63d74b378e8bd8 miarmy: "5.3.11": 01255ad4bb83bc5369d162d3af5fdec2 renderman-maya: @@ -1136,6 +1148,7 @@ package_ids: "2.1.0": 410fe5d88fe65eb62b746b5905681c6a "2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9 "2.1.0.2": 90f7878cad296e0813028fc37e6de448 + "3.0.0": cd7c9d84d9fcece9cd743a838597cd05 renderman-maya: "21.3": 1e7b1b9c5f011b76555d8d751a49cc32 "21.4": 128a1a2126250f5413db08ab19c1de25 @@ -1194,6 +1207,7 @@ package_ids: "2.1.0": 410fe5d88fe65eb62b746b5905681c6a "2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9 "2.1.0.2": 90f7878cad296e0813028fc37e6de448 + "3.0.0": cd7c9d84d9fcece9cd743a838597cd05 renderman-maya: "21.3": 1e7b1b9c5f011b76555d8d751a49cc32 "21.4": 128a1a2126250f5413db08ab19c1de25 @@ -1252,6 +1266,7 @@ package_ids: "2.1.0": 410fe5d88fe65eb62b746b5905681c6a "2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9 "2.1.0.2": 90f7878cad296e0813028fc37e6de448 + "3.0.0": cd7c9d84d9fcece9cd743a838597cd05 renderman-maya: "21.3": 1e7b1b9c5f011b76555d8d751a49cc32 "21.4": 128a1a2126250f5413db08ab19c1de25 @@ -1310,6 +1325,7 @@ package_ids: "2.1.0": 410fe5d88fe65eb62b746b5905681c6a "2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9 "2.1.0.2": 90f7878cad296e0813028fc37e6de448 + "3.0.0": cd7c9d84d9fcece9cd743a838597cd05 renderman-maya: "21.3": 1e7b1b9c5f011b76555d8d751a49cc32 "21.4": 128a1a2126250f5413db08ab19c1de25 @@ -1368,6 +1384,7 @@ package_ids: "2.1.0": 410fe5d88fe65eb62b746b5905681c6a "2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9 "2.1.0.2": 90f7878cad296e0813028fc37e6de448 + "3.0.0": cd7c9d84d9fcece9cd743a838597cd05 renderman-maya: "21.3": 1e7b1b9c5f011b76555d8d751a49cc32 "21.4": 128a1a2126250f5413db08ab19c1de25 @@ -1410,6 +1427,7 @@ package_ids: "2.1.0": cbe8d96aea67a3f39d17918f69312432 "2.1.0.1": 95a2241e0efdba008eec42deef13c24d "2.1.0.2": 0c4af05117037150629d43e7a85c8389 + "3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba renderman-maya: "21.6": 914263d130a20364759697ba1aef15f1 yeti: @@ -1434,6 +1452,7 @@ package_ids: "2.1.0": cbe8d96aea67a3f39d17918f69312432 "2.1.0.1": 95a2241e0efdba008eec42deef13c24d "2.1.0.2": 0c4af05117037150629d43e7a85c8389 + "3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba renderman-maya: "21.6": 914263d130a20364759697ba1aef15f1 yeti: @@ -1458,6 +1477,7 @@ package_ids: "2.1.0": cbe8d96aea67a3f39d17918f69312432 "2.1.0.1": 95a2241e0efdba008eec42deef13c24d "2.1.0.2": 0c4af05117037150629d43e7a85c8389 + "3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba renderman-maya: "21.6": 914263d130a20364759697ba1aef15f1 yeti: