Skip to content
This repository was archived by the owner on Jul 21, 2022. It is now read-only.

Ct 181 new mto a #189

Open
easythrees wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Ct 181 new mto a #189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
20b87d3
adding profiler hooks and collection to uploader
easythrees Mar 16, 2018
a5d4a77
Removing duplicate import
easythrees Mar 16, 2018
3b993a3
Adding SIGTERM handler
easythrees Mar 25, 2018
3388f6d
Adding new MtoA Package ID for 3.0.0
easythrees Apr 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion conductor/lib/uploader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
import datetime
import time
import json
import logging
import os
import Queue
import signal
import sys
import tempfile
import thread
from threading import Thread
import time
import traceback

import yappi # yappi is a profiler specializing in multi-threading

from conductor import CONFIG
from conductor.lib import api_client, common, worker, client_db, loggeria

Expand Down Expand Up @@ -583,10 +587,60 @@ def handle_upload_response(self, project, upload_files, upload_id=None, md5_only
except:
return traceback.format_exc()

def profiling_start(self):
# for profiling, grab the environment variable's value
# to determine where to put the profiling data

if "CONDUCTOR_PROFILE_DATA_PATH" not in os.environ:
profile_path = tempfile.gettempdir()
else:
profile_path = os.environ["CONDUCTOR_PROFILE_DATA_PATH"]

if not profile_path:
profile_path = tempfile.gettempdir()

timestr = time.strftime("%Y%m%d-%H%M%S")
func_filename = '{}_func_profile.dmp'.format(timestr)
thread_filename = '{}_thread_profile.txt'.format(timestr)

self.profile_func_filename = os.path.join(profile_path, func_filename)
self.profile_thread_filename = os.path.join(profile_path, thread_filename)

# start profiling
logger.info('starting profiling')
yappi.start()
return True

def profiling_stop(self):
logger.info('stopping profiling')
yappi.stop()

# There are two collections we want
# the function stats and the thread stats
func_stats = yappi.get_func_stats()
pstats_func = yappi.convert2pstats(func_stats)
pstats_func.dump_stats(self.profile_func_filename)

thread_stats = yappi.get_thread_stats()
thread_stats_file = open(self.profile_thread_filename, 'w')
thread_stats.print_all(thread_stats_file)
thread_stats_file.close()

def sigterm_handler(self, signal, frame):
logger.info("Caught SIGTERM, exiting...")
self.profiling_stop()
sys.exit(0)

def main(self, run_one_loop=False):
logger.info('Uploader Started. Checking for uploads...')

# in some cases, this uploader will be terminated with a SIGTERM
# in which case, we need to exit gracefully via a handler
signal.signal(signal.SIGTERM, self.sigterm_handler)

# begin profiling
self.profiling_start()

while not common.SIGINT_EXIT:
try:
# TODO: we should pass args as url params, not http data
Expand Down Expand Up @@ -629,6 +683,9 @@ def main(self, run_one_loop=False):

except KeyboardInterrupt:
logger.info("ctrl-c exit")

# stop profiling here
self.profiling_stop()
break
except:
logger.exception('Caught exception:\n')
Expand Down
20 changes: 20 additions & 0 deletions conductor/resources/resources.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -409,6 +410,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -481,6 +483,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -554,6 +557,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -627,6 +631,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -700,6 +705,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -773,6 +779,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": 376c8d7c44c638545ee6f587a9503baa
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -846,6 +853,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": eafd6b0c0e336266fd63d74b378e8bd8
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -919,6 +927,7 @@ package_ids:
"2.1.0": ee7336e8a1799fc479617d5774bd43c7
"2.1.0.1": a0eddec3d5716765c8a11086abf88624
"2.1.0.2": d0e9caeba57ff51e0e9d1ef5e66cd6cf
"3.0.0": eafd6b0c0e336266fd63d74b378e8bd8
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -975,6 +984,7 @@ package_ids:
"2.1.0": ad6d64e2abb7877bcac1838e84d1a91d
"2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f
"2.1.0.2": bc992ba2e26a9820ade73266a6f300d8
"3.0.0": eafd6b0c0e336266fd63d74b378e8bd8
renderman-maya:
"21.3": c691fab2d2770adc01d614e16b8d6e64
"21.4": ebf7f44b9e0b119b29c2768c1092c4c4
Expand Down Expand Up @@ -1028,6 +1038,7 @@ package_ids:
"2.1.0": ad6d64e2abb7877bcac1838e84d1a91d
"2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f
"2.1.0.2": bc992ba2e26a9820ade73266a6f300d8
"3.0.0": eafd6b0c0e336266fd63d74b378e8bd8
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -1083,6 +1094,7 @@ package_ids:
"2.1.0": ad6d64e2abb7877bcac1838e84d1a91d
"2.1.0.1": 50123b3cd11dbc64fdb062415ff1a20f
"2.1.0.2": bc992ba2e26a9820ade73266a6f300d8
"3.0.0": eafd6b0c0e336266fd63d74b378e8bd8
miarmy:
"5.3.11": 01255ad4bb83bc5369d162d3af5fdec2
renderman-maya:
Expand Down Expand Up @@ -1136,6 +1148,7 @@ package_ids:
"2.1.0": 410fe5d88fe65eb62b746b5905681c6a
"2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9
"2.1.0.2": 90f7878cad296e0813028fc37e6de448
"3.0.0": cd7c9d84d9fcece9cd743a838597cd05
renderman-maya:
"21.3": 1e7b1b9c5f011b76555d8d751a49cc32
"21.4": 128a1a2126250f5413db08ab19c1de25
Expand Down Expand Up @@ -1194,6 +1207,7 @@ package_ids:
"2.1.0": 410fe5d88fe65eb62b746b5905681c6a
"2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9
"2.1.0.2": 90f7878cad296e0813028fc37e6de448
"3.0.0": cd7c9d84d9fcece9cd743a838597cd05
renderman-maya:
"21.3": 1e7b1b9c5f011b76555d8d751a49cc32
"21.4": 128a1a2126250f5413db08ab19c1de25
Expand Down Expand Up @@ -1252,6 +1266,7 @@ package_ids:
"2.1.0": 410fe5d88fe65eb62b746b5905681c6a
"2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9
"2.1.0.2": 90f7878cad296e0813028fc37e6de448
"3.0.0": cd7c9d84d9fcece9cd743a838597cd05
renderman-maya:
"21.3": 1e7b1b9c5f011b76555d8d751a49cc32
"21.4": 128a1a2126250f5413db08ab19c1de25
Expand Down Expand Up @@ -1310,6 +1325,7 @@ package_ids:
"2.1.0": 410fe5d88fe65eb62b746b5905681c6a
"2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9
"2.1.0.2": 90f7878cad296e0813028fc37e6de448
"3.0.0": cd7c9d84d9fcece9cd743a838597cd05
renderman-maya:
"21.3": 1e7b1b9c5f011b76555d8d751a49cc32
"21.4": 128a1a2126250f5413db08ab19c1de25
Expand Down Expand Up @@ -1368,6 +1384,7 @@ package_ids:
"2.1.0": 410fe5d88fe65eb62b746b5905681c6a
"2.1.0.1": 379b3514603cac841e36a7b0cc06f4d9
"2.1.0.2": 90f7878cad296e0813028fc37e6de448
"3.0.0": cd7c9d84d9fcece9cd743a838597cd05
renderman-maya:
"21.3": 1e7b1b9c5f011b76555d8d751a49cc32
"21.4": 128a1a2126250f5413db08ab19c1de25
Expand Down Expand Up @@ -1410,6 +1427,7 @@ package_ids:
"2.1.0": cbe8d96aea67a3f39d17918f69312432
"2.1.0.1": 95a2241e0efdba008eec42deef13c24d
"2.1.0.2": 0c4af05117037150629d43e7a85c8389
"3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba
renderman-maya:
"21.6": 914263d130a20364759697ba1aef15f1
yeti:
Expand All @@ -1434,6 +1452,7 @@ package_ids:
"2.1.0": cbe8d96aea67a3f39d17918f69312432
"2.1.0.1": 95a2241e0efdba008eec42deef13c24d
"2.1.0.2": 0c4af05117037150629d43e7a85c8389
"3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba
renderman-maya:
"21.6": 914263d130a20364759697ba1aef15f1
yeti:
Expand All @@ -1458,6 +1477,7 @@ package_ids:
"2.1.0": cbe8d96aea67a3f39d17918f69312432
"2.1.0.1": 95a2241e0efdba008eec42deef13c24d
"2.1.0.2": 0c4af05117037150629d43e7a85c8389
"3.0.0": 8d60ca7e0d3ff0b0fd91e3dd2bcb21ba
renderman-maya:
"21.6": 914263d130a20364759697ba1aef15f1
yeti:
Expand Down