MSSQL Improvements and addition of a domain field lookup

client/bugLogService.cfc

@@ -12,7 +12,9 @@
 	<cfset variables.postToRESTasJSON = false>
 	<cfset variables.userAgent = "bugloghq-coldfusion-client">
 	<cfset variables.checkpointsKey = "__buglog_checkpoints__">
-	<cfset variables.writeToCFLog = true>	<!--- indicates whether to write to the local log or not ---->
+	<cfset variables.writeToCFLog = false>	<!--- indicates whether to write to the local log or not ---->
+	<cfset variables.hostname = "">
+	<cfset variables.sensitiveFieldNames = "">
 
 	<!--- Handle cases in which the application scope is not defined (Fix contributed by Morgan Dennithorne) --->
 	<cfif isDefined("application.applicationName")>
@@ -33,6 +35,7 @@
 		<cfargument name="appName" type="string" required="false" default="#variables.appName#">
 		<cfargument name="maxDumpDepth" type="numeric" required="false" default="#variables.maxDumpDepth#">
 		<cfargument name="writeToCFLog" type="boolean" required="false" default="#variables.writeToCFLog#">
+		<cfargument name="sensitiveFieldNames" type="string" required="false" default="#variables.sensitiveFieldNames#">
 
 		<cfscript>
 			var wsParams = structNew();
@@ -60,12 +63,13 @@
 				variables.protocol = "CFC";
 
 			// store settings
-			variables.bugLogListener = arguments.bugLogListener;
-			variables.bugEmailSender = arguments.bugEmailSender;
-			variables.bugEmailRecipients = arguments.bugEmailRecipients;
-			variables.apikey = arguments.apikey;
-			variables.maxDumpDepth = arguments.maxDumpDepth;
-			variables.writeToCFLog = arguments.writeToCFLog;
+			variables.bugLogListener		= arguments.bugLogListener;
+			variables.bugEmailSender		= arguments.bugEmailSender;
+			variables.bugEmailRecipients	= arguments.bugEmailRecipients;
+			variables.apikey				= arguments.apikey;
+			variables.maxDumpDepth			= arguments.maxDumpDepth;
+			variables.writeToCFLog			= arguments.writeToCFLog;
+			variables.sensitiveFieldNames	= arguments.sensitiveFieldNames;
 			if(arguments.appName neq "")
 				variables.appName = arguments.appName;
 
@@ -128,7 +132,8 @@
 		<cfargument name="maxDumpDepth" type="numeric" required="false" default="#variables.maxDumpDepth#">
 		<cfargument name="writeToCFLog" type="boolean" required="false" default="#variables.writeToCFLog#">
 		<cfargument name="AppName" type="string" required="false" default="#variables.appName#">
-
+		<cfargument name="domain" type="string" required="false" default="">
+
 		<cfset var shortMessage = "">
 		<cfset var longMessage = "">
 		<cfset var tmpCFID = "">
@@ -148,6 +153,12 @@
 			<cfset tmpCFTOKEN = cftoken>
 		</cfif>
 
+		<cfif isSimpleValue(arguments.extraInfo)>
+			<cfset arguments.extraInfo = sanitizeSensitiveData(arguments.extraInfo)>
+		<cfelseif isStruct(arguments.extraInfo)>
+			<cfset arguments.extraInfo = sanitizeKeyData(arguments.extraInfo)>
+		</cfif>
+
 		<cftry>
 			<!--- if we are tracking checkpoints, then add the buglog call as the last checkpoint --->
 			<cfif arrayLen(getCheckpoints())>
@@ -167,6 +178,7 @@
 						"hostName" = variables.hostName,
 						"exceptionMessage" = arguments.exception.message,
 						"exceptionDetails" = arguments.exception.detail,
+						"domain" = arguments.domain,
 						"CFID" = tmpCFID,
 						"CFTOKEN" = tmpCFTOKEN,
 						"userAgent" = cgi.HTTP_USER_AGENT,
@@ -204,7 +216,8 @@
 																data.userAgent,
 																data.templatePath,
 																sanitizeForXML(data.HTMLReport),
-																data.apikey)>
+																data.apikey,
+																cgi.server_name)>
 				</cfif>
 			<cfelse>
 				<cfif variables.bugEmailRecipients neq "">
@@ -513,4 +526,28 @@
 		<cfreturn out>
 	</cffunction>
 
-</cfcomponent>
+	<cffunction name="sanitizeKeyData" access="private" returntype="struct" hint="removes values from structures for known fields that hold sensitive data">
+		<cfargument name="data" type="struct">
+		<cfset var local = {}>
+		<cfloop list="#structKeyList(arguments.data)#" index="local.i">
+			<cfif isStruct(arguments.data[local.i])>
+				<cfset arguments.data[local.i] = sanitizeKeyData(arguments.data[local.i])>
+			<cfelse>
+				<cfif listFindNoCase(variables.sensitiveFieldNames,local.i)>
+					<cfset arguments.data[local.i] = "***POTENTIAL_SENSITIVE_VALUE_REMOVED***">
+				</cfif>
+			</cfif>
+		</cfloop>
+		<cfreturn arguments.data>
+	</cffunction>
+
+
+	<cffunction name="sanitizeSensitiveData" access="private" returntype="string" hint="Removes known formats for credit card and S3 passwords">
+		<cfargument name="data" type="string" required="true">
+		<cfset arguments.data = ReReplace(arguments.data, '[45][0-9]{3}-?[0-9]{4}-?[0-9]{4}-?([0-9]{4})', '***VISA/MASTERCARD_REMOVED*** (\1)', 'ALL')>
+		<cfset arguments.data = ReReplace(arguments.data, '3[47][0-9]{2}-?[0-9]{6}-?([0-9]{5})', '***AMEX_REMOVED*** (\1)', 'ALL')>
+		<cfset arguments.data = ReReplace(arguments.data, 's3:\/\/[^@]+@', 's3://***S3_PASSWORD_REMOVED***@', 'ALL')>
+		<cfreturn arguments.data>
+	</cffunction>
+
+</cfcomponent>

components/bugLogListener.cfc

@@ -41,6 +41,7 @@
 			variables.oHostFinder = createObject("component","bugLog.components.hostFinder").init( variables.oDAOFactory.getDAO("host") );
 			variables.oSeverityFinder = createObject("component","bugLog.components.severityFinder").init( variables.oDAOFactory.getDAO("severity") );
 			variables.oSourceFinder = createObject("component","bugLog.components.sourceFinder").init( variables.oDAOFactory.getDAO("source") );
+			variables.oDomainFinder = createObject("component","bugLog.components.domainFinder").init( variables.oDAOFactory.getDAO("domain") );
 			variables.oUserFinder = createObject("component","bugLog.components.userFinder").init( variables.oDAOFactory.getDAO("user") );
 
 			// load the rule processor
@@ -54,6 +55,7 @@
 			variables.oHostCache = createObject("component","bugLog.components.lib.cache.cacheService").init(50, cacheTTL, false);
 			variables.oSeverityCache = createObject("component","bugLog.components.lib.cache.cacheService").init(10, cacheTTL, false);
 			variables.oSourceCache = createObject("component","bugLog.components.lib.cache.cacheService").init(5, cacheTTL, false);
+			variables.oDomainCache = createObject("component","bugLog.components.lib.cache.cacheService").init(50, cacheTTL, false);
 			variables.oUserCache = createObject("component","bugLog.components.lib.cache.cacheService").init(50, cacheTTL, false);
 
 			// load scheduler
@@ -91,12 +93,15 @@
 			var autoCreateHost = allowAutoCreate("host");
 			var autoCreateSeverity = allowAutoCreate("severity");
 			var autoCreateSource = allowAutoCreate("source");
-
+			var autoCreateDomain = true; //TODO change this to a setting
+
 			// extract related objects from bean
 			oApp = getApplicationFromBean( bean, autoCreateApp );
 			oHost = getHostFromBean( bean, autoCreateHost );
 			oSeverity = getSeverityFromBean( bean, autoCreateSeverity );
 			oSource = getSourceFromBean( bean, autoCreateSource );
+			oDomain = getDomainFromBean( bean, autoCreateDomain );
+
 
 			// create entry
 			oEntry = createObject("component","bugLog.components.entry").init( oDF.getDAO("entry") );
@@ -108,6 +113,7 @@
 			oEntry.setHostID(oHost.getHostID());
 			oEntry.setExceptionMessage(bean.getexceptionMessage());
 			oEntry.setExceptionDetails(bean.getexceptionDetails());
+			oEntry.setDomainId(oDomain.getDomainID());
 			oEntry.setCFID(bean.getcfid());
 			oEntry.setCFTOKEN(bean.getcftoken());
 			oEntry.setUserAgent(bean.getuserAgent());
@@ -383,6 +389,42 @@
 		<cfreturn oSource>
 	</cffunction>
 
+	<cffunction name="getDomainFromBean" access="private" returntype="domain" hint="Uses the information on the rawEntryBean to retrieve the corresponding Domain object">
+		<cfargument name="entryBean" type="rawEntryBean" required="true">
+		<cfargument name="createIfNeeded" type="boolean" default="false">
+		<cfscript>
+			var key = "";
+			var bean = arguments.entryBean;
+			var oDomain = 0;
+			var oDF = variables.oDAOFactory;
+
+			key = bean.getDomain();
+
+			try {
+				// first we try to get it from the cache
+				oDomain = variables.oDomainCache.retrieve( key );
+
+			} catch(cacheService.itemNotFound e) {
+				// entry not in cache, so we get it from DB
+				try {
+					oDomain = variables.oDomainFinder.findByName( key );
+				} catch(domainFinderException.domainNotFound e) {
+					// code does not exist, so we need to create it (if autocreate enabled)
+					if(!arguments.createIfNeeded){
+						throw(message="Invalid Domain",type="invalidDomain");
+					}
+					oDomain = createObject("component","bugLog.components.domain").init( oDF.getDAO("domain") );
+					oDomain.setDomain( key );
+					oDomain.save();
+				}
+
+				// store entry in cache
+				variables.oDomainCache.store( key, oDomain );
+			}
+		</cfscript>
+		<cfreturn oDomain>
+	</cffunction>
+
 	<cffunction name="loadRules" access="private" returntype="void" hint="this method loads the rules into the rule processor">
 		<cfscript>
 			var oRule = 0;

components/db/domainDAO.cfc

@@ -0,0 +1,10 @@
+<cfcomponent extends="bugLog.components.lib.dao.DAO">
+
+	<cffunction name="initTableParams" access="package" returntype="void" hint="setup table specific settings">
+		<cfset setTableName("bl_Domain")>
+		<cfset setPrimaryKey("DomainID","cf_sql_numeric")>
+		<cfset setLabelField("domain","cf_sql_varchar")>
+		<cfset addColumn("domain", "cf_sql_varchar")>
+	</cffunction>
+
+</cfcomponent>

components/db/entryDAO.cfc

@@ -11,6 +11,7 @@
 		<cfset addColumn("sourceID", "cf_sql_numeric")>
 		<cfset addColumn("severityID", "cf_sql_numeric")>
 		<cfset addColumn("hostID", "cf_sql_numeric")>
+		<cfset addColumn("domainID", "cf_sql_numeric")>
 		<cfset addColumn("exceptionMessage", "cf_sql_varchar")>
 		<cfset addColumn("CFID", "cf_sql_varchar")>
 		<cfset addColumn("CFTOKEN", "cf_sql_varchar")>
@@ -102,5 +103,30 @@
 				WHERE severityID = <cfqueryparam cfsqltype="cf_sql_numeric" value="#arguments.fromSeverityID#">
 		</cfquery>	
 	</cffunction>
+
+	<cffunction name="deleteByDomainID" access="public" returntype="void" hint="Deletes all entries that belong to the given DomainId">
+		<cfargument name="domainId" type="numeric" required="true">
+		<cfset var dsn = getDataProvider().getConfig().getDSN()>
+		<cfset var username = getDataProvider().getConfig().getUsername()>
+		<cfset var password = getDataProvider().getConfig().getPassword()>
+		<cfquery name="qry" datasource="#dsn#" username="#username#" password="#password#">
+			DELETE
+				FROM #getTableName()#
+				WHERE domainId = <cfqueryparam cfsqltype="cf_sql_numeric" value="#arguments.domainId#">
+		</cfquery>	
+	</cffunction>
+
+	<cffunction name="updateDomainID" access="public" returntype="void" hint="Updates all entries that belong to the given DomainId to a new DomainId">
+		<cfargument name="fromDomainId" type="numeric" required="true">
+		<cfargument name="toDomainId" type="numeric" required="true">
+		<cfset var dsn = getDataProvider().getConfig().getDSN()>
+		<cfset var username = getDataProvider().getConfig().getUsername()>
+		<cfset var password = getDataProvider().getConfig().getPassword()>
+		<cfquery name="qry" datasource="#dsn#" username="#username#" password="#password#">
+			UPDATE #getTableName()#
+				SET domainId = <cfqueryparam cfsqltype="cf_sql_numeric" value="#arguments.toDomainId#">
+				WHERE domainId = <cfqueryparam cfsqltype="cf_sql_numeric" value="#arguments.fromDomainId#">
+		</cfquery>	
+	</cffunction>
 
 </cfcomponent>

components/domain.cfc

@@ -0,0 +1,30 @@
+<cfcomponent>
+
+	<cfscript>
+		variables.oDAO = 0;
+		variables.instance.domainId = 0;
+		variables.instance.domain = "";
+
+		function setDomainId(data) {variables.instance.domainId = arguments.data;}
+		function setDomain(data) {variables.instance.domain = arguments.data;}
+
+		function getDomainId() {return variables.instance.domainId;}
+		function getDomain() {return variables.instance.domain;}
+
+		function getID() {return getDomainId();}
+	</cfscript>
+
+	<cffunction name="init" access="public" returnType="domain">
+		<cfargument name="dao" type="bugLog.components.db.domainDAO" required="true">
+		<cfset variables.oDAO = arguments.dao>
+		<cfreturn this>
+	</cffunction>
+
+	<cffunction name="save" access="public">
+		<cfset var rtn = 0>
+		<cfset variables.instance.ID = getId()>
+		<cfset rtn = variables.oDAO.save(argumentCollection = variables.instance)>
+		<cfset setDomainId(rtn)>
+	</cffunction>
+
+</cfcomponent>

components/domainFinder.cfc

@@ -0,0 +1,42 @@
+<cfcomponent extends="finder">
+
+	<cffunction name="findByID" returnType="domain" access="public">
+		<cfargument name="id" type="numeric" required="true">
+		<cfscript>
+			var qry = variables.oDAO.get(arguments.id);
+			var o = 0;
+
+			if(qry.recordCount gt 0) {
+				o = createObject("component","bugLog.components.domain").init( variables.oDAO );
+				o.setDomainID(qry.domainID);
+				o.setDomain(qry.domain);
+				return o;
+			} else {
+				throw("ID not found","domainFinderException.IDNotFound");
+			}
+		</cfscript>
+	</cffunction>
+
+	<cffunction name="findByName" returnType="domain" access="public">
+		<cfargument name="name" type="string" required="true">
+		<cfscript>
+			var qry = variables.oDAO.getByLabel(arguments.name);
+			var o = 0;
+
+			if(qry.recordCount gt 0) {
+				o = createObject("component","bugLog.components.domain").init( variables.oDAO );
+				o.setDomainID(qry.domainID);
+				o.setDomain(qry.domain);
+				return o;
+			} else {
+				throw("domain not found","domainFinderException.domainNotFound");
+			}
+		</cfscript>
+	</cffunction>
+
+	<cffunction name="findByCode" returnType="domain" access="public">
+		<cfargument name="code" type="string" required="true">
+		<cfreturn findByName(arguments.code)>
+	</cffunction>
+
+</cfcomponent>

components/entry.cfc

@@ -17,6 +17,7 @@
 		variables.instance.templatePath = "";
 		variables.instance.HTMLReport = "";
 		variables.instance.createdOn = now();
+		variables.instance.domainId = 0;
 
 		function setEntryID(data) {variables.instance.ID = arguments.data;}
 		function setDateTime(data) {variables.instance.mydateTime = arguments.data;}
@@ -33,6 +34,7 @@
 		function setTemplatePath(data) {variables.instance.templatePath = left(arguments.data,500);}
 		function setHTMLReport(data) {variables.instance.HTMLReport = arguments.data;}
 		function setCreatedOn(data) {variables.instance.createdOn = arguments.data;}
+		function setDomainId(data) {variables.instance.domainId = arguments.data;}
 
 		function getEntryID() {return variables.instance.ID;}
 		function getDateTime() {return variables.instance.mydateTime;}
@@ -49,6 +51,7 @@
 		function getTemplate_Path() {return variables.instance.templatePath;}
 		function getHTMLReport() {return variables.instance.HTMLReport;}
 		function getCreatedOn() {return variables.instance.createdOn;}
+		function getDomainId() {return variables.instance.domainId;}
 
 		function getID() {return variables.instance.ID;}
 	</cfscript>
@@ -89,4 +92,10 @@
 		<cfreturn createObject("component","bugLog.components.severityFinder").init( oSeverityDAO ).findByID(variables.instance.severityID)>
 	</cffunction>
 
+	<cffunction name="getDomain" access="public" returntype="domain" hint="Returns the domain object">
+		<cfset var oDataProvider = variables.oDAO.getDataProvider()>
+		<cfset var oDomainDAO = createObject("component","bugLog.components.db.domainDAO").init( oDataProvider )>
+		<cfreturn createObject("component","bugLog.components.domainFinder").init( oDomainDAO ).findByID(variables.instance.domainId)>
+	</cffunction>
+
 </cfcomponent>