htmlscraper.n

// Super fast and non validating HTML scraper
class HTMLParser{

	// parser callbacks, overload in your class 	
	text(value){} // text node
	comment(value){} // comment node <!-- -->
	cdata(value){} // cdata section <![CDATA[ ]]>
	process(value){} // process <? ?>
	begin(name){} // begin tag
	end(){} // end tag, not called when self closing
	close(name){} // close tag
	attr(name){} // attribute name
	value(val){} // attribute value

	// HTML parser function, 110 loc and very forgiving.
	parse(source){
		if(typeof source != 'string') source = source.toString()
		var len = source.length
		var pos = 0
		var start = pos
		while(pos < len){
			var ch = source.charCodeAt(pos++)
			if(ch == 60){ // <
				// lets emit textnode since last
				if(start != pos - 1 && text) text(source.slice(start, pos - 1))
				var next = source.charCodeAt(pos)
				if(next == 33){ // <!
					after = source.charCodeAt(pos+1)
					if(after == 45){ // <!- comment
						pos += 2
						while(pos < len){
							ch = source.charCodeAt(pos++)
							if(ch == 45 && source.charCodeAt(pos) == 45 &&
							    source.charCodeAt(pos + 1) == 62){
								pos += 2
								comment(source.slice(start, pos - 3))
								break
							}
						}
						start = pos
						continue
					}
					if(after == 91){ // <![ probably followed by CDATA[ just parse to ]]>
						pos += 8
						start = pos
						while(pos < len){
							ch = source.charCodeAt(pos++)
							if(ch == 93 && source.charCodeAt(pos) == 93 &&
							    source.charCodeAt(pos + 1) == 62){
								pos += 2
								cdata(source.slice(start, pos - 3))
								break
							}
						}
						start = pos
						continue
					}
				}
				if(next == 63){ // <? command
					pos++
					start = pos
					while(pos < len){
						ch = source.charCodeAt(pos++)
						if(ch == 63 && source.charCodeAt(pos) == 62){
							pos++
							process(source.slice(start, pos - 2))
							break
						}
					}
					start = pos
					continue
				}
				if(next == 47){ // </ closing tag
					start = pos + 1
					while(pos < len){
						ch = source.charCodeAt(pos++)
						if(ch == 62){
							close(source.slice(start, pos - 1))
							break
						}
					}
					start = pos
					continue
				}
				start = pos // try to parse a tag
				var tag = true // first name encountered is tagname
				while(pos < len){
					ch = source.charCodeAt(pos++)
					// whitespace, end of tag or assign
					if(ch == 62 || ch == 47 || ch == 10 || ch == 12 || ch ==32 || ch == 61){
						if(start != pos - 1){
							if(tag){ // lets emit the tagname
								begin(source.slice(start, pos - 1))
								tag = false
							}// emit attribute name
							else attr(source.slice(start, pos - 1))
						}
						start = pos
						if(ch == 62){ // >
							end()
							break
						}
						else if(ch == 47 && source.charCodeAt(pos) == 62){ // />
							pos++
							close()
							break
						}
					}
					else if(ch == 34 || ch == 39){ // " or '
						start = pos
						var end = ch
						while(pos < len){
							ch = source.charCodeAt(pos++)
							if(ch == end){
								value(source.slice(start, pos - 1))
								break
							}
						}
						start = pos
					}
				}
				start = pos
			} 
		}
	}
}

// Builds a JS object tree accessible like json: tag_1.tag[2].attr
class HTMLScraper extends HTMLParser{

	// void tags
	enum void_tag{
		area, base, br, col, embed, hr, img, input, keygen, link, menuitem, meta, param, source, track, wbr
	}

	class HTMLNode{

		// poor mans selector query for tagname == tagname, prop == value
		whereOne(tagname, prop, value, out){
			var keys = Object.keys(this)
			for(var k from keys){
				if(k !== 'owner' && k !== 'parent' && k !== 'tagname'){
					var val = this[k]
					if(this.tagname == tagname && k == prop && val == value){
						if(out) out.push(this)
						else return this
					}
					if(val && typeof val.dump == 'function'){
						var ret = val.where(tagname, prop, value, out)
						if(!out && ret !== undefined) return ret
					}
				}
			}
		}

		// selector query to return all nodes matching
		whereAll(tagname, prop, value){
			var ret = []
			whereOne(tagname, prop, value, ret)
			return ret
		}

		// log all values on this node
		logValues(){
			var keys = Object.keys(this)
			for(var k from keys){
				val = this[k]
				if(typeof val == 'string'){
					log(k+' = '+val)
				}
			}
		}

		// log entire tree
		logTree(s = ''){
			var keys = Object.keys(this)
			for(var k from keys){
				if(k !== 'owner' && k !== 'parent' && k !== 'tagname'){
					var val = this[k]
					if(Array.isArray(val)) continue
					if(val && typeof val.logTree == 'function'){
						val.logTree(s+k+'.')
						continue
					}
					if(typeof val == 'string'){
						if(val.length > 40)
							log(s+k+' = ' + val.slice(0,40).replace(/[\r\n]/g,'\\n')+'...')
						else
							log(s+k+' = ' + val.replace(/[\r\n]/g,'\\n'))
					}
				}
			}
		}
		
		// insert a new tag by name
		appendChild(name, value){
			if(arguments.length == 1){
				var cpy = name
				value = new HTMLNode()
				name = cpy.tagname
				for(var k in cpy) value[k] = cpy[k]
			}
			var old = this[name]
			if(old !== undefined){
				if(Array.isArray(old)){
					this[name + '_' + old.length] = value
					old.push(value)
				}
				else{
					this[name + '_0'] = old
					this[name + '_1'] = value
					this[name] = [old, value]
				}
			}
			else this[name] = value
		}

		// serialize HTML again, it does auto indenting. with tabs.
		toString(indent = ''){
			var keys = Object.keys(this)
			var ret = ''
			var hastag = this.tagname!== undefined
			if(hastag) ret += '<' + this.tagname
			var tags = []
			for(var k from keys){
				if(k !== 'owner' && k !== 'parent' && k !== 'tagname'){
					var val = this[k]
					if(k.charCodeAt(0) == 36){ // attribute
						if(ret[ret.length - 1] != ' ') ret += ' '
						ret += k.slice(1)
						var delim = "'"
						if(val !== 1){
							if(val.indexOf(delim) !== -1) delim = '"'
							ret += '=' + delim + val + delim
						}
						continue
					} 
					if(Array.isArray(val)) continue
					if(val && typeof val.logTree == 'function'){
						tags.push(val)
						continue
					}
					if(typeof val == 'string'){
						if(k.indexOf('text') == 0){
							tags.push(val)
							continue
						}
						if(k.indexOf('cdata') == 0){
							tags.push('<![CDATA['+val+']]>')
							continue
						}
						if(k.indexOf('comment') == 0){
							tags.push('<!--'+val+'-->')
							continue
						}
						throw new Error('Value is string, but not attribute or known tag '+k+' = '+val)
					}
				}
			}
			// check if we are selfclosing
			if(hastag && this.tagname in void_tag){
				ret += '/>'
				return ret
			}
			if(hastag) ret += '>\n'
			var sub = hastag? indent + '\t': indent
			for(var tag from tags){
				if(typeof tag == 'string'){
					// TODO! reindent textblocks
					ret += sub + tag + '\n'
				}
				else{
					ret += sub + tag.toString(sub) + '\n'
				}
			}
			if(hastag) ret += indent + '</'+this.tagname+'>'
			return ret
		}
	}

	// destructured named arguments with default values
	// and assign-on-this
	parse(source, @stripempty){
		node = new HTMLNode()
		HTMLParser.parse.call(this, source)
		return node
	}

	// todo, add decoding of these entity things
	var entities = {
		"amp":"&","gt":">","lt":"<","quot":"\"","apos":"'","AElig":198,"Aacute":193,"Acirc":194,
		"Agrave":192,"Aring":197,"Atilde":195,"Auml":196,"Ccedil":199,"ETH":208,"Eacute":201,"Ecirc":202,
		"Egrave":200,"Euml":203,"Iacute":205,"Icirc":206,"Igrave":204,"Iuml":207,"Ntilde":209,"Oacute":211,
		"Ocirc":212,"Ograve":210,"Oslash":216,"Otilde":213,"Ouml":214,"THORN":222,"Uacute":218,"Ucirc":219,
		"Ugrave":217,"Uuml":220,"Yacute":221,"aacute":225,"acirc":226,"aelig":230,"agrave":224,"aring":229,
		"atilde":227,"auml":228,"ccedil":231,"eacute":233,"ecirc":234,"egrave":232,"eth":240,"euml":235,
		"iacute":237,"icirc":238,"igrave":236,"iuml":239,"ntilde":241,"oacute":243,"ocirc":244,"ograve":242,
		"oslash":248,"otilde":245,"ouml":246,"szlig":223,"thorn":254,"uacute":250,"ucirc":251,"ugrave":249,
		"uuml":252,"yacute":253,"yuml":255,"copy":169,"reg":174,"nbsp":160,"iexcl":161,"cent":162,"pound":163,
		"curren":164,"yen":165,"brvbar":166,"sect":167,"uml":168,"ordf":170,"laquo":171,"not":172,"shy":173,
		"macr":175,"deg":176,"plusmn":177,"sup1":185,"sup2":178,"sup3":179,"acute":180,"micro":181,"para":182,
		"middot":183,"cedil":184,"ordm":186,"raquo":187,"frac14":188,"frac12":189,"frac34":190,"iquest":191,
		"times":215,"divide":247,"OElig":338,"oelig":339,"Scaron":352,"scaron":353,"Yuml":376,"fnof":402,
		"circ":710,"tilde":732,"Alpha":913,"Beta":914,"Gamma":915,"Delta":916,"Epsilon":917,"Zeta":918,
		"Eta":919,"Theta":920,"Iota":921,"Kappa":922,"Lambda":923,"Mu":924,"Nu":925,"Xi":926,"Omicron":927,
		"Pi":928,"Rho":929,"Sigma":931,"Tau":932,"Upsilon":933,"Phi":934,"Chi":935,"Psi":936,"Omega":937,
		"alpha":945,"beta":946,"gamma":947,"delta":948,"epsilon":949,"zeta":950,"eta":951,"theta":952,
		"iota":953,"kappa":954,"lambda":955,"mu":956,"nu":957,"xi":958,"omicron":959,"pi":960,"rho":961,
		"sigmaf":962,"sigma":963,"tau":964,"upsilon":965,"phi":966,"chi":967,"psi":968,"omega":969,
		"thetasym":977,"upsih":978,"piv":982,"ensp":8194,"emsp":8195,"thinsp":8201,"zwnj":8204,"zwj":8205,
		"lrm":8206,"rlm":8207,"ndash":8211,"mdash":8212,"lsquo":8216,"rsquo":8217,"sbquo":8218,"ldquo":8220,
		"rdquo":8221,"bdquo":8222,"dagger":8224,"Dagger":8225,"bull":8226,"hellip":8230,"permil":8240,
		"prime":8242,"Prime":8243,"lsaquo":8249,"rsaquo":8250,"oline":8254,"frasl":8260,"euro":8364,
		"image":8465,"weierp":8472,"real":8476,"trade":8482,"alefsym":8501,"larr":8592,"uarr":8593,
		"rarr":8594,"darr":8595,"harr":8596,"crarr":8629,"lArr":8656,"uArr":8657,"rArr":8658,"dArr":8659,
		"hArr":8660,"forall":8704,"part":8706,"exist":8707,"empty":8709,"nabla":8711,"isin":8712,
		"notin":8713,"ni":8715,"prod":8719,"sum":8721,"minus":8722,"lowast":8727,"radic":8730,"prop":8733,
		"infin":8734,"ang":8736,"and":8743,"or":8744,"cap":8745,"cup":8746,"int":8747,"there4":8756,"sim":8764,
		"cong":8773,"asymp":8776,"ne":8800,"equiv":8801,"le":8804,"ge":8805,"sub":8834,"sup":8835,"nsub":8836,
		"sube":8838,"supe":8839,"oplus":8853,"otimes":8855,"perp":8869,"sdot":8901,"lceil":8968,"rceil":8969,
		"lfloor":8970,"rfloor":8971,"lang":9001,"rang":9002,"loz":9674,"spades":9824,"clubs":9827,"hearts":9829,
		"diams":9830
	}

	// begin a tag
	begin(name){
		var newnode = new HTMLNode()	
		newnode.parent = node
		node.appendChild(name, newnode)
		newnode.tagname = name
		node = newnode
	}
	
	var isempty = /^[\r\n\s]+$/ // discard empty textnodes

	text(value){
		if(!value.match(isempty)) node.appendChild('text', value)
	}

	comment(val) ->node.appendChild('comment', value)
	cdata(val) ->node.appendChild('cdata', value)

	attr(name){
		if(name in node) last_attr = undefined
		else {
			node[last_attr = '$'+name] = 1
		}
	}

	value(val){
		if(last_attr !== undefined){
			node[last_attr] = val
		}
	}

	end(){
		last_attr = undefined
		if(node.tagname in void_tag || node.tagname.charCodeAt(0) == 33){
			node = node.parent
		}
	}
	
	close(name){
		last_attr = undefined
		while(node && name !== undefined && node.tagname != name && node.parent){
			node = node.parent
		}
		if(node.parent)	node = node.parent
	}
}