From 70eaee4de5344ade788fe4f6e874e3bb7409515d Mon Sep 17 00:00:00 2001 From: Ovyerus Date: Sat, 25 Mar 2017 20:49:05 +1100 Subject: [PATCH 01/12] Rewrite to ES6 and remove dependency on underscore and other things --- Markov.js | 770 +++++++++++++++++++++++++++++++++++++++ WeightedList.js | 239 +++++++++++++ cmarkov.js | 855 -------------------------------------------- js-weighted-list.js | 245 ------------- package.json | 6 +- test.js | 51 +-- 6 files changed, 1029 insertions(+), 1137 deletions(-) create mode 100644 Markov.js create mode 100644 WeightedList.js delete mode 100644 cmarkov.js delete mode 100644 js-weighted-list.js diff --git a/Markov.js b/Markov.js new file mode 100644 index 0000000..da12fab --- /dev/null +++ b/Markov.js @@ -0,0 +1,770 @@ +const natural = require('natural'); +const pstack = require('pstack'); +const md5File = require('md5-file'); +const NGrams = natural.NGrams; +const pos = require('pos'); +const tbl = require('cli-table'); +const seraph = require("seraph"); +const Tagger = natural.BrillPOSTagger; +const tokenizer = require("node-tokenizer"); +const wlist = require("./WeightedList"); +const range = require('python-range'); +const fs = require('fs'); +const Promise = require('bluebird'); + +class Markov { + constructor(options) { + this.options = Object.assign({ + db: 'http://localhost:7473', + name: 'dev', + user: 'neo4j', + pass: 'neo4j', + depth: [1,3], + weight: 1.2, + depthWeight: 2, + certainty: 1 + }, options); + } + + tokenize(text) { + let cues = ['?', '!', '.', ',', ':', ';', '\t', '\n', '\r']; + let tokens = text.split(' '); + + if (this.options.debug) console.log('tokens', tokens); + + cues.forEach(cue => { + tokens = tokens.map(token => { + let parts = token.split(cue); + let l = parts.length; + + if (l === 1) return token; + + let buffer = []; + parts.forEach((part, i) => { + buffer.push(part); + if (i < l - 1) { + buffer.push(cue); + } + }); + + return buffer; + }); + + tokens = tokens.flatten(); + tokens = tokens.filter(item => item !== ''); + if (this.options.debug) console.log('tokens', tokens); + }); + + return tokens; + } + + test() { + let text = 'I would like to talk today about: how to develop a new foreign policy direction for our country? one that replaces! randomness with purpose, ideology with strategy, and chaos with peace.\n\nIt is time to shake the rust off of America\'s foreign policy. It\'s time to invite new voices and new visions into the fold.'; + let text2 = 'i started i did it , i was, oh, that first couple of weeks with illegal immigration and mexico and all of this stuff , right ? and all of a sudden , people are coming over , and i say the wall and now they\'re starting to look elsewhere for help . i started i did it , i was , oh , that first couple of weeks with illegal immigration'; + + console.log(this.beautify(text2)); + } + + beautify(text) { + text = text.replace(/\s(\.|,|:|;|!|\?)\s/gmi, (match, punc) => punc + ' '); + text.replace(/(\.|!|\?|\n)(\s)([a-z])/gmi, (match, punc, space, letter) => punc + space + letter.toUpperCase()); + text = text.substr(0, 1).toUpperCase() + text.substr(1); + + return text; + } + + test2() { + this.open(() => { + this.graph.query('MATCH p=(x:`ngrams-trump` {gram:{x}})-[:then*1..50]->(y:`ngrams-trump` {gram:{y}}) RETURN p LIMIT 50', {x: 'world', y: 'trump'}, (err, result) => { + if (err) throw err; + if (result) { + result.forEach(item => { + let nodes = item.nodes.map(node => node.split('/').pop()); + + let stack = new pstack(); + let grams = []; + + nodes.forEach(node => { + stack.add(done => { + this.graph.read(node, (err, node) => { + if (err) throw err; + if (node) grams.push(node.gram); + done(); + }); + }); + }); + + stack.start(() => { + console.log("----------------------\n",grams.join(' ')); + }); + if (this.options.debug) console.log('nodes', nodes); + }); + } + + if (this.options.debug) console.log('result', result); + }); + }); + + return this; + } + + open() { + return new Promise((resolve, reject) => { + this.graph = seraph({ + server: 'http://localhost:7474', + user: this.options.user, + pass: this.options.pass + }); + + this.graph.constraints.uniqueness.createIfNone(`ngrams-${this.options.name}`, 'gram', (err, constraint) => { + if (err) { + reject(err); + } else { + this.graph.constraints.uniqueness.createIfNone(`pos-${this.options.name}`, 'gram', (err, constraint) => { + let baseFolder = './node_modules/natural/lib/natural/brill_pos_tagger/data/English'; + let rulesFile = `${baseFolder}/tr_from_posjs.txt`; + let lexiconFile = `${baseFolder}/lexicon_from_posjs.json`; + let defaultCategory = 'N'; + + let tagger = new Tagger(lexiconFile, rulesFile, defaultCategory, err => { + if (err) { + reject(err); + } else { + this.tagger = tagger; + resolve(); + } + }); + }); + } + }); + }); + } + + read(filename, callback) { + let start = new Date().getTime(); + + this.open(() => { + md5File(filename, (err, sum) => { + /* if (this.data.docs[sum]) { + console.log(`Training already done on that document, on ${this.data.docs[sum]}`); + callback(this.data.grams); + return false; + }*/ + + fstool.file.read(filename, text => { + /* text = new pos.Lexer().lex(test); + let tokenizer = new natural.RegexpTokenizer({pattern: / /}); + text = tokenizer.tokenize(text);*/ + text = this.tokenize(text); + // console.log('text', text); + + // Generate the n-grams. + let grams = {}; + let unique = {}; + let nodeIndex = {}; + + let stackNgram = new pstack({ + progress: 'Reading the N-grams...', + reportInterval: 100, + batch: 10 + }); + + let stackRel = new pstack({ + progress: 'Mapping...', + reportInterval: 100 + }); + + for (let depth of range(this.options.depth[0], this.options.depth[1]+ 1)) { + console.log(`Starting Depth ${depth}`); + console.log('Generating the ngrams.'); + + grams[depth] = NGrams.ngrams(text, depth); + + console.log(`Stringification of ${grams[depth].length} ngrams`); + + let ngramObj = {}; + + grams[depth].forEach(item => { + ngramObj[item.join('|')] = true; + }); + + let uniqueNgrams = Object.keys(ngramObj).length; + + console.log('Graphing'); + + // Save the gram + for (let gram in ngramObj) { + let v = ngramObj[gram]; + gram = gram.toLowerCase(); + stackNgram.add(done => { + this.graph.save({gram, depth}, `ngrams-${this.options.name}`, (err, node) => { + if (err) { + this.graph.find({gram, depth}, false, `ngrams-${this.options.name}`, (err, response) => { + if (err) throw err; + if (response && response.length > 0) { + nodeIndex[gram] = response[0].idl + // console.log('Node:\t\t', '[found]');; + } + + done(); + }); + return false; + } else { + // console.log('Node:\t\t', '[created]'); + nodeIndex[gram] = node.id; + done(); + } + }); + }); + } + + // Process the gram graph + for (let n in grams[depth]) { + if (n === 0) return false; + + let current = grams[depth][n - 1].join('|').toLowerCase(); + let next = grams[depth][n].slice(-1).join('').toLowerCase(); + + stackRel.add(done => { + // console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); + this.graph.relationships(nodeIndex[current], 'out', 'then', (err, relationships) => { + if (err) { + // console.log('Relationship:\t', '[failed]'); + return false; + } else { + if (relationships) { + // Look for the relationship + let relationship = relationships.find(rel => rel.start == nodeIndex[current] && rel.send == nodeIndex[next]); + + if (relationship) { + // Update the relationship weight + relationship.properties.weight += 1; + this.graph.rel.update(relationship, err => { + if (err) // console.log('Relationship:\t', '[failed]'); + // console.log('Relationship:\t', '[updated]'); + done(); + }); + } else { + this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngrame: next, idx: nodeIndex[next]}, err => { + if (err); // console.log('Relationship:\t', '[failed]'); + else; // console.log('Relationship:\t', '[created]); + done(); + }); + } + } else { + // console.log('Relationship:\t', '[failed]'); + done(); + } + } + }); + }); + } + } + + stackNgram.start(() => { + stackRel.start(() => { + // console.log('>>>>>>>>>> nodeIndex', nodeIndex); + + let end = new Date().getTime(); + + console.log('==========================='); + console.log(`== Training time: ${(end - start)/(1000 * 60)} ==`); + console.log('==========================='); + + callback(grams); + }); + }); + }); + }); + }); + } + + readPOS(filename, callback) { + let start = new Date().getTime(); + + this.open(() => { + md5File(filename, (err, sum) => { + if (err) throw err; + + fs.readFile(filename, 'utf8', text => { + text = this.tokenize(text); + + text = this.tagger.tag(text).map(item => item[1]); + + if (this.options.debug) console.log('text', text); + + // Generate the n-grams; + let grams = {}; + let unique = {}; + let nodeIndex = {}; + + let stackNgram = new pstack({ + progess: 'Reading the N-grams...', + reportInterval: 100, + batch: 10 + }); + + let stackRel = new pstack({ + progress: 'Mapping...', + reportInterval: 100 + }); + + for (let depth of range(this.options.depth[0], this.options.depth[1] + 1)) { + console.log(`Starting Depth ${depth}`); + console.log('Generating the POS ngrams'); + + grams[depth] = NGrams.ngrams(text, depth); + + console.log(`Stringification of ${grams[depth].length} POS ngrams.`); + + let ngramObj = {}; + + grams[depth].forEach(item => { + ngramObj[item.join('|')] = true; + }); + + let uniqueNgrams = Object.keys(ngramObj).length; + + console.log('Graphing'); + + for (let gram of ngramObj) { + gram = gram.toLowerCase(); + + stackNgram.add(done => { + this.graph.save({gram, depth}, `pos-${this.options.name}`, (err, node) => { + if (err) { + // Read it + this.graph.find({gram, depth}, false, `pos-${this.options.name}`, (err, reponse) => { + if (err) throw err; + if (response && response.length > 0) { + nodeIndex[gram] = response[0].id; + if (this.options.debug) console.log('Node:\t\t', '[found]'); + } + + done(); + }); + + return false; + } else { + // consoel.log('Node:\t\t', '[created]'); + nodeIndex[gram] = node.id; + done(); + } + }); + }); + } + + // Process the gram graph + for (let n in grams[depth]) { + if (n === 0) return false; + + let current = grams[depth][n - 1].join('|').toLowerCase(); + let next = grams[depth][n].slice(-1).join('').toLowerCase(); + + stackRel.add(done => { + if (this.options.debug) console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); + this.graph.relationships(nodeIndex[current], 'out', 'then', (err, relationships) => { + if (err) { + if (this.options.debug) console.log('Relationship:\t', '[failed]'); + return false; + } else { + if (relationships) { + // Look for the relationship + let relationship = relationships.find(rel => rel.start == nodeIndex[current] && rel.end == nodeIndex[next]); + + if (relationship) { + // Update the relationship weight + relationship.properties.wieght += 1; + + this.graph.rel.update(relationship, err => { + if (err); if (this.options.debug) console.log('Relationship:\t', '[failed]'); + else; if (this.options.debug) console.log('Relationship:\t', '[updated]'); + done(); + }); + } else { + this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngram: next, idx: nodeIndex[next]}, (err) => { + if (err); if (this.options.debug) console.log('Relationship:\t', '[failed]'); + else; if (this.options.debug) console.log('Relationship:\t', '[updated]'); + done(); + }); + } + } else { + if (this.options.debug) console.log('Relationship:\t', '[failed]'); + done(); + } + } + }); + }); + } + } + + stackNgram.start(() => { + stackRel.start(() => { + if (this.options.debug) console.log('>>>>>>>>> nodeIndex', nodeIndex); + + let end = new Date().getTime(); + + console.log('==========================='); + console.log(`== POS Training time: ${(end - start)/(1000*60)} ==`); + console.log('==========================='); + }); + }); + }); + }); + }); + } + + getNext(chain, callback) { + let ngrams = {}; + let nodes = {}; + let posngrams = {}; + let posnodes = {}; + + if (this.options.debug) console.log('chain', chain); + + this.open(() => { + let stack = new pstack(); + let buffer = false; + + // Generate the option list, with weights + for (let depth of range(this.options.depth[1], this.options.depth[0]-1, -1)) { + stack.add(done => { + if (buffer && depth <= this.options.lowpri) { + if (this.options.debug) console.log('Skipped.'); + done(); + return false; + } + + // Generate the ngram to lookup (last N words in the chain) + ngrams[depth] = chain.slice(-depth).join('|').toLowerCase(); + + // Get the nodes + // let localNodes = this.getNodes(ngrams[depth]); + this.graph.find({gram: ngrams[depth]}, false, `ngram-${this.options.name}`, (err, response) => { + if (this.options.debug) console.log('response', ngrams[depth], response); + if (err) throw err; + if (response && response.length > 0) { + this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { + if (err) throw err; + if (relationships.length > 0) { + buffer = true; + if (this.options.debug) console.log('>>>> Depth: ', depth, relationships.length); + } + + // buffer[depth] = relationships; + if (false && depth >= 3 && relationships.length === 1) { + // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... + if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); + nodes = {}; + nodes[relationships[0].properties.ngram] = 1; + done(); + } else { + if (this.options.debug) console.log('relationships', ngrams[depth], relationships); + relationships.forEach(relationship => { + if (nodes[relationship.properties.ngram]) { + // No cumulation! + // nodes[gramId] += count; //*depth*this.options.depthWeight; + } else { + if (this.options.depthWeight) { + nodes[relationship.properties.name] = relationship.properties.weight * Math.pow(depth, this.options.depthweight); + } else { + nodes[relationship.properties.ngram] = relationship.properties.weight; + } + } + }); + if (Object.keys(nodes).lenth > 0) buffer = true; + done(); + } + }); + } else { + if (this.options.debug) console.log('ngram not found: ', ngrams[depth]); + done(); + } + }); + }); + } + + if (this.options.pos) { + // For each node option, build the POS + stack.add(done => { + let substack = new pstack(); + + // Check the POs for the current chain + if (this.options.debug) console.log('>>', chain); + let tags = this.tagger.tag(chain.slice(-20)).map(item => item[1]); + + for (let depth of range(this.options.depth[1], this.options.depth[0] - 1, -1)) { + substack.add(subdone => { + // Generate the ngram to lookup (last N words in the chain) + posngrams[depth] = tags.slice(-depth).join('|').toLowerCase(); + if (this.options.debug) console.log('>>>> Depth: ', depth); + + // Get the nodes + // let localeNodes = this.getNodes(ngrams[depth]); + this.graph.find({gram: posngrams[depth]}, false, `pos-${this.options.name}`, (err, response) => { + if (this.options.debug) console.log('response', posngrams[depth], response); + if (err) throw err; + if (response && response.length > 0) { + this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { + if (depth >= 3 && relationships.length === 1) { + // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... + if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); + + posnodes = {}; + posnodes[relationships[0].properties.ngram] = 1; + subdone(); + } else { + if (this.options.debug) console.log('relationships', posngrams[depth], relationships); + relationships.forEach(relationship => { + if (posnodes[relationship.properties.ngram]) { + // No cumulation! + // nodes[gramId] += count; //*depth*this.options.depthWeight; + } else { + if (this.options.depthWeight) { + posnodes[relationship.properties.ngram] = relationship.properties.weight * Math.pow(depth, this.options.depthweight); + } else { + posnodes[relationship.properties.ngram] = relationship.properties.weight; + } + } + }); + + subdone(); + } + }); + } else { + if (this.options.debug) console.log('ngram not found: ', ngrams[depth]); + subdone(); + } + }); + }); + } + + // Check the pos structure of each node option + + for (let k in nodes) { + substack.add(subdone => { + // Get the POS tag + let text = chain.slice(0); + text.push(k); + if (this.options.debug) console.log('text', text); + + let tags = this.tagger.tag(text.slice(-10)).map(item => { + return item[1]; + }); + + let tag = tags.slice(-1)[0].toLowerCase(0); + if (this.options.debug) console.log(`[${tag}] `, chain.join(' '), '->', k, posnodes[tag]); + + if (posnodes[tag]) { + nodes[k] += posnodes[tag]; + nodes[k] /= 2; + } + + subdone(); + }); + } + + substack.start(() => { + if (this.options.debug) console.log('nodes', nodes); + if (this.options.debug) console.log('posnodes', posnodes); + if (this.options.debug) consolee.log('-------------------------'); + done(); + }); + }); + } + + stack.start(() => { + if (this.options.debug) { + this.printEdges(nodes, 'Count'); + console.log(chain.join('|')); + } + + // Calculate the total + let total = 0; + for (let count of nodes) total += count; + + let minP = 1; + + // Calculate the probabilities + + for (let gramId in nodes) { + nodes[gramId] = count/total; + if (nodes[gramId] < minP) minP = nodes[gramId]; + } + + if (this.options.certainty) { + if (minP < this.options.certainty) { + // The probabilities are way too low to filter. We need to remove the least probable options + let nodeArray = []; + for (let id in nodes) nodeArray.push({id, p: nodes[id]}); + + // Sort and slice + nodeArray.sort((a, b) => b.p - a.p); + nodeArray = nodeArray.slice(0, 100/(this.options.certainty*100)); + + // Convert back to an object + nodes = {}; + nodeArray.forEach(item => nodes[item.id] = item.p); + if (this.options.debug) console.log('nodes', nodes); + } else { + // Remove the lowest probabilities + for (let gramId in nodes) { + if (nodes[gramId] < this.options.certainty) delete nodes[gramId]; + } + } + } + + // Recalculate the total + total = 0; + for (let p of nodes) { + total += p; + } + + // Recalculate the probabilities + for (let gramId in nodes) { + nodes[gramId] = nodes[gramId]/total; + } + + let rn = Math.random(); + + let _nodes = []; + for (let k in nodes) _nodes.push([k, nodes[k]]); + _nodes.sort((a, b) => a[1] - b[1]); + + let wl = new wlist(_nodes); + + let sample = wl.peek()[0]; + + // if (sample === '.') this.printEdges(nodes, 'Count'); + /* + let choices = []; + for (let gramId in nodes) { + let p = nodes[gramId]; + let count = p*100; + count = Math.ceil(Math.pow(count, this.options.weight)); + for (let n of range(0, count)) choices.push(gramId); + } + + if (this.options.debug) console.log('choices', choices); + let sample = choices[Math.floor(Math.random() * choices.length)]; + + if (chain[chain.length - 1] === "'") { + // this.printEdges(nodes, 'Count'); + // this.printEdges(nodes, `Probabilities: \033[37m\033[44m${chain.slice(-3).join(' ')} ______`); + if (this.options.debug) console.log('sample: ', sample); + if (this.options.debug) console.log('buffer: ', JSON.stringify(buffer, null, 4)); + } + + if (!sample) console.log('!!!!!!!', choices); + */ + + callback(sample); + }); + }); + + // return this.data.grams[choices[Math.floor(Math.random() * choices.length)]]; + } + + generate(start, count, callback) { + //let chain = new pos.Lexer().lex(start); + /*let tokenizer = new natural.RegexpTokenizer({pattern: / /}); + let chain = tokenizer.tokenize(start.toLowerCase());*/ + let chain = this.tokenize(start.toLowerCase()); + + this.addToChain(chain, callback, count); + + /* + this.getNext(chain); + for (let n of range(0, count)) chain.push(this.getNext(chain)); + return this.beautify(chain.join(' ));*/ + } + + addToChain(chain, callback, limit) { + if (this.options.debug) console.log('>', chain.length, limit); + + if (chain.length === limit) { + callback(this.beautify(chain.join(' '))); + } else { + this.getNext(chain, ngram => { + if (!ngram) { + callback(scope.beautify(chain.join(' '))); + return false; + } + chain.push(ngram); + this.addToChain(chain, callback, limit); + }); + } + } + + printEdges(nodes, title) { + nodes = []; + for (let id in nodes) _nodes.push({id, p: nodes[id]}); + + // Sort and slice + nodes.sort((a, b) => b.p - a.p); + console.log(`\n\u{1b}[32m ${title}\u{1b}[37m\u{1b}[40m`); + this.table(nodes, { + 'Word': 'word', + 'Probability': 'p' + }); + } + + table(array, cols) { + let table = new tbl({head: Object.keys(cols)}); + array.forEach(item => { + let row = []; + for (let k in cols) _nodes.push(item[cols[k]]); + table.push(row); + }); + console.log(table.toString()); + } +} + +module.exports = Markov; + +/* + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'world'}) +RETURN p + + +MATCH p=shortestPath((x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'clinton'})) +RETURN p + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) +RETURN p LIMIT 50 + + +MATCH (x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->()-[:TO|:CC|:BCC]->(person) +RETURN distinct person + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) +RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight + +*/ + +Array.prototype.flatten = function() { + const flat = []; + + this.forEach(item => { + if (Array.isArray(item)) { + flat.push([].concat.apply([], item)); + } else { + flat.push(item); + } + }); + + return [].concat.apply([], flat); +} + +function readFile(filename) { + return new Promise((resolve, reject) => { + fs.readFile(filename, 'utf8', (err, text) => { + if (err) { + reject(err); + } else { + resolve(text); + } + }); + }); +} \ No newline at end of file diff --git a/WeightedList.js b/WeightedList.js new file mode 100644 index 0000000..677540a --- /dev/null +++ b/WeightedList.js @@ -0,0 +1,239 @@ +/** +* js-weighted-list.js +* +* version 0.3 +* +* This file is licensed under the MIT License, please see MIT-LICENSE.txt for details. +* +* https://github.com/timgilbert/js-weighted-list is its home. +*/ + +class WeightedList { + constructor(initial) { + this.weights = {}; + this.data = {}; + this.length = 0; + this.hasData = false; + + initial = initial != undefined ? initial : []; + + if (Array.isArray(initial)) { + for (let i in initial) { + this.push(initial[i]); + } + } else { + throw new Error(`Invalid type of initial passed to WeightedList constructor. Type: ${initial.constructor.name}' (expected array or nothing)`); + } + } + + /** + * Add a single item to the list. The parameter passed in represents a single + * key, with a weight and optionally some data attached. + * + * @param {Array|Object} element Either a 2/3 element array of [key, weight, data] (data i optional), or an object with {key: k, weight: w, data: d} where data is optional. + */ + push(element) { + // Catch undefineds or empty arrays. + if (!element) throw new Error('element is not an array or object or is empty.'); + let key, weight, data; + + if (Array.isArray(element)) { + key = element[0]; + weight = element[1]; + data = element[2]; + + // e.g. wl.push([]) + if (!key || typeof key !== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); + // I suppose we could default to 1 here, but the API is already too forgiving. + if (!weight || typeof weight !== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); + } else if (typeof element === 'object') { + // We expect {key: 'zombies', weight: 10, data: {fast: true}} + key = element.key; + weight = element.weight; + data = element.data; + + if (!key || typeof key !== 'string') throw new Error('element.key is not defined or is not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('element.weight is not defined or is not a number.'); + } else { + // If it somehow got through the first catcher + throw new Error('element is not a supported type. Expected [key, weight] or {key: k, weight: w}') + } + + return this._pushValues(key, weight, data); + } + + /** + * Add an item to the WeightedList + * + * @access private + * @param {String} key The key under which the item is stored. + * @param {Number} weight The weight to assign to the item. + * @param {?Object} data Any optional data for the item. + */ + _pushValues(key, weight, data) { + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + if (this.weights[key]) throw new Error(`An item with the key '${key}' already exists.`); + if (weight <= 0) throw new Error(`weight must be higher than 0, got ${weight}`); + + this.weights[key] = weight; + if (data) { + this.hasData = true; + this.data[key] = data; + } + + this.length++; + } + + /** + * Add the given weight to the list item with the give key. This operation + * will silently create the key if it does not already exist + * + * @todo Might be nice to have a version of this that would throw an error on an unknown key. + * + * @param {String} key Key to add weight onto. + * @param {Number} weight Weight to add. + */ + addWeight(key, weight) { + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + + this.weights[key] += weight; + } + + /** + * Select `n` elements (without replacement). + * If `remove` is true, removes the elements from the list. + * + * @param {Number} [n=1] Amount of elements to get + * @param {Boolean} [remove=false] Remove the elements from the list or not. + * @returns {Array} + */ + peek(n=1, remove=false) { + if (!n || typeof n !== 'number') throw new Error('n is undefined or not a number.'); + if (typeof remove !== 'boolean') throw new Error('remove is undefined or not a boolean.'); + if (this.length - n < 0) throw new Error(`Stack underflow! Tried to retrieve ${n} element(s) from a list of ${this.length}`); + + let heap = this._buildWeightedHeap(); + let result = []; + + for (let i = 0; i < n; i++) { + let key = heap.pop(); + + result.push(this.hasData ? {key, data: this.data[key]} : key); + + if (remove) { + delete this.weights[key]; + delete this.data[key]; + this.length--; + } + } + + return result; + } + + /** + * Return the entire list in a random order. Does not edit the list. + * + * @returns {Array} + */ + suffle() { + return this.peek(this.length); + } + + /** + * Removes an item/number of items from the start of the list. + * + * @param {Number} [n=1] Amount of items to pop + */ + pop(n=1) { + return this.peek(n, true); + } + + /** + * Builds a WeightedHeap out of the data in the list. + */ + _buildWeightedHeap() { + let items = []; + + for (let key in this.weights) { + if (this.weights.hasOwnProperty(key)) items.push([key, this.weights[key]]); + } + + return new WeightedHeap(items); + } +} + +/** + * A JavaScript implementation of the algorithm described by Jasen Orendorff here: http://stackoverflow.com/a/2149533/87990 + * + * @prop {Number} weight + * @prop {Number} value + * @prop {Number} total + */ +class HeapNode { + constructor(weight, value, total) { + this.weight = weight; + this.value = value; + this.total = total; // Total weight of this node and its children. + } +} + +class WeightedHeap { + /** + * Construct a WeightedHeap + * + * Note, we're using a heap structure here for its tree properties, not as a + * classic binary heap. A node heap[i] has children at heap[i<<1] and at + * heap[(i<<1)+1]. Its parent is at h[i>>1]. Heap[0] is vacant. + */ + constructor(items) { + this.heap = [null]; // Math is easier to read if we index array from 1 + + // First put everything on the heap + for (let i in items) { + let weight = items[i][1]; + let value = items[i][0]; + this.heap.push(new HeadNode(weight, value, weight)); + } + + // Now go through the heap and add each node's weight to its parent + for (let i = this.heap.length - 1; i > 1; i--) this.heap[i >> 1].total += this.heap[i].total; + } + + pop() { + // Start with a random amount of gas + let gas = this.heap[i].total * Math.random(); + + // Start driving at the root node; + let i = 1; + + // While we have enough gas to keep going past i + while (gas > this.heap[i].weight) { + gas -= this.heap[i].weight; // Drive past i + i <<= 1; // Move to first Child + + if (gas > this.heap[i].total) { + gas -= this.heap[i].total; // Drive past firstchild and its descendants + i++; // Move on to second child + } + } + + // Out of gas - i is our selected node + let value = this.heap[i].value; + let selectedWeight = this.heap[i].weight; + + this.heap[i].weight = 0; // Make sure i isn't chosen again + + while (i > 0) { + this.heap[i].total -= selectedWeight // Remove the weight from its parent's total + i >>= 1; // Move to the next parent + } + + return value; + } +} + +// NB: another binary heap implementation is at http://eloquentjavascript.net/appendix2.html + +module.exports = WeightedList; \ No newline at end of file diff --git a/cmarkov.js b/cmarkov.js deleted file mode 100644 index cc9ac95..0000000 --- a/cmarkov.js +++ /dev/null @@ -1,855 +0,0 @@ -var _ = require('underscore'); -var fstool = require('fs-tool'); -var natural = require('natural'); -var pstack = require('pstack'); -var md5File = require('md5-file'); -var progressbar = require('progress'); -var NGrams = natural.NGrams; -var pos = require('pos'); -var tbl = require('cli-table'); -var seraph = require("seraph"); -var Tagger = require("natural").BrillPOSTagger; -var tokenizer = require("node-tokenizer"); -var wlist = require("./js-weighted-list"); - -var markov = function(options) { - this.options = _.extend({ - db: 'http://localhost:7474', - name: 'dev', - depth: [1,3], - weight: 1.2, - depthWeight: 2, - certainty: 1 - }, options); -} - -markov.prototype.tokenize = function(text) { - - var cues = ["?","!",".",",",":",";","\t","\n","\r"]; - //var cues = [":"]; - var tokens = text.split(' '); - - //console.log("tokens",tokens); - - _.each(cues, function(cue) { - tokens = _.map(tokens, function(token) { - var parts = token.split(cue); - var l = parts.length; - if (l==0) { - return token; - } - var buffer = []; - _.each(parts, function(part, n) { - buffer.push(part); - if (n(y:`ngrams-trump` {gram:{y}}) RETURN p LIMIT 50", {x: 'world', y:'trump'}, function(err, result) { - if (result) { - _.each(result, function(item) { - var nodes = _.map(item.nodes, function(node) { - return node.split('/').pop(); - }); - - var stack = new pstack(); - var grams = []; - - _.each(nodes, function(node) { - stack.add(function(done) { - scope.graph.read(node, function(err, node) { - if (node) { - grams.push(node.gram); - } - done(); - }); - }); - }); - - stack.start(function() { - console.log("----------------------\n",grams.join(' ')); - }); - - - //console.log("nodes",nodes); - }); - } - //console.log("result",result); - }); - }); - - - return this; -} - -markov.prototype.open = function(callback) { - var scope = this; - scope.graph = seraph({ - server: "http://localhost:7474", - user: "neo4j", - pass: "pwd" - }); - scope.graph.constraints.uniqueness.createIfNone('ngrams-'+scope.options.name, 'gram', function(err, constraint) { - scope.graph.constraints.uniqueness.createIfNone('pos-'+scope.options.name, 'gram', function(err, constraint) { - var base_folder = "./node_modules/natural/lib/natural/brill_pos_tagger/data/English"; - var rules_file = base_folder + "/tr_from_posjs.txt"; - var lexicon_file = base_folder + "/lexicon_from_posjs.json"; - var default_category = 'N'; - - var tagger; - tagger = new Tagger(lexicon_file, rules_file, default_category, function(error) { - if (error) { - console.log(error); - } else { - scope.tagger = tagger; - callback(); - } - }); - - }); - }); - - return this; -} - -markov.prototype.read = function(filename, callback) { - var scope = this; - - var start = new Date().getTime(); - - this.open(function() { - md5File(filename, function (error, sum) { - /*if (scope.data.docs[sum]) { - console.log("Training already done on that document, on "+scope.data.docs[sum]); - callback(scope.data.grams); - return false; - } - - scope.data.docs[sum] = new Date(); - */ - fstool.file.read(filename, function(text) { - - //text = new pos.Lexer().lex(text); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - text = tokenizer.tokenize(text);*/ - text = scope.tokenize(text); - //console.log("text", text); - - // Generate the n-grams - var grams = {}; - var unique = {}; - var nodeIndex = {}; - - - var stack_ngram = new pstack({ - progress: 'Reading the N-grams...', - reportInterval: 100, - batch: 10 - }); - - var stack_rel = new pstack({ - progress: 'Mapping...', - reportInterval: 100 - }); - - _.each(_.range(scope.options.depth[0], scope.options.depth[1]+1), function(depth) { - - console.log("Starting Depth "+depth); - - console.log("Generating the ngrams"); - - grams[depth] = NGrams.ngrams(text, depth); - - console.log("Stringification of "+grams[depth].length+" ngrams"); - var ngramObj = {}; - _.each(grams[depth], function(item) { - ngramObj[item.join('|')] = true; - }); - - var uniqueNgams = _.size(ngramObj); - - console.log("Graphing"); - - // Save the gram - _.each(ngramObj, function(v, gram) { - gram = gram.toLowerCase(); - stack_ngram.add(function(done) { - scope.graph.save({ - gram: gram, - depth: depth - }, 'ngrams-'+scope.options.name, function(err, node) { - if (err) { - // Read it - scope.graph.find({ - gram: gram, - depth: depth - }, false, 'ngrams-'+scope.options.name, function (err, response) { - if (response && response.length > 0) { - nodeIndex[gram] = response[0].id; - //console.log("Node:\t\t", "[found]"); - } - done(); - }); - return false; - } else { - //console.log("Node:\t\t", "[created]"); - nodeIndex[gram] = node.id; - done(); - } - }); - }); - }); - - // Process the gram graph - _.each(grams[depth], function(str, n) { - if (n==0) { - return false; - } - var current = grams[depth][n-1].join('|').toLowerCase(); - var next = grams[depth][n].slice(-1).join('').toLowerCase(); - - stack_rel.add(function(done) { - //console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); - - scope.graph.relationships(nodeIndex[current], 'out', 'then', function(err, relationships) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - return false; - } else { - if (relationships) { - // Look for the relationship - var relationship = _.find(relationships, function(rel) { - return rel.start == nodeIndex[current] && rel.end == nodeIndex[next]; - }); - - if (relationship) { - // Update the relationship weight - relationship.properties.weight += 1; - scope.graph.rel.update(relationship, function(err) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } - //console.log("Relationship:\t", "[updated]"); - done(); - }); - } else { - scope.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight:1, ngram:next, idx:nodeIndex[next]}, function(err, relationship) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } else { - //console.log("Relationship:\t", "[created]"); - } - done(); - }); - } - - } else { - //console.log("Relationship:\t", "[failed]"); - done(); - } - } - }); - - }); - }); - - }); - - stack_ngram.start(function() { - stack_rel.start(function() { - //console.log(">>>>>>>>> nodeIndex",nodeIndex); - - var end = new Date().getTime(); - - - console.log("==========================="); - console.log("== Training time: ",(end-start)/(1000*60)," =="); - console.log("==========================="); - - callback(grams); - }); - }); - }); - }); - }); -} - -markov.prototype.readPOS = function(filename, callback) { - var scope = this; - - var start = new Date().getTime(); - - this.open(function() { - md5File(filename, function (error, sum) { - fstool.file.read(filename, function(text) { - - //text = new pos.Lexer().lex(text); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - text = tokenizer.tokenize(text);*/ - text = scope.tokenize(text); - - text = _.map(scope.tagger.tag(text), function(item) { - return item[1]; - }); - - //console.log("text", text); - - // Generate the n-grams - var grams = {}; - var unique = {}; - var nodeIndex = {}; - - - var stack_ngram = new pstack({ - progress: 'Reading the N-grams...', - reportInterval: 100, - batch: 10 - }); - - var stack_rel = new pstack({ - progress: 'Mapping...', - reportInterval: 100 - }); - - - _.each(_.range(scope.options.depth[0], scope.options.depth[1]+1), function(depth) { - - console.log("Starting Depth "+depth); - - console.log("Generating the POS ngrams"); - - grams[depth] = NGrams.ngrams(text, depth); - - console.log("Stringification of "+grams[depth].length+" POS ngrams"); - var ngramObj = {}; - _.each(grams[depth], function(item) { - ngramObj[item.join('|')] = true; - }); - - var uniqueNgams = _.size(ngramObj); - - console.log("Graphing"); - - // Save the gram - _.each(ngramObj, function(v, gram) { - gram = gram.toLowerCase(); - stack_ngram.add(function(done) { - scope.graph.save({ - gram: gram, - depth: depth - }, 'pos-'+scope.options.name, function(err, node) { - if (err) { - // Read it - scope.graph.find({ - gram: gram, - depth: depth - }, false, 'pos-'+scope.options.name, function (err, response) { - if (response && response.length > 0) { - nodeIndex[gram] = response[0].id; - //console.log("Node:\t\t", "[found]"); - } - done(); - }); - return false; - } else { - //console.log("Node:\t\t", "[created]"); - nodeIndex[gram] = node.id; - done(); - } - }); - }); - }); - - // Process the gram graph - _.each(grams[depth], function(str, n) { - if (n==0) { - return false; - } - var current = grams[depth][n-1].join('|').toLowerCase(); - var next = grams[depth][n].slice(-1).join('').toLowerCase(); - - stack_rel.add(function(done) { - //console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); - - scope.graph.relationships(nodeIndex[current], 'out', 'then', function(err, relationships) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - return false; - } else { - if (relationships) { - // Look for the relationship - var relationship = _.find(relationships, function(rel) { - return rel.start == nodeIndex[current] && rel.end == nodeIndex[next]; - }); - - if (relationship) { - // Update the relationship weight - relationship.properties.weight += 1; - scope.graph.rel.update(relationship, function(err) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } - //console.log("Relationship:\t", "[updated]"); - done(); - }); - } else { - scope.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight:1, ngram:next, idx:nodeIndex[next]}, function(err, relationship) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } else { - //console.log("Relationship:\t", "[created]"); - } - done(); - }); - } - - } else { - //console.log("Relationship:\t", "[failed]"); - done(); - } - } - }); - - }); - }); - }); - - stack_ngram.start(function() { - stack_rel.start(function() { - //console.log(">>>>>>>>> nodeIndex",nodeIndex); - - var end = new Date().getTime(); - - - console.log("==========================="); - console.log("== POS Training time: ",(end-start)/(1000*60)," =="); - console.log("==========================="); - - callback(grams); - }); - }); - - }); - }); - }); -} - - - -markov.prototype.getNext = function(chain, callback) { - var scope = this; - var ngrams = {}; - var nodes = {}; - var posngrams = {}; - var posnodes = {}; - - //console.log("chain",chain); - - this.open(function() { - - var stack = new pstack(); - var buffer = false; - - - // Generate the option list, with weights - _.each(_.range(scope.options.depth[1], scope.options.depth[0]-1, -1), function(depth) { - stack.add(function(done) { - if (buffer && depth <= scope.options.lowpri) { - //console.log("skiped."); - done(); - return false; - } - - - // Generate the ngram to lookup (last N words in the chain) - ngrams[depth] = chain.slice(-depth).join('|').toLowerCase(); - - // Get the nodes - //var localNodes = scope.getNodes(ngrams[depth]); - scope.graph.find({ - gram: ngrams[depth] - }, false, 'ngrams-'+scope.options.name, function (err, response) { - //console.log("response",ngrams[depth], response); - if (response && response.length>0) { - scope.graph.relationships(response[0].id, 'out', 'then', function(err, relationships) { - - if (relationships.length==0) { - - } else { - buffer = true; - //console.log(">>>> Depth: ", depth, relationships.length); - } - - //buffer[depth] = relationships; - if (false && depth>=3 && relationships.length==1) { - // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... - //console.log("Going with ",relationships[0].properties.ngram); - nodes = {}; - nodes[relationships[0].properties.ngram] = 1; - done(); - } else { - //console.log("relationships",ngrams[depth], relationships); - _.each(relationships, function(relationship) { - - if (nodes[relationship.properties.ngram]) { - // No cumulation! - //nodes[gramId] += count;//*depth*scope.options.depthWeight; - } else { - if (scope.options.depthWeight) { - nodes[relationship.properties.ngram] = relationship.properties.weight*Math.pow(depth, scope.options.depthWeight); - } else { - nodes[relationship.properties.ngram] = relationship.properties.weight; - } - } - }); - if (_.size(nodes)>0) { - buffer = true; - } - done(); - } - - - }); - } else { - //console.log("ngram not found: ", ngrams[depth]); - done(); - } - - }); - }); - }); - - if (scope.options.pos) { - // For each node option, build the POS - stack.add(function(done) { - - var substack = new pstack(); - - // Check the POS for the current chain - //console.log(">>",chain); - var tags = _.map(scope.tagger.tag(chain.slice(-20)), function(item) { - return item[1]; - }); - - - // Generate the option list, with weights - _.each(_.range(scope.options.depth[1], scope.options.depth[0]-1, -1), function(depth) { - substack.add(function(subdone) { - // Generate the ngram to lookup (last N words in the chain) - posngrams[depth] = tags.slice(-depth).join('|').toLowerCase(); - //console.log(">>>> Depth: ",depth); - // Get the nodes - //var localNodes = scope.getNodes(ngrams[depth]); - scope.graph.find({ - gram: posngrams[depth] - }, false, 'pos-'+scope.options.name, function (err, response) { - //console.log("response",posngrams[depth], response); - if (response && response.length>0) { - scope.graph.relationships(response[0].id, 'out', 'then', function(err, relationships) { - - - if (depth>=3 && relationships.length==1) { - // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... - //console.log("Going with ",relationships[0].properties.ngram); - posnodes = {}; - posnodes[relationships[0].properties.ngram] = 1; - subdone(); - } else { - //console.log("relationships",posngrams[depth], relationships); - _.each(relationships, function(relationship) { - - if (posnodes[relationship.properties.ngram]) { - // No cumulation! - //nodes[gramId] += count;//*depth*scope.options.depthWeight; - } else { - if (scope.options.depthWeight) { - posnodes[relationship.properties.ngram] = relationship.properties.weight*Math.pow(depth, scope.options.depthWeight); - } else { - posnodes[relationship.properties.ngram] = relationship.properties.weight; - } - } - }); - subdone(); - } - }); - } else { - //console.log("ngram not found: ", ngrams[depth]); - subdone(); - } - - }); - }); - }); - - // Check the pos structure of each node option - _.each(nodes, function(w,k) { - substack.add(function(subdone) { - // Get the POS tag - var text = chain.slice(0); - text.push(k); - //console.log("text",text); - var tags = _.map(scope.tagger.tag(text.slice(-10)), function(item) { - return item[1]; - }); - - var tag = tags.slice(-1)[0].toLowerCase(); - - //console.log("["+tag+"] ",chain.join(' '),'->',k, posnodes[tag]); - if (posnodes[tag]) { - nodes[k] += posnodes[tag]; - nodes[k] /= 2; - } - subdone(); - }); - }); - - - - substack.start(function() { - //console.log("nodes",nodes); - //console.log("posnodes",posnodes); - //console.log("-------------------------"); - done(); - }); - - }); - } - - - stack.start(function() { - - - if (scope.options.debug) { - scope.print_edges(nodes, 'Count'); - console.log(chain.join('|')); - } - - // Calculate the total - var total = 0; - _.each(nodes, function(count, gramId) { - total += count; - }); - - var minP = 1; - - // Calculate the probabilities - _.each(nodes, function(count, gramId) { - nodes[gramId] = count/total; - if (nodes[gramId] < minP) { - minP = nodes[gramId]; - } - }); - - if (scope.options.certainty) { - if (minP < scope.options.certainty) { - // The probabilities are way too low to filter. We need to remove the least probable options - var nodeArray = _.map(nodes, function(p, gramId) { - return { - id: gramId, - p: p - }; - }); - // Sort and slice - nodeArray.sort(function(a, b) { - return b.p-a.p; - }); - nodeArray = nodeArray.slice(0, 100/(scope.options.certainty*100)); - - // Convert back to an object - nodes = {}; - _.each(nodeArray, function(item) { - nodes[item.id] = item.p; - }); - //console.log("nodes",nodes); - } else { - // Remove the lowest probabilities - nodes = _.omit(nodes, function(p, gramId) { - return p < scope.options.certainty - }); - } - } - - // Recalculate the total - var total = 0; - _.each(nodes, function(p, gramId) { - total += p; - }); - - // Recalculate the probabilities - _.each(nodes, function(p, gramId) { - nodes[gramId] = p/total; - }); - - - var rn = Math.random(); - - var _nodes = _.map(nodes, function(v,k) {return [k,v]}); - _nodes.sort(function(a,b) { - return a[1]-b[1]; - }); - - var wl = new wlist(_nodes); - - - var sample = wl.peek()[0]; - /* - if (sample=='.') { - scope.print_edges(nodes, 'Count'); - } - */ - /* - var choices = []; - _.each(nodes, function(p, gramId) { - var count = p*100; - count = Math.ceil(Math.pow(count, scope.options.weight)); - _.each(_.range(0,count), function(n) { - choices.push(gramId); - }); - }); - - //console.log("choices",choices); - var sample = _.sample(choices); - if (chain[chain.length-1]=='\'') { - //scope.print_edges(nodes, 'Count'); - //scope.print_edges(nodes, 'Probabilities: \033[37m\033[44m'+chain.slice(-3).join(' ')+' ______'); - //console.log("sample: ",sample); - //console.log("buffer:",JSON.stringify(buffer,null,4)); - } - - if (!sample) { - //console.log("!!!!!!!",choices); - } - */ - callback(sample); - }); - - }); - - //return this.data.grams[_.sample(choices)]; -} - -markov.prototype.generate = function(start, count, callback) { - var scope = this; - //var chain = new pos.Lexer().lex(start); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - var chain = tokenizer.tokenize(start.toLowerCase());*/ - chain = scope.tokenize(start.toLowerCase()); - - this.addToChain(chain, callback, count); - /* - this.getNext(chain) - _.each(_.range(0,count), function(n) { - chain.push(scope.getNext(chain)); - }); - return scope.beautify(chain.join(' '));*/ -} -markov.prototype.addToChain = function(chain, callback, limit) { - var scope = this; - //console.log(">",chain.length,limit); - if (chain.length==limit) { - callback(scope.beautify(chain.join(' '))); - } else { - this.getNext(chain, function(ngram) { - if (!ngram) { - callback(scope.beautify(chain.join(' '))); - return false; - } - chain.push(ngram); - scope.addToChain(chain, callback, limit); - }); - } -} - -markov.prototype.print_edges = function(nodes, title) { - var scope = this; - nodes = _.map(nodes, function(p, gramId) { - return { - word: gramId, - p: p - }; - }); - // Sort and slice - nodes.sort(function(a, b) { - return b.p-a.p; - }); - console.log("\n\033[32m "+title+"\033[37m\033[40m"); - scope.table(nodes, { - 'Word': 'word', - 'Probability': 'p' - }); -} -markov.prototype.table = function(array, cols) { - var scope = this; - var table = new tbl({ - head: _.keys(cols) - }); - _.each(array, function(item) { - var row = _.map(cols, function(v,k) { - return item[v]; - }); - table.push(row); - }); - console.log(table.toString()); -} - -module.exports = markov; - - -/* - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'world'}) -RETURN p - - -MATCH p=shortestPath((x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'clinton'})) -RETURN p - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) -RETURN p LIMIT 50 - - -MATCH (x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->()-[:TO|:CC|:BCC]->(person) -RETURN distinct person - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) -RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight - -*/ \ No newline at end of file diff --git a/js-weighted-list.js b/js-weighted-list.js deleted file mode 100644 index 2de1d72..0000000 --- a/js-weighted-list.js +++ /dev/null @@ -1,245 +0,0 @@ -/** -* js-weighted-list.js -* -* version 0.2 -* -* This file is licensed under the MIT License, please see MIT-LICENSE.txt for details. -* -* https://github.com/timgilbert/js-weighted-list is its home. -*/ - -var WeightedList = (function() { - - function _WeightedList(initial) { - this.weights = {}; - this.data = {}; - this.length = 0; - this.hasData = false; - - initial = typeof initial !== 'undefined' ? initial : []; - - if (Array.isArray(initial)) { - for (var i = 0; i < initial.length; i++) { - //var item = initial[i]; - //this.push(item[0], item[1], item[2]); - this.push(initial[i]); - } - } else { - throw new Error('Unknown object "' + initial.toString() + '" passed to ' + - 'WeightedList constructor! (Expected array or nothing)'); - } - } - - _WeightedList.prototype = { - /** - * Add a single item to the list. The parameter passed in represents a single - * key, with a weight and optionally some data attached. - * - * The parameter to this function can either be a 2-3 element array of - * [k, w, d] for key, weight and data (data is optional) or an object with the - * values {'key': k, 'weight': w, 'data': d} where d is optional. - */ - push: function(element) { - var key, weight, data; - - if (Array.isArray(element)) { - key = element[0], weight = element[1], data = element[2]; - if (typeof key === 'undefined') { - // Eg, wl.push([]) - throw new Error('In WeightedList.push([ ... ]), need at least two elements'); - } else if (typeof weight === 'undefined') { - // I suppose we could default to 1 here, but the API is already too forgiving - throw new Error('In array passed to WeightedList.push([ ... ]), second ' + - 'element is undefined!'); - } - } else if (typeof element === 'object') { - // We expect {"key": "zombies", "weight": 10, "data": {"fast": true}} - key = element.key, weight = element.weight, data = element.data; - if (typeof key === 'undefined') { - throw new Error("In WeightedList.push({ ... }), no {'key': 'xyzzy'} pair found"); - } else if (typeof weight === 'undefined') { - // I suppose we could default to 1 here, but the API is already too forgiving - throw new Error('In array passed to WeightedList.push({ ... }), no ' + - "{'weight': 42} pair found"); - } - } else { - // else what the heck were you trying to give me? - throw new Error('WeightedList.push() passed unknown type "' + typeof element + - '", expected [key, weight] or {"key": k, "weight": w}'); - } - return this._push_values(key, weight, data); - - }, - /** - * Add an item to the list - * @access private - * @param {String} key the key under which this item is stored - * @param {number} weight the weight to assign to this key - * @param {?Object} data any optional data associated wth this key - */ - _push_values: function(key, weight, data) { - //console.debug('k:', key, 'w:', weight, 'd:', data); - - if (this.weights[key]) { - throw new Error(''); - } - if (typeof weight !== typeof 1) { - throw new Error('Weight must be numeric (got ' + weight.toString() + ')'); - } - if (weight <= 0) { - throw new Error('Weight must be >= 0 (got ' + weight + ')'); - } - - this.weights[key] = weight; - - if (typeof data !== 'undefined') { - this.hasData = true; - this.data[key] = data; - } - this.length++; - }, - - /** - * Add the given weight to the list item with the given key. Note that if - * the key does not already exist, this operation will silently create it. - * - * @todo might be nice to have a version of this that would throw an error - * on an unknown key. - */ - addWeight: function(key, weight) { - this.weights[key] += weight; - }, - - /** - * Select n random elements (without replacement), default 1. - * If andRemove is true (default false), remove the elements - * from the list. (This is what the pop() method does.) - */ - peek: function(n, andRemove) { - if (typeof n === 'undefined') { - n = 1; - } - andRemove = !!andRemove; - - if (this.length - n < 0) { - throw new Error('Stack underflow! Tried to retrieve ' + n + - ' element' + (n === 1 ? '' : 's') + - ' from a list of ' + this.length); - } - - var heap = this._buildWeightedHeap(); - //console.debug('heap:', heap); - var result = []; - - for (var i = 0; i < n; i++) { - var key = heap.pop(); - //console.debug('k:', key); - if (this.hasData) { - result.push({key: key, data: this.data[key]}); - } else { - result.push(key); - } - if (andRemove) { - delete this.weights[key]; - delete this.data[key]; - this.length--; - } - } - return result; - }, - - /** - * Return the entire list in a random order (note that this does not mutate the list) - */ - shuffle: function() { - return this.peek(this.length); - }, - - /** - * - */ - pop: function(n) { - return this.peek(n, true); - }, - - /** - * Build a WeightedHeap instance based on the data we've got - */ - _buildWeightedHeap: function() { - var items = []; - for (var key in this.weights) if (this.weights.hasOwnProperty(key)) { - items.push([key, this.weights[key]]); - } - //console.log('items',items); - return new _WeightedHeap(items); - } - }; - - /** - * This is a javascript implementation of the algorithm described by - * Jason Orendorff here: http://stackoverflow.com/a/2149533/87990 - */ - function _HeapNode(weight, value, total) { - this.weight = weight; - this.value = value; - this.total = total; // Total weight of this node and its children - } - /** - * Note, we're using a heap structure here for its tree properties, not as a - * classic binary heap. A node heap[i] has children at heap[i<<1] and at - * heap[(i<<1)+1]. Its parent is at h[i>>1]. Heap[0] is vacant. - */ - function _WeightedHeap(items) { - this.heap = [null]; // Math is easier to read if we index array from 1 - - // First put everything on the heap - for (var i = 0; i < items.length; i++) { - var weight = items[i][1]; - var value = items[i][0]; - this.heap.push(new _HeapNode(weight, value, weight)); - } - // Now go through the heap and add each node's weight to its parent - for (i = this.heap.length - 1; i > 1; i--) { - this.heap[i>>1].total += this.heap[i].total; - } - //console.debug('_Wh heap', this.heap); - } - - _WeightedHeap.prototype = { - pop: function() { - // Start with a random amount of gas - var gas = this.heap[1].total * Math.random(); - - // Start driving at the root node - var i = 1; - - // While we have enough gas to keep going past i: - while (gas > this.heap[i].weight) { - gas -= this.heap[i].weight; // Drive past i - i <<= 1; // Move to first child - if (gas > this.heap[i].total) { - gas -= this.heap[i].total; // Drive past first child and its descendants - i++; // Move on to second child - } - } - // Out of gas - i is our selected node. - var value = this.heap[i].value; - var selectedWeight = this.heap[i].weight; - - this.heap[i].weight = 0; // Make sure i isn't chosen again - while (i > 0) { - // Remove the weight from its parent's total - this.heap[i].total -= selectedWeight; - i >>= 1; // Move to the next parent - } - return value; - } - }; - - // NB: another binary heap implementation is at - // http://eloquentjavascript.net/appendix2.html - - return _WeightedList; -})(); - -module.exports = WeightedList; \ No newline at end of file diff --git a/package.json b/package.json index 8d7e4fe..19c364c 100644 --- a/package.json +++ b/package.json @@ -23,17 +23,15 @@ "node": "*" }, "dependencies": { - "argcli": "latest", "cli-table": "^0.3.1", - "fs-tool": "latest", "md5-file": "^3.1.0", "natural": "latest", "node-tokenizer": "0.0.0", "pos": "^0.3.0", - "progress": "^1.1.8", "pstack": "latest", "seraph": "^0.14.4", - "underscore": "latest" + "python-range": "0.3.0", + "bluebird": "latest" }, "readmeFilename": "readme.md", "bugs": { diff --git a/test.js b/test.js index b14ac53..8f714c3 100644 --- a/test.js +++ b/test.js @@ -1,36 +1,21 @@ -var cmarkov = require('./cmarkov'); +const Markov = require('./Markov'); -var bot = new cmarkov({ - name: 'trump', - depth: [1,5], - lowpri: 3, - weight: 1, - depthWeight: 1, - certainty: 0.1, - pos: true, - debug: false +const readMe = './training-data/bible.txt'; +const bot = new Markov({ + name: 'bible', + depth: [1, 5], + lowpri: 3, + weight: 1, + depthWeight: 1, + certainty: 0.1, + pos: true, + debug: true }); -//bot.test(); - - -bot.read("training-data/trump.txt", function() { - bot.readPOS("training-data/trump.txt", function() { - bot.generate("I would like to talk today about", 200, function(str) { - console.log(str); - }); - }); -}); - - -/* -var start = new Date().getTime(); -bot.generate("I would like to talk today about", 200, function(str) { - //console.trace(); - console.log(str); - - var end = new Date().getTime(); - var total = (end-start)/(1000*60); - console.log("Time: ",total); -}); -*/ \ No newline at end of file +bot.read(readMe, () => { + console.log('read file'); + bot.readPOS(readMe, () => { + console.log('read pos'); + bot.generate('god', 200, console.log); + }); +}); \ No newline at end of file From 8fa6bd9f2baa9d7aca5556cc1a7c3822ae6f5912 Mon Sep 17 00:00:00 2001 From: sr229 Date: Fri, 28 Apr 2017 18:30:56 +0800 Subject: [PATCH 02/12] add linters --- .eslintrc.json | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++ .tern-project | 6 ++++ Markov.js | 2 +- package.json | 8 ++++- 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 .eslintrc.json create mode 100644 .tern-project diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..48dd7a4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,84 @@ +{ + "env": { + "es6": true, + "node": true + }, + + "extends": "eslint:recommended", + "parserOptions": { + "sourceType": "module", + "ecmaVersion": 6, + "ecmaFeatures": { + "jsx": false + } + }, + + "ignore": [ + "*.min.js" + ], + + "rules": { + "no-console": 0, + "global-require":"off", + + "quotes": ["warn", "single", {"avoidEscape": true, "allowTemplateLiterals": true}], + "semi": "warn", + "indent": "error", + "no-unsafe-negation": "error", + "eqeqeq": ["warn", "always", {"null": "ignore"}], + "no-alert": "error", + "no-useless-return": "error", + "array-bracket-spacing": "error", + "brace-style": ["error", "1tbs", {"allowSingleLine": true}], + "comma-dangle": "error", + "comma-spacing": "error", + "comma-style": "error", + "computed-property-spacing": "error", + "func-call-spacing": "error", + "key-spacing": "error", + "keyword-spacing": ["error", {"overrides": {"catch": {"after": false}}}], + "new-parens": "error", + "no-array-constructor": "warn", + "no-new-object": "warn", + "no-whitespace-before-property": "error", + "object-curly-spacing": "error", + "one-var-declaration-per-line": "error", + "quote-props": ["error", "as-needed"], + "semi-spacing": "error", + "space-before-blocks": ["error", "always"], + "space-before-function-paren": ["error", "never"], + "space-in-parens": "error", + "space-unary-ops": ["error", {"words": true, "nonwords": false}], + "arrow-parens": ["warn", "as-needed"], + "arrow-spacing": "error", + "no-useless-rename": "error", + "prefer-arrow-callback": "error", + "template-curly-spacing": "error", + "valid-jsdoc": [ + "warn", + { + "prefer": { + "return": "returns", + "arg": "param", + "argument": "param", + "augments": "extends", + "property": "prop" + }, + + "preferType": { + "object": "Object", + "string": "String", + "number": "Number", + "boolean": "Boolean", + "rromise": "Promise", + "array": "Array", + "undefined":" Undefined", + "null": "Null", + "function": "Function" + }, + + "requireReturn": false + } + ] + } +} diff --git a/.tern-project b/.tern-project new file mode 100644 index 0000000..dcf878b --- /dev/null +++ b/.tern-project @@ -0,0 +1,6 @@ +{ + "plugins": { + "node": {}, + }, + "ecmaVersion": 6, +} \ No newline at end of file diff --git a/Markov.js b/Markov.js index da12fab..508d206 100644 --- a/Markov.js +++ b/Markov.js @@ -567,7 +567,7 @@ class Markov { substack.start(() => { if (this.options.debug) console.log('nodes', nodes); if (this.options.debug) console.log('posnodes', posnodes); - if (this.options.debug) consolee.log('-------------------------'); + if (this.options.debug) console.log('-------------------------'); done(); }); }); diff --git a/package.json b/package.json index 19c364c..b48628b 100644 --- a/package.json +++ b/package.json @@ -37,5 +37,11 @@ "bugs": { "url": "https://github.com/26medias/context-aware-markov-chains/issues" }, - "scripts": {} + "scripts": {}, + "devDependencies": { + "babel-eslint": "^7.2.3", + "eslint": "^3.19.0", + "eslint-config-defaults": "^9.0.0", + "eslint-plugin-react": "^6.10.3" + } } From 898231723d9339702df59c90f7d84cec7ffc3b03 Mon Sep 17 00:00:00 2001 From: sr229 Date: Fri, 28 Apr 2017 18:32:56 +0800 Subject: [PATCH 03/12] ESLint --- Markov.js | 60 ++++++++++++++++++++++++------------------------- WeightedList.js | 22 +++++++++--------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Markov.js b/Markov.js index 508d206..d71f7a2 100644 --- a/Markov.js +++ b/Markov.js @@ -4,10 +4,10 @@ const md5File = require('md5-file'); const NGrams = natural.NGrams; const pos = require('pos'); const tbl = require('cli-table'); -const seraph = require("seraph"); +const seraph = require('seraph'); const Tagger = natural.BrillPOSTagger; -const tokenizer = require("node-tokenizer"); -const wlist = require("./WeightedList"); +const tokenizer = require('node-tokenizer'); +const wlist = require('./WeightedList'); const range = require('python-range'); const fs = require('fs'); const Promise = require('bluebird'); @@ -19,7 +19,7 @@ class Markov { name: 'dev', user: 'neo4j', pass: 'neo4j', - depth: [1,3], + depth: [1, 3], weight: 1.2, depthWeight: 2, certainty: 1 @@ -37,7 +37,7 @@ class Markov { let parts = token.split(cue); let l = parts.length; - if (l === 1) return token; + if (l ==== 1) return token; let buffer = []; parts.forEach((part, i) => { @@ -51,7 +51,7 @@ class Markov { }); tokens = tokens.flatten(); - tokens = tokens.filter(item => item !== ''); + tokens = tokens.filter(item => item !=== ''); if (this.options.debug) console.log('tokens', tokens); }); @@ -95,7 +95,7 @@ class Markov { }); stack.start(() => { - console.log("----------------------\n",grams.join(' ')); + console.log('----------------------\n', grams.join(' ')); }); if (this.options.debug) console.log('nodes', nodes); }); @@ -170,7 +170,7 @@ class Markov { }); let stackRel = new pstack({ - progress: 'Mapping...', + progress: 'Mapping...', reportInterval: 100 }); @@ -202,7 +202,7 @@ class Markov { this.graph.find({gram, depth}, false, `ngrams-${this.options.name}`, (err, response) => { if (err) throw err; if (response && response.length > 0) { - nodeIndex[gram] = response[0].idl + nodeIndex[gram] = response[0].idl; // console.log('Node:\t\t', '[found]');; } @@ -220,7 +220,7 @@ class Markov { // Process the gram graph for (let n in grams[depth]) { - if (n === 0) return false; + if (n ==== 0) return false; let current = grams[depth][n - 1].join('|').toLowerCase(); let next = grams[depth][n].slice(-1).join('').toLowerCase(); @@ -234,7 +234,7 @@ class Markov { } else { if (relationships) { // Look for the relationship - let relationship = relationships.find(rel => rel.start == nodeIndex[current] && rel.send == nodeIndex[next]); + let relationship = relationships.find(rel => rel.start ==== nodeIndex[current] && rel.send ==== nodeIndex[next]); if (relationship) { // Update the relationship weight @@ -242,7 +242,7 @@ class Markov { this.graph.rel.update(relationship, err => { if (err) // console.log('Relationship:\t', '[failed]'); // console.log('Relationship:\t', '[updated]'); - done(); + done(); }); } else { this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngrame: next, idx: nodeIndex[next]}, err => { @@ -267,9 +267,9 @@ class Markov { let end = new Date().getTime(); - console.log('==========================='); - console.log(`== Training time: ${(end - start)/(1000 * 60)} ==`); - console.log('==========================='); + console.log('========================================'); + console.log(`=== Training time: ${(end - start)/(1000 * 60)} ===`); + console.log('========================================'); callback(grams); }); @@ -356,13 +356,13 @@ class Markov { // Process the gram graph for (let n in grams[depth]) { - if (n === 0) return false; + if (n ==== 0) return false; let current = grams[depth][n - 1].join('|').toLowerCase(); let next = grams[depth][n].slice(-1).join('').toLowerCase(); stackRel.add(done => { - if (this.options.debug) console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); + if (this.options.debug) console.log('>> ', current, ' >>> ', next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); this.graph.relationships(nodeIndex[current], 'out', 'then', (err, relationships) => { if (err) { if (this.options.debug) console.log('Relationship:\t', '[failed]'); @@ -370,7 +370,7 @@ class Markov { } else { if (relationships) { // Look for the relationship - let relationship = relationships.find(rel => rel.start == nodeIndex[current] && rel.end == nodeIndex[next]); + let relationship = relationships.find(rel => rel.start ==== nodeIndex[current] && rel.end === nodeIndex[next]); if (relationship) { // Update the relationship weight @@ -382,7 +382,7 @@ class Markov { done(); }); } else { - this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngram: next, idx: nodeIndex[next]}, (err) => { + this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngram: next, idx: nodeIndex[next]}, err => { if (err); if (this.options.debug) console.log('Relationship:\t', '[failed]'); else; if (this.options.debug) console.log('Relationship:\t', '[updated]'); done(); @@ -404,9 +404,9 @@ class Markov { let end = new Date().getTime(); - console.log('==========================='); - console.log(`== POS Training time: ${(end - start)/(1000*60)} ==`); - console.log('==========================='); + console.log('========================================'); + console.log(`=== POS Training time: ${(end - start)/(1000*60)} ===`); + console.log('========================================'); }); }); }); @@ -452,7 +452,7 @@ class Markov { } // buffer[depth] = relationships; - if (false && depth >= 3 && relationships.length === 1) { + if (false && depth >= 3 && relationships.length ==== 1) { // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); nodes = {}; @@ -506,7 +506,7 @@ class Markov { if (err) throw err; if (response && response.length > 0) { this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { - if (depth >= 3 && relationships.length === 1) { + if (depth >= 3 && relationships.length ==== 1) { // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); @@ -635,7 +635,7 @@ class Markov { let sample = wl.peek()[0]; - // if (sample === '.') this.printEdges(nodes, 'Count'); + // if (sample ==== '.') this.printEdges(nodes, 'Count'); /* let choices = []; for (let gramId in nodes) { @@ -648,7 +648,7 @@ class Markov { if (this.options.debug) console.log('choices', choices); let sample = choices[Math.floor(Math.random() * choices.length)]; - if (chain[chain.length - 1] === "'") { + if (chain[chain.length - 1] ==== "'") { // this.printEdges(nodes, 'Count'); // this.printEdges(nodes, `Probabilities: \033[37m\033[44m${chain.slice(-3).join(' ')} ______`); if (this.options.debug) console.log('sample: ', sample); @@ -682,7 +682,7 @@ class Markov { addToChain(chain, callback, limit) { if (this.options.debug) console.log('>', chain.length, limit); - if (chain.length === limit) { + if (chain.length ==== limit) { callback(this.beautify(chain.join(' '))); } else { this.getNext(chain, ngram => { @@ -704,8 +704,8 @@ class Markov { nodes.sort((a, b) => b.p - a.p); console.log(`\n\u{1b}[32m ${title}\u{1b}[37m\u{1b}[40m`); this.table(nodes, { - 'Word': 'word', - 'Probability': 'p' + Word: 'word', + Probability: 'p' }); } @@ -755,7 +755,7 @@ Array.prototype.flatten = function() { }); return [].concat.apply([], flat); -} +}; function readFile(filename) { return new Promise((resolve, reject) => { diff --git a/WeightedList.js b/WeightedList.js index 677540a..9287d3f 100644 --- a/WeightedList.js +++ b/WeightedList.js @@ -43,17 +43,17 @@ class WeightedList { data = element[2]; // e.g. wl.push([]) - if (!key || typeof key !== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); + if (!key || typeof key !=== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); // I suppose we could default to 1 here, but the API is already too forgiving. - if (!weight || typeof weight !== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); - } else if (typeof element === 'object') { + if (!weight || typeof weight !=== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); + } else if (typeof element ==== 'object') { // We expect {key: 'zombies', weight: 10, data: {fast: true}} key = element.key; weight = element.weight; data = element.data; - if (!key || typeof key !== 'string') throw new Error('element.key is not defined or is not a string.'); - if (!weight || typeof weight !== 'number') throw new Error('element.weight is not defined or is not a number.'); + if (!key || typeof key !=== 'string') throw new Error('element.key is not defined or is not a string.'); + if (!weight || typeof weight !=== 'number') throw new Error('element.weight is not defined or is not a number.'); } else { // If it somehow got through the first catcher throw new Error('element is not a supported type. Expected [key, weight] or {key: k, weight: w}') @@ -71,8 +71,8 @@ class WeightedList { * @param {?Object} data Any optional data for the item. */ _pushValues(key, weight, data) { - if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); - if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + if (!key || typeof key !=== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !=== 'number') throw new Error('weight is undefined or not a number.'); if (this.weights[key]) throw new Error(`An item with the key '${key}' already exists.`); if (weight <= 0) throw new Error(`weight must be higher than 0, got ${weight}`); @@ -95,8 +95,8 @@ class WeightedList { * @param {Number} weight Weight to add. */ addWeight(key, weight) { - if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); - if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + if (!key || typeof key !=== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !=== 'number') throw new Error('weight is undefined or not a number.'); this.weights[key] += weight; } @@ -110,8 +110,8 @@ class WeightedList { * @returns {Array} */ peek(n=1, remove=false) { - if (!n || typeof n !== 'number') throw new Error('n is undefined or not a number.'); - if (typeof remove !== 'boolean') throw new Error('remove is undefined or not a boolean.'); + if (!n || typeof n !=== 'number') throw new Error('n is undefined or not a number.'); + if (typeof remove !=== 'boolean') throw new Error('remove is undefined or not a boolean.'); if (this.length - n < 0) throw new Error(`Stack underflow! Tried to retrieve ${n} element(s) from a list of ${this.length}`); let heap = this._buildWeightedHeap(); From c2f9d3950d24e037f75220713f8fba5cd622edef Mon Sep 17 00:00:00 2001 From: sr229 Date: Fri, 28 Apr 2017 18:41:20 +0800 Subject: [PATCH 04/12] pet me ovy --- WeightedList.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/WeightedList.js b/WeightedList.js index 9287d3f..08a5790 100644 --- a/WeightedList.js +++ b/WeightedList.js @@ -43,17 +43,17 @@ class WeightedList { data = element[2]; // e.g. wl.push([]) - if (!key || typeof key !=== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); + if (!key || typeof key !== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); // I suppose we could default to 1 here, but the API is already too forgiving. - if (!weight || typeof weight !=== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); - } else if (typeof element ==== 'object') { + if (!weight || typeof weight !== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); + } else if (typeof element === 'object') { // We expect {key: 'zombies', weight: 10, data: {fast: true}} key = element.key; weight = element.weight; data = element.data; - if (!key || typeof key !=== 'string') throw new Error('element.key is not defined or is not a string.'); - if (!weight || typeof weight !=== 'number') throw new Error('element.weight is not defined or is not a number.'); + if (!key || typeof key !== 'string') throw new Error('element.key is not defined or is not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('element.weight is not defined or is not a number.'); } else { // If it somehow got through the first catcher throw new Error('element is not a supported type. Expected [key, weight] or {key: k, weight: w}') @@ -71,8 +71,8 @@ class WeightedList { * @param {?Object} data Any optional data for the item. */ _pushValues(key, weight, data) { - if (!key || typeof key !=== 'string') throw new Error('key is undefined or not a string.'); - if (!weight || typeof weight !=== 'number') throw new Error('weight is undefined or not a number.'); + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); if (this.weights[key]) throw new Error(`An item with the key '${key}' already exists.`); if (weight <= 0) throw new Error(`weight must be higher than 0, got ${weight}`); @@ -95,8 +95,8 @@ class WeightedList { * @param {Number} weight Weight to add. */ addWeight(key, weight) { - if (!key || typeof key !=== 'string') throw new Error('key is undefined or not a string.'); - if (!weight || typeof weight !=== 'number') throw new Error('weight is undefined or not a number.'); + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); this.weights[key] += weight; } @@ -110,8 +110,8 @@ class WeightedList { * @returns {Array} */ peek(n=1, remove=false) { - if (!n || typeof n !=== 'number') throw new Error('n is undefined or not a number.'); - if (typeof remove !=== 'boolean') throw new Error('remove is undefined or not a boolean.'); + if (!n || typeof n !== 'number') throw new Error('n is undefined or not a number.'); + if (typeof remove !== 'boolean') throw new Error('remove is undefined or not a boolean.'); if (this.length - n < 0) throw new Error(`Stack underflow! Tried to retrieve ${n} element(s) from a list of ${this.length}`); let heap = this._buildWeightedHeap(); @@ -226,7 +226,7 @@ class WeightedHeap { this.heap[i].weight = 0; // Make sure i isn't chosen again while (i > 0) { - this.heap[i].total -= selectedWeight // Remove the weight from its parent's total + this.heap[i].total -= selectedWeight; // Remove the weight from its parent's total i >>= 1; // Move to the next parent } From f773f86d4fde917caea1b398b5e79c26b1cbb658 Mon Sep 17 00:00:00 2001 From: Capuccino Date: Sat, 29 Jul 2017 05:28:03 +0000 Subject: [PATCH 05/12] ovy fix shitcode you heck --- Markov.js | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Markov.js b/Markov.js index 7553c2c..4efba6a 100644 --- a/Markov.js +++ b/Markov.js @@ -37,7 +37,7 @@ class Markov { let parts = token.split(cue); let l = parts.length; - if (l ==== 1) return token; + if (l === 1) return token; let buffer = []; parts.forEach((part, i) => { @@ -51,7 +51,7 @@ class Markov { }); tokens = tokens.flatten(); - tokens = tokens.filter(item => item !=== ''); + tokens = tokens.filter(item => item !== ''); if (this.options.debug) console.log('tokens', tokens); }); @@ -64,15 +64,9 @@ class Markov { console.log(this.beautify(text2)); } - @@ -852,4 +853,4 @@ RETURN distinct person - MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) - RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight - - -*/ - +*/ - - - console.log(this.beautify(text2)); + /* MATCH p =(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) + RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight + console.log(this.beautify(text2)); */ } beautify(text) { From 5d1109f54f67f3ab014bcfb92eedfe9b64c634de Mon Sep 17 00:00:00 2001 From: Capuccino Date: Sat, 29 Jul 2017 13:30:46 +0800 Subject: [PATCH 06/12] nya --- Markov.js | 1 - 1 file changed, 1 deletion(-) diff --git a/Markov.js b/Markov.js index 4efba6a..b67b3d1 100644 --- a/Markov.js +++ b/Markov.js @@ -67,7 +67,6 @@ class Markov { /* MATCH p =(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight console.log(this.beautify(text2)); */ - } beautify(text) { text = text.replace(/\s(\.|,|:|;|!|\?)\s/gmi, (match, punc) => punc + ' '); From 6b9f905400b215f05e7883711051e05e19250dae Mon Sep 17 00:00:00 2001 From: Capuccino Date: Sat, 29 Jul 2017 13:32:10 +0800 Subject: [PATCH 07/12] fix some really shitty equalities --- Markov.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Markov.js b/Markov.js index b67b3d1..bd427ec 100644 --- a/Markov.js +++ b/Markov.js @@ -223,7 +223,7 @@ class Markov { // Process the gram graph for (let n in grams[depth]) { - if (n ==== 0) return false; + if (n === 0) return false; let current = grams[depth][n - 1].join('|').toLowerCase(); let next = grams[depth][n].slice(-1).join('').toLowerCase(); @@ -237,7 +237,7 @@ class Markov { } else { if (relationships) { // Look for the relationship - let relationship = relationships.find(rel => rel.start ==== nodeIndex[current] && rel.send ==== nodeIndex[next]); + let relationship = relationships.find(rel => rel.start === nodeIndex[current] && rel.send === nodeIndex[next]); if (relationship) { // Update the relationship weight @@ -270,9 +270,9 @@ class Markov { let end = new Date().getTime(); - console.log('========================================'); + console.log('=================='); console.log(`=== Training time: ${(end - start)/(1000 * 60)} ===`); - console.log('========================================'); + console.log('=================='); callback(grams); }); @@ -359,7 +359,7 @@ class Markov { // Process the gram graph for (let n in grams[depth]) { - if (n ==== 0) return false; + if (n === 0) return false; let current = grams[depth][n - 1].join('|').toLowerCase(); let next = grams[depth][n].slice(-1).join('').toLowerCase(); @@ -373,7 +373,7 @@ class Markov { } else { if (relationships) { // Look for the relationship - let relationship = relationships.find(rel => rel.start ==== nodeIndex[current] && rel.end === nodeIndex[next]); + let relationship = relationships.find(rel => rel.start === nodeIndex[current] && rel.end === nodeIndex[next]); if (relationship) { // Update the relationship weight @@ -407,9 +407,9 @@ class Markov { let end = new Date().getTime(); - console.log('========================================'); + console.log('=================='); console.log(`=== POS Training time: ${(end - start)/(1000*60)} ===`); - console.log('========================================'); + console.log('=================='); }); }); }); @@ -455,7 +455,7 @@ class Markov { } // buffer[depth] = relationships; - if (false && depth >= 3 && relationships.length ==== 1) { + if (false && depth >= 3 && relationships.length === 1) { // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); nodes = {}; @@ -509,7 +509,7 @@ class Markov { if (err) throw err; if (response && response.length > 0) { this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { - if (depth >= 3 && relationships.length ==== 1) { + if (depth >= 3 && relationships.length === 1) { // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); @@ -638,7 +638,7 @@ class Markov { let sample = wl.peek()[0]; - // if (sample ==== '.') this.printEdges(nodes, 'Count'); + // if (sample === '.') this.printEdges(nodes, 'Count'); /* let choices = []; for (let gramId in nodes) { @@ -651,7 +651,7 @@ class Markov { if (this.options.debug) console.log('choices', choices); let sample = choices[Math.floor(Math.random() * choices.length)]; - if (chain[chain.length - 1] ==== "'") { + if (chain[chain.length - 1] === "'") { // this.printEdges(nodes, 'Count'); // this.printEdges(nodes, `Probabilities: \033[37m\033[44m${chain.slice(-3).join(' ')} ______`); if (this.options.debug) console.log('sample: ', sample); @@ -685,7 +685,7 @@ class Markov { addToChain(chain, callback, limit) { if (this.options.debug) console.log('>', chain.length, limit); - if (chain.length ==== limit) { + if (chain.length === limit) { callback(this.beautify(chain.join(' '))); } else { this.getNext(chain, ngram => { From 1d041368147b5ffa42fe832ed73834583efe23ea Mon Sep 17 00:00:00 2001 From: sr229 Date: Sat, 29 Jul 2017 13:40:22 +0800 Subject: [PATCH 08/12] fix tests --- test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.js b/test.js index 38b16a9..c33a3f0 100644 --- a/test.js +++ b/test.js @@ -1,4 +1,4 @@ -const Markov = new(require('./Markov')); +const Markov = new (require('./Markov'))(); const readMe = './training-data/bible.txt'; const bot = new Markov({ From 695937a9d033cc321e2acb3afa018ba4f98f7b7a Mon Sep 17 00:00:00 2001 From: sr229 Date: Sat, 29 Jul 2017 13:43:36 +0800 Subject: [PATCH 09/12] I thonk I did this correct --- Markov.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Markov.js b/Markov.js index bd427ec..753d796 100644 --- a/Markov.js +++ b/Markov.js @@ -5,7 +5,7 @@ const NGrams = natural.NGrams; const pos = require('pos'); const tbl = require('cli-table'); const seraph = require('seraph'); -const Tagger = require("brill-pos-tagger"); +const Tagger = require('brill-pos-tagger'); const tokenizer = require('node-tokenizer'); const wlist = require('./WeightedList'); const range = require('python-range'); @@ -61,9 +61,9 @@ class Markov { test() { let text = 'I would like to talk today about: how to develop a new foreign policy direction for our country? one that replaces! randomness with purpose, ideology with strategy, and chaos with peace.\n\nIt is time to shake the rust off of America�s foreign policy. It\'s time to invite new voices and new visions into the fold.'; let text2 = 'i started � i did it , i was, oh, that first couple of weeks with illegal immigration and mexico and all of this stuff , right ? and all of a sudden , people are coming over , and i say the wall and now they�re starting to look elsewhere for help . i started � i did it , i was , oh , that first couple of weeks with illegal immigration'; - - console.log(this.beautify(text2)); - } + + console.log(this.beautify(text2)); + } /* MATCH p =(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight console.log(this.beautify(text2)); */ @@ -154,7 +154,7 @@ class Markov { return false; }*/ - fstool.file.read(filename, text => { + fs.readFileSync(filename, text => { /* text = new pos.Lexer().lex(test); let tokenizer = new natural.RegexpTokenizer({pattern: / /}); text = tokenizer.tokenize(text);*/ From 566815758f3aad0d22a8ccbddb45428c1d64e60d Mon Sep 17 00:00:00 2001 From: sr229 Date: Sat, 29 Jul 2017 13:45:46 +0800 Subject: [PATCH 10/12] fix things here --- WeightedList.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WeightedList.js b/WeightedList.js index 08a5790..48aab77 100644 --- a/WeightedList.js +++ b/WeightedList.js @@ -194,7 +194,7 @@ class WeightedHeap { for (let i in items) { let weight = items[i][1]; let value = items[i][0]; - this.heap.push(new HeadNode(weight, value, weight)); + this.heap.push(new this.HeadNode(weight, value, weight)); } // Now go through the heap and add each node's weight to its parent @@ -236,4 +236,4 @@ class WeightedHeap { // NB: another binary heap implementation is at http://eloquentjavascript.net/appendix2.html -module.exports = WeightedList; \ No newline at end of file +module.exports = {WeightedList, HeapNode}; \ No newline at end of file From ebfc5729f83b338fd7f868dc0d734ea6ffb19118 Mon Sep 17 00:00:00 2001 From: sr229 Date: Sat, 29 Jul 2017 13:47:45 +0800 Subject: [PATCH 11/12] someone forgot to add this even tho repo says the LCIENSE is MIT --- LICENSE | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..44ce85d --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2016 Twenty-Six medias, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From 7caf6191e6eddeb294b2dc3ab4e3e42cc4e3c1ac Mon Sep 17 00:00:00 2001 From: sr229 Date: Sat, 29 Jul 2017 14:09:49 +0800 Subject: [PATCH 12/12] fix referencing issues --- Markov.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Markov.js b/Markov.js index 753d796..0d653de 100644 --- a/Markov.js +++ b/Markov.js @@ -770,4 +770,6 @@ function readFile(filename) { } }); }); -} \ No newline at end of file +} + +module.exports = Markov; \ No newline at end of file