diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..48dd7a4 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,84 @@ +{ + "env": { + "es6": true, + "node": true + }, + + "extends": "eslint:recommended", + "parserOptions": { + "sourceType": "module", + "ecmaVersion": 6, + "ecmaFeatures": { + "jsx": false + } + }, + + "ignore": [ + "*.min.js" + ], + + "rules": { + "no-console": 0, + "global-require":"off", + + "quotes": ["warn", "single", {"avoidEscape": true, "allowTemplateLiterals": true}], + "semi": "warn", + "indent": "error", + "no-unsafe-negation": "error", + "eqeqeq": ["warn", "always", {"null": "ignore"}], + "no-alert": "error", + "no-useless-return": "error", + "array-bracket-spacing": "error", + "brace-style": ["error", "1tbs", {"allowSingleLine": true}], + "comma-dangle": "error", + "comma-spacing": "error", + "comma-style": "error", + "computed-property-spacing": "error", + "func-call-spacing": "error", + "key-spacing": "error", + "keyword-spacing": ["error", {"overrides": {"catch": {"after": false}}}], + "new-parens": "error", + "no-array-constructor": "warn", + "no-new-object": "warn", + "no-whitespace-before-property": "error", + "object-curly-spacing": "error", + "one-var-declaration-per-line": "error", + "quote-props": ["error", "as-needed"], + "semi-spacing": "error", + "space-before-blocks": ["error", "always"], + "space-before-function-paren": ["error", "never"], + "space-in-parens": "error", + "space-unary-ops": ["error", {"words": true, "nonwords": false}], + "arrow-parens": ["warn", "as-needed"], + "arrow-spacing": "error", + "no-useless-rename": "error", + "prefer-arrow-callback": "error", + "template-curly-spacing": "error", + "valid-jsdoc": [ + "warn", + { + "prefer": { + "return": "returns", + "arg": "param", + "argument": "param", + "augments": "extends", + "property": "prop" + }, + + "preferType": { + "object": "Object", + "string": "String", + "number": "Number", + "boolean": "Boolean", + "rromise": "Promise", + "array": "Array", + "undefined":" Undefined", + "null": "Null", + "function": "Function" + }, + + "requireReturn": false + } + ] + } +} diff --git a/.tern-project b/.tern-project new file mode 100644 index 0000000..dcf878b --- /dev/null +++ b/.tern-project @@ -0,0 +1,6 @@ +{ + "plugins": { + "node": {}, + }, + "ecmaVersion": 6, +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..44ce85d --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2016 Twenty-Six medias, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Markov.js b/Markov.js new file mode 100644 index 0000000..0d653de --- /dev/null +++ b/Markov.js @@ -0,0 +1,775 @@ +const natural = require('natural'); +const pstack = require('pstack'); +const md5File = require('md5-file'); +const NGrams = natural.NGrams; +const pos = require('pos'); +const tbl = require('cli-table'); +const seraph = require('seraph'); +const Tagger = require('brill-pos-tagger'); +const tokenizer = require('node-tokenizer'); +const wlist = require('./WeightedList'); +const range = require('python-range'); +const fs = require('fs'); +const Promise = require('bluebird'); + +class Markov { + constructor(options) { + this.options = Object.assign({ + db: 'http://localhost:7473', + name: 'dev', + user: 'neo4j', + pass: 'neo4j', + depth: [1, 3], + weight: 1.2, + depthWeight: 2, + certainty: 1 + }, options); + } + + tokenize(text) { + let cues = ['?', '!', '.', ',', ':', ';', '\t', '\n', '\r']; + let tokens = text.split(' '); + + if (this.options.debug) console.log('tokens', tokens); + + cues.forEach(cue => { + tokens = tokens.map(token => { + let parts = token.split(cue); + let l = parts.length; + + if (l === 1) return token; + + let buffer = []; + parts.forEach((part, i) => { + buffer.push(part); + if (i < l - 1) { + buffer.push(cue); + } + }); + + return buffer; + }); + + tokens = tokens.flatten(); + tokens = tokens.filter(item => item !== ''); + if (this.options.debug) console.log('tokens', tokens); + }); + + return tokens; + } + + test() { + let text = 'I would like to talk today about: how to develop a new foreign policy direction for our country? one that replaces! randomness with purpose, ideology with strategy, and chaos with peace.\n\nIt is time to shake the rust off of America�s foreign policy. It\'s time to invite new voices and new visions into the fold.'; + let text2 = 'i started � i did it , i was, oh, that first couple of weeks with illegal immigration and mexico and all of this stuff , right ? and all of a sudden , people are coming over , and i say the wall and now they�re starting to look elsewhere for help . i started � i did it , i was , oh , that first couple of weeks with illegal immigration'; + + console.log(this.beautify(text2)); + } + /* MATCH p =(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) + RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight + console.log(this.beautify(text2)); */ + + beautify(text) { + text = text.replace(/\s(\.|,|:|;|!|\?)\s/gmi, (match, punc) => punc + ' '); + text.replace(/(\.|!|\?|\n)(\s)([a-z])/gmi, (match, punc, space, letter) => punc + space + letter.toUpperCase()); + text = text.substr(0, 1).toUpperCase() + text.substr(1); + + return text; + } + + test2() { + this.open(() => { + this.graph.query('MATCH p=(x:`ngrams-trump` {gram:{x}})-[:then*1..50]->(y:`ngrams-trump` {gram:{y}}) RETURN p LIMIT 50', {x: 'world', y: 'trump'}, (err, result) => { + if (err) throw err; + if (result) { + result.forEach(item => { + let nodes = item.nodes.map(node => node.split('/').pop()); + + let stack = new pstack(); + let grams = []; + + nodes.forEach(node => { + stack.add(done => { + this.graph.read(node, (err, node) => { + if (err) throw err; + if (node) grams.push(node.gram); + done(); + }); + }); + }); + + stack.start(() => { + console.log('----------------------\n', grams.join(' ')); + }); + if (this.options.debug) console.log('nodes', nodes); + }); + } + + if (this.options.debug) console.log('result', result); + }); + }); + + return this; + } + + open() { + return new Promise((resolve, reject) => { + this.graph = seraph({ + server: 'http://localhost:7474', + user: this.options.user, + pass: this.options.pass + }); + + this.graph.constraints.uniqueness.createIfNone(`ngrams-${this.options.name}`, 'gram', (err, constraint) => { + if (err) { + reject(err); + } else { + this.graph.constraints.uniqueness.createIfNone(`pos-${this.options.name}`, 'gram', (err, constraint) => { + let baseFolder = './node_modules/natural/lib/natural/brill_pos_tagger/data/English'; + let rulesFile = `${baseFolder}/tr_from_posjs.txt`; + let lexiconFile = `${baseFolder}/lexicon_from_posjs.json`; + let defaultCategory = 'N'; + + let tagger = new Tagger(lexiconFile, rulesFile, defaultCategory, err => { + if (err) { + reject(err); + } else { + this.tagger = tagger; + resolve(); + } + }); + }); + } + }); + }); + } + + read(filename, callback) { + let start = new Date().getTime(); + + this.open(() => { + md5File(filename, (err, sum) => { + /* if (this.data.docs[sum]) { + console.log(`Training already done on that document, on ${this.data.docs[sum]}`); + callback(this.data.grams); + return false; + }*/ + + fs.readFileSync(filename, text => { + /* text = new pos.Lexer().lex(test); + let tokenizer = new natural.RegexpTokenizer({pattern: / /}); + text = tokenizer.tokenize(text);*/ + text = this.tokenize(text); + // console.log('text', text); + + // Generate the n-grams. + let grams = {}; + let unique = {}; + let nodeIndex = {}; + + let stackNgram = new pstack({ + progress: 'Reading the N-grams...', + reportInterval: 100, + batch: 10 + }); + + let stackRel = new pstack({ + progress: 'Mapping...', + reportInterval: 100 + }); + + for (let depth of range(this.options.depth[0], this.options.depth[1]+ 1)) { + console.log(`Starting Depth ${depth}`); + console.log('Generating the ngrams.'); + + grams[depth] = NGrams.ngrams(text, depth); + + console.log(`Stringification of ${grams[depth].length} ngrams`); + + let ngramObj = {}; + + grams[depth].forEach(item => { + ngramObj[item.join('|')] = true; + }); + + let uniqueNgrams = Object.keys(ngramObj).length; + + console.log('Graphing'); + + // Save the gram + for (let gram in ngramObj) { + let v = ngramObj[gram]; + gram = gram.toLowerCase(); + stackNgram.add(done => { + this.graph.save({gram, depth}, `ngrams-${this.options.name}`, (err, node) => { + if (err) { + this.graph.find({gram, depth}, false, `ngrams-${this.options.name}`, (err, response) => { + if (err) throw err; + if (response && response.length > 0) { + nodeIndex[gram] = response[0].idl; + // console.log('Node:\t\t', '[found]');; + } + + done(); + }); + return false; + } else { + // console.log('Node:\t\t', '[created]'); + nodeIndex[gram] = node.id; + done(); + } + }); + }); + } + + // Process the gram graph + for (let n in grams[depth]) { + if (n === 0) return false; + + let current = grams[depth][n - 1].join('|').toLowerCase(); + let next = grams[depth][n].slice(-1).join('').toLowerCase(); + + stackRel.add(done => { + // console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); + this.graph.relationships(nodeIndex[current], 'out', 'then', (err, relationships) => { + if (err) { + // console.log('Relationship:\t', '[failed]'); + return false; + } else { + if (relationships) { + // Look for the relationship + let relationship = relationships.find(rel => rel.start === nodeIndex[current] && rel.send === nodeIndex[next]); + + if (relationship) { + // Update the relationship weight + relationship.properties.weight += 1; + this.graph.rel.update(relationship, err => { + if (err) // console.log('Relationship:\t', '[failed]'); + // console.log('Relationship:\t', '[updated]'); + done(); + }); + } else { + this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngrame: next, idx: nodeIndex[next]}, err => { + if (err); // console.log('Relationship:\t', '[failed]'); + else; // console.log('Relationship:\t', '[created]); + done(); + }); + } + } else { + // console.log('Relationship:\t', '[failed]'); + done(); + } + } + }); + }); + } + } + + stackNgram.start(() => { + stackRel.start(() => { + // console.log('>>>>>>>>>> nodeIndex', nodeIndex); + + let end = new Date().getTime(); + + console.log('=================='); + console.log(`=== Training time: ${(end - start)/(1000 * 60)} ===`); + console.log('=================='); + + callback(grams); + }); + }); + }); + }); + }); + } + + readPOS(filename, callback) { + let start = new Date().getTime(); + + this.open(() => { + md5File(filename, (err, sum) => { + if (err) throw err; + + fs.readFile(filename, 'utf8', text => { + text = this.tokenize(text); + + text = this.tagger.tag(text).map(item => item[1]); + + if (this.options.debug) console.log('text', text); + + // Generate the n-grams; + let grams = {}; + let unique = {}; + let nodeIndex = {}; + + let stackNgram = new pstack({ + progess: 'Reading the N-grams...', + reportInterval: 100, + batch: 10 + }); + + let stackRel = new pstack({ + progress: 'Mapping...', + reportInterval: 100 + }); + + for (let depth of range(this.options.depth[0], this.options.depth[1] + 1)) { + console.log(`Starting Depth ${depth}`); + console.log('Generating the POS ngrams'); + + grams[depth] = NGrams.ngrams(text, depth); + + console.log(`Stringification of ${grams[depth].length} POS ngrams.`); + + let ngramObj = {}; + + grams[depth].forEach(item => { + ngramObj[item.join('|')] = true; + }); + + let uniqueNgrams = Object.keys(ngramObj).length; + + console.log('Graphing'); + + for (let gram of ngramObj) { + gram = gram.toLowerCase(); + + stackNgram.add(done => { + this.graph.save({gram, depth}, `pos-${this.options.name}`, (err, node) => { + if (err) { + // Read it + this.graph.find({gram, depth}, false, `pos-${this.options.name}`, (err, reponse) => { + if (err) throw err; + if (response && response.length > 0) { + nodeIndex[gram] = response[0].id; + if (this.options.debug) console.log('Node:\t\t', '[found]'); + } + + done(); + }); + + return false; + } else { + // consoel.log('Node:\t\t', '[created]'); + nodeIndex[gram] = node.id; + done(); + } + }); + }); + } + + // Process the gram graph + for (let n in grams[depth]) { + if (n === 0) return false; + + let current = grams[depth][n - 1].join('|').toLowerCase(); + let next = grams[depth][n].slice(-1).join('').toLowerCase(); + + stackRel.add(done => { + if (this.options.debug) console.log('>> ', current, ' >>> ', next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); + this.graph.relationships(nodeIndex[current], 'out', 'then', (err, relationships) => { + if (err) { + if (this.options.debug) console.log('Relationship:\t', '[failed]'); + return false; + } else { + if (relationships) { + // Look for the relationship + let relationship = relationships.find(rel => rel.start === nodeIndex[current] && rel.end === nodeIndex[next]); + + if (relationship) { + // Update the relationship weight + relationship.properties.wieght += 1; + + this.graph.rel.update(relationship, err => { + if (err); if (this.options.debug) console.log('Relationship:\t', '[failed]'); + else; if (this.options.debug) console.log('Relationship:\t', '[updated]'); + done(); + }); + } else { + this.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight: 1, ngram: next, idx: nodeIndex[next]}, err => { + if (err); if (this.options.debug) console.log('Relationship:\t', '[failed]'); + else; if (this.options.debug) console.log('Relationship:\t', '[updated]'); + done(); + }); + } + } else { + if (this.options.debug) console.log('Relationship:\t', '[failed]'); + done(); + } + } + }); + }); + } + } + + stackNgram.start(() => { + stackRel.start(() => { + if (this.options.debug) console.log('>>>>>>>>> nodeIndex', nodeIndex); + + let end = new Date().getTime(); + + console.log('=================='); + console.log(`=== POS Training time: ${(end - start)/(1000*60)} ===`); + console.log('=================='); + }); + }); + }); + }); + }); + } + + getNext(chain, callback) { + let ngrams = {}; + let nodes = {}; + let posngrams = {}; + let posnodes = {}; + + if (this.options.debug) console.log('chain', chain); + + this.open(() => { + let stack = new pstack(); + let buffer = false; + + // Generate the option list, with weights + for (let depth of range(this.options.depth[1], this.options.depth[0]-1, -1)) { + stack.add(done => { + if (buffer && depth <= this.options.lowpri) { + if (this.options.debug) console.log('Skipped.'); + done(); + return false; + } + + // Generate the ngram to lookup (last N words in the chain) + ngrams[depth] = chain.slice(-depth).join('|').toLowerCase(); + + // Get the nodes + // let localNodes = this.getNodes(ngrams[depth]); + this.graph.find({gram: ngrams[depth]}, false, `ngram-${this.options.name}`, (err, response) => { + if (this.options.debug) console.log('response', ngrams[depth], response); + if (err) throw err; + if (response && response.length > 0) { + this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { + if (err) throw err; + if (relationships.length > 0) { + buffer = true; + if (this.options.debug) console.log('>>>> Depth: ', depth, relationships.length); + } + + // buffer[depth] = relationships; + if (false && depth >= 3 && relationships.length === 1) { + // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... + if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); + nodes = {}; + nodes[relationships[0].properties.ngram] = 1; + done(); + } else { + if (this.options.debug) console.log('relationships', ngrams[depth], relationships); + relationships.forEach(relationship => { + if (nodes[relationship.properties.ngram]) { + // No cumulation! + // nodes[gramId] += count; //*depth*this.options.depthWeight; + } else { + if (this.options.depthWeight) { + nodes[relationship.properties.name] = relationship.properties.weight * Math.pow(depth, this.options.depthweight); + } else { + nodes[relationship.properties.ngram] = relationship.properties.weight; + } + } + }); + if (Object.keys(nodes).lenth > 0) buffer = true; + done(); + } + }); + } else { + if (this.options.debug) console.log('ngram not found: ', ngrams[depth]); + done(); + } + }); + }); + } + + if (this.options.pos) { + // For each node option, build the POS + stack.add(done => { + let substack = new pstack(); + + // Check the POs for the current chain + if (this.options.debug) console.log('>>', chain); + let tags = this.tagger.tag(chain.slice(-20)).map(item => item[1]); + + for (let depth of range(this.options.depth[1], this.options.depth[0] - 1, -1)) { + substack.add(subdone => { + // Generate the ngram to lookup (last N words in the chain) + posngrams[depth] = tags.slice(-depth).join('|').toLowerCase(); + if (this.options.debug) console.log('>>>> Depth: ', depth); + + // Get the nodes + // let localeNodes = this.getNodes(ngrams[depth]); + this.graph.find({gram: posngrams[depth]}, false, `pos-${this.options.name}`, (err, response) => { + if (this.options.debug) console.log('response', posngrams[depth], response); + if (err) throw err; + if (response && response.length > 0) { + this.graph.relationships(response[0].id, 'out', 'then', (err, relationships) => { + if (depth >= 3 && relationships.length === 1) { + // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... + if (this.options.debug) console.log('Going with ', relationships[0].properties.ngram); + + posnodes = {}; + posnodes[relationships[0].properties.ngram] = 1; + subdone(); + } else { + if (this.options.debug) console.log('relationships', posngrams[depth], relationships); + relationships.forEach(relationship => { + if (posnodes[relationship.properties.ngram]) { + // No cumulation! + // nodes[gramId] += count; //*depth*this.options.depthWeight; + } else { + if (this.options.depthWeight) { + posnodes[relationship.properties.ngram] = relationship.properties.weight * Math.pow(depth, this.options.depthweight); + } else { + posnodes[relationship.properties.ngram] = relationship.properties.weight; + } + } + }); + + subdone(); + } + }); + } else { + if (this.options.debug) console.log('ngram not found: ', ngrams[depth]); + subdone(); + } + }); + }); + } + + // Check the pos structure of each node option + + for (let k in nodes) { + substack.add(subdone => { + // Get the POS tag + let text = chain.slice(0); + text.push(k); + if (this.options.debug) console.log('text', text); + + let tags = this.tagger.tag(text.slice(-10)).map(item => { + return item[1]; + }); + + let tag = tags.slice(-1)[0].toLowerCase(0); + if (this.options.debug) console.log(`[${tag}] `, chain.join(' '), '->', k, posnodes[tag]); + + if (posnodes[tag]) { + nodes[k] += posnodes[tag]; + nodes[k] /= 2; + } + + subdone(); + }); + } + + substack.start(() => { + if (this.options.debug) console.log('nodes', nodes); + if (this.options.debug) console.log('posnodes', posnodes); + if (this.options.debug) console.log('-------------------------'); + done(); + }); + }); + } + + stack.start(() => { + if (this.options.debug) { + this.printEdges(nodes, 'Count'); + console.log(chain.join('|')); + } + + // Calculate the total + let total = 0; + for (let count of nodes) total += count; + + let minP = 1; + + // Calculate the probabilities + + for (let gramId in nodes) { + nodes[gramId] = count/total; + if (nodes[gramId] < minP) minP = nodes[gramId]; + } + + if (this.options.certainty) { + if (minP < this.options.certainty) { + // The probabilities are way too low to filter. We need to remove the least probable options + let nodeArray = []; + for (let id in nodes) nodeArray.push({id, p: nodes[id]}); + + // Sort and slice + nodeArray.sort((a, b) => b.p - a.p); + nodeArray = nodeArray.slice(0, 100/(this.options.certainty*100)); + + // Convert back to an object + nodes = {}; + nodeArray.forEach(item => nodes[item.id] = item.p); + if (this.options.debug) console.log('nodes', nodes); + } else { + // Remove the lowest probabilities + for (let gramId in nodes) { + if (nodes[gramId] < this.options.certainty) delete nodes[gramId]; + } + } + } + + // Recalculate the total + total = 0; + for (let p of nodes) { + total += p; + } + + // Recalculate the probabilities + for (let gramId in nodes) { + nodes[gramId] = nodes[gramId]/total; + } + + let rn = Math.random(); + + let _nodes = []; + for (let k in nodes) _nodes.push([k, nodes[k]]); + _nodes.sort((a, b) => a[1] - b[1]); + + let wl = new wlist(_nodes); + + let sample = wl.peek()[0]; + + // if (sample === '.') this.printEdges(nodes, 'Count'); + /* + let choices = []; + for (let gramId in nodes) { + let p = nodes[gramId]; + let count = p*100; + count = Math.ceil(Math.pow(count, this.options.weight)); + for (let n of range(0, count)) choices.push(gramId); + } + + if (this.options.debug) console.log('choices', choices); + let sample = choices[Math.floor(Math.random() * choices.length)]; + + if (chain[chain.length - 1] === "'") { + // this.printEdges(nodes, 'Count'); + // this.printEdges(nodes, `Probabilities: \033[37m\033[44m${chain.slice(-3).join(' ')} ______`); + if (this.options.debug) console.log('sample: ', sample); + if (this.options.debug) console.log('buffer: ', JSON.stringify(buffer, null, 4)); + } + + if (!sample) console.log('!!!!!!!', choices); + */ + + callback(sample); + }); + }); + + // return this.data.grams[choices[Math.floor(Math.random() * choices.length)]]; + } + + generate(start, count, callback) { + //let chain = new pos.Lexer().lex(start); + /*let tokenizer = new natural.RegexpTokenizer({pattern: / /}); + let chain = tokenizer.tokenize(start.toLowerCase());*/ + let chain = this.tokenize(start.toLowerCase()); + + this.addToChain(chain, callback, count); + + /* + this.getNext(chain); + for (let n of range(0, count)) chain.push(this.getNext(chain)); + return this.beautify(chain.join(' ));*/ + } + + addToChain(chain, callback, limit) { + if (this.options.debug) console.log('>', chain.length, limit); + + if (chain.length === limit) { + callback(this.beautify(chain.join(' '))); + } else { + this.getNext(chain, ngram => { + if (!ngram) { + callback(scope.beautify(chain.join(' '))); + return false; + } + chain.push(ngram); + this.addToChain(chain, callback, limit); + }); + } + } + + printEdges(nodes, title) { + nodes = []; + for (let id in nodes) _nodes.push({id, p: nodes[id]}); + + // Sort and slice + nodes.sort((a, b) => b.p - a.p); + console.log(`\n\u{1b}[32m ${title}\u{1b}[37m\u{1b}[40m`); + this.table(nodes, { + Word: 'word', + Probability: 'p' + }); + } + + table(array, cols) { + let table = new tbl({head: Object.keys(cols)}); + array.forEach(item => { + let row = []; + for (let k in cols) _nodes.push(item[cols[k]]); + table.push(row); + }); + console.log(table.toString()); + } +} + +module.exports = Markov; + +/* + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'world'}) +RETURN p + + +MATCH p=shortestPath((x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'clinton'})) +RETURN p + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) +RETURN p LIMIT 50 + + +MATCH (x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->()-[:TO|:CC|:BCC]->(person) +RETURN distinct person + +MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) +RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight + +*/ + +Array.prototype.flatten = function() { + const flat = []; + + this.forEach(item => { + if (Array.isArray(item)) { + flat.push([].concat.apply([], item)); + } else { + flat.push(item); + } + }); + + return [].concat.apply([], flat); +}; + +function readFile(filename) { + return new Promise((resolve, reject) => { + fs.readFile(filename, 'utf8', (err, text) => { + if (err) { + reject(err); + } else { + resolve(text); + } + }); + }); +} + +module.exports = Markov; \ No newline at end of file diff --git a/WeightedList.js b/WeightedList.js new file mode 100644 index 0000000..48aab77 --- /dev/null +++ b/WeightedList.js @@ -0,0 +1,239 @@ +/** +* js-weighted-list.js +* +* version 0.3 +* +* This file is licensed under the MIT License, please see MIT-LICENSE.txt for details. +* +* https://github.com/timgilbert/js-weighted-list is its home. +*/ + +class WeightedList { + constructor(initial) { + this.weights = {}; + this.data = {}; + this.length = 0; + this.hasData = false; + + initial = initial != undefined ? initial : []; + + if (Array.isArray(initial)) { + for (let i in initial) { + this.push(initial[i]); + } + } else { + throw new Error(`Invalid type of initial passed to WeightedList constructor. Type: ${initial.constructor.name}' (expected array or nothing)`); + } + } + + /** + * Add a single item to the list. The parameter passed in represents a single + * key, with a weight and optionally some data attached. + * + * @param {Array|Object} element Either a 2/3 element array of [key, weight, data] (data i optional), or an object with {key: k, weight: w, data: d} where data is optional. + */ + push(element) { + // Catch undefineds or empty arrays. + if (!element) throw new Error('element is not an array or object or is empty.'); + let key, weight, data; + + if (Array.isArray(element)) { + key = element[0]; + weight = element[1]; + data = element[2]; + + // e.g. wl.push([]) + if (!key || typeof key !== 'string') throw new Error('element needs at least two elements. First element is undefined or not a string.'); + // I suppose we could default to 1 here, but the API is already too forgiving. + if (!weight || typeof weight !== 'number') throw new Error('element needs at least two elements. Second element is undefined or not a number.'); + } else if (typeof element === 'object') { + // We expect {key: 'zombies', weight: 10, data: {fast: true}} + key = element.key; + weight = element.weight; + data = element.data; + + if (!key || typeof key !== 'string') throw new Error('element.key is not defined or is not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('element.weight is not defined or is not a number.'); + } else { + // If it somehow got through the first catcher + throw new Error('element is not a supported type. Expected [key, weight] or {key: k, weight: w}') + } + + return this._pushValues(key, weight, data); + } + + /** + * Add an item to the WeightedList + * + * @access private + * @param {String} key The key under which the item is stored. + * @param {Number} weight The weight to assign to the item. + * @param {?Object} data Any optional data for the item. + */ + _pushValues(key, weight, data) { + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + if (this.weights[key]) throw new Error(`An item with the key '${key}' already exists.`); + if (weight <= 0) throw new Error(`weight must be higher than 0, got ${weight}`); + + this.weights[key] = weight; + if (data) { + this.hasData = true; + this.data[key] = data; + } + + this.length++; + } + + /** + * Add the given weight to the list item with the give key. This operation + * will silently create the key if it does not already exist + * + * @todo Might be nice to have a version of this that would throw an error on an unknown key. + * + * @param {String} key Key to add weight onto. + * @param {Number} weight Weight to add. + */ + addWeight(key, weight) { + if (!key || typeof key !== 'string') throw new Error('key is undefined or not a string.'); + if (!weight || typeof weight !== 'number') throw new Error('weight is undefined or not a number.'); + + this.weights[key] += weight; + } + + /** + * Select `n` elements (without replacement). + * If `remove` is true, removes the elements from the list. + * + * @param {Number} [n=1] Amount of elements to get + * @param {Boolean} [remove=false] Remove the elements from the list or not. + * @returns {Array} + */ + peek(n=1, remove=false) { + if (!n || typeof n !== 'number') throw new Error('n is undefined or not a number.'); + if (typeof remove !== 'boolean') throw new Error('remove is undefined or not a boolean.'); + if (this.length - n < 0) throw new Error(`Stack underflow! Tried to retrieve ${n} element(s) from a list of ${this.length}`); + + let heap = this._buildWeightedHeap(); + let result = []; + + for (let i = 0; i < n; i++) { + let key = heap.pop(); + + result.push(this.hasData ? {key, data: this.data[key]} : key); + + if (remove) { + delete this.weights[key]; + delete this.data[key]; + this.length--; + } + } + + return result; + } + + /** + * Return the entire list in a random order. Does not edit the list. + * + * @returns {Array} + */ + suffle() { + return this.peek(this.length); + } + + /** + * Removes an item/number of items from the start of the list. + * + * @param {Number} [n=1] Amount of items to pop + */ + pop(n=1) { + return this.peek(n, true); + } + + /** + * Builds a WeightedHeap out of the data in the list. + */ + _buildWeightedHeap() { + let items = []; + + for (let key in this.weights) { + if (this.weights.hasOwnProperty(key)) items.push([key, this.weights[key]]); + } + + return new WeightedHeap(items); + } +} + +/** + * A JavaScript implementation of the algorithm described by Jasen Orendorff here: http://stackoverflow.com/a/2149533/87990 + * + * @prop {Number} weight + * @prop {Number} value + * @prop {Number} total + */ +class HeapNode { + constructor(weight, value, total) { + this.weight = weight; + this.value = value; + this.total = total; // Total weight of this node and its children. + } +} + +class WeightedHeap { + /** + * Construct a WeightedHeap + * + * Note, we're using a heap structure here for its tree properties, not as a + * classic binary heap. A node heap[i] has children at heap[i<<1] and at + * heap[(i<<1)+1]. Its parent is at h[i>>1]. Heap[0] is vacant. + */ + constructor(items) { + this.heap = [null]; // Math is easier to read if we index array from 1 + + // First put everything on the heap + for (let i in items) { + let weight = items[i][1]; + let value = items[i][0]; + this.heap.push(new this.HeadNode(weight, value, weight)); + } + + // Now go through the heap and add each node's weight to its parent + for (let i = this.heap.length - 1; i > 1; i--) this.heap[i >> 1].total += this.heap[i].total; + } + + pop() { + // Start with a random amount of gas + let gas = this.heap[i].total * Math.random(); + + // Start driving at the root node; + let i = 1; + + // While we have enough gas to keep going past i + while (gas > this.heap[i].weight) { + gas -= this.heap[i].weight; // Drive past i + i <<= 1; // Move to first Child + + if (gas > this.heap[i].total) { + gas -= this.heap[i].total; // Drive past firstchild and its descendants + i++; // Move on to second child + } + } + + // Out of gas - i is our selected node + let value = this.heap[i].value; + let selectedWeight = this.heap[i].weight; + + this.heap[i].weight = 0; // Make sure i isn't chosen again + + while (i > 0) { + this.heap[i].total -= selectedWeight; // Remove the weight from its parent's total + i >>= 1; // Move to the next parent + } + + return value; + } +} + +// NB: another binary heap implementation is at http://eloquentjavascript.net/appendix2.html + +module.exports = {WeightedList, HeapNode}; \ No newline at end of file diff --git a/cmarkov.js b/cmarkov.js deleted file mode 100644 index a8cdc55..0000000 --- a/cmarkov.js +++ /dev/null @@ -1,856 +0,0 @@ -var _ = require('underscore'); -var fstool = require('fs-tool'); -var natural = require('natural'); -var pstack = require('pstack'); -var md5File = require('md5-file'); -var progressbar = require('progress'); -var NGrams = natural.NGrams; -var pos = require('pos'); -var tbl = require('cli-table'); -var seraph = require("seraph"); -// var Tagger = require("natural").BrillPOSTagger; -var Tagger = require("brill-pos-tagger"); -var tokenizer = require("node-tokenizer"); -var wlist = require("./js-weighted-list"); - -var markov = function(options) { - this.options = _.extend({ - db: 'http://localhost:7474', - name: 'dev', - depth: [1,3], - weight: 1.2, - depthWeight: 2, - certainty: 1 - }, options); -} - -markov.prototype.tokenize = function(text) { - - var cues = ["?","!",".",",",":",";","\t","\n","\r"]; - //var cues = [":"]; - var tokens = text.split(' '); - - //console.log("tokens",tokens); - - _.each(cues, function(cue) { - tokens = _.map(tokens, function(token) { - var parts = token.split(cue); - var l = parts.length; - if (l==0) { - return token; - } - var buffer = []; - _.each(parts, function(part, n) { - buffer.push(part); - if (n(y:`ngrams-trump` {gram:{y}}) RETURN p LIMIT 50", {x: 'world', y:'trump'}, function(err, result) { - if (result) { - _.each(result, function(item) { - var nodes = _.map(item.nodes, function(node) { - return node.split('/').pop(); - }); - - var stack = new pstack(); - var grams = []; - - _.each(nodes, function(node) { - stack.add(function(done) { - scope.graph.read(node, function(err, node) { - if (node) { - grams.push(node.gram); - } - done(); - }); - }); - }); - - stack.start(function() { - console.log("----------------------\n",grams.join(' ')); - }); - - - //console.log("nodes",nodes); - }); - } - //console.log("result",result); - }); - }); - - - return this; -} - -markov.prototype.open = function(callback) { - var scope = this; - scope.graph = seraph({ - server: "http://localhost:7474", - user: "neo4j", - pass: "pwd" - }); - scope.graph.constraints.uniqueness.createIfNone('ngrams-'+scope.options.name, 'gram', function(err, constraint) { - scope.graph.constraints.uniqueness.createIfNone('pos-'+scope.options.name, 'gram', function(err, constraint) { - var base_folder = "./node_modules/natural/lib/natural/brill_pos_tagger/data/English"; - var rules_file = base_folder + "/tr_from_posjs.txt"; - var lexicon_file = base_folder + "/lexicon_from_posjs.json"; - var default_category = 'N'; - - var tagger; - tagger = new Tagger(lexicon_file, rules_file, default_category, function(error) { - if (error) { - console.log(error); - } else { - scope.tagger = tagger; - callback(); - } - }); - - }); - }); - - return this; -} - -markov.prototype.read = function(filename, callback) { - var scope = this; - - var start = new Date().getTime(); - - this.open(function() { - md5File(filename, function (error, sum) { - /*if (scope.data.docs[sum]) { - console.log("Training already done on that document, on "+scope.data.docs[sum]); - callback(scope.data.grams); - return false; - } - - scope.data.docs[sum] = new Date(); - */ - fstool.file.read(filename, function(text) { - - //text = new pos.Lexer().lex(text); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - text = tokenizer.tokenize(text);*/ - text = scope.tokenize(text); - //console.log("text", text); - - // Generate the n-grams - var grams = {}; - var unique = {}; - var nodeIndex = {}; - - - var stack_ngram = new pstack({ - progress: 'Reading the N-grams...', - reportInterval: 100, - batch: 10 - }); - - var stack_rel = new pstack({ - progress: 'Mapping...', - reportInterval: 100 - }); - - _.each(_.range(scope.options.depth[0], scope.options.depth[1]+1), function(depth) { - - console.log("Starting Depth "+depth); - - console.log("Generating the ngrams"); - - grams[depth] = NGrams.ngrams(text, depth); - - console.log("Stringification of "+grams[depth].length+" ngrams"); - var ngramObj = {}; - _.each(grams[depth], function(item) { - ngramObj[item.join('|')] = true; - }); - - var uniqueNgams = _.size(ngramObj); - - console.log("Graphing"); - - // Save the gram - _.each(ngramObj, function(v, gram) { - gram = gram.toLowerCase(); - stack_ngram.add(function(done) { - scope.graph.save({ - gram: gram, - depth: depth - }, 'ngrams-'+scope.options.name, function(err, node) { - if (err) { - // Read it - scope.graph.find({ - gram: gram, - depth: depth - }, false, 'ngrams-'+scope.options.name, function (err, response) { - if (response && response.length > 0) { - nodeIndex[gram] = response[0].id; - //console.log("Node:\t\t", "[found]"); - } - done(); - }); - return false; - } else { - //console.log("Node:\t\t", "[created]"); - nodeIndex[gram] = node.id; - done(); - } - }); - }); - }); - - // Process the gram graph - _.each(grams[depth], function(str, n) { - if (n==0) { - return false; - } - var current = grams[depth][n-1].join('|').toLowerCase(); - var next = grams[depth][n].slice(-1).join('').toLowerCase(); - - stack_rel.add(function(done) { - //console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); - - scope.graph.relationships(nodeIndex[current], 'out', 'then', function(err, relationships) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - return false; - } else { - if (relationships) { - // Look for the relationship - var relationship = _.find(relationships, function(rel) { - return rel.start == nodeIndex[current] && rel.end == nodeIndex[next]; - }); - - if (relationship) { - // Update the relationship weight - relationship.properties.weight += 1; - scope.graph.rel.update(relationship, function(err) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } - //console.log("Relationship:\t", "[updated]"); - done(); - }); - } else { - scope.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight:1, ngram:next, idx:nodeIndex[next]}, function(err, relationship) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } else { - //console.log("Relationship:\t", "[created]"); - } - done(); - }); - } - - } else { - //console.log("Relationship:\t", "[failed]"); - done(); - } - } - }); - - }); - }); - - }); - - stack_ngram.start(function() { - stack_rel.start(function() { - //console.log(">>>>>>>>> nodeIndex",nodeIndex); - - var end = new Date().getTime(); - - - console.log("==========================="); - console.log("== Training time: ",(end-start)/(1000*60)," =="); - console.log("==========================="); - - callback(grams); - }); - }); - }); - }); - }); -} - -markov.prototype.readPOS = function(filename, callback) { - var scope = this; - - var start = new Date().getTime(); - - this.open(function() { - md5File(filename, function (error, sum) { - fstool.file.read(filename, function(text) { - - //text = new pos.Lexer().lex(text); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - text = tokenizer.tokenize(text);*/ - text = scope.tokenize(text); - - text = _.map(scope.tagger.tag(text), function(item) { - return item[1]; - }); - - //console.log("text", text); - - // Generate the n-grams - var grams = {}; - var unique = {}; - var nodeIndex = {}; - - - var stack_ngram = new pstack({ - progress: 'Reading the N-grams...', - reportInterval: 100, - batch: 10 - }); - - var stack_rel = new pstack({ - progress: 'Mapping...', - reportInterval: 100 - }); - - - _.each(_.range(scope.options.depth[0], scope.options.depth[1]+1), function(depth) { - - console.log("Starting Depth "+depth); - - console.log("Generating the POS ngrams"); - - grams[depth] = NGrams.ngrams(text, depth); - - console.log("Stringification of "+grams[depth].length+" POS ngrams"); - var ngramObj = {}; - _.each(grams[depth], function(item) { - ngramObj[item.join('|')] = true; - }); - - var uniqueNgams = _.size(ngramObj); - - console.log("Graphing"); - - // Save the gram - _.each(ngramObj, function(v, gram) { - gram = gram.toLowerCase(); - stack_ngram.add(function(done) { - scope.graph.save({ - gram: gram, - depth: depth - }, 'pos-'+scope.options.name, function(err, node) { - if (err) { - // Read it - scope.graph.find({ - gram: gram, - depth: depth - }, false, 'pos-'+scope.options.name, function (err, response) { - if (response && response.length > 0) { - nodeIndex[gram] = response[0].id; - //console.log("Node:\t\t", "[found]"); - } - done(); - }); - return false; - } else { - //console.log("Node:\t\t", "[created]"); - nodeIndex[gram] = node.id; - done(); - } - }); - }); - }); - - // Process the gram graph - _.each(grams[depth], function(str, n) { - if (n==0) { - return false; - } - var current = grams[depth][n-1].join('|').toLowerCase(); - var next = grams[depth][n].slice(-1).join('').toLowerCase(); - - stack_rel.add(function(done) { - //console.log(">> ",current,' >>> ',next, ' -> ', nodeIndex[current], ' >>> ', nodeIndex[next]); - - scope.graph.relationships(nodeIndex[current], 'out', 'then', function(err, relationships) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - return false; - } else { - if (relationships) { - // Look for the relationship - var relationship = _.find(relationships, function(rel) { - return rel.start == nodeIndex[current] && rel.end == nodeIndex[next]; - }); - - if (relationship) { - // Update the relationship weight - relationship.properties.weight += 1; - scope.graph.rel.update(relationship, function(err) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } - //console.log("Relationship:\t", "[updated]"); - done(); - }); - } else { - scope.graph.relate(nodeIndex[current], 'then', nodeIndex[next], {weight:1, ngram:next, idx:nodeIndex[next]}, function(err, relationship) { - if (err) { - //console.log("Relationship:\t", "[failed]"); - } else { - //console.log("Relationship:\t", "[created]"); - } - done(); - }); - } - - } else { - //console.log("Relationship:\t", "[failed]"); - done(); - } - } - }); - - }); - }); - }); - - stack_ngram.start(function() { - stack_rel.start(function() { - //console.log(">>>>>>>>> nodeIndex",nodeIndex); - - var end = new Date().getTime(); - - - console.log("==========================="); - console.log("== POS Training time: ",(end-start)/(1000*60)," =="); - console.log("==========================="); - - callback(grams); - }); - }); - - }); - }); - }); -} - - - -markov.prototype.getNext = function(chain, callback) { - var scope = this; - var ngrams = {}; - var nodes = {}; - var posngrams = {}; - var posnodes = {}; - - //console.log("chain",chain); - - this.open(function() { - - var stack = new pstack(); - var buffer = false; - - - // Generate the option list, with weights - _.each(_.range(scope.options.depth[1], scope.options.depth[0]-1, -1), function(depth) { - stack.add(function(done) { - if (buffer && depth <= scope.options.lowpri) { - //console.log("skiped."); - done(); - return false; - } - - - // Generate the ngram to lookup (last N words in the chain) - ngrams[depth] = chain.slice(-depth).join('|').toLowerCase(); - - // Get the nodes - //var localNodes = scope.getNodes(ngrams[depth]); - scope.graph.find({ - gram: ngrams[depth] - }, false, 'ngrams-'+scope.options.name, function (err, response) { - //console.log("response",ngrams[depth], response); - if (response && response.length>0) { - scope.graph.relationships(response[0].id, 'out', 'then', function(err, relationships) { - - if (relationships.length==0) { - - } else { - buffer = true; - //console.log(">>>> Depth: ", depth, relationships.length); - } - - //buffer[depth] = relationships; - if (false && depth>=3 && relationships.length==1) { - // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... - //console.log("Going with ",relationships[0].properties.ngram); - nodes = {}; - nodes[relationships[0].properties.ngram] = 1; - done(); - } else { - //console.log("relationships",ngrams[depth], relationships); - _.each(relationships, function(relationship) { - - if (nodes[relationship.properties.ngram]) { - // No cumulation! - //nodes[gramId] += count;//*depth*scope.options.depthWeight; - } else { - if (scope.options.depthWeight) { - nodes[relationship.properties.ngram] = relationship.properties.weight*Math.pow(depth, scope.options.depthWeight); - } else { - nodes[relationship.properties.ngram] = relationship.properties.weight; - } - } - }); - if (_.size(nodes)>0) { - buffer = true; - } - done(); - } - - - }); - } else { - //console.log("ngram not found: ", ngrams[depth]); - done(); - } - - }); - }); - }); - - if (scope.options.pos) { - // For each node option, build the POS - stack.add(function(done) { - - var substack = new pstack(); - - // Check the POS for the current chain - //console.log(">>",chain); - var tags = _.map(scope.tagger.tag(chain.slice(-20)), function(item) { - return item[1]; - }); - - - // Generate the option list, with weights - _.each(_.range(scope.options.depth[1], scope.options.depth[0]-1, -1), function(depth) { - substack.add(function(subdone) { - // Generate the ngram to lookup (last N words in the chain) - posngrams[depth] = tags.slice(-depth).join('|').toLowerCase(); - //console.log(">>>> Depth: ",depth); - // Get the nodes - //var localNodes = scope.getNodes(ngrams[depth]); - scope.graph.find({ - gram: posngrams[depth] - }, false, 'pos-'+scope.options.name, function (err, response) { - //console.log("response",posngrams[depth], response); - if (response && response.length>0) { - scope.graph.relationships(response[0].id, 'out', 'then', function(err, relationships) { - - - if (depth>=3 && relationships.length==1) { - // Only one edge. We should go with it, to keep the structure of i'm, it's, they're... - //console.log("Going with ",relationships[0].properties.ngram); - posnodes = {}; - posnodes[relationships[0].properties.ngram] = 1; - subdone(); - } else { - //console.log("relationships",posngrams[depth], relationships); - _.each(relationships, function(relationship) { - - if (posnodes[relationship.properties.ngram]) { - // No cumulation! - //nodes[gramId] += count;//*depth*scope.options.depthWeight; - } else { - if (scope.options.depthWeight) { - posnodes[relationship.properties.ngram] = relationship.properties.weight*Math.pow(depth, scope.options.depthWeight); - } else { - posnodes[relationship.properties.ngram] = relationship.properties.weight; - } - } - }); - subdone(); - } - }); - } else { - //console.log("ngram not found: ", ngrams[depth]); - subdone(); - } - - }); - }); - }); - - // Check the pos structure of each node option - _.each(nodes, function(w,k) { - substack.add(function(subdone) { - // Get the POS tag - var text = chain.slice(0); - text.push(k); - //console.log("text",text); - var tags = _.map(scope.tagger.tag(text.slice(-10)), function(item) { - return item[1]; - }); - - var tag = tags.slice(-1)[0].toLowerCase(); - - //console.log("["+tag+"] ",chain.join(' '),'->',k, posnodes[tag]); - if (posnodes[tag]) { - nodes[k] += posnodes[tag]; - nodes[k] /= 2; - } - subdone(); - }); - }); - - - - substack.start(function() { - //console.log("nodes",nodes); - //console.log("posnodes",posnodes); - //console.log("-------------------------"); - done(); - }); - - }); - } - - - stack.start(function() { - - - if (scope.options.debug) { - scope.print_edges(nodes, 'Count'); - console.log(chain.join('|')); - } - - // Calculate the total - var total = 0; - _.each(nodes, function(count, gramId) { - total += count; - }); - - var minP = 1; - - // Calculate the probabilities - _.each(nodes, function(count, gramId) { - nodes[gramId] = count/total; - if (nodes[gramId] < minP) { - minP = nodes[gramId]; - } - }); - - if (scope.options.certainty) { - if (minP < scope.options.certainty) { - // The probabilities are way too low to filter. We need to remove the least probable options - var nodeArray = _.map(nodes, function(p, gramId) { - return { - id: gramId, - p: p - }; - }); - // Sort and slice - nodeArray.sort(function(a, b) { - return b.p-a.p; - }); - nodeArray = nodeArray.slice(0, 100/(scope.options.certainty*100)); - - // Convert back to an object - nodes = {}; - _.each(nodeArray, function(item) { - nodes[item.id] = item.p; - }); - //console.log("nodes",nodes); - } else { - // Remove the lowest probabilities - nodes = _.omit(nodes, function(p, gramId) { - return p < scope.options.certainty - }); - } - } - - // Recalculate the total - var total = 0; - _.each(nodes, function(p, gramId) { - total += p; - }); - - // Recalculate the probabilities - _.each(nodes, function(p, gramId) { - nodes[gramId] = p/total; - }); - - - var rn = Math.random(); - - var _nodes = _.map(nodes, function(v,k) {return [k,v]}); - _nodes.sort(function(a,b) { - return a[1]-b[1]; - }); - - var wl = new wlist(_nodes); - - - var sample = wl.peek()[0]; - /* - if (sample=='.') { - scope.print_edges(nodes, 'Count'); - } - */ - /* - var choices = []; - _.each(nodes, function(p, gramId) { - var count = p*100; - count = Math.ceil(Math.pow(count, scope.options.weight)); - _.each(_.range(0,count), function(n) { - choices.push(gramId); - }); - }); - - //console.log("choices",choices); - var sample = _.sample(choices); - if (chain[chain.length-1]=='\'') { - //scope.print_edges(nodes, 'Count'); - //scope.print_edges(nodes, 'Probabilities: \033[37m\033[44m'+chain.slice(-3).join(' ')+' ______'); - //console.log("sample: ",sample); - //console.log("buffer:",JSON.stringify(buffer,null,4)); - } - - if (!sample) { - //console.log("!!!!!!!",choices); - } - */ - callback(sample); - }); - - }); - - //return this.data.grams[_.sample(choices)]; -} - -markov.prototype.generate = function(start, count, callback) { - var scope = this; - //var chain = new pos.Lexer().lex(start); - /*var tokenizer = new natural.RegexpTokenizer({pattern: / /}); - var chain = tokenizer.tokenize(start.toLowerCase());*/ - chain = scope.tokenize(start.toLowerCase()); - - this.addToChain(chain, callback, count); - /* - this.getNext(chain) - _.each(_.range(0,count), function(n) { - chain.push(scope.getNext(chain)); - }); - return scope.beautify(chain.join(' '));*/ -} -markov.prototype.addToChain = function(chain, callback, limit) { - var scope = this; - //console.log(">",chain.length,limit); - if (chain.length==limit) { - callback(scope.beautify(chain.join(' '))); - } else { - this.getNext(chain, function(ngram) { - if (!ngram) { - callback(scope.beautify(chain.join(' '))); - return false; - } - chain.push(ngram); - scope.addToChain(chain, callback, limit); - }); - } -} - -markov.prototype.print_edges = function(nodes, title) { - var scope = this; - nodes = _.map(nodes, function(p, gramId) { - return { - word: gramId, - p: p - }; - }); - // Sort and slice - nodes.sort(function(a, b) { - return b.p-a.p; - }); - console.log("\n\033[32m "+title+"\033[37m\033[40m"); - scope.table(nodes, { - 'Word': 'word', - 'Probability': 'p' - }); -} -markov.prototype.table = function(array, cols) { - var scope = this; - var table = new tbl({ - head: _.keys(cols) - }); - _.each(array, function(item) { - var row = _.map(cols, function(v,k) { - return item[v]; - }); - table.push(row); - }); - console.log(table.toString()); -} - -module.exports = markov; - - -/* - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'world'}) -RETURN p - - -MATCH p=shortestPath((x:`ngrams-trump` {gram:'obama'})-[:then*1..20]->(y:`ngrams-trump` {gram:'clinton'})) -RETURN p - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) -RETURN p LIMIT 50 - - -MATCH (x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->()-[:TO|:CC|:BCC]->(person) -RETURN distinct person - -MATCH p=(x:`ngrams-trump` {gram:'obama'})-[:then*1..5]->(y:`ngrams-trump` {gram:'clinton'}) -RETURN p AS shortestPath, reduce(weight=0, r in rels : weight+r.weight) AS totalWeight - -*/ diff --git a/index.js b/index.js new file mode 100644 index 0000000..5c92d1f --- /dev/null +++ b/index.js @@ -0,0 +1 @@ +module.exports = require('./Markov'); \ No newline at end of file diff --git a/js-weighted-list.js b/js-weighted-list.js deleted file mode 100644 index 2de1d72..0000000 --- a/js-weighted-list.js +++ /dev/null @@ -1,245 +0,0 @@ -/** -* js-weighted-list.js -* -* version 0.2 -* -* This file is licensed under the MIT License, please see MIT-LICENSE.txt for details. -* -* https://github.com/timgilbert/js-weighted-list is its home. -*/ - -var WeightedList = (function() { - - function _WeightedList(initial) { - this.weights = {}; - this.data = {}; - this.length = 0; - this.hasData = false; - - initial = typeof initial !== 'undefined' ? initial : []; - - if (Array.isArray(initial)) { - for (var i = 0; i < initial.length; i++) { - //var item = initial[i]; - //this.push(item[0], item[1], item[2]); - this.push(initial[i]); - } - } else { - throw new Error('Unknown object "' + initial.toString() + '" passed to ' + - 'WeightedList constructor! (Expected array or nothing)'); - } - } - - _WeightedList.prototype = { - /** - * Add a single item to the list. The parameter passed in represents a single - * key, with a weight and optionally some data attached. - * - * The parameter to this function can either be a 2-3 element array of - * [k, w, d] for key, weight and data (data is optional) or an object with the - * values {'key': k, 'weight': w, 'data': d} where d is optional. - */ - push: function(element) { - var key, weight, data; - - if (Array.isArray(element)) { - key = element[0], weight = element[1], data = element[2]; - if (typeof key === 'undefined') { - // Eg, wl.push([]) - throw new Error('In WeightedList.push([ ... ]), need at least two elements'); - } else if (typeof weight === 'undefined') { - // I suppose we could default to 1 here, but the API is already too forgiving - throw new Error('In array passed to WeightedList.push([ ... ]), second ' + - 'element is undefined!'); - } - } else if (typeof element === 'object') { - // We expect {"key": "zombies", "weight": 10, "data": {"fast": true}} - key = element.key, weight = element.weight, data = element.data; - if (typeof key === 'undefined') { - throw new Error("In WeightedList.push({ ... }), no {'key': 'xyzzy'} pair found"); - } else if (typeof weight === 'undefined') { - // I suppose we could default to 1 here, but the API is already too forgiving - throw new Error('In array passed to WeightedList.push({ ... }), no ' + - "{'weight': 42} pair found"); - } - } else { - // else what the heck were you trying to give me? - throw new Error('WeightedList.push() passed unknown type "' + typeof element + - '", expected [key, weight] or {"key": k, "weight": w}'); - } - return this._push_values(key, weight, data); - - }, - /** - * Add an item to the list - * @access private - * @param {String} key the key under which this item is stored - * @param {number} weight the weight to assign to this key - * @param {?Object} data any optional data associated wth this key - */ - _push_values: function(key, weight, data) { - //console.debug('k:', key, 'w:', weight, 'd:', data); - - if (this.weights[key]) { - throw new Error(''); - } - if (typeof weight !== typeof 1) { - throw new Error('Weight must be numeric (got ' + weight.toString() + ')'); - } - if (weight <= 0) { - throw new Error('Weight must be >= 0 (got ' + weight + ')'); - } - - this.weights[key] = weight; - - if (typeof data !== 'undefined') { - this.hasData = true; - this.data[key] = data; - } - this.length++; - }, - - /** - * Add the given weight to the list item with the given key. Note that if - * the key does not already exist, this operation will silently create it. - * - * @todo might be nice to have a version of this that would throw an error - * on an unknown key. - */ - addWeight: function(key, weight) { - this.weights[key] += weight; - }, - - /** - * Select n random elements (without replacement), default 1. - * If andRemove is true (default false), remove the elements - * from the list. (This is what the pop() method does.) - */ - peek: function(n, andRemove) { - if (typeof n === 'undefined') { - n = 1; - } - andRemove = !!andRemove; - - if (this.length - n < 0) { - throw new Error('Stack underflow! Tried to retrieve ' + n + - ' element' + (n === 1 ? '' : 's') + - ' from a list of ' + this.length); - } - - var heap = this._buildWeightedHeap(); - //console.debug('heap:', heap); - var result = []; - - for (var i = 0; i < n; i++) { - var key = heap.pop(); - //console.debug('k:', key); - if (this.hasData) { - result.push({key: key, data: this.data[key]}); - } else { - result.push(key); - } - if (andRemove) { - delete this.weights[key]; - delete this.data[key]; - this.length--; - } - } - return result; - }, - - /** - * Return the entire list in a random order (note that this does not mutate the list) - */ - shuffle: function() { - return this.peek(this.length); - }, - - /** - * - */ - pop: function(n) { - return this.peek(n, true); - }, - - /** - * Build a WeightedHeap instance based on the data we've got - */ - _buildWeightedHeap: function() { - var items = []; - for (var key in this.weights) if (this.weights.hasOwnProperty(key)) { - items.push([key, this.weights[key]]); - } - //console.log('items',items); - return new _WeightedHeap(items); - } - }; - - /** - * This is a javascript implementation of the algorithm described by - * Jason Orendorff here: http://stackoverflow.com/a/2149533/87990 - */ - function _HeapNode(weight, value, total) { - this.weight = weight; - this.value = value; - this.total = total; // Total weight of this node and its children - } - /** - * Note, we're using a heap structure here for its tree properties, not as a - * classic binary heap. A node heap[i] has children at heap[i<<1] and at - * heap[(i<<1)+1]. Its parent is at h[i>>1]. Heap[0] is vacant. - */ - function _WeightedHeap(items) { - this.heap = [null]; // Math is easier to read if we index array from 1 - - // First put everything on the heap - for (var i = 0; i < items.length; i++) { - var weight = items[i][1]; - var value = items[i][0]; - this.heap.push(new _HeapNode(weight, value, weight)); - } - // Now go through the heap and add each node's weight to its parent - for (i = this.heap.length - 1; i > 1; i--) { - this.heap[i>>1].total += this.heap[i].total; - } - //console.debug('_Wh heap', this.heap); - } - - _WeightedHeap.prototype = { - pop: function() { - // Start with a random amount of gas - var gas = this.heap[1].total * Math.random(); - - // Start driving at the root node - var i = 1; - - // While we have enough gas to keep going past i: - while (gas > this.heap[i].weight) { - gas -= this.heap[i].weight; // Drive past i - i <<= 1; // Move to first child - if (gas > this.heap[i].total) { - gas -= this.heap[i].total; // Drive past first child and its descendants - i++; // Move on to second child - } - } - // Out of gas - i is our selected node. - var value = this.heap[i].value; - var selectedWeight = this.heap[i].weight; - - this.heap[i].weight = 0; // Make sure i isn't chosen again - while (i > 0) { - // Remove the weight from its parent's total - this.heap[i].total -= selectedWeight; - i >>= 1; // Move to the next parent - } - return value; - } - }; - - // NB: another binary heap implementation is at - // http://eloquentjavascript.net/appendix2.html - - return _WeightedList; -})(); - -module.exports = WeightedList; \ No newline at end of file diff --git a/package.json b/package.json index 149ef93..5976a4e 100644 --- a/package.json +++ b/package.json @@ -26,19 +26,24 @@ "argcli": "latest", "brill-pos-tagger": "0.0.4", "cli-table": "^0.3.1", - "fs-tool": "latest", "md5-file": "^3.1.0", "natural": "latest", "node-tokenizer": "0.0.0", "pos": "^0.3.0", - "progress": "^1.1.8", "pstack": "latest", "seraph": "^0.14.4", - "underscore": "latest" + "python-range": "0.3.0", + "bluebird": "latest" }, "readmeFilename": "readme.md", "bugs": { "url": "https://github.com/26medias/context-aware-markov-chains/issues" }, - "scripts": {} + "scripts": {}, + "devDependencies": { + "babel-eslint": "^7.2.3", + "eslint": "^3.19.0", + "eslint-config-defaults": "^9.0.0", + "eslint-plugin-react": "^6.10.3" + } } diff --git a/test.js b/test.js index b14ac53..c33a3f0 100644 --- a/test.js +++ b/test.js @@ -1,36 +1,21 @@ -var cmarkov = require('./cmarkov'); +const Markov = new (require('./Markov'))(); -var bot = new cmarkov({ - name: 'trump', - depth: [1,5], - lowpri: 3, - weight: 1, - depthWeight: 1, - certainty: 0.1, - pos: true, - debug: false +const readMe = './training-data/bible.txt'; +const bot = new Markov({ + name: 'bible', + depth: [1, 5], + lowpri: 3, + weight: 1, + depthWeight: 1, + certainty: 0.1, + pos: true, + debug: true }); -//bot.test(); - - -bot.read("training-data/trump.txt", function() { - bot.readPOS("training-data/trump.txt", function() { - bot.generate("I would like to talk today about", 200, function(str) { - console.log(str); - }); - }); -}); - - -/* -var start = new Date().getTime(); -bot.generate("I would like to talk today about", 200, function(str) { - //console.trace(); - console.log(str); - - var end = new Date().getTime(); - var total = (end-start)/(1000*60); - console.log("Time: ",total); -}); -*/ \ No newline at end of file +bot.read(readMe, () => { + console.log('read file'); + bot.readPOS(readMe, () => { + console.log('read pos'); + bot.generate('god', 200, console.log); + }); +}); \ No newline at end of file