diff --git a/.gitignore b/.gitignore index 0bdf831..fe1e35b 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ apps/models # Files generated by dask apps/dask-worker-space + +# vscode config +.vscode/ diff --git a/deploy/templates/docker-compose.jin b/deploy/templates/docker-compose.jin index 019a186..ab86ef7 100644 --- a/deploy/templates/docker-compose.jin +++ b/deploy/templates/docker-compose.jin @@ -58,6 +58,16 @@ services: - JAGERENV={{environ.JAGERENV}} - ACCESS_KEY={{services.obj_storage.credentials.access_key}} - SECRET_KEY={{services.obj_storage.credentials.secret_key}} + prometheus: + {%- if build %} + build: "{{services.prometheus.buildpath}}" + {%- endif %} + image: "jagereye/prometheus:{{services.prometheus.version}}" + network_mode: "{{services.prometheus.network_mode}}" + ports: + {%- for key, port in services.prometheus.ports.items() %} + - "{{port}}" + {%- endfor %} {%- endif %} {%- if apps %} {%- for app, content in apps.items() %} diff --git a/services/api/analyzers.js b/services/api/analyzers.js index 3c6abd0..09b6eca 100644 --- a/services/api/analyzers.js +++ b/services/api/analyzers.js @@ -16,6 +16,19 @@ const MAX_ANALYZERS = 8 const NUM_OF_BRAINS = 1 const DEFAULT_REQUEST_TIMEOUT = 15000 +// setup for prometheus +const prometheusClient = require('prom-client') +const register = prometheusClient.register +const Gauge = prometheusClient.Gauge +const g = new Gauge({ + name: 'analyzer_status', + help: 'Analyzer status', + labelNames: ['analyzer'] +}); + +// status mapping +const analyzerStatus = ['created','starting','running','source_down','stopped'] + /* * Projections */ @@ -444,6 +457,37 @@ function getAnalyzerPipeline(req, res, next) { }) } +function getMetrics(req, res, next) { + models['analyzers'].find({}, getConfProjection, (err, list) => { + if (err) { return next(createError(500, null, err)) } + if (list.length === 0) { return res.status(200).send([]) } + const request = JSON.stringify({ + command: 'READ', + params: list.map(x => x['_id']) + }) + requestBackend(request, (reply, isLastReply, closeResponse) => { + if (reply['code'] && reply['code'] === NATS.REQ_TIMEOUT) { + let error = new Error('Timeout Error: Request: getting analyzers') + return next(createError(500, null, error)) + } + if (reply['error']) { + closeResponse() + return next(createError(500, reply['error']['message'])) + } + closeResponse() + result = list.map(x => { + let status = reply['result'][x['_id']] + g.labels(x['name']).set(analyzerStatus.indexOf(status)) + return + }) + + res.set('Content-Type', register.contentType); + res.status(200).send(register.metrics()) + return + }) + }) +} + /* * Routing Table */ @@ -467,5 +511,9 @@ routesWithAuth( ['post', '/analyzer/:id/start', startAnalyzer], ['post', '/analyzer/:id/stop', stopAnalyzer], ) +routesWithAuth( + router, + ['get', '/metrics', getMetrics] +) module.exports = router diff --git a/services/api/app.js b/services/api/app.js index 1fc4596..7631805 100644 --- a/services/api/app.js +++ b/services/api/app.js @@ -6,6 +6,7 @@ const cors = require('cors') const config = require('./config') const { users, createAdminUser } = require('./users') const analyzers = require('./analyzers') +const status = require('./status') const events = require('./events') const helpers = require('./helpers') @@ -20,6 +21,7 @@ app.use(expressValidator()) const API_ENTRY = `/${config.services.api.base_url}` app.use(API_ENTRY, users) app.use(API_ENTRY, analyzers) +app.use(API_ENTRY, status) app.use(API_ENTRY, events) app.use(API_ENTRY, helpers) diff --git a/services/api/package-lock.json b/services/api/package-lock.json index 9f231f4..1df2fa4 100644 --- a/services/api/package-lock.json +++ b/services/api/package-lock.json @@ -42,6 +42,16 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "asn1": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, "async": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", @@ -55,6 +65,20 @@ "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz", + "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w==" "aws-sdk": { "version": "2.245.1", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.245.1.tgz", @@ -81,6 +105,20 @@ "resolved": "https://registry.npmjs.org/base64url/-/base64url-2.0.0.tgz", "integrity": "sha1-6sFuA+oUOO/5Qj1puqNiYu0fcLs=" }, + "bcrypt-pbkdf": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", + "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", + "optional": true, + "requires": { + "tweetnacl": "0.14.5" + } + }, + "bintrees": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", + "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" + }, "bluebird": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.0.tgz", @@ -133,6 +171,24 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" + }, + "combined-stream": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", + "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", + "requires": { + "delayed-stream": "1.0.0" + } + }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -167,6 +223,14 @@ "vary": "1.1.2" } }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "1.0.0" + } + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -175,6 +239,11 @@ "ms": "2.0.0" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -185,6 +254,15 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "ecc-jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", + "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", + "optional": true, + "requires": { + "jsbn": "0.1.1" + } + }, "ecdsa-sig-formatter": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.9.tgz", @@ -309,6 +387,16 @@ } } }, + "extend": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", + "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, "fast-deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", @@ -349,6 +437,21 @@ "which": "1.3.0" } }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", + "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", + "requires": { + "asynckit": "0.4.0", + "combined-stream": "1.0.6", + "mime-types": "2.1.18" + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -359,6 +462,41 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "1.0.0" + } + }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", + "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", + "requires": { + "ajv": "5.5.2", + "har-schema": "2.0.0" + }, + "dependencies": { + "ajv": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", + "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "requires": { + "co": "4.6.0", + "fast-deep-equal": "1.1.0", + "fast-json-stable-stringify": "2.0.0", + "json-schema-traverse": "0.3.1" + } + } + } + }, "hooks-fixed": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hooks-fixed/-/hooks-fixed-2.0.2.tgz", @@ -382,6 +520,16 @@ } } }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "1.0.0", + "jsprim": "1.4.1", + "sshpk": "1.14.1" + } + }, "iconv-lite": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", @@ -402,6 +550,11 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz", "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=" }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -412,6 +565,10 @@ "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", @@ -426,11 +583,27 @@ "esprima": "4.0.0" } }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "optional": true + }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, "json-schema-traverse": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, "jsonwebtoken": { "version": "8.2.1", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.2.1.tgz", @@ -455,6 +628,17 @@ } } }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "jwa": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.5.tgz", @@ -4519,6 +4703,11 @@ "resolved": "https://registry.npmjs.org/nuid/-/nuid-0.6.14.tgz", "integrity": "sha1-oiXItyDkbFY62NE2QTaGafbCW0g=" }, + "oauth-sign": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -4575,11 +4764,24 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha1-HUCLP9t2kjuVQ9lvtMnf1TXZy10=" }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, + "prom-client": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.0.0.tgz", + "integrity": "sha512-UM4uYDwmA7x9yTq+AZcL4lU/XF11RkbQWbIouFaVMLxdV4qBB5CEmEosQlR1lGvduBuS1IWonHFh1WBtFSoZ3A==", + "requires": { + "tdigest": "0.1.1" + } + }, "proxy-addr": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz", @@ -4639,6 +4841,51 @@ "resolved": "https://registry.npmjs.org/regexp-clone/-/regexp-clone-0.0.1.tgz", "integrity": "sha1-p8LgmJH9vzj7sQ03b7cwA+aKxYk=" }, + "request": { + "version": "2.87.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.87.0.tgz", + "integrity": "sha512-fcogkm7Az5bsS6Sl0sibkbhcKsnyon/jV1kF3ajGmF0c8HrttdKTPRT9hieOaQHA5HEq6r8OyWOo/o781C1tNw==", + "requires": { + "aws-sign2": "0.7.0", + "aws4": "1.7.0", + "caseless": "0.12.0", + "combined-stream": "1.0.6", + "extend": "3.0.1", + "forever-agent": "0.6.1", + "form-data": "2.3.2", + "har-validator": "5.0.3", + "http-signature": "1.2.0", + "is-typedarray": "1.0.0", + "isstream": "0.1.2", + "json-stringify-safe": "5.0.1", + "mime-types": "2.1.18", + "oauth-sign": "0.8.2", + "performance-now": "2.1.0", + "qs": "6.5.1", + "safe-buffer": "5.1.1", + "tough-cookie": "2.3.4", + "tunnel-agent": "0.6.0", + "uuid": "3.2.1" + } + }, + "request-promise-core": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", + "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=", + "requires": { + "lodash": "4.17.5" + } + }, + "request-promise-native": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.5.tgz", + "integrity": "sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU=", + "requires": { + "request-promise-core": "1.1.1", + "stealthy-require": "1.1.1", + "tough-cookie": "2.3.4" + } + }, "require_optional": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", @@ -4721,11 +4968,31 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, + "sshpk": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", + "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=", + "requires": { + "asn1": "0.2.3", + "assert-plus": "1.0.0", + "bcrypt-pbkdf": "1.0.1", + "dashdash": "1.14.1", + "ecc-jsbn": "0.1.1", + "getpass": "0.1.7", + "jsbn": "0.1.1", + "tweetnacl": "0.14.5" + } + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, "string_decoder": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", @@ -4734,6 +5001,43 @@ "safe-buffer": "5.1.1" } }, + "tdigest": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", + "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "requires": { + "bintrees": "1.0.1" + } + }, + "tough-cookie": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", + "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", + "requires": { + "punycode": "1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "5.1.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "optional": true + }, "type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", @@ -4788,15 +5092,25 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", + "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "1.3.0" + } + }, "which": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz", diff --git a/services/api/package.json b/services/api/package.json index 39e798a..91f5d33 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -30,6 +30,9 @@ "npm": "^5.8.0", "passport": "^0.4.0", "passport-jwt": "^4.0.0", + "prom-client": "^11.0.0", + "request": "^2.87.0", + "request-promise-native": "^1.0.5", "ws": "3.3.2" } } diff --git a/services/api/status.js b/services/api/status.js new file mode 100644 index 0000000..bcefff4 --- /dev/null +++ b/services/api/status.js @@ -0,0 +1,53 @@ +const express = require('express') +const router = express.Router() +const { routesWithAuth } = require('./auth') +const { createError } = require('./utils') +const request = require('request-promise-native'); + +// status mapping +const analyzerStatus = ['created','starting','running','source_down','stopped'] + +async function getStatus(req, res, next) { + let analyzers, targets + try { + analyzers = await request('http://localhost:9090/api/v1/query?query=analyzer_status', { json: true }) + targets = await request('http://localhost:9090/api/v1/targets', { json: true }) + } catch(e) { + return next(createError(500, e)) + } + let result = { + analyzers: [], + services: [] + } + // analyzers + if (analyzers['status'] !== 'success') { return next(createError(500, err)) } + analyzers = analyzers['data']['result'] + for(let i = 0 ; i < analyzers.length ; i++) { + let statusCode = parseInt(analyzers[i]['value'][1]) + result.analyzers.push({ + analyzer : analyzers[i]['metric']['analyzer'], + status : statusCode === -1 ? 'unknown' : analyzerStatus[statusCode] + }) + } + // targets + if (targets['status'] !== 'success') { return next(createError(500, err)) } + targets = targets['data']['activeTargets'] + for(let i = 0 ; i < targets.length ; i++) { + result.services.push({ + service: targets[i].labels.job, + status: targets[i].health + }) + } + + return res.send(result) +} + +/* + * Routing Table + */ +routesWithAuth( + router, + ['get', '/status', getStatus], +) + +module.exports = router \ No newline at end of file diff --git a/services/database/Dockerfile b/services/database/Dockerfile index f8112e8..c641bfc 100644 --- a/services/database/Dockerfile +++ b/services/database/Dockerfile @@ -3,3 +3,12 @@ FROM mongo:3.6.0 MAINTAINER uniray7 uniray7@gmail.com # TODO: need add access control setting + +COPY mongodb_exporter-linux-amd64 /mongodb_exporter +COPY start.sh /start.sh + +RUN chmod +x /mongodb_exporter && chmod +x /start.sh + +EXPOSE 27017 9001 + +CMD ["/start.sh"] diff --git a/services/database/mongodb_exporter-linux-amd64 b/services/database/mongodb_exporter-linux-amd64 new file mode 100644 index 0000000..71f3401 Binary files /dev/null and b/services/database/mongodb_exporter-linux-amd64 differ diff --git a/services/database/start.sh b/services/database/start.sh new file mode 100644 index 0000000..4becd7f --- /dev/null +++ b/services/database/start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +/mongodb_exporter & +mongod \ No newline at end of file diff --git a/services/messaging/Dockerfile b/services/messaging/Dockerfile index 46f197f..68d346d 100644 --- a/services/messaging/Dockerfile +++ b/services/messaging/Dockerfile @@ -1,8 +1,29 @@ -FROM scratch + + +### prometheus exporter stage ### +# Golang binary building stage +FROM golang:1.9.4 + +# download the source +WORKDIR /go/src/github.com/nats-io/prometheus-nats-exporter +RUN git clone --branch v0.1.0 https://github.com/nats-io/prometheus-nats-exporter.git . + +# build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -a -tags netgo -installsuffix netgo -ldflags "-s -w" + + +### final image stage ### +FROM bash + +COPY --from=0 /go/src/github.com/nats-io/prometheus-nats-exporter/prometheus-nats-exporter /prometheus-nats-exporter COPY gnatsd /gnatsd COPY gnatsd.conf gnatsd.conf +COPY start.sh /start.sh + +RUN chmod +x /start.sh + +EXPOSE 7777 -# Run via the configuration file -ENTRYPOINT ["/gnatsd"] -CMD ["-c", "gnatsd.conf"] +ENTRYPOINT ["/usr/local/bin/bash"] +CMD ["-c", "/start.sh"] \ No newline at end of file diff --git a/services/messaging/start.sh b/services/messaging/start.sh new file mode 100644 index 0000000..c0dda2b --- /dev/null +++ b/services/messaging/start.sh @@ -0,0 +1,7 @@ +#!/usr/local/bin/bash + +# start the prometheus exporter +/prometheus-nats-exporter -varz "http://localhost:7777" & + +# Run via the configuration file +/gnatsd -c gnatsd.conf diff --git a/services/obj_storage/Dockerfile b/services/obj_storage/Dockerfile index 31b7246..195fa52 100644 --- a/services/obj_storage/Dockerfile +++ b/services/obj_storage/Dockerfile @@ -1,4 +1,4 @@ -FROM minio/minio:RELEASE.2018-04-04T05-20-54Z +FROM minio/minio:RELEASE.2018-05-11T00-29-24Z MAINTAINER SuJiaKuan feabries@gmail.com diff --git a/services/prometheus/Dockerfile b/services/prometheus/Dockerfile new file mode 100644 index 0000000..c6b8106 --- /dev/null +++ b/services/prometheus/Dockerfile @@ -0,0 +1,10 @@ +FROM prom/prometheus:v2.2.1 + +LABEL maintainer="blair1226@gmail.com" + +COPY prometheus.yml /etc/prometheus/prometheus.yml + +EXPOSE 9090 +# Override the original entrypoint. +ENTRYPOINT ["prometheus"] +CMD ["--config.file=/etc/prometheus/prometheus.yml"] \ No newline at end of file diff --git a/services/prometheus/prometheus.yml b/services/prometheus/prometheus.yml new file mode 100644 index 0000000..ab2ec7d --- /dev/null +++ b/services/prometheus/prometheus.yml @@ -0,0 +1,26 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] + - job_name: 'node' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9100'] + - job_name: 'mongodb' + static_configs: + - targets: ['localhost:9001'] + - job_name: 'minio' + metrics_path: /minio/prometheus/metrics + static_configs: + - targets: ['localhost:9000'] + - job_name: 'nats' + static_configs: + - targets: ['localhost:7777'] + - job_name: 'analyzers' + metrics_path: /api/v1/metrics + static_configs: + - targets: ['localhost:5000'] \ No newline at end of file diff --git a/shared/config.development.yml b/shared/config.development.yml index cefc853..d35b078 100644 --- a/shared/config.development.yml +++ b/shared/config.development.yml @@ -18,14 +18,16 @@ services: network_mode: host ports: client: "27017" + prometheus: "9001" messaging: version: "nats-1.0.4" network_mode: host ports: client: "4222" monitor: "8222" + prometheus: "7777" obj_storage: - version: "minio-RELEASE.2018-04-04T05-20-54Z" + version: "minio-RELEASE.2018-05-11T00-29-24Z" network_mode: host ports: client: "9000" @@ -39,6 +41,12 @@ services: credentials: access_key: 'jagereye' secret_key: 'jagereye' + prometheus: + version: "prometheus-v2.2.1" + network_mode: host + ports: + client: "9090" + expiration: version: "0.0.1" network_mode: host