mirror of
https://github.com/BreizhHardware/express-prom-bundle.git
synced 2026-01-20 09:47:23 +01:00
Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5b1aa494cb | ||
|
|
65549a769b | ||
|
|
de83ac09a0 | ||
|
|
52865dfb02 | ||
|
|
5c6ed64a31 | ||
|
|
d8c6492163 | ||
|
|
c92b85ae96 | ||
|
|
48f8b992fd | ||
|
|
61e4343a8c | ||
|
|
7b89690d3b | ||
|
|
40db5cacbd | ||
|
|
43334b923f | ||
|
|
20eb668e36 | ||
|
|
53c4505378 | ||
|
|
b0aa05d42b | ||
|
|
1e9300ebf3 | ||
|
|
190bf0be31 | ||
|
|
c918b44c2b | ||
|
|
b374e7f54c | ||
|
|
08d98b450c | ||
|
|
3675c516a9 | ||
|
|
42197ac478 | ||
|
|
66e1fca8dd | ||
|
|
36c40daa5d | ||
|
|
8beadcb434 |
82
.eslintrc
82
.eslintrc
@@ -1,42 +1,54 @@
|
||||
{
|
||||
"env": {
|
||||
"node": true,
|
||||
"es6": true
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 6,
|
||||
"sourceType": "module"
|
||||
},
|
||||
|
||||
"extends": "eslint:recommended",
|
||||
"globals": {
|
||||
"app": true,
|
||||
"fetch": true
|
||||
},
|
||||
|
||||
"rules": {
|
||||
"no-cond-assign": 0,
|
||||
"no-constant-condition": 0,
|
||||
"no-empty": 0,
|
||||
"no-fallthrough": 0,
|
||||
"no-unused-vars": 1,
|
||||
"no-console": 1,
|
||||
"env": {
|
||||
"node": true,
|
||||
"es6": true
|
||||
},
|
||||
|
||||
"semi": 2,
|
||||
"curly": 2,
|
||||
"consistent-this": [2, "self"],
|
||||
"indent": [ 2, 4, { "SwitchCase": 1 } ],
|
||||
"linebreak-style": [2, "unix"],
|
||||
"no-nested-ternary": 2,
|
||||
"extends": "eslint:recommended",
|
||||
|
||||
"new-parens": 2,
|
||||
"no-dupe-class-members": 2,
|
||||
"require-yield": 2,
|
||||
"arrow-spacing": 1,
|
||||
"no-var": 2,
|
||||
"rules": {
|
||||
"array-bracket-spacing": [2, "never"],
|
||||
"block-scoped-var": 2,
|
||||
"brace-style": [2, "1tbs"],
|
||||
"computed-property-spacing": [2, "never"],
|
||||
"curly": 2,
|
||||
"eol-last": 2,
|
||||
"eqeqeq": [2, "smart"],
|
||||
"max-depth": [1, 3],
|
||||
"new-cap": 1,
|
||||
"no-extend-native": 2,
|
||||
"no-mixed-spaces-and-tabs": 2,
|
||||
"no-trailing-spaces": 1,
|
||||
"no-unused-vars": 1,
|
||||
"no-use-before-define": [2, "nofunc"],
|
||||
"object-curly-spacing": [2, "never"],
|
||||
"quotes": [1, "single", "avoid-escape"],
|
||||
"semi": [2, "always"],
|
||||
"keyword-spacing": [2, {"before": true, "after": true}],
|
||||
"space-unary-ops": 2,
|
||||
"no-console": [1, { allow: ["info", "warn", "error"] }],
|
||||
|
||||
"no-multi-spaces": 1,
|
||||
"space-return-throw-case": 0,
|
||||
"space-infix-ops": [1, {"int32Hint": false}],
|
||||
"brace-style": 1,
|
||||
"space-before-blocks": 1,
|
||||
"operator-linebreak": [1, "before"],
|
||||
"no-unneeded-ternary": 1,
|
||||
"no-lonely-if": 1,
|
||||
"key-spacing": 1,
|
||||
"quotes": [1, "double", "avoid-escape"],
|
||||
"no-trailing-spaces": [1, { "skipBlankLines": true }]
|
||||
}
|
||||
"max-len": [1, 120],
|
||||
"max-statements": [1, 50],
|
||||
|
||||
"consistent-this": [2, "self"],
|
||||
"no-var": 2,
|
||||
"no-dupe-class-members": 2,
|
||||
"operator-linebreak": [1, "before"],
|
||||
"no-unneeded-ternary": [1, {"defaultAssignment": false}],
|
||||
"no-lonely-if": 1,
|
||||
"linebreak-style": [2, "unix"],
|
||||
"no-nested-ternary": 2,
|
||||
"require-yield": 2
|
||||
}
|
||||
}
|
||||
|
||||
145
README.md
145
README.md
@@ -1,16 +1,17 @@
|
||||
[](https://travis-ci.org/jochen-schweizer/express-prom-bundle) [](https://coveralls.io/github/jochen-schweizer/express-prom-bundle?branch=master) [](https://www.tldrlegal.com/l/mit)
|
||||
[](https://travis-ci.org/jochen-schweizer/express-prom-bundle) [](https://coveralls.io/github/jochen-schweizer/express-prom-bundle?branch=master) [](https://www.tldrlegal.com/l/mit) [](http://badge.fury.io/js/express-prom-bundle)
|
||||
|
||||
# express prometheus bundle
|
||||
|
||||
express middleware with popular prometheus metrics in one bundle.
|
||||
Express middleware with popular prometheus metrics in one bundle. It's also compatible with koa v1 (see below).
|
||||
|
||||
Internally it uses **prom-client**. See: https://github.com/siimon/prom-client
|
||||
|
||||
Included metrics:
|
||||
|
||||
|
||||
* `up`: normally is just 1
|
||||
* `nodejs_memory_heap_total_bytes` and `nodejs_memory_heap_used_bytes`
|
||||
* `http_request_seconds`: http latency histogram labeled with `status_code`
|
||||
* `http_request_duration_seconds`: http latency histogram labeled with `status_code`, `method` and `path`
|
||||
|
||||
**Please note version 2.x is NOT backwards compatible with 1.x**
|
||||
|
||||
## Install
|
||||
|
||||
@@ -21,8 +22,9 @@ npm install express-prom-bundle
|
||||
## Usage
|
||||
|
||||
```javascript
|
||||
const promBundle = require("express-prom-bundle"),
|
||||
const promBundle = require("express-prom-bundle");
|
||||
const metricsMiddleware = promBundle({/* options */ });
|
||||
const app = require("express")();
|
||||
|
||||
app.use(metricsMiddleware);
|
||||
app.use(/* your middleware */);
|
||||
@@ -34,7 +36,7 @@ app.listen(3000);
|
||||
|
||||
**ALERT!**
|
||||
|
||||
The order in wich the routes are registered is important, since
|
||||
The order in which the routes are registered is important, since
|
||||
**only the routes registered after the express-prom-bundle will be measured**
|
||||
|
||||
You can use this to your advantage to bypass some of the routes.
|
||||
@@ -42,34 +44,68 @@ See the example below.
|
||||
|
||||
## Options
|
||||
|
||||
* **prefix**: prefix added to every metric name
|
||||
* **whitelist**, **blacklist**: array of strings or regexp specifying which metrics to include/exclude
|
||||
* **buckets**: buckets used for `http_request_seconds` histogram
|
||||
* **excludeRoutes**: array of strings or regexp specifying which routes should be skipped for `http_request_seconds` metric. It uses `req.path` as subject when checking
|
||||
* **autoregister**: boolean. If `/metrics` endpoint should be registered. It is **true** by default
|
||||
* **buckets**: buckets used for `http_request_seconds` histogram
|
||||
* **includeMethod**: include HTTP method (GET, PUT, ...) as a label to `http_request_duration_seconds`
|
||||
* **includePath**: include URL path as a label (see below)
|
||||
* **normalizePath**: boolean or `function(req)` - path normalization for `includePath` option
|
||||
* **autoregister**: if `/metrics` endpoint should be registered. (Default: **true**)
|
||||
* **whitelist**, **blacklist**: array of strings or regexp specifying which metrics to include/exclude
|
||||
* **excludeRoutes**: (deprecated) array of strings or regexp specifying which routes should be skipped for `http_request_duration_seconds` metric. It uses `req.originalUrl` as subject when checking. You want normally use express or meddleware features instead of this options.
|
||||
|
||||
## Example
|
||||
### More details on includePath option
|
||||
|
||||
The goal is to have separate latency statistics by URL path, e.g. `/my-app/user/`, `/products/by-category` etc.
|
||||
|
||||
Just taking `req.path` as a label value won't work as IDs are often part of the URL,
|
||||
like `/user/12352/profile`. So what we actually need is a path template.
|
||||
The module tries to figure out what parts of the path are values or IDs,
|
||||
and what is an actual path. The example mentioned before would be
|
||||
normalized to `/user/#val/profile` and that will become the value for the label.
|
||||
|
||||
You can override this magical behavior and define your own function by
|
||||
providing an optional callback using **normalizePath** option.
|
||||
You can also replace the default **normalizePath** function globally.
|
||||
This is handy if the rest of the middleware is done elsewhere
|
||||
e.g. via `kraken.js meddleware`.
|
||||
|
||||
```javascript
|
||||
app.use(promBundle(/* options? */));
|
||||
|
||||
// let's reuse the existing one and just add some
|
||||
// functionality on top
|
||||
const originalNormalize = promBunle.normalizePath;
|
||||
promBunle.normalizePath = (req, opts) => {
|
||||
const path = originalNormalize(req, opts);
|
||||
// count all docs (no matter which file) as a single path
|
||||
return path.match(/^\/docs/) ? '/docs/*' : path;
|
||||
};
|
||||
```
|
||||
|
||||
For more details:
|
||||
* [url-value-parser](https://www.npmjs.com/package/url-value-parser) - magic behind automatic path normalization
|
||||
* [normalizePath.js](https://github.com/jochen-schweizer/express-prom-bundle/blob/master/src/normalizePath.js) - source code for path processing
|
||||
|
||||
|
||||
|
||||
## express example
|
||||
|
||||
setup std. metrics but exclude `up`-metric:
|
||||
|
||||
```javascript
|
||||
"use strict";
|
||||
|
||||
const express = require("express"),
|
||||
app = express(),
|
||||
promBundle = require("express-prom-bundle");
|
||||
|
||||
const express = require("express");
|
||||
const app = express();
|
||||
const promBundle = require("express-prom-bundle");
|
||||
|
||||
// calls to this route will not appear in metrics
|
||||
// because it's applied before promBundle
|
||||
app.get("/status", (req, res) => res.send("i am healthy"));
|
||||
|
||||
app.use(promBundle({
|
||||
prefix: "demo_app:something",
|
||||
excludeRoutes: ["/foo"]
|
||||
}));
|
||||
// register metrics collection for all routes
|
||||
// ... except those starting with /foo
|
||||
app.use("/((?!foo))*", promBundle({includePath: true}));
|
||||
|
||||
// this call will NOT appear in metrics, because it matches excludeRoutes
|
||||
// this call will NOT appear in metrics,
|
||||
// because express will skip the metrics middleware
|
||||
app.get("/foo", (req, res) => res.send("bar"));
|
||||
|
||||
// calls to this route will appear in metrics
|
||||
@@ -80,6 +116,65 @@ app.listen(3000);
|
||||
|
||||
See an [advanced example on github](https://github.com/jochen-schweizer/express-prom-bundle/blob/master/advanced-example.js)
|
||||
|
||||
## koa v1 example
|
||||
|
||||
```javascript
|
||||
const promBundle = require("express-prom-bundle");
|
||||
const koa = require("koa");
|
||||
const c2k = require("koa-connect");
|
||||
const metricsMiddleware = promBundle({/* options */ });
|
||||
|
||||
const app = koa();
|
||||
|
||||
app.use(c2k(metricsMiddleware));
|
||||
app.use(/* your middleware */);
|
||||
app.listen(3000);
|
||||
```
|
||||
|
||||
## using with kraken.js
|
||||
|
||||
Here is meddleware config sample, which can be used in a standard **kraken.js** application:
|
||||
|
||||
```json
|
||||
{
|
||||
"middleware": {
|
||||
"expressPromBundle": {
|
||||
"route": "/((?!status|favicon.ico|robots.txt))*",
|
||||
"priority": 0,
|
||||
"module": {
|
||||
"name": "express-prom-bundle",
|
||||
"arguments": [
|
||||
{
|
||||
"includeMethod": true,
|
||||
"buckets": [0.1, 1, 5]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Changelog
|
||||
|
||||
* **2.1.0**
|
||||
* deprecate **excludeRoutes**, use **req.originalPath** instead of **req.path**
|
||||
* **2.0.0**
|
||||
* the reason for the version lift were:
|
||||
* compliance to official naming recommendation: https://prometheus.io/docs/practices/naming/
|
||||
* stopping promotion of an anti-pattern - see https://groups.google.com/d/msg/prometheus-developers/XjlOnDCK9qc/ovKzV3AIBwAJ
|
||||
* dealing with **prom-client** being a singleton with a built-in registry
|
||||
* main histogram metric renamed from `http_request_seconds` to `http_request_duration_seconds`
|
||||
* options removed: **prefix**, **keepDefaultMetrics**
|
||||
* factory removed (as the only reason of it was adding the prefix)
|
||||
* upgrade prom-client to 6.3.0
|
||||
* code style changed to the one closer to express
|
||||
|
||||
|
||||
* **1.2.1**
|
||||
* upgrade prom-client to 6.1.2
|
||||
* add options: includeMethod, includePath, keepDefaultMetrics
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
MIT
|
||||
|
||||
@@ -1,38 +1,44 @@
|
||||
"use strict";
|
||||
'use strict';
|
||||
|
||||
const express = require("express");
|
||||
const express = require('express');
|
||||
const app = express();
|
||||
const promBundle = require("express-prom-bundle");
|
||||
const promBundle = require('express-prom-bundle');
|
||||
|
||||
// here we want to remove default metrics provided in prom-client
|
||||
// this must be done before initializing promBundle
|
||||
clearInterval(promBundle.promClient.defaultMetrics());
|
||||
promBundle.promClient.register.clear();
|
||||
|
||||
const bundle = promBundle({
|
||||
prefix: "demo_app:something:",
|
||||
blacklist: [/up/],
|
||||
buckets: [0.1, 0.4, 0.7]
|
||||
blacklist: [/up/],
|
||||
buckets: [0.1, 0.4, 0.7],
|
||||
includeMethod: true,
|
||||
includePath: true
|
||||
});
|
||||
|
||||
app.use(bundle);
|
||||
|
||||
// native prom-client metric (no prefix)
|
||||
const c1 = new bundle.promClient.Counter("c1", "c1 help");
|
||||
const c1 = new bundle.promClient.Counter('c1', 'c1 help');
|
||||
c1.inc(10);
|
||||
|
||||
// create metric using factory (w/ prefix)
|
||||
const c2 = bundle.factory.newCounter("c2", "c2 help");
|
||||
c2.inc(20);
|
||||
|
||||
app.get("/foo", (req, res) => {
|
||||
setTimeout(() => {
|
||||
res.send("foo response\n");
|
||||
}, 500);
|
||||
app.get('/foo/:id', (req, res) => {
|
||||
setTimeout(() => {
|
||||
res.send('foo response\n');
|
||||
}, 500);
|
||||
});
|
||||
app.get("/bar", (req, res) => res.send("bar response\n"));
|
||||
app.delete('/foo/:id', (req, res) => {
|
||||
setTimeout(() => {
|
||||
res.send('foo deleted\n');
|
||||
}, 300);
|
||||
});
|
||||
app.get('/bar', (req, res) => res.send('bar response\n'));
|
||||
|
||||
app.listen(3000, () => console.log("listening on 3000")); // eslint-disable-line
|
||||
|
||||
/*
|
||||
test in shell console:
|
||||
|
||||
curl localhost:3000/foo
|
||||
curl localhost:3000/bar
|
||||
curl localhost:3000/metrics
|
||||
*/
|
||||
app.listen(3000, () => console.info( // eslint-disable-line
|
||||
'listening on 3000\n'
|
||||
+ 'test in shell console\n\n'
|
||||
+ 'curl localhost:3000/foo/1234\n'
|
||||
+ 'curl -X DELETE localhost:3000/foo/5432\n'
|
||||
+ 'curl localhost:3000/bar\n'
|
||||
+ 'curl localhost:3000/metrics\n'
|
||||
));
|
||||
|
||||
23
package.json
23
package.json
@@ -1,13 +1,14 @@
|
||||
{
|
||||
"name": "express-prom-bundle",
|
||||
"version": "1.1.7",
|
||||
"version": "2.2.0",
|
||||
"description": "express middleware with popular prometheus metrics in one bundle",
|
||||
"main": "src/index.js",
|
||||
"keywords": [
|
||||
"prometheus",
|
||||
"metrics",
|
||||
"express",
|
||||
"bundle"
|
||||
"path",
|
||||
"method"
|
||||
],
|
||||
"scripts": {
|
||||
"test": "node_modules/jasme/run.js"
|
||||
@@ -16,15 +17,19 @@
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"on-finished": "^2.3.0",
|
||||
"prom-client": "^3.4.6"
|
||||
"prom-client": "^6.3.0",
|
||||
"url-value-parser": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"coveralls": "^2.11.12",
|
||||
"eslint": "^2.13.1",
|
||||
"express": "^4.13.4",
|
||||
"istanbul": "^0.4.4",
|
||||
"jasme": "^4.1.2",
|
||||
"supertest": "^1.2.0"
|
||||
"coveralls": "^2.11.15",
|
||||
"eslint": "^3.11.1",
|
||||
"express": "^4.14.0",
|
||||
"istanbul": "^0.4.5",
|
||||
"jasme": "^5.2.0",
|
||||
"koa": "^1.2.4",
|
||||
"koa-connect": "^1.0.0",
|
||||
"supertest": "^2.0.1",
|
||||
"supertest-koa-agent": "^0.3.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
"use strict";
|
||||
/* eslint-env jasmine */
|
||||
const PromFactory = require("../src/PromFactory");
|
||||
|
||||
describe("PromFactory", () => {
|
||||
let factory;
|
||||
beforeEach(() => {
|
||||
factory = new PromFactory();
|
||||
});
|
||||
it("creates Counter", () => {
|
||||
const metric = factory.newCounter(
|
||||
"test1",
|
||||
"help for test1",
|
||||
["label1", "label2"]
|
||||
);
|
||||
expect(metric.name).toBe("test1");
|
||||
expect(metric.help).toBe("help for test1");
|
||||
expect(metric.labelNames).toEqual(["label1", "label2"]);
|
||||
});
|
||||
it("throws on duplicate names", () => {
|
||||
factory.newCounter("n","h");
|
||||
expect(() => factory.newCounter("n","h2")).toThrow();
|
||||
});
|
||||
it("creates Gauge", () => {
|
||||
const metric = factory.newGauge(
|
||||
"test2",
|
||||
"help for test2",
|
||||
["label1", "label2"]
|
||||
);
|
||||
expect(metric.name).toBe("test2");
|
||||
expect(metric.help).toBe("help for test2");
|
||||
expect(metric.labelNames).toEqual(["label1", "label2"]);
|
||||
});
|
||||
it("creates Histogram with labels", () => {
|
||||
const metric = factory.newHistogram(
|
||||
"test3",
|
||||
"help for test3",
|
||||
["label1", "label2"],
|
||||
{buckets: [1, 2, 3]}
|
||||
);
|
||||
expect(metric.name).toBe("test3");
|
||||
expect(metric.help).toBe("help for test3");
|
||||
expect(metric.labelNames).toEqual(["label1", "label2"]);
|
||||
expect(metric.bucketValues).toEqual({"1": 0, "2": 0, "3": 0});
|
||||
});
|
||||
it("creates Summary without labels", () => {
|
||||
const metric = factory.newSummary(
|
||||
"test4",
|
||||
"help for test4",
|
||||
{percentiles: [0.1, 0.5]}
|
||||
);
|
||||
expect(metric.name).toBe("test4");
|
||||
expect(metric.help).toBe("help for test4");
|
||||
expect(metric.percentiles).toEqual([0.1, 0.5]);
|
||||
});
|
||||
|
||||
});
|
||||
221
spec/index.spec.js
Normal file
221
spec/index.spec.js
Normal file
@@ -0,0 +1,221 @@
|
||||
'use strict';
|
||||
/* eslint-env jasmine */
|
||||
|
||||
const express = require('express');
|
||||
const supertest = require('supertest');
|
||||
const bundle = require('../');
|
||||
const koa = require('koa');
|
||||
const c2k = require('koa-connect');
|
||||
const supertestKoa = require('supertest-koa-agent');
|
||||
const promClient = require('prom-client');
|
||||
|
||||
describe('index', () => {
|
||||
beforeEach(() => {
|
||||
promClient.register.clear();
|
||||
});
|
||||
|
||||
it('metrics returns up=1', done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
whitelist: ['up']
|
||||
});
|
||||
app.use(bundled);
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get('/test').end(() => {
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.text).toMatch(/up\s1/);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('metrics should be attached to /metrics by default', done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
whitelist: ['up']
|
||||
});
|
||||
app.use(bundled);
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('metrics can be attached to /metrics programatically', done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
autoregister: false
|
||||
});
|
||||
app.use(bundled.metricsMiddleware);
|
||||
app.use(bundled);
|
||||
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('metrics can be filtered using exect match', () => {
|
||||
const instance = bundle({blacklist: ['up']});
|
||||
expect(instance.metrics.up).not.toBeDefined();
|
||||
expect(instance.metrics.http_request_duration_seconds).toBeDefined();
|
||||
});
|
||||
it('metrics can be filtered using regex', () => {
|
||||
const instance = bundle({blacklist: [/http/]});
|
||||
expect(instance.metrics.up).toBeDefined();
|
||||
expect(instance.metrics.http_request_duration_seconds).not.toBeDefined();
|
||||
});
|
||||
it('metrics can be whitelisted', () => {
|
||||
const instance = bundle({whitelist: [/^up$/]});
|
||||
expect(instance.metrics.up).toBeDefined();
|
||||
expect(instance.metrics.nodejs_memory_heap_total_bytes).not.toBeDefined();
|
||||
expect(instance.metrics.http_request_duration_seconds).not.toBeDefined();
|
||||
});
|
||||
it('throws on both white and blacklist', () => {
|
||||
expect(() => {
|
||||
bundle({whitelist: [/up/], blacklist: [/up/]});
|
||||
}).toThrow();
|
||||
});
|
||||
it('returns error 500 on incorrect middleware usage', done => {
|
||||
const app = express();
|
||||
app.use(bundle);
|
||||
supertest(app)
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(500);
|
||||
done();
|
||||
});
|
||||
});
|
||||
it('http latency gets counted', done => {
|
||||
const app = express();
|
||||
const instance = bundle();
|
||||
app.use(instance);
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get('/test')
|
||||
.end(() => {
|
||||
const metricHashMap = instance.metrics.http_request_duration_seconds.hashMap;
|
||||
expect(metricHashMap['status_code:200']).toBeDefined();
|
||||
const labeled = metricHashMap['status_code:200'];
|
||||
expect(labeled.count).toBe(1);
|
||||
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('filters out the excludeRoutes', done => {
|
||||
const app = express();
|
||||
const instance = bundle({
|
||||
excludeRoutes: ['/test']
|
||||
});
|
||||
app.use(instance);
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get('/test')
|
||||
.end(() => {
|
||||
const metricHashMap = instance.metrics.http_request_duration_seconds.hashMap;
|
||||
expect(metricHashMap['status_code:200']).not.toBeDefined();
|
||||
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('complains about deprecated options', () => {
|
||||
expect(() => bundle({prefix: 'hello'})).toThrow();
|
||||
});
|
||||
|
||||
it('tolerates includePath, includeMethod', done => {
|
||||
const app = express();
|
||||
const instance = bundle({
|
||||
includePath: true,
|
||||
includeMethod: true
|
||||
});
|
||||
app.use(instance);
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get('/test')
|
||||
.end(() => {
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizePath can be replaced', done => {
|
||||
const app = express();
|
||||
const original = bundle.normalizePath;
|
||||
bundle.normalizePath = () => 'dummy';
|
||||
const instance = bundle({
|
||||
includePath: true,
|
||||
});
|
||||
app.use(instance);
|
||||
app.use('/test', (req, res) => res.send('it worked'));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get('/test')
|
||||
.end(() => {
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.text).toMatch(/"dummy"/m);
|
||||
bundle.normalizePath = original;
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('Koa: metrics returns up=1', done => {
|
||||
const app = koa();
|
||||
const bundled = bundle({
|
||||
whitelist: ['up']
|
||||
});
|
||||
app.use(c2k(bundled));
|
||||
|
||||
app.use(function*(next) {
|
||||
if (this.path !== 'test') {
|
||||
return yield next;
|
||||
}
|
||||
this.body = 'it worked';
|
||||
});
|
||||
|
||||
const agent = supertestKoa(app);
|
||||
agent.get('/test').end(() => {
|
||||
agent
|
||||
.get('/metrics')
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.text).toMatch(/^up\s1/m);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,140 +0,0 @@
|
||||
"use strict";
|
||||
/* eslint-env jasmine */
|
||||
|
||||
let express = require("express"),
|
||||
supertest = require("supertest"),
|
||||
bundle = require("../");
|
||||
|
||||
describe("index", () => {
|
||||
it("metrics returns up=1", done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
prefix: "hello:",
|
||||
whitelist: ["up"]
|
||||
});
|
||||
app.use(bundled);
|
||||
|
||||
app.use("/test", (req, res) => res.send("it worked"));
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get("/test").end(() => {
|
||||
agent
|
||||
.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.text).toMatch(/hello:up\s1/);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("metrics should be attached to /metrics by default", done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
prefix: "hello:",
|
||||
whitelist: ["up"]
|
||||
});
|
||||
app.use(bundled);
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it("metrics can be attached to /metrics programatically", done => {
|
||||
const app = express();
|
||||
const bundled = bundle({
|
||||
autoregister: false
|
||||
});
|
||||
app.use(bundled.metricsMiddleware);
|
||||
app.use(bundled);
|
||||
|
||||
app.use("/test", (req, res) => res.send("it worked"));
|
||||
|
||||
const agent = supertest(app);
|
||||
agent.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it("metrics can be filtered using exect match", () => {
|
||||
const instance = bundle({blacklist: ["up"]});
|
||||
expect(instance.metrics.up).not.toBeDefined();
|
||||
expect(instance.metrics.nodejs_memory_heap_total_bytes).toBeDefined();
|
||||
});
|
||||
it("metrics can be filtered using regex", () => {
|
||||
const instance = bundle({blacklist: [/memory/]});
|
||||
expect(instance.metrics.up).toBeDefined();
|
||||
expect(instance.metrics.nodejs_memory_heap_total_bytes).not.toBeDefined();
|
||||
});
|
||||
it("metrics can be whitelisted", () => {
|
||||
const instance = bundle({whitelist: [/^up$/]});
|
||||
expect(instance.metrics.up).toBeDefined();
|
||||
expect(instance.metrics.nodejs_memory_heap_total_bytes).not.toBeDefined();
|
||||
expect(instance.metrics.http_request_seconds).not.toBeDefined();
|
||||
});
|
||||
it("throws on both white and blacklist", () => {
|
||||
expect(() => {
|
||||
bundle({whitelist: [/up/], blacklist: [/up/]});
|
||||
}).toThrow();
|
||||
});
|
||||
it("returns error 500 on incorrect middleware usage", done => {
|
||||
const app = express();
|
||||
app.use(bundle);
|
||||
supertest(app)
|
||||
.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(500);
|
||||
done();
|
||||
});
|
||||
});
|
||||
it("http latency gets counted", done => {
|
||||
const app = express();
|
||||
const instance = bundle();
|
||||
app.use(instance);
|
||||
app.use("/test", (req, res) => res.send("it worked"));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get("/test")
|
||||
.end(() => {
|
||||
const metricHashMap = instance.metrics.http_request_seconds.hashMap;
|
||||
expect(metricHashMap["status_code:200"]).toBeDefined();
|
||||
const labeled = metricHashMap["status_code:200"];
|
||||
expect(labeled.count).toBe(1);
|
||||
|
||||
agent
|
||||
.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
it("filters out the excludeRoutes", done => {
|
||||
const app = express();
|
||||
const instance = bundle({
|
||||
excludeRoutes: ["/test"]
|
||||
});
|
||||
app.use(instance);
|
||||
app.use("/test", (req, res) => res.send("it worked"));
|
||||
const agent = supertest(app);
|
||||
agent
|
||||
.get("/test")
|
||||
.end(() => {
|
||||
const metricHashMap = instance.metrics.http_request_seconds.hashMap;
|
||||
expect(metricHashMap["status_code:200"]).not.toBeDefined();
|
||||
|
||||
agent
|
||||
.get("/metrics")
|
||||
.end((err, res) => {
|
||||
expect(res.status).toBe(200);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
28
spec/normalizePath.spec.js
Normal file
28
spec/normalizePath.spec.js
Normal file
@@ -0,0 +1,28 @@
|
||||
'use strict';
|
||||
/* eslint-env jasmine */
|
||||
|
||||
const normalizePath = require('../src/normalizePath');
|
||||
|
||||
describe('normalizePath', () => {
|
||||
it('returns original if disabled in opts', () => {
|
||||
expect(
|
||||
normalizePath({originalUrl: '/a/12345'}, {normalizePath: false})
|
||||
).toBe('/a/12345');
|
||||
});
|
||||
|
||||
it('returns run callback if configured', () => {
|
||||
expect(
|
||||
normalizePath(
|
||||
{originalUrl: '/a/12345'},
|
||||
{
|
||||
normalizePath: req => req.originalUrl + '-ok'
|
||||
}
|
||||
)
|
||||
).toBe('/a/12345-ok');
|
||||
});
|
||||
|
||||
it('uses UrlValueParser by default', () => {
|
||||
expect(normalizePath({originalUrl: '/a/12345'}))
|
||||
.toBe('/a/#val');
|
||||
});
|
||||
});
|
||||
23
spec/normalizeStatusCode.spec.js
Normal file
23
spec/normalizeStatusCode.spec.js
Normal file
@@ -0,0 +1,23 @@
|
||||
'use strict';
|
||||
/* eslint-env jasmine */
|
||||
|
||||
const normalizeStatusCode = require('../src/normalizeStatusCode');
|
||||
|
||||
describe('normalizeStatusCode', () => {
|
||||
it('returns original if disabled in opts', () => {
|
||||
expect(
|
||||
normalizeStatusCode({status_code: 404}, {normalizeStatusCode: false})
|
||||
).toBe(404);
|
||||
});
|
||||
|
||||
it('returns run callback if configured', () => {
|
||||
expect(
|
||||
normalizeStatusCode(
|
||||
{status_code: 500},
|
||||
{
|
||||
formatStatusCode: res => String(res.status_code).slice(0, -2) + 'xx'
|
||||
}
|
||||
)
|
||||
).toBe('5xx');
|
||||
});
|
||||
});
|
||||
@@ -1,53 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
module.exports = class {
|
||||
constructor(opts) {
|
||||
this.opts = opts || {};
|
||||
this.promClient = this.opts.promClient || require("prom-client");
|
||||
this.metrics = {};
|
||||
}
|
||||
|
||||
metricExists(name) {
|
||||
return !!this.metrics[name];
|
||||
}
|
||||
|
||||
checkDuplicate(name) {
|
||||
if (this.metricExists(name)) {
|
||||
throw new Error("trying to add already existing metric: " + name);
|
||||
}
|
||||
}
|
||||
|
||||
makeRealName(name) {
|
||||
return (this.opts.prefix || "") + name;
|
||||
}
|
||||
|
||||
makeMetric(TheClass, args) {
|
||||
// convert pseudo-array
|
||||
const applyParams = Array.prototype.slice.call(args);
|
||||
const name = applyParams[0];
|
||||
this.checkDuplicate(name);
|
||||
const realName = this.makeRealName(name);
|
||||
applyParams[0] = realName;
|
||||
applyParams.unshift(null); // add some dummy context for apply
|
||||
|
||||
// call constructor with variable params
|
||||
this.metrics[name] = new (Function.prototype.bind.apply(TheClass, applyParams));
|
||||
return this.metrics[name];
|
||||
}
|
||||
|
||||
newCounter() {
|
||||
return this.makeMetric(this.promClient.Counter, arguments);
|
||||
}
|
||||
|
||||
newGauge() {
|
||||
return this.makeMetric(this.promClient.Gauge, arguments);
|
||||
}
|
||||
|
||||
newHistogram() {
|
||||
return this.makeMetric(this.promClient.Histogram, arguments);
|
||||
}
|
||||
|
||||
newSummary() {
|
||||
return this.makeMetric(this.promClient.Summary, arguments);
|
||||
}
|
||||
};
|
||||
238
src/index.js
238
src/index.js
@@ -1,137 +1,149 @@
|
||||
"use strict";
|
||||
|
||||
const
|
||||
PromFactory = require("./PromFactory"),
|
||||
onFinished = require("on-finished");
|
||||
'use strict';
|
||||
const onFinished = require('on-finished');
|
||||
const promClient = require('prom-client');
|
||||
const normalizePath = require('./normalizePath');
|
||||
const normalizeStatusCode = require('./normalizeStatusCode');
|
||||
|
||||
function matchVsRegExps(element, regexps) {
|
||||
for (let regexp of regexps) {
|
||||
if (regexp instanceof RegExp) {
|
||||
if (element.match(regexp)) {
|
||||
return true;
|
||||
}
|
||||
} else if (element == regexp) {
|
||||
return true;
|
||||
}
|
||||
for (let regexp of regexps) {
|
||||
if (regexp instanceof RegExp) {
|
||||
if (element.match(regexp)) {
|
||||
return true;
|
||||
}
|
||||
} else if (element === regexp) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
function filterArrayByRegExps(array, regexps) {
|
||||
return array.filter(element => {
|
||||
return matchVsRegExps(element, regexps);
|
||||
});
|
||||
return array.filter(element => {
|
||||
return matchVsRegExps(element, regexps);
|
||||
});
|
||||
}
|
||||
|
||||
function prepareMetricNames(opts, metricTemplates) {
|
||||
const names = Object.keys(metricTemplates);
|
||||
if (opts.whitelist) {
|
||||
if (opts.blacklist) {
|
||||
throw new Error("you cannot have whitelist and blacklist at the same time");
|
||||
}
|
||||
return filterArrayByRegExps(names, opts.whitelist);
|
||||
}
|
||||
const names = Object.keys(metricTemplates);
|
||||
if (opts.whitelist) {
|
||||
if (opts.blacklist) {
|
||||
const blacklisted = filterArrayByRegExps(names, opts.blacklist);
|
||||
return names.filter(name => blacklisted.indexOf(name) === -1);
|
||||
throw new Error('you cannot have whitelist and blacklist at the same time');
|
||||
}
|
||||
return names;
|
||||
return filterArrayByRegExps(names, opts.whitelist);
|
||||
}
|
||||
if (opts.blacklist) {
|
||||
const blacklisted = filterArrayByRegExps(names, opts.blacklist);
|
||||
return names.filter(name => blacklisted.indexOf(name) === -1);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function main(opts) {
|
||||
opts = Object.assign({ autoregister: true }, opts || {} );
|
||||
if (arguments[2] && arguments[1] && arguments[1].send) {
|
||||
arguments[1].status(500)
|
||||
.send("<h1>500 Error</h1>\n"
|
||||
+ "<p>Unexpected 3rd param in express-prom-bundle.\n"
|
||||
+ "<p>Did you just put express-prom-bundle into app.use "
|
||||
+ "without calling it as a function first?");
|
||||
return;
|
||||
opts = Object.assign({autoregister: true}, opts);
|
||||
if (arguments[2] && arguments[1] && arguments[1].send) {
|
||||
arguments[1].status(500)
|
||||
.send('<h1>500 Error</h1>\n'
|
||||
+ '<p>Unexpected 3rd param in express-prom-bundle.\n'
|
||||
+ '<p>Did you just put express-prom-bundle into app.use '
|
||||
+ 'without calling it as a function first?');
|
||||
return;
|
||||
}
|
||||
|
||||
if (opts.prefix || opts.keepDefaultMetrics !== undefined) {
|
||||
throw new Error(
|
||||
'express-prom-bundle detected obsolete options:'
|
||||
+ 'prefix and/or keepDefaultMetrics. '
|
||||
+ 'Please refer to oficial docs. '
|
||||
+ 'Most likely you upgraded the module without necessary code changes'
|
||||
);
|
||||
}
|
||||
|
||||
const httpMtricName = opts.httpDurationMetricName || 'http_request_duration_seconds';
|
||||
|
||||
const metricTemplates = {
|
||||
'up': () => new promClient.Gauge(
|
||||
'up',
|
||||
'1 = up, 0 = not up'
|
||||
),
|
||||
'http_request_duration_seconds': () => {
|
||||
const labels = ['status_code'];
|
||||
if (opts.includeMethod) {
|
||||
labels.push('method');
|
||||
}
|
||||
if (opts.includePath) {
|
||||
labels.push('path');
|
||||
}
|
||||
const metric = new promClient.Histogram(
|
||||
httpMtricName,
|
||||
'duration histogram of http responses labeled with: ' + labels.join(', '),
|
||||
labels,
|
||||
{
|
||||
buckets: opts.buckets || [0.003, 0.03, 0.1, 0.3, 1.5, 10]
|
||||
}
|
||||
);
|
||||
return metric;
|
||||
}
|
||||
};
|
||||
|
||||
const metrics = {};
|
||||
const names = prepareMetricNames(opts, metricTemplates);
|
||||
|
||||
for (let name of names) {
|
||||
metrics[name] = metricTemplates[name]();
|
||||
}
|
||||
|
||||
if (metrics.up) {
|
||||
metrics.up.set(1);
|
||||
}
|
||||
|
||||
const metricsMiddleware = function(req, res) {
|
||||
res.writeHead(200, {'Content-Type': 'text/plain'});
|
||||
res.end(promClient.register.metrics());
|
||||
};
|
||||
|
||||
const middleware = function (req, res, next) {
|
||||
const path = req.originalUrl;
|
||||
let labels;
|
||||
|
||||
if (opts.autoregister && path.match(/^\/metrics\/?$/)) {
|
||||
return metricsMiddleware(req, res);
|
||||
}
|
||||
|
||||
const factory = new PromFactory(opts);
|
||||
|
||||
const metricTemplates = {
|
||||
"up": () => factory.newGauge(
|
||||
"up",
|
||||
"1 = up, 0 = not up"
|
||||
),
|
||||
"nodejs_memory_heap_total_bytes": () => factory.newGauge(
|
||||
"nodejs_memory_heap_total_bytes",
|
||||
"value of process.memoryUsage().heapTotal"
|
||||
),
|
||||
"nodejs_memory_heap_used_bytes": () => factory.newGauge(
|
||||
"nodejs_memory_heap_used_bytes",
|
||||
"value of process.memoryUsage().heapUsed"
|
||||
),
|
||||
"http_request_seconds": () => {
|
||||
const metric = factory.newHistogram(
|
||||
"http_request_seconds",
|
||||
"number of http responses labeled with status code",
|
||||
["status_code"],
|
||||
{
|
||||
buckets: opts.buckets || [0.003, 0.03, 0.1, 0.3, 1.5, 10]
|
||||
}
|
||||
);
|
||||
return metric;
|
||||
}
|
||||
};
|
||||
|
||||
const
|
||||
metrics = {},
|
||||
names = prepareMetricNames(opts, metricTemplates);
|
||||
|
||||
|
||||
for (let name of names) {
|
||||
metrics[name] = metricTemplates[name]();
|
||||
if (opts.excludeRoutes && matchVsRegExps(path, opts.excludeRoutes)) {
|
||||
return next();
|
||||
}
|
||||
|
||||
if (metrics.up) {
|
||||
metrics.up.set(1);
|
||||
if (metrics[httpMtricName]) {
|
||||
labels = {'status_code': 0};
|
||||
let timer = metrics[httpMtricName].startTimer(labels);
|
||||
onFinished(res, () => {
|
||||
if (opts.normalizeStatusCode) {
|
||||
labels.status_code = main.normalizeStatusCode(res, opts);
|
||||
} else {
|
||||
labels.status_code = res.statusCode;
|
||||
}
|
||||
|
||||
if (opts.includeMethod) {
|
||||
labels.method = req.method;
|
||||
}
|
||||
if (opts.includePath) {
|
||||
labels.path = main.normalizePath(req, opts);
|
||||
}
|
||||
timer();
|
||||
});
|
||||
}
|
||||
|
||||
const metricsMiddleware = function(req,res) {
|
||||
let memoryUsage = process.memoryUsage();
|
||||
if (metrics["nodejs_memory_heap_total_bytes"]) {
|
||||
metrics["nodejs_memory_heap_total_bytes"].set(memoryUsage.heapTotal);
|
||||
}
|
||||
if (metrics["nodejs_memory_heap_used_bytes"]) {
|
||||
metrics["nodejs_memory_heap_used_bytes"].set(memoryUsage.heapUsed);
|
||||
}
|
||||
next();
|
||||
};
|
||||
|
||||
res.contentType("text/plain").send(factory.promClient.register.metrics());
|
||||
return;
|
||||
};
|
||||
|
||||
const middleware = function (req, res, next) {
|
||||
if (opts.autoregister && req.path == "/metrics") {
|
||||
return metricsMiddleware(req,res);
|
||||
}
|
||||
|
||||
if (opts.excludeRoutes && matchVsRegExps(req.path, opts.excludeRoutes)) {
|
||||
return next();
|
||||
}
|
||||
|
||||
let labels;
|
||||
if (metrics["http_request_seconds"]) {
|
||||
labels = {"status_code": 0};
|
||||
let timer = metrics["http_request_seconds"].startTimer(labels);
|
||||
onFinished(res, () => {
|
||||
labels["status_code"] = res.statusCode;
|
||||
timer();
|
||||
});
|
||||
}
|
||||
|
||||
next();
|
||||
};
|
||||
|
||||
middleware.factory = factory;
|
||||
middleware.metricTemplates = metricTemplates;
|
||||
middleware.metrics = metrics;
|
||||
middleware.promClient = factory.promClient;
|
||||
middleware.metricsMiddleware = metricsMiddleware;
|
||||
return middleware;
|
||||
middleware.metricTemplates = metricTemplates;
|
||||
middleware.metrics = metrics;
|
||||
middleware.promClient = promClient;
|
||||
middleware.metricsMiddleware = metricsMiddleware;
|
||||
return middleware;
|
||||
}
|
||||
|
||||
main.promClient = promClient;
|
||||
main.normalizePath = normalizePath;
|
||||
main.normalizeStatusCode = normalizeStatusCode;
|
||||
module.exports = main;
|
||||
|
||||
26
src/normalizePath.js
Normal file
26
src/normalizePath.js
Normal file
@@ -0,0 +1,26 @@
|
||||
'use strict';
|
||||
|
||||
const UrlValueParser = require('url-value-parser');
|
||||
const url = require('url');
|
||||
let urlValueParser;
|
||||
|
||||
module.exports = function(req, opts) {
|
||||
opts = opts || {};
|
||||
|
||||
// originalUrl is taken, because url and path can be changed
|
||||
// by middlewares such as 'router'. Note: this function is called onFinish
|
||||
/// i.e. always in the tail of the middleware chain
|
||||
const path = url.parse(req.originalUrl).pathname;
|
||||
|
||||
if (opts.normalizePath !== undefined && !opts.normalizePath) {
|
||||
return path;
|
||||
}
|
||||
if (typeof opts.normalizePath === 'function') {
|
||||
return opts.normalizePath(req, opts);
|
||||
}
|
||||
|
||||
if (!urlValueParser) {
|
||||
urlValueParser = new UrlValueParser();
|
||||
}
|
||||
return urlValueParser.replacePathValues(path);
|
||||
};
|
||||
11
src/normalizeStatusCode.js
Normal file
11
src/normalizeStatusCode.js
Normal file
@@ -0,0 +1,11 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function(res, opts) {
|
||||
opts = opts || {};
|
||||
|
||||
if (typeof opts.formatStatusCode === 'function') {
|
||||
return opts.formatStatusCode(res, opts);
|
||||
}
|
||||
|
||||
return res.status_code;
|
||||
};
|
||||
Reference in New Issue
Block a user