limiter.test.js
2.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
/**
*
* While using nock, suppose all response return almost in no time.
* In order to measure limiter effects, set Crawler rateLimit to 500ms.
*
*/
'use strict';
var Crawler = require('../lib/crawler');
var expect = require('chai').expect;
var nock = require('nock');
var c;
var tsArrs = [];
describe('Limiter tests', function () {
this.timeout(10000);
before(function () {
nock.cleanAll();
});
beforeEach(function () {
nock('http://nockHost').get(uri => uri.indexOf('status') >= 0).times(5).reply(200, 'Yes');
c = new Crawler({
jquery: false,
rateLimit: 500,
callback: function (err, result, done) {
expect(err).to.be.equal(null);
expect(result.statusCode).to.equal(200);
done();
},
});
c.on('request', () => tsArrs.push(Date.now()));
});
afterEach(function () {
c = {};
tsArrs = [];
});
it('One limiter, tasks should execute one by one', function (done) {
for (var i = 0; i < 5; i++) {
// limiter is 'default', if not assigned explicitly
c.queue({ uri: 'http://nockHost/status/200' });
}
c.on('drain', function () {
expect(tsArrs.length).to.equal(5);
// setTimeout in nodejs is delayed
// 4 rateLimit +- 50ms = 4 * 500 +- 50
expect(tsArrs[4] - tsArrs[0]).to.be.least(1950);
expect(tsArrs[4] - tsArrs[0]).to.be.most(2050);
done();
});
});
it('Multiple limiters, tasks should execute in parallel', function (done) {
for (var i = 0; i < 5; i++) {
c.queue({ uri: 'http://nockHost/status/200', limiter: i });
}
c.on('drain', function () {
expect(tsArrs.length).to.equal(5);
// setTimeout in nodejs is delayed
// request sent almost at same time
expect(tsArrs[4] - tsArrs[0]).to.be.most(50);
done();
});
});
it('Multiple limiters are mutual independent', function (done) {
for (var i = 0; i < 5; i++) {
var limiter = i === 4 ? 'second' : 'default';
c.queue({ uri: 'http://nockHost/status/200', limiter: limiter });
}
c.on('drain', function () {
expect(tsArrs.length).to.equal(5);
// setTimeout in nodejs is delayed
// 3 rateLimit +- 50ms = 3 * 500 +- 50
expect(tsArrs[4] - tsArrs[0]).to.be.least(1450);
expect(tsArrs[4] - tsArrs[0]).to.be.most(1550);
done();
});
});
});