cacheOption.test.js
1.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
/*jshint expr:true */
'use strict';
var Crawler = require('../lib/crawler');
var expect = require('chai').expect;
var nock = require('nock');
// var sinon = require('sinon');
var httpTarget = 'http://target.com';
var c;
var scope;
describe('Cache features tests', function() {
describe('Skip Duplicate active', function() {
beforeEach(function () {
scope = nock('http://target.com');
});
afterEach(function () {
c = {};
});
it('should not skip one single url', function (done) {
var call = scope.get('/').reply(200);
c = new Crawler({
jquery: false,
skipDuplicates: true,
callback: function (error, result) {
expect(error).to.be.null;
expect(result.statusCode).to.equal(200);
expect(call.isDone()).to.be.true;
done();
},
});
c.queue(httpTarget);
});
it('should notify the callback when an error occurs and "retries" is disabled', function (done) {
var koScope = scope.get('/').replyWithError('too bad');
c = new Crawler({
jquery: false,
skipDuplicates: true,
retries: 0,
callback: function (error) {
expect(error).to.exist;
expect(koScope.isDone()).to.be.true;
done();
},
});
c.queue(httpTarget);
});
it('should retry and notify the callback when an error occurs and "retries" is enabled', function (done) {
var koScope = scope.get('/').replyWithError('too bad').persist();
c = new Crawler({
jquery: false,
skipDuplicates: true,
retries: 1,
retryTimeout: 10,
callback: function (error) {
expect(error).to.exist;
expect(koScope.isDone()).to.be.true;
scope.persist(false);
done();
},
});
c.queue(httpTarget);
});
//it('should skip previous crawled urls', function (done) {});
});
});