web-crawler/script.js at master · qacheck/web-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
jQuery(function($){
	function arrUnique(value, index, self) {
	    return self.indexOf(value) === index;
	}

	var ajax_scan = [],
		results = $('#webcrl-results'),
		message = $('#webcrl-message'),
		btn_do = $('#webcrl_do'),
		btn_save = $('#webcrl_save'),
		btn_stop = $('#webcrl_stop');

	function web_crawler(source_url, source_type) {
		if(source_url.length>0) {
			btn_do.html('Scanning...');
			btn_do.prop('disabled', true);
			btn_save.prop('disabled', true);
			btn_stop.prop('disabled', false);

			var crawl_url = source_url.pop();

			var ajax = $.ajax({
					url: webcrl.ajax_url+'?action=webcrl_scan',
					type: 'POST',
					data: {su:crawl_url,st:source_type},
					dataType: 'json',
					success: function(res) {
						var msg = (res['error']!='')?res['crawled_url']+' - '+res['error'] : res['crawled_url'];
						message.html(msg);
						//console.log(res);
						var next_crawl = source_url.concat(res['next_crawl']);
						next_crawl = next_crawl.filter(arrUnique);

						web_crawler(next_crawl,source_type);
					},
					error: function(xhr,status,err) {
						web_crawler(source_url,source_type);
					}

				});
			ajax_scan.push(ajax);
		} else {
			btn_do.html('Get products');
			btn_do.prop('disabled', false);
			btn_save.prop('disabled', false);
			btn_stop.prop('disabled', true);
		}
	}

	btn_do.on('click', function(e){
		var su = [$('#webcrl_su').val()],
			ua = $('#webcrl_ua').val(),
			st = $('#webcrl_source_type').val(); //source type
		// btn_save.prop('disabled', true);
		// btn_stop.prop('disabled', false);
		if(su!='' && st!='') {
			$.ajax({
				url: webcrl.ajax_url+'?action=webcrl_remove_crawled',
				data: {ua:ua,su:su[0],st:st},
				success: function(res) {
					web_crawler(su, st);
				}
			});
		} else {
			alert('Input your source url!');
		}
	});

	btn_stop.on('click', function(e){
		// if(ajax_scan.length>0) {
		// 	ajax_scan.each(function(index, ajax){
		// 		ajax.abort();
		// 	});
		// 	ajax_scan = [];
		// }
		while(ajax_scan.length>0) {
			var ajax = ajax_scan.pop();
			ajax.abort();
		}

		btn_do.html('Get products');
		btn_do.prop('disabled', false);
		btn_save.prop('disabled', false);
		btn_stop.prop('disabled', true);
	});

	btn_save.on('click', function(e){
		btn_save.html('Loading...');
		message.html('');
		$.ajax({
			url: webcrl.ajax_url+'?action=webcrl_view_crawled',
			//data: {ua:ua,su:su[0]},
			success: function(res) {
				$('#webcrl-view-products').html(res);
				btn_save.html('View');
			},
			error: function(xhr,status,err) {
				$('#webcrl-view-products').html(err);
				btn_save.html('Fail!');
			}
		});
	});
});