看着自己写的并发控制,虽然有效果,但。。。太蹩脚了。。。自己都想笑 ^^ ^^ ,你们有什么好的方法让我借鉴一下?
downloadList.push(url);
downloadDir.push(dir);
console.log("待下载数量:"+downloadList.length);
var down=function(){
console.log("正在下载数量"+streamNumber);
while(downloadList.length&&(streamNumber<streamLimit)){
streamNumber++;
var url=downloadList.pop();
download(url,downloadDir.pop(),Math.floor(Math.random()*100000) + url.substr(-4,4),function(err){
if(err){
console.log(err);
}
streamNumber--;
console.log("下载了"+(downloadAll++)+"张图片");
});
}
}
var download = function(url, dir, filename,cb){
var stream=fs.createWriteStream(dir + "/" + filename);
stream.on("error",function(err){
if(err){
cb(err+"ssssssssssssssssss");
}
});
var r=request.get({
url: url,
headers: {
'referer': 'http://www.dgtle.com/'
}
});
r.on("error",function(err){
if(err){
cb(err+"rrrrrrrrrrrrrrrrrrr");
}
});
r.pipe(stream);
r.on("end",function(){
cb(null)
});
};
setInterval(function(){
console.log("--------------------");
down();
},1000);
8 回复
async.mapLimit控制, 可以看这里 https://github.com/alsotang/node-lessons/tree/master/lesson5 我写的一个很简单的蜘蛛也用到 https://github.com/luoyjx/tag-spider/blob/master/index.js#L112