var PORT = 80;
var webroot = './webroot::80';

var crypto = require('crypto');
var http = require('http');
var request = require('request');
var url=require('url');
var fs=require('fs');
var path=require('path');
var querystring=require('querystring');




//setInterval(function(){console.log('dddddddd');},500);



function _md5(str){
    var md5 = crypto.createHash('md5');
    md5.update(str);
    return md5.digest('hex');
}
//md5.update('123456')
//console.log(md5.digest('hex'));

var URI_pool = ['http://www.oschina.net/'];

function getData(URI){
    console.log(URI);
    
    http.get(URI , (res) => {
        res.setEncoding('utf8');
        
        
        var file_name = _md5(URI);
        try{
            fs.accessSync('./html/'+ file_name +'.html');
            console.log('ddddd');
            
            return false;
        }catch(e){
            //console.log(e);
        }
        
        
        res.on('data', (chunk) => {
            
                //console.log(chunk);
                
                var hrefs = chunk.match(/href=['"](http:\/\/[^" ']+)/g);
                
                for(var i in hrefs){
                    if(hrefs[i].indexOf('http://www.oschina.net') > -1)
                    URI_pool.push(hrefs[i].replace(/href=['"]/,''));
                }
                //console.log(URI_pool);return;
                //
                
                
                    //写入文件
                    fs.open('./html/'+ file_name +'.html','a',  (err , fd) => {
                        if(fd){
                            fs.write(fd , chunk);
                        }else
                        console.log(err);
                    });
                    
                    
                    return false;
                    
                
                
        }); 
    });

}


//var URI = '';
setInterval(function(){
    if(URI_pool.length)
    getData(URI_pool.shift());
},100);