<?php
/*
 |--------------------------------------------------------------------------
 | SkyCaiji (蓝天采集器)
 |--------------------------------------------------------------------------
 | Copyright (c) 2018 https://www.skycaiji.com All rights reserved.
 |--------------------------------------------------------------------------
 | 使用协议  https://www.skycaiji.com/licenses
 |--------------------------------------------------------------------------
 */

namespace skycaiji\admin\event;
use skycaiji\admin\model\CacheModel;
class Cpattern extends CpatternBase{
	/*采集,return false表示终止采集*/
	public function collect($num=10){
		if(!defined('IS_COLLECTING')){
			define('IS_COLLECTING', 1);
		}
	
		if(!$this->show_opened_tools){
			$opened_tools=array();
			
			if($GLOBALS['_sc']['c']['caiji']['robots']){
				$opened_tools[]='遵守robots协议';
			}
			if($this->config['page_render']){
				$opened_tools[]='页面渲染';
			}
			if($GLOBALS['_sc']['c']['download_img']['download_img']){
				$opened_tools[]='图片本地化';
			}
			if($GLOBALS['_sc']['c']['proxy']['open']){
				$opened_tools[]='代理';
			}
			if($GLOBALS['_sc']['c']['translate']['open']){
				$opened_tools[]='翻译';
			}
			if(!empty($opened_tools)){
				$this->echo_msg('开启功能：'.implode('、', $opened_tools),'black');
			}
			if($num>0){
				$this->echo_msg('预计采集'.$num.'条数据','black');
			}
				
			$this->show_opened_tools=true;
		}
	
		$this->collect_num=$num;
		$this->collected_field_list=array();
		
		$source_is_url=intval($this->config['source_is_url']);
		if(!isset($this->original_source_urls)){
			
			$this->original_source_urls=array();
			foreach ( $this->config ['source_url'] as $k => $v ) {
				if(empty($v)){
					continue;
				}
				$return_s_urls = $this->convert_source_url ( $v );
				if (is_array ( $return_s_urls )) {
					foreach ($return_s_urls as $r_s_u){
						$this->original_source_urls[md5($r_s_u)]=$r_s_u;
					}
				} else {
					$this->original_source_urls[md5($return_s_urls)]=$return_s_urls;
				}
			}
		}
		if(empty($this->original_source_urls)){
			$this->echo_msg('没有起始页网址！');
			return 'completed';
		}
	
		if($source_is_url){
			
			if(isset($this->used_source_urls['_source_is_url_'])){
				$this->echo_msg('所有起始页采集完毕！','green');
				return 'completed';
			}
		}else{
			if(count($this->original_source_urls)<=count($this->used_source_urls)){
				$this->echo_msg('所有起始页采集完毕！','green');
				return 'completed';
			}
		}
	
		$source_interval=$GLOBALS['_sc']['c']['caiji']['interval']*60;
		$time_interval_list=array();
	
		$source_urls=array();
		$mcacheSource=CacheModel::getInstance('source_url');
		if($source_is_url){
			
			$source_urls=$this->original_source_urls;
		}else{
			$cacheSources=$mcacheSource->db()->where(array('cname'=>array('in',array_keys($this->original_source_urls))))->column('dateline','cname');
			if(!empty($cacheSources)){
				$count_db_used=0;
				$sortSources=array('undb'=>array(),'db'=>array());
				
				foreach ($this->original_source_urls as $sKey=>$sVal){
					if(!isset($cacheSources[$sKey])){
						
						$sortSources['undb'][$sKey]=$sVal;
					}else{
						
						$time_interval=abs(NOW_TIME-$cacheSources[$sKey]);
						if($time_interval<$source_interval){
							
							$this->used_source_urls[$sVal]=1;
							$count_db_used++;
							$time_interval_list[]=$time_interval;
						}else{
							$sortSources['db'][$sKey]=$sVal;
						}
					}
				}
				if($count_db_used>0){
					$this->echo_msg($count_db_used.'条已采集起始网址被过滤，下次采集需等待'.($source_interval-max($time_interval_list)).'秒，<a href="'
							.url('Admin/Setting/caiji').'" target="_blank">设置间隔</a>','black');
					if(count($this->original_source_urls)<=count($this->used_source_urls)){
						$this->echo_msg('所有起始页采集完毕！','green');
						return 'completed';
					}
				}
				$source_urls=array_merge($sortSources['undb'],$sortSources['db']);
				unset($sortSources);
				unset($cacheSources);
			}else{
				$source_urls=$this->original_source_urls;
			}
		}
		$mcollected=model('Collected');
		
		if($source_is_url){
			
			$this->cont_urls_list['_source_is_url_']=array_values($source_urls);
			$source_urls=array('_source_is_url_'=>'_source_is_url_');
		}
	
	
		foreach ($source_urls as $key_source_url=>$source_url){
			$this->cur_source_url=$source_url;
			if(array_key_exists($source_url,$this->used_source_urls)){
				
				continue;
			}
			if($source_is_url){
				$this->echo_msg("起始页已转换为内容页网址",'black');
			}else{
				$this->echo_msg("采集起始页：<a href='{$source_url}' target='_blank'>{$source_url}</a>",'green');
			}
			if($source_is_url){
				
				$this->_collect_fields();
			}else{
				
				if(!empty($this->config['level_urls'])){
					
					
					$this->echo_msg('开始分析多级网址','black');
					$return_msg=$this->_collect_level($source_url,1);
					if($return_msg=='completed'){
						return $return_msg;
					}
				}else{
					
					$cont_urls=$this->getContUrls($source_url);
					$this->cont_urls_list[$source_url]=$this->_collect_unused_cont_urls($cont_urls);
					$this->_collect_fields();
				}
			}
				
			if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){
				break;
			}
		}
		
		
		return $this->collected_field_list;
	}
	/**
	 * 优化设置页面post过来的config
	 * @param unknown $config
	 */
	public function setConfig($config){
		$config['url_complete']=intval($config['url_complete']);
	
		$config['url_reverse']=intval($config['url_reverse']);
		$config['page_render']=intval($config['page_render']);
		$config['url_repeat']=intval($config['url_repeat']);
		
		if(!empty($config['request_headers'])){
			if(!is_array($config['request_headers']['custom_names'])){
				$config['request_headers']['custom_names']=array();
			}
			if(!is_array($config['request_headers']['custom_vals'])){
				$config['request_headers']['custom_vals']=array();
			}
				
			foreach ($config['request_headers']['custom_names'] as $k=>$v){
				if(empty($v)){
					
					unset($config['request_headers']['custom_names'][$k]);
					unset($config['request_headers']['custom_vals'][$k]);
				}
			}
			$config['request_headers']['custom_names']=array_values($config['request_headers']['custom_names']);
			$config['request_headers']['custom_vals']=array_values($config['request_headers']['custom_vals']);
		}
	
		
		foreach ($config['source_url'] as $k=>$v){
			if(preg_match('/[\r\n]/', $v)){
				
				if(preg_match_all('/^\w+\:\/\/[^\r\n]+/im',$v,$v_urls)){
					
					$v_urls=array_unique($v_urls[0]);
					$v_urls=array_values($v_urls);
					$config['source_url'][$k]=implode("\r\n", $v_urls);
				}else{
					unset($config['source_url'][$k]);
				}
			}else{
				
				if(!preg_match('/^\w+\:\/\/.+/i', $v)){
					
					unset($config['source_url'][$k]);
				}
			}
		}
		
		$config['source_url']=array_unique($config['source_url']);
		$config['source_url']=array_filter($config['source_url']);
		$config['source_url']=array_values($config['source_url']);
	
	
		if(!empty($config['field_list'])){
			
			foreach ($config['field_list'] as $k=>$v){
				$config['field_list'][$k]=json_decode(url_b64decode($v),true);
			}
		}
		if(!empty($config['field_process'])){
			
			foreach ($config['field_process'] as $k=>$v){
				$config['field_process'][$k]=json_decode(url_b64decode($v),true);
				$config['field_process'][$k]=$this->setProcess($config['field_process'][$k]);
			}
		}
		$config['common_process']=input('process/a',null,'trim');
		$config['common_process']=$this->setProcess($config['common_process']);
	
		
		if(!empty($config['paging_fields'])){
			foreach ($config['paging_fields'] as $k=>$v){
				$config['paging_fields'][$k]=json_decode(url_b64decode($v),true);
			}
		}
		
		if(!empty($config['level_urls'])){
			
			foreach ($config['level_urls'] as $k=>$v){
				$config['level_urls'][$k]=json_decode(url_b64decode($v),true);
			}
		}
		
		if(!empty($config['relation_urls'])){
			
			foreach ($config['relation_urls'] as $k=>$v){
				$config['relation_urls'][$k]=json_decode(url_b64decode($v),true);
			}
		}
	
		
		$config['url_post']=intval($config['url_post']);
		
		if(!empty($config['url_posts'])){
			if(!is_array($config['url_posts']['names'])){
				$config['url_posts']['names']=array();
			}
			if(!is_array($config['url_posts']['vals'])){
				$config['url_posts']['vals']=array();
			}
			if(!empty($config['url_posts']['names'])){
				foreach ($config['url_posts']['names'] as $k=>$v){
					if(empty($v)){
						
						unset($config['url_posts']['names'][$k]);
						unset($config['url_posts']['vals'][$k]);
					}
				}
			}
			$config['url_posts']['names']=array_values($config['url_posts']['names']);
			$config['url_posts']['vals']=array_values($config['url_posts']['vals']);
		}
	
		return $config;
	}
	public function init($collData){
		$collData['config']=unserialize($collData['config']);
		$this->collector=$collData;
		$releData=model('Release')->where(array('task_id'=>$collData['task_id']))->find();
		if(!empty($releData)){
			$releData=$releData->toArray();
		}
		$this->release=$releData;
		
		$keyConfig='collector_config_'.$collData['id'];
		$cacheConfig=cache($keyConfig);
		if(empty($cacheConfig)||$cacheConfig['update_time']!=$collData['uptime']){
			
			$config=$this->initConfig($collData['config']);
			cache($keyConfig,array('update_time'=>$collData['uptime'],'config'=>$config));
		}else{
			$config=$cacheConfig['config'];
		}
		
		if($config['request_headers']['open']){
			
			$taskHeaders=array();
			if(!empty($config['request_headers']['useragent'])){
				$taskHeaders['useragent']=$config['request_headers']['useragent'];
			}
			if(!empty($config['request_headers']['cookie'])){
				$taskHeaders['cookie']=$config['request_headers']['cookie'];
			}
			if(!empty($config['request_headers']['referer'])){
				$taskHeaders['referer']=$config['request_headers']['referer'];
			}
			
			if(!empty($config['request_headers']['custom_names'])){
				foreach ($config['request_headers']['custom_names'] as $k=>$v){
					if(!empty($v)){
						$taskHeaders[$v]=$config['request_headers']['custom_vals'][$k];
					}
				}
			}
			
			$GLOBALS['_sc']['task_request_headers']=array(
				'request_headers'=>$config['request_headers'],
				'headers'=>$taskHeaders
			);
		}else{
			
			$GLOBALS['_sc']['task_request_headers']=null;
		}
		
		$this->config=$config;
	}
	
	public function initConfig($config){
		$newConfig=array();
		$newConfig['charset'] = $config['charset']=='custom' ? $config ['charset_custom'] : $config ['charset'];
		$newConfig['charset']= empty($newConfig['charset'])?'auto':$newConfig['charset'];
		
		if(!empty($config['area'])){
			
			if(empty($config['area_module'])){
				
				$newConfig['reg_source_cont']=$this->convert_sign_match($config['area']);
			}
		}elseif(!empty($config['area_start'])||!empty($config['area_end'])) {
			
			$newConfig['reg_source_cont'] = $config['area_start'] . (!empty($config['area_end']) ? '(?P<match>[\s\S]+?)' : '(?P<match>[\s\S]+)') . $config['area_end'];
		}
		if(!empty($newConfig['reg_source_cont'])){
			
			$newConfig['reg_source_cont'] = str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont'] );
			$newConfig['reg_source_cont'] = preg_replace ( '/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont'] );
		}elseif(!empty($config['area_module'])){
			
			$newConfig['reg_source_cont']=$config['area'];
		}
	
		
		if(empty($config['url_rule_module'])){
			
			if(!empty($config['url_rule'])){
				$newConfig['reg_source_cont_url']=$this->convert_sign_match($config['url_rule']);
				$newConfig['reg_source_cont_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $newConfig['reg_source_cont_url']);
				$newConfig['reg_source_cont_url']=str_replace ( '(*)', '[\s\S]*?', $newConfig['reg_source_cont_url'] );
			}else{
				
				$newConfig['reg_source_cont_url']='\bhref=[\'\"](?P<match>[^\'\"\<\>]+?)[\'\"]';
			}
			
			$config['url_merge']=$this->set_merge_default($newConfig['reg_source_cont_url'], $config['url_merge']);
		}elseif('xpath'==$config['url_rule_module']){
			if(!empty($config['url_rule'])){
				$newConfig['reg_source_cont_url']=$config['url_rule'];
			}else{
				
				$newConfig['reg_source_cont_url']='//a';
			}
			
			$config['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['url_merge']);
		}elseif('json'==$config['url_rule_module']){
			$newConfig['reg_source_cont_url']=$config['url_rule'];
			
			$config['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['url_merge']);
		}
		
		if(!empty($config['url_must'])){
			
			$newConfig['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_must']);
			$newConfig['url_must']=str_replace('(*)', '[\s\S]*?', $newConfig['url_must']); 
		}
	
		
		if(!empty($config['url_ban'])){
			
			$newConfig['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['url_ban']);
			$newConfig['url_ban']=str_replace('(*)', '[\s\S]*?', $newConfig['url_ban']); 
		}
		
		if(!empty($config['level_urls'])){
			$config['new_level_urls']=array();
			foreach ($config['level_urls'] as $luk=>$luv){
				
				if(!empty($luv['area'])){
					if(empty($luv['area_module'])){
						
						$luv['reg_area']=$this->convert_sign_match($luv['area']);
						$luv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_area']);
						$luv['reg_area']=str_replace('(*)', '[\s\S]*?', $luv['reg_area']); 
					}else{
						
						$luv['reg_area']=$luv['area'];
					}
					$luv['reg_area_module']=$luv['area_module'];
				}
				
				if(empty($luv['url_rule_module'])){
					
					if(!empty($luv['url_rule'])){
						$luv['reg_url']=$this->convert_sign_match($luv['url_rule']);
						$luv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['reg_url']);
						$luv['reg_url']=str_replace ( '(*)', '[\s\S]*?', $luv['reg_url'] );
					}else{
						
						$luv['reg_url']='\bhref=[\'\"](?P<match>[^\'\"\<\>]+?)[\'\"]';
					}
					
					$luv['url_merge']=$this->set_merge_default($luv['reg_url'], $luv['url_merge']);
				}elseif('xpath'==$luv['url_rule_module']){
					if(!empty($luv['url_rule'])){
						$luv['reg_url']=$luv['url_rule'];
					}else{
						
						$luv['reg_url']='//a';
					}
					
					$luv['url_merge']=$this->set_merge_default('(?P<match>.+)', $luv['url_merge']);
				}elseif('json'==$luv['url_rule_module']){
					$luv['reg_url']=$luv['url_rule'];
					
					$luv['url_merge']=$this->set_merge_default('(?P<match>.+)', $luv['url_merge']);
				}
				$luv['reg_url_module']=$luv['url_rule_module'];
	
				
				if(!empty($luv['url_must'])){
					
					$luv['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_must']);
					$luv['url_must']=str_replace('(*)', '[\s\S]*?', $luv['url_must']); 
				}
	
				
				if(!empty($luv['url_ban'])){
					
					$luv['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $luv['url_ban']);
					$luv['url_ban']=str_replace('(*)', '[\s\S]*?', $luv['url_ban']); 
				}
	
				$config['level_urls'][$luk]=$luv;
				$config['new_level_urls'][$luv['name']]=$luv;
			}
		}
		
		$relation_urls=array();
		if(!empty($config['relation_urls'])){
			foreach ($config['relation_urls'] as $ruv){
				
				if(!empty($ruv['area'])){
					if(empty($ruv['area_module'])){
						
						$ruv['reg_area']=$this->convert_sign_match($ruv['area']);
						$ruv['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_area']);
						$ruv['reg_area']=str_replace('(*)', '[\s\S]*?', $ruv['reg_area']); 
					}else{
						
						$ruv['reg_area']=$ruv['area'];
					}
					$ruv['reg_area_module']=$ruv['area_module'];
				}
				
				if(empty($ruv['url_rule_module'])){
					
					$ruv['reg_url']=$this->convert_sign_match($ruv['url_rule']);
					$ruv['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $ruv['reg_url']);
					$ruv['reg_url']=str_replace('(*)', '[\s\S]*?', $ruv['reg_url']); 
					
					$ruv['url_merge']=$this->set_merge_default($ruv['reg_url'], $ruv['url_merge']);
				}elseif(in_array($ruv['url_rule_module'],array('xpath','json'))){
					$ruv['reg_url']=$ruv['url_rule'];
					
					$ruv['url_merge']=$this->set_merge_default('(?P<match>.+)', $ruv['url_merge']);
				}
				$ruv['reg_url_module']=$ruv['url_rule_module'];
	
				$relation_urls[$ruv['name']]=$ruv;
			}
		}
		$relation_depth_urls=array();
		foreach ($relation_urls as $ruv){
			$rDepth=0;
			$rFuName=$ruv['page'];
			if(empty($rFuName)){
				
				$rDepth=0;
			}else{
				
				$passRelation=false;
				$rFuPage=$rFuName;
				do{
					if(empty($relation_urls[$rFuPage])){
						
						$passRelation=true;
						break;
					}
					$rFuPage=$relation_urls[$rFuPage]['page'];
					if($rFuPage==$rFuName){
						
						$passRelation=true;
						break;
					}
					$rDepth++;
				}while(!empty($rFuPage));
	
				if($passRelation){
					
					continue;
				}
			}
			$relation_depth_urls[$rDepth][$ruv['name']]=$ruv;
		}
		ksort($relation_depth_urls);
		$config['new_relation_urls']=array();
		foreach ($relation_depth_urls as $rurls){
			
			if(is_array($rurls)){
				$config['new_relation_urls']=array_merge($config['new_relation_urls'],$rurls);
			}
		}
	
		
		if(!empty($config['field_list'])){
			foreach ($config['field_list'] as $fk=>$fv){
				if('rule'==$fv['module']){
					
					$fv['reg_rule']=$this->convert_sign_match($fv['rule']);
					$fv['reg_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $fv['reg_rule']);
					$fv['reg_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_rule']); 
					
					$fv['rule_merge']=$this->set_merge_default($fv['reg_rule'], $fv['rule_merge']);
				}elseif('extract'==$fv['module']){
					
					if(!empty($fv['extract_rule'])){
						
						$fv['reg_extract_rule']=$this->convert_sign_match($fv['extract_rule']);
						$fv['reg_extract_rule']=preg_replace('/\\\*([\'\/])/', "\\\\$1",$fv['reg_extract_rule']);
						$fv['reg_extract_rule']=str_replace('(*)', '[\s\S]*?', $fv['reg_extract_rule']); 
						
						$fv['extract_rule_merge']=$this->set_merge_default($fv['reg_extract_rule'], '');
					}
				}
				$config['field_list'][$fk]=$fv;
			}
		}
		
		if(!empty($config['field_process'])){
			foreach ($config['field_process'] as $k=>$v){
				$config['field_process'][$k]=$this->initProcess($v);
			}
		}
		
		if(!empty($config['common_process'])){
			$config['common_process']=$this->initProcess($config['common_process']);
		}
		
		
		if(!empty($config['paging']['area'])){
			if(empty($config['paging']['area_module'])){
				$config['paging']['reg_area']=$this->convert_sign_match($config['paging']['area']);
				$config['paging']['reg_area']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_area']);
				$config['paging']['reg_area']=str_replace('(*)', '[\s\S]*?', $config['paging']['reg_area']); 
				
				$config['paging']['reg_area_merge']=$this->set_merge_default($config['paging']['reg_area'], '');
			}else{
				
				$config['paging']['reg_area']=$config['paging']['area'];
			}
			$config['paging']['reg_area_module']=$config['paging']['area_module'];
		}
		
		if(!empty($config['paging']['url_rule'])){
			if(empty($config['paging']['url_rule_module'])){
				
				$config['paging']['reg_url']=$this->convert_sign_match($config['paging']['url_rule']);
				$config['paging']['reg_url']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['reg_url']);
				$config['paging']['reg_url']=str_replace ( '(*)', '[\s\S]*?', $config['paging']['reg_url'] );
	
				
				$config['paging']['url_merge']=$this->set_merge_default($config['paging']['reg_url'], $config['paging']['url_merge']);
				if(empty($config['paging']['url_merge'])){
					
					$config['paging']['url_merge']=cp_sign('match');
				}
			}else{
				
				$config['paging']['reg_url']=$config['paging']['url_rule'];
				
				$config['paging']['url_merge']=$this->set_merge_default('(?P<match>.+)', $config['paging']['url_merge']);
			}
			$config['paging']['reg_url_module']=$config['paging']['url_rule_module'];
		}
	
		
		if(!empty($config['paging']['url_must'])){
			
			$config['paging']['url_must']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_must']);
			$config['paging']['url_must']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_must']); 
		}
	
		
		if(!empty($config['paging']['url_ban'])){
			
			$config['paging']['url_ban']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $config['paging']['url_ban']);
			$config['paging']['url_ban']=str_replace('(*)', '[\s\S]*?', $config['paging']['url_ban']); 
		}
	
	
		
		$module_normal_fields=array();
		$module_extract_fields=array();
		$module_merge_fields=array();
		if(!empty($config['field_list'])){
			foreach ($config['field_list'] as $fk=>$fv){
				$fieldModule=strtolower($fv['module']);
				$fieldConfig=array('field'=>$fv,'process'=>$config['field_process'][$fk]);
				if('extract'==$fieldModule){
					
					$module_extract_fields[$fv['name']]=$fieldConfig;
				}elseif('merge'==$fieldModule){
					
					$module_merge_fields[$fv['name']]=$fieldConfig;
				}else{
					
					$module_normal_fields[$fv['name']]=$fieldConfig;
				}
			}
		}
		
		$config['new_field_list']=array_merge($module_normal_fields,$module_extract_fields,$module_merge_fields);
	
	
		
		$new_paging_fields=array(
				'normal'=>array(),
				'extract'=>array(),
				'merge'=>array(),
		);
		if(!empty($config['paging_fields'])){
			$pagingFields=array();
			foreach ($config['paging_fields'] as $pfield){
				
				$pagingFields[$pfield['field']]=$pfield;
			}
			if(!empty($pagingFields['::all'])){
				
				$fieldAllParams=$pagingFields['::all'];
				unset($pagingFields['::all']);
				foreach ($config['new_field_list'] as $k=>$v){
					
					if(empty($pagingFields[$k])){
						
						$fieldAllParams['field']=$k;
						$pagingFields[$k]=$fieldAllParams;
					}
				}
			}
			$config['paging_fields']=$pagingFields;
			unset($pagingFields);
			
			foreach ($config['paging_fields'] as $pfk=>$pfield){
				$pfield['delimiter']=str_replace(array('\r','\n'), array("\r","\n"), $pfield['delimiter']);
				$config['paging_fields'][$pfk]=$pfield;
				if(!empty($module_normal_fields[$pfield['field']])){
					
					$new_paging_fields['normal'][$pfield['field']]=$pfield;
				}elseif(!empty($module_extract_fields[$pfield['field']])){
					
					$new_paging_fields['extract'][$pfield['field']]=$pfield;
				}elseif(!empty($module_merge_fields[$pfield['field']])){
					
					$new_paging_fields['merge'][$pfield['field']]=$pfield;
				}
			}
		}
		
		$config['new_paging_fields']=array_merge($new_paging_fields['normal'],$new_paging_fields['extract'],$new_paging_fields['merge']);
	
		$config=array_merge($config,$newConfig);
		return $config;
	}
	/*采集级别网址*/
	public function collLevelUrls($source_url,$curLevel=1){
		$curLevel=$curLevel>0?$curLevel:0;
		if($curLevel>0){
			
			$nextLevel=0;
			if(!empty($this->config['level_urls'])){
				
				if(!empty($this->config['level_urls'][$curLevel-1])){
					
					if(!empty($this->config['level_urls'][$curLevel])){
						
						$nextLevel=$curLevel+1;
					}
				}
			}
			
			$cont_urls=$this->getLevelUrls($source_url,$curLevel);
		}else{
			
			$cont_urls=$this->getContUrls($source_url);
		}
		
		return array('urls'=>$cont_urls,'levelName'=>$this->config['level_urls'][$curLevel-1]['name'],'nextLevel'=>$nextLevel);
	}
	
	/*获取内容网址*/
	public function getContUrls($source_url){
		if(empty($source_url)){
			return $this->error('请输入起始网址');
		}
		
		$config=array(
				'reg_area'=>$this->config['reg_source_cont'],
				'reg_area_module'=>$this->config['area_module'],
				'reg_url'=>$this->config['reg_source_cont_url'],
				'reg_url_module'=>$this->config['url_rule_module'],
				'url_merge'=>$this->config['url_merge'],
				'url_must'=>$this->config['url_must'],
				'url_ban'=>$this->config['url_ban'],
		);
		return $this->_get_urls($source_url, $config);
	}
	/*获取多级网址*/
	public function getLevelUrls($parent_url,$level=1){
		$level=$level>1?$level:1;
		$config=$this->config['level_urls'][$level-1];
		if(empty($config)){
			return $this->error('没有'.($level).'级网址规则');
		}
		if(empty($config['reg_url'])){
			return $this->error('必须填写多级“提取网址规则”');
		}
	
		if(empty($parent_url)){
			return $this->error('请输入父级网址');
		}
		return $this->_get_urls($parent_url, $config,true);
	}

	/*获取分页链接*/
	public function getPagingUrls($from_url,$html,$is_test=false){
		$paging_urls=array();
		if($this->config['paging']['open']){
			
			if(empty($html)){
				$html=$this->get_html($from_url);
			}
				
			if(!empty($this->config['paging']['reg_url'])){
				
				if(!empty($this->config['new_paging_fields'])){
					
					$base_url=$this->match_base_url($from_url, $html);
					$domain_url=$this->match_domain_url($from_url);
						
					$paging_area='';
					if(!empty($this->config['paging']['reg_area'])){
						
						if(empty($this->config['paging']['reg_area_module'])){
							
							$sign_match=$this->sign_addslashes(cp_sign('match','(?P<num>\d*)'));
							if(preg_match_all('/'.$sign_match.'/i', $this->config['paging']['reg_area_merge'],$match_signs)){
								
								if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){
									
									$re_match=array();
									foreach($match_signs['num'] as $ms_k=>$ms_v){
										$re_match[$ms_k]=$m_paging_area['match'.$ms_v];
									}
									$paging_area=str_replace($match_signs[0], $re_match, $this->config['paging']['reg_area_merge']);
								}
							}else{
								if(preg_match('/'.$this->config['paging']['reg_area'].'/i',$html,$m_paging_area)){
									
									$paging_area=$m_paging_area[0];
								}
							}
						}elseif('json'==$this->config['paging']['reg_area_module']){
							$paging_area=$this->rule_module_json_data(array('json'=>$this->config['paging']['reg_area'],'json_arr'=>'jsonencode'),$html);
						}elseif('xpath'==$this->config['paging']['reg_area_module']){
							$paging_area=$this->rule_module_xpath_data(array('xpath'=>$this->config['paging']['reg_area'],'xpath_attr'=>'outerHtml'),$html);
						}
					}else{
						
						$paging_area=$html;
					}
					if(!empty($paging_area)){
						
	
						
						if(!empty($this->config['paging']['url_complete'])){
							
							$paging_area=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche_p_a) use ($base_url,$domain_url){
								
								return \skycaiji\admin\event\Cpattern::create_complete_url($matche_p_a[1], $base_url, $domain_url);
							},$paging_area);
						}
	
						$m_paging_urls=$this->rule_match_urls($this->config['paging'],$paging_area,true);
	
						
						foreach ($m_paging_urls as $purl){
							if(!empty($this->config['paging']['url_must'])){
								
								if(!preg_match('/'.$this->config['paging']['url_must'].'/i', $purl)){
									continue;
								}
							}
							if(!empty($this->config['paging']['url_ban'])){
								
								if(preg_match('/'.$this->config['paging']['url_ban'].'/i', $purl)){
									continue;
								}
							}
								
							if($from_url==$purl){
								
								continue;
							}
								
							if(strpos($purl,' ')==false){
								
								$paging_urls[]=$purl;
							}
								
						}
	
						
						if(!empty($paging_urls)){
							$paging_urls=array_filter($paging_urls);
							$paging_urls=array_unique($paging_urls);
							$paging_urls=array_values($paging_urls);
							
						}else{
							if($is_test){
								return $this->error('未获取到分页链接，请检查分页链接规则');
							}
						}
					}else{
						if($is_test){
							return $this->error('未获取到分页区域，请检查分页区域规则');
						}
					}
				}else{
					if($is_test){
						return $this->error('请添加分页内容字段');
					}
				}
			}else{
				if($is_test){
					return $this->error('必须填写分页链接规则');
				}
			}
		}else{
			if($is_test){
				return $this->error('未开启分页');
			}
		}
		return $paging_urls;
	}
	
	/*设置字段值*/
	public function setField($field_config,$cur_url,$html,$cont_url){
		$cur_url_md5=md5($cur_url);

		$field_process=$field_config['process'];
		$field_params=$field_config['field'];
		$module=strtolower($field_params['module']);
	
		if(!empty($field_params['source'])&&in_array($module, array('rule','xpath','json','auto'))){
			
			$field_source_url='';
			$source_echo_msg='——采集';
			if('source_url'==$field_params['source']){
				
				$field_source_url=$this->cur_source_url;
				$source_echo_msg.='起始页';
			}elseif(preg_match('/^relation_url:(.+)$/i', $field_params['source'],$relationName)){
				
				$relationName=$relationName[1];
				$field_source_url=$this->getRelationUrl($relationName, $cur_url, $html);
				$source_echo_msg.="关联页“{$relationName}”";
			}elseif(preg_match('/^level_url:(.+)$/i', $field_params['source'],$levelName)){
				
				$levelName=$levelName[1];
				if(empty($this->config['new_level_urls'][$levelName])){
					
					return;
				}
				if(empty($this->cur_level_urls[$levelName])){
					
					return;
				}
				$field_source_url=$this->cur_level_urls[$levelName];
				$source_echo_msg.="多级页“{$levelName}”";
			}
			if(empty($field_source_url)){
				
				return;
			}
			
			if($field_source_url!=$cur_url){
				$cur_url=$field_source_url;
				$this->echo_msg($source_echo_msg."：<a href='{$field_source_url}' target='_blank'>{$field_source_url}</a>",'black');
				$html=$this->get_html($field_source_url,true);
			}
		}
		static $fieldArr1=array('words','num','time','list');
		static $fieldArr2=array('auto','json');
		static $baseUrls=array();
		static $domainUrls=array();
	
		$urlMd5=md5($cur_url);
		if(empty($baseUrls[$urlMd5])){
			$baseUrls[$urlMd5]=$this->match_base_url($cur_url, $html);
		}
		if(empty($domainUrls[$urlMd5])){
			$domainUrls[$urlMd5]=$this->match_domain_url($cur_url);
		}
		$base_url=$baseUrls[$urlMd5];
		$domain_url=$domainUrls[$urlMd5];
	
		$val='';
		$field_func='field_module_'.$module;
		if(method_exists($this, $field_func)){
			
			if('extract'==$module){
				
				
				if(is_array($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5])){
					
					$val=array();
					foreach ($this->field_val_list[$field_params['extract']]['values'][$cur_url_md5] as $k=>$v){
						$extract_field_val=array(
							'value'=>$v,
							'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5][$k],
						);
						$val[$k]=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url);
					}
				}else{
					
					$extract_field_val=array(
						'value'=>$this->field_val_list[$field_params['extract']]['values'][$cur_url_md5],
						'img'=>$this->field_val_list[$field_params['extract']]['imgs'][$cur_url_md5],
					);
					$val=$this->field_module_extract($field_params, $extract_field_val, $base_url, $domain_url);
				}
			}elseif('merge'==$module){
				
				if(empty($this->first_loop_field)){
					
					$cur_field_val_list=array();
					foreach ($this->field_val_list as $k=>$v){
						$cur_field_val_list[$k]=array(
							'value'=>$v['values'][$cur_url_md5],
							'img'=>$v['imgs'][$cur_url_md5]
						);
					}
					$val=$this->field_module_merge($field_params,$cur_field_val_list);
				}else{
					
					$val=array();
					
					foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){
						$cur_field_val_list=array();
						foreach ($this->field_val_list as $k=>$v){
							$cur_field_val_list[$k]=array(
								'value'=>(is_array($v['values'][$cur_url_md5])?$v['values'][$cur_url_md5][$v_k]:$v['values'][$cur_url_md5]),
								'img'=>(is_array($v['imgs'][$cur_url_md5][$v_k])?$v['imgs'][$cur_url_md5][$v_k]:$v['imgs'][$cur_url_md5])
							);
						}
						$val[$v_k]=$this->field_module_merge($field_params,$cur_field_val_list);
					}
				}
			}elseif(in_array($module,$fieldArr1)){
				
				if(empty($this->first_loop_field)){
					
					$val=$this->$field_func($field_params);
				}else{
					
					$val=array();
					
					foreach ($this->field_val_list[$this->first_loop_field]['values'][$cur_url_md5] as $v_k=>$v_v){
						$val[$v_k]=$this->$field_func($field_params);
					}
				}
			}elseif(in_array($module,$fieldArr2)){
				$val=$this->$field_func($field_params,$html,$cur_url);
			}else{
				$val=$this->$field_func($field_params,$html);
			}
		}
	
		$vals=null;
		if(is_array($val)){
			
			$is_loop=true;
			$vals=array_values($val);
		}else{
			
			$is_loop=false;
			$vals=array($val);
		}
	
		$field_name=$field_params['name'];
		if(!isset($this->field_val_list[$field_name])){
			
			$this->field_val_list[$field_name]=array('values'=>array(),'imgs'=>array());
		}

		$cont_url_md5=empty($cont_url)?$cur_url_md5:md5($cont_url);
		
		foreach ($vals as $v_k=>$val){
			$loopIndex=$is_loop?$v_k:-1;
			if(!empty($field_process)){
				
				$val=$this->process_field($val,$field_process,$cur_url_md5,$loopIndex,$cont_url_md5);
			}
			if(!empty($this->config['common_process'])){
				
				$val=$this->process_field($val,$this->config['common_process'],$cur_url_md5,$loopIndex,$cont_url_md5);
			}
			if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5])){
				
				if(empty($this->first_loop_field)){
					
					foreach ($this->field_val_list as $f_k=>$f_v){
						
						unset($this->field_val_list[$f_k]['values'][$cur_url_md5]);
						unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]);
					}
					return;
				}else{
					
					if(isset($this->exclude_cont_urls[$cont_url_md5][$cur_url_md5][$loopIndex])){
						
						if(!$is_loop){
							
							foreach ($this->field_val_list as $f_k=>$f_v){
								
								unset($this->field_val_list[$f_k]['values'][$cur_url_md5]);
								unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5]);
							}
							return;
						}else{
							
							foreach ($this->field_val_list as $f_k=>$f_v){
								
								if(is_array($this->field_val_list[$f_k]['values'][$cur_url_md5])){
									
									unset($this->field_val_list[$f_k]['values'][$cur_url_md5][$v_k]);
								}
								if(is_array($this->field_val_list[$f_k]['imgs'][$cur_url_md5])){
									
									unset($this->field_val_list[$f_k]['imgs'][$cur_url_md5][$v_k]);
								}
							}
							continue;
						}
					}
				}
			}
	
			
			$val=preg_replace_callback('/(?<=\bhref\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){
				
				return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url);
			},$val);
			$val=preg_replace_callback('/(?<=\bsrc\=[\'\"])([^\'\"]*)(?=[\'\"])/i',function($matche) use ($base_url,$domain_url){
				return \skycaiji\admin\event\Cpattern::create_complete_url($matche[1], $base_url, $domain_url);
			},$val);
					
			if($is_loop){
				
				if(!isset($this->field_val_list[$field_name]['values'][$cur_url_md5])){
					$this->field_val_list[$field_name]['values'][$cur_url_md5]=array();
					$this->field_val_list[$field_name]['imgs'][$cur_url_md5]=array();
				}
				$this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$val;
			}else{
				
				$this->field_val_list[$field_name]['values'][$cur_url_md5]=$val;
			}
			if(!empty($GLOBALS['_sc']['c']['download_img']['download_img'])&&!empty($val)){
				
				$valImgs=array();
				if(preg_match_all('/<img[^<>]*\bsrc=[\'\"]*(\w+\:\/\/[^\'\"\s]+)[\'\"]*/i',$val,$imgUrls)){
					
					$valImgs=is_array($imgUrls[1])?$imgUrls[1]:array();
				}
				if('extract'==$module&&'cover'==$field_params['extract_module']){
					
					$valImgs=array_merge($valImgs,array($val));
				}
				
				$noImgVal=preg_replace_callback('/\{\[img\]\}(http[s]{0,1}\:\/\/[^\s]+?)\{\[\/img\]\}/i',function($matche) use (&$valImgs){
					$valImgs[]=$matche[1];
					return $matche[1];
				},$val);
	
				if($noImgVal!=$val){
					
					if($is_loop){
						$this->field_val_list[$field_name]['values'][$cur_url_md5][$v_k]=$noImgVal;
					}else{
						$this->field_val_list[$field_name]['values'][$cur_url_md5]=$noImgVal;
					}
				}

				if(!empty($valImgs)){
					$valImgs=array_unique($valImgs);
					$valImgs=array_values($valImgs);
					if($is_loop){
						
						$this->field_val_list[$field_name]['imgs'][$cur_url_md5][$v_k]=$valImgs;
					}else{
						
						$this->field_val_list[$field_name]['imgs'][$cur_url_md5]=$valImgs;
					}
				}
			}
		}
	}
	/*设置分页的字段列表值*/
	public function setPagingFields($cont_url,$page_url){
		$contMd5=md5($cont_url);
		$pageMd5=md5($page_url);
	
		if(empty($page_url)){
			return $this->error('请输入分页网址');
		}
		if(!preg_match('/^\w+\:\/\//',$page_url)){
			return $this->error($page_url.'网址不完整');
		}
		if(empty($this->config['paging']['max'])||(count((array)$this->used_paging_urls[$contMd5])<$this->config['paging']['max'])){
			
			$this->set_html_interval();
			$this->echo_msg("——采集分页：<a href='{$page_url}' target='_blank'>{$page_url}</a>",'black');
				
			$html=$this->get_html($page_url);
			if(empty($html)){
				return $this->error('分页获取失败：'.$page_url);
			}
				
			if(!isset($this->used_paging_urls[$contMd5][$pageMd5])){
				
				$this->used_paging_urls[$contMd5][$pageMd5]=$page_url;
				
				foreach ($this->config['new_paging_fields'] as $v){
					$this->setField($this->config['new_field_list'][$v['field']],$page_url,$html,$cont_url);
				}
			}
			
			$paging_urls=$this->getPagingUrls($page_url,$html);
			if(!empty($paging_urls)){
				
				$nextUrl='';
				foreach ($paging_urls as $purl){
					if(!isset($this->used_paging_urls[$contMd5][md5($purl)])&&$cont_url!=$purl){
						
						$nextUrl=$purl;
						break;
					}
				}
				if(!empty($nextUrl)){
					$this->setPagingFields($cont_url,$nextUrl);
				}
			}
		}
	}

	/**
	 * 获取关联页网址
	 * @param unknown $name 关联页名称
	 * @param unknown $cont_url 内容页网址
	 * @param unknown $html 内容页源码
	 * @return string
	 */
	public function getRelationUrl($name,$cont_url,$html){
		if(empty($html)){
			$html=$this->get_html($cont_url,true);
		}
		if(empty($html)){
			
			return '';
		}
		$relation_url=$this->config['new_relation_urls'][$name];
		if(empty($relation_url)){
			
			return '';
		}
	
		if(empty($relation_url['page'])){
			
			if(!isset($this->relation_url_list[$cont_url][$name])){
				$html=$this->rule_match_area($relation_url, $html);
				if(empty($html)){
					
					return '';
				}
				$relationUrl=$this->rule_match_urls($relation_url, $html);
				$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
				$this->relation_url_list[$cont_url][$name]=$relationUrl;
			}else{
				$relationUrl=$this->relation_url_list[$cont_url][$name];
			}
		}else{
			
			$page=$relation_url['page'];
			$pass=false;
			$depth_pages=array();
			$depth=0;
			while(!empty($page)){
				
				if($page==$name){
					
					$pass=true;
					break;
				}
				if(empty($this->config['new_relation_urls'][$page])){
					
					$pass=true;
					break;
				}
				$depth++;
				$depth_pages[$depth]=$page;
				$page=$this->config['new_relation_urls'][$page]['page'];
			}
			if($pass){
				
				return '';
			}
				
			krsort($depth_pages);
			$contPage=reset($depth_pages);
			$relationUrl='';
			if(isset($contPage)){
				
				if(!isset($this->relation_url_list[$cont_url][$contPage])){
					$html=$this->rule_match_area($this->config['new_relation_urls'][$contPage], $html);
					if(empty($html)){
						
						return '';
					}
					$relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$contPage], $html);
					$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
					$this->relation_url_list[$cont_url][$contPage]=$relationUrl;
				}else{
					$relationUrl=$this->relation_url_list[$cont_url][$contPage];
				}
			}
			$depth_pages=array_slice($depth_pages, 1);
			$depth_pages=is_array($depth_pages)?$depth_pages:array();
			$depth_pages[]=$relation_url['name'];
	
			foreach ($depth_pages as $page){
				if(empty($relationUrl)){
					
					return '';
				}
				if(!isset($this->relation_url_list[$cont_url][$page])){
					
					$relationHtml=$this->get_html($relationUrl,true);
					$relationHtml=$this->rule_match_area($this->config['new_relation_urls'][$page], $relationHtml);
					if(empty($relationHtml)){
						
						return '';
					}
					$relationUrl=$this->rule_match_urls($this->config['new_relation_urls'][$page],$relationHtml);
					$relationUrl=(is_array($relationUrl)&&!empty($relationUrl))?reset($relationUrl):'';
	
					$this->relation_url_list[$cont_url][$page]=$relationUrl;
				}else{
					$relationUrl=$this->relation_url_list[$cont_url][$page];
				}
			}
		}
	
		
		
		
		
		

		return $relationUrl;
	}
	
	/*获取内容页字段列表，这里是入口*/
	public function getFields($cont_url){
		$this->field_val_list=array();
		$this->first_loop_field=null;
	
		if(empty($cont_url)){
			return $this->error('请输入内容页网址');
		}
		if(!preg_match('/^\w+\:\/\//',$cont_url)){
			return $this->error($cont_url.'网址不完整');
		}
		$html=$this->get_html($cont_url,false,$this->config['url_post']);
		if(empty($html)){
			return $this->error('抓取页面失败');
		}
		foreach($this->config['new_field_list'] as $field_config){
			$this->setField($field_config,$cont_url,$html,$cont_url);
		}
		$paging_urls=$this->getPagingUrls($cont_url,$html);
		if(!empty($paging_urls)){
			
			$this->setPagingFields($cont_url,reset($paging_urls));
		}
		
		$val_list=array();
		if(!empty($this->field_val_list)){
			if(empty($this->first_loop_field)){
				
				foreach ($this->field_val_list as $fieldName=>$fieldVal){
					$val_values=array_filter_keep0($fieldVal['values']);
					$val_values=implode($this->config['new_paging_fields'][$fieldName]['delimiter'], $val_values);

					$val_imgs=array();
					if(!empty($fieldVal['imgs'])){
						foreach ($fieldVal['imgs'] as $v){
							if(!empty($v)){
								if(is_array($v)){
									$val_imgs=array_merge($val_imgs,$v);
								}else{
									$val_imgs[]=$v;
								}
							}
						}
						if(!empty($val_imgs)){
							$val_imgs=array_unique($val_imgs);
							$val_imgs=array_filter($val_imgs);
							$val_imgs=array_values($val_imgs);
						}
					}
					$val_list[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs);
				}
			}else{
				
				
				foreach ($this->field_val_list[$this->first_loop_field]['values'] as $page_key=>$page_vals){
					
					if(empty($page_vals)){
						
						continue;
					}
					foreach ($page_vals as $loop_index=>$loop_val){
						
						$vals=array();
						foreach ($this->field_val_list as $fieldName=>$fieldVals){
							if(is_array($fieldVals['values'][$page_key])){
								
								$val_values=$fieldVals['values'][$page_key][$loop_index];
								$val_imgs=$fieldVals['imgs'][$page_key][$loop_index];
							}else{
								
								$val_values=$fieldVals['values'][$page_key];
								$val_imgs=$fieldVals['imgs'][$page_key];
							}
							if(!empty($val_imgs)){
								$val_imgs=array_unique($val_imgs);
								$val_imgs=array_filter($val_imgs);
								$val_imgs=array_values($val_imgs);
							}
							$vals[$fieldName]=array('value'=>$val_values,'img'=>$val_imgs);
						}
						$val_list[]=$vals;
					}
				}
			}
		}
		return $val_list?$val_list:array();
	}
	/*设置数据处理，保存config时使用*/
	public function setProcess($processList){
		if(is_array($processList)){
			$processList=array_array_map('trim',$processList);
			foreach ($processList as $k=>$v){
				$v['module']=strtolower($v['module']);
				if(!empty($v['title'])){
					$v['title']=str_replace(array("'",'"'),'',strip_tags($v['title']));
				}
				if('html'==$v['module']){
					$v['html_allow']=$this->clear_tags($v['html_allow']);
					$v['html_allow']=implode(',', $v['html_allow']);
					$v['html_filter']=$this->clear_tags($v['html_filter']);
					$v['html_filter']=implode(',', $v['html_filter']);
				}elseif('filter'==$v['module']){
					if(preg_match_all('/[^\r\n]+/', $v['filter_list'],$filterList)){
						$filterList=array_filter(array_unique($filterList[0]));
						$v['filter_list']=implode("\r\n",$filterList);
					}
					$v['filter_list']=trim($v['filter_list']);
				}
				$processList[$k]=$v;
			}
			$processList=array_values($processList);
		}
		return $processList;
	}
	/*初始化数据处理，初始化config时使用*/
	public function initProcess($processList){
		if(!empty($processList)){
			foreach ($processList as $k=>$v){
				if('replace'==$v['module']){
					$v['replace_from']=preg_replace('/\\\*([\'\/])/', "\\\\$1", $v['replace_from']);
					$v['replace_from']=str_replace('(*)', '[\s\S]*?', $v['replace_from']); 
				}
				$processList[$k]=$v;
			}
		}
		return $processList;
	}
	
	/*统一：获取网址列表*/
	public function _get_urls($source_url,$config,$is_level=false){
		$is_level=$is_level?'多级':'';
	
		$html=$this->get_html($source_url);
		if(empty($html)){
			return $this->error($is_level.'页面为空');
		}
		$base_url=$this->match_base_url($source_url, $html);
		$domain_url=$this->match_domain_url($source_url);
		
		$html=$this->rule_match_area($config, $html);
		if(empty($html)){
			return $this->error("未提取到{$is_level}区域内容！");
		}
	
		$cont_urls=$this->rule_match_urls($config, $html);
		$cont_urls1=array();
	
		
		if(isset($this->config['url_op'])){
			
			$op_not_complete=in_array('not_complete',$this->config['url_op'])?true:false;
		}else{
			if(isset($this->config['url_complete'])){
				
				$op_not_complete=$this->config['url_complete']?false:true;
			}else{
				
				$op_not_complete=false;
			}
		}
	
		foreach ($cont_urls as $cont_url){
			if(!$op_not_complete){
				
				$cont_url=$this->create_complete_url($cont_url, $base_url, $domain_url);
			}
			if(!empty($config['url_must'])){
				
				if(!preg_match('/'.$config['url_must'].'/i', $cont_url)){
					continue;
				}
			}
	
			if(!empty($config['url_ban'])){
				
				if(preg_match('/'.$config['url_ban'].'/i', $cont_url)){
					continue;
				}
			}
			if(!empty($cont_url)){
				if(strpos($cont_url,' ')==false){
					
					
					
					$cont_urls1[]=$cont_url;
				}
			}
		}
		$cont_urls=$cont_urls1;
		unset($cont_urls1);
	
		if(empty($cont_urls)){
			return $this->error("未获取到".($is_level?$is_level:'内容')."网址！");
		}else{
			if(!empty($this->config['url_reverse'])){
				
				$cont_urls=array_reverse($cont_urls);
			}
			if(!empty($this->config['url_post'])){
				
				$postParams=array();
				if(!empty($this->config['url_posts']['names'])){
					foreach ($this->config['url_posts']['names'] as $k=>$v){
						if (!empty($v)){
							$postParams[]=$v.'='.rawurlencode($this->config['url_posts']['vals'][$k]);
						}
					}
				}
				if(!empty($postParams)){
					
					$postParams=implode('&', $postParams);
					foreach ($cont_urls as $k=>$v){
						$v.=strpos($v,'?')===false?'?':'&';
						$v.=$postParams;
						$cont_urls[$k]=$v;
					}
				}
			}
			return array_values($cont_urls);
		}
	}
	/*执行采集返回未使用的网址*/
	public function _collect_unused_cont_urls($cont_urls=array(),$echo_str=''){
		$mcollected=model('Collected');
		$count_conts=count((array)$cont_urls);
		if($this->config['url_repeat']){
			
			$db_cont_urls=array();
		}else{
			
			$db_cont_urls=$mcollected->getUrlByUrl($cont_urls);
		}
		$unused_cont_urls=array();
		$count_used=0;
		if(!empty($cont_urls)){
			foreach ($cont_urls as $cont_url){
				if(array_key_exists(md5($cont_url), $this->used_cont_urls)){
					
					$count_used++;
				}elseif(in_array($cont_url, $db_cont_urls)){
					
					$count_used++;
				}else{
					
					$unused_cont_urls[md5($cont_url)]=$cont_url;
				}
			}
		}
		if($count_used>0){
			$count_used=min(count($cont_urls),$count_used);
			$this->echo_msg($echo_str.'采集到'.$count_conts.'条网址，<span style="color:orange">'.$count_used.'</span>条重复，<span style="color:green">'.(count($unused_cont_urls)).'</span>条有效','black');
		}else{
			$this->echo_msg($echo_str.'采集到<span style="color:green">'.$count_conts.'</span>条有效网址','black');
		}
		return $unused_cont_urls;
	}
	/*执行级别采集*/
	public function _collect_level($source_url,$level=1){
		$end_echo='</div>';
	
		$level=max(1,$level);
		$level_str='';
		for($i=1;$i<$level;$i++){
			
		}
		$next_level_str=$level_str;
		if($level<=1){
			
			$this->cur_level_urls=array();
		}
		$this->echo_msg('','',true,'<div style="padding-left:30px;">');
	
		$level_data=$this->collLevelUrls($source_url,$level);
		$this->echo_msg($level_str.'抓取到'.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址'.count((array)$level_data['urls']).'条','black');
	
		$mcollected=model('Collected');
		$mcacheLevel=CacheModel::getInstance('level_url');
	
		if(!empty($level_data['urls'])){
			
			$level_urls=array();
			foreach ($level_data['urls'] as $level_url){
				$level_urls["level_{$level}:{$level_url}"]=$level_url;
			}
				
			$level_interval=$GLOBALS['_sc']['c']['caiji']['interval']*60;
			$time_interval_list=array();
			
			$cacheLevels=$mcacheLevel->db()->where(array('cname'=>array('in',array_map('md5', array_keys($level_urls)))))->column('dateline','cname');
	
			if(!empty($cacheLevels)){
				$count_db_used=0;
				$sortLevels=array('undb'=>array(),'db'=>array());
				
				foreach ($level_urls as $level_key=>$level_url){
					$md5_level_key=md5($level_key);
					if(!isset($cacheLevels[$md5_level_key])){
						
						$sortLevels['undb'][$level_key]=$level_url;
					}else{
						
						$time_interval=abs(NOW_TIME-$cacheLevels[$md5_level_key]);
						if($time_interval<$level_interval){
							
							$this->used_level_urls[$level_key]=1;
							$count_db_used++;
							$time_interval_list[]=$time_interval;
						}else{
							$sortLevels['db'][$level_key]=$level_url;
						}
					}
				}
				if($count_db_used>0){
					$this->echo_msg($level_str.$count_db_used.'条已采集网址被过滤，下次采集需等待'.($level_interval-max($time_interval_list)).'秒，<a href="'
							.url('Admin/Setting/caiji').'" target="_blank">设置间隔</a>','black');
					if(count($level_urls)<=$count_db_used){
						$this->echo_msg($level_str.$level.'级“'.$this->config['level_urls'][$level-1]['name'].'”网址采集完毕！','green',true,$end_echo);
						return 'completed';
					}
				}
				$level_urls=array_merge($sortLevels['undb'],$sortLevels['db']);
				unset($sortLevels);
				unset($cacheLevels);
			}
			$level_data['urls']=$level_urls;
		}
	
		
		$finished_source=true;
		$cur_level_i=0;;
		if(!empty($level_data['urls'])){
			foreach ($level_data['urls'] as $level_key=>$level_url){
				$cur_level_i++;
				if(array_key_exists($level_key,$this->used_level_urls)){
					
					continue;
				}
				$this->cur_level_urls[$this->config['level_urls'][$level-1]['name']]=$level_url;
				$this->echo_msg("{$next_level_str}分析第{$level}级：<a href='{$level_url}' target='_blank'>{$level_url}</a>",'black');
				if($level_data['nextLevel']>0){
					
					$return_msg=$this->_collect_level($level_url,$level_data['nextLevel']);
					if($return_msg=='completed'){
						$this->echo_msg('','',true,$end_echo);
						return $return_msg;
					}
				}else{
					
					$cont_urls=$this->getContUrls($level_url);
					$cont_urls=$this->_collect_unused_cont_urls($cont_urls,$next_level_str);
					$this->cont_urls_list[$level_key]=$cont_urls;
					$this->_collect_fields($next_level_str);
				}
				if($this->collect_num>0){
					
					if(count($this->collected_field_list)>=$this->collect_num){
						
						if($cur_level_i<count((array)$level_data['urls'])){
							$finished_source=false;
						}
						break;
					}
				}
			}
		}
		if($finished_source){
			
			$source_key='level_'.($level-1).':'.$source_url;
			$this->used_level_urls[$source_key]=1;
			$mcacheLevel->setCache(md5($source_key),$source_key);
			if($level<=1){
				
				$mcacheSource=CacheModel::getInstance('source_url');
				$this->used_source_urls[$source_url]=1;
				$mcacheSource->setCache(md5($source_url),$source_url);
			}
		}
		$this->echo_msg('','',true,$end_echo);
	}
	/*采集字段列表*/
	public function _collect_fields($echo_str=''){
		$mcollected=model('Collected');
		$mcacheSource=CacheModel::getInstance('source_url');
		$mcacheLevel=CacheModel::getInstance('level_url');
		$mcacheCont=CacheModel::getInstance('cont_url');
		
		
		foreach ($this->cont_urls_list as $cont_key=>$cont_urls){
			$source_type=0;
			if('_source_is_url_'==$cont_key){
				$source_type=0;
			}elseif(strpos($cont_key,'level_')===0){
				$source_type=2;
			}else{
				$source_type=1;
			}
				
			if($source_type==2){
				if(array_key_exists($cont_key,$this->used_level_urls)){
					
					continue;
				}
			}else{
				if(array_key_exists($cont_key,$this->used_source_urls)){
					
					continue;
				}
			}
				
			$finished_cont=true;
			$cur_c_i=0;
			foreach ($cont_urls as $cont_url){
				$cur_c_i+=1;
				$md5_cont_url=md5($cont_url);
				if(array_key_exists($md5_cont_url,$this->used_cont_urls)){
					
					continue;
				}
				if($this->config['url_repeat']||$mcollected->getCountByUrl($cont_url)<=0){
					
					if(!empty($this->collected_field_list)){
						
						
						if($this->set_html_interval()===true){
							
							if(!$this->config['url_repeat']&&$mcollected->getCountByUrl($cont_url)>0){
								$this->used_cont_urls[$md5_cont_url]=1;
								continue;
							}
						}
					}
						
					
					if(input('?backstage')){
						
						$backstageDate=CacheModel::getInstance('backstage_task')->db()->field('dateline')->where('cname',$this->collector['task_id'])->find();
						if(empty($backstageDate)||$GLOBALS['_sc']['backstage_task_runtime']<$backstageDate['dateline']){
							
							unset($GLOBALS['_sc']['backstage_task_ids'][$this->collector['task_id']]);
							exit('终止进程');
						}
					}
						
					if($mcacheCont->getCount($md5_cont_url)>0){
						
						$this->used_cont_urls[$md5_cont_url]=1;
						continue;
					}
					$mcacheCont->setCache($md5_cont_url, 1);
						
					$this->echo_msg($echo_str."采集内容页：<a href='{$cont_url}' target='_blank'>{$cont_url}</a>",'black');
					$field_vals_list=$this->getFields($cont_url);
	
					$is_loop=empty($this->first_loop_field)?false:true;
					if(!empty($field_vals_list)){
						$is_real_time=false;
						if(!empty($GLOBALS['_sc']['c']['caiji']['real_time'])&&!empty($GLOBALS['_sc']['real_time_release'])){
							
							$is_real_time=true;
						}
						if(!$is_loop){
							
							$field_vals_list=array($field_vals_list);
						}else{
							
							
							$loop_cont_urls=array();
							foreach ($field_vals_list as $k=>$field_vals){
								$loop_cont_urls[$k]=$cont_url.'#'.md5(serialize($field_vals));
							}
							if(!empty($loop_cont_urls)){
								$loop_exists_urls=$mcollected->getUrlByUrl($loop_cont_urls);
								if(!empty($loop_exists_urls)){
									
									$loop_exists_urls=array_flip($loop_exists_urls);
									foreach ($loop_cont_urls as $k=>$loop_cont_url){
										if(isset($loop_exists_urls[$loop_cont_url])){
											
											unset($field_vals_list[$k]);
										}
									}
									$this->echo_msg($echo_str.'已过滤'.count((array)$loop_exists_urls).'条重复数据','black');
								}
							}
							if(isset($this->exclude_cont_urls[$md5_cont_url])){
								
								$excludeNum=0;
								foreach($this->exclude_cont_urls[$md5_cont_url] as $k=>$v){
									
									$excludeNum+=count((array)$v);
								}
								
								$this->echo_msg($echo_str.'通过数据处理排除了'.$excludeNum.'条数据','black');
							}
							$field_vals_list=array_values($field_vals_list);
						}
	
						foreach ($field_vals_list as $field_vals){
							$collected_error='';
							$collected_data=array('url'=>$cont_url,'fields'=>$field_vals);
							if($is_loop){
								
								$collected_data['url'].='#'.md5(serialize($field_vals));
							}else{
								
								if(isset($this->exclude_cont_urls[$md5_cont_url])){
									
									$collected_error=reset($this->exclude_cont_urls[$md5_cont_url]);
									$collected_error=$this->exclude_url_msg($collected_error);
								}
							}
							if(empty($collected_error)){
								if(!empty($this->config['field_title'])){
									
									$collected_data['title']=$field_vals[$this->config['field_title']]['value'];
								}
								if(!empty($collected_data['title'])){
									
									if($mcollected->getCountByTitle($collected_data['title'])>0){
										
										$collected_error='标题重复：'.mb_substr($collected_data['title'],0,300,'utf-8');
									}
								}
							}
							if(empty($collected_error)){
								
								if($is_real_time){
									
									
									$GLOBALS['_sc']['real_time_release']->export(array($collected_data));
										
									unset($collected_data['fields']);
									unset($collected_data['title']);
								}
								
								$this->collected_field_list[]=$collected_data;
							}else{
								
								if(!$this->config['url_repeat']){
									
									controller('ReleaseBase','event')->record_collected($collected_data['url'],
										array('id'=>0,'error'=>$collected_error),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module'])
									);
								}else{
									
									$this->echo_msg($collected_error);
								}
							}
						}
					}
					
					if($is_loop){
						
						
						controller('ReleaseBase','event')->record_collected(
						$cont_url,array('id'=>1,'target'=>'','desc'=>'循环入库'),array('task_id'=>$this->collector['task_id'],'module'=>$this->release['module']),null,false
						);
					}
				}else{
					
					$this->echo_msg('已采集过该<a href="'.$cont_url.'" target="_blank">网址</a>','black');
				}
				$this->used_cont_urls[$md5_cont_url]=1;
	
				if($this->collect_num>0){
					
					if(count($this->collected_field_list)>=$this->collect_num){
						
						if($cur_c_i<count($cont_urls)){
							
							$finished_cont=false;
						}
						break;
					}
				}
			}
				
			if($finished_cont){
				
				if($source_type==1){
					
					$mcacheSource->setCache(md5($cont_key),$cont_key);
				}elseif($source_type==2){
					
					$mcacheLevel->setCache(md5($cont_key),$cont_key);
				}
	
				if($source_type==2){
					
					$this->used_level_urls[$cont_key]=1;
				}else{
					
					$this->used_source_urls[$cont_key]=1;
				}
			}
			
			if($this->collect_num>0&&count($this->collected_field_list)>=$this->collect_num){
				break;
			}
		}
	}
}
?>