<?php
// 采集模型
class Spider extends Base {
	private static $getInitial = array();
	public $fields = array(); //采集的字段及对应的名称 (全部)
	public $defaultFields = array('title', 'content'); //默认要采集的字段
	public $addFields = array(); //发布时需要增加的字段
	public $param = array(); //发布时设置的参数
	/**
	 * 蜘蛛初始化函数
	 *
	 * @param  $rule array
	 * @access public
	 */
	public function __construct($rule) {
		$this -> rule = $rule;
		$this -> username = get_cookie('username');
		$this -> charset = strtolower(C('APP_CHARSET', 1));
		$this -> baseUrl = $rule['set']['baseurl'];
		// 采集字段初始化
		$L = L('', 'spider');
		$this -> fields = array('title' => array('name' => $L['TITLE'], 'value' => ''),
			'keywords' => array('name' => $L['KEYWORDS'], 'value' => ''),
			'description' => array('name' => $L['DESCRIPTION'], 'value' => ''),
			'content' => array('name' => $L['CONTENT'], 'value' => ''),
			'author' => array('name' => $L['AUTHOR'], 'value' => get_cookie('username')),
			'copyfrom' => array('name' => $L['COPYFROM'], 'value' => $L['INTERNET']),
			'addtime' => array('name' => $L['ADDTIME'], 'value' => '{TIME}')
			);
		// 回调方法 初始化蜘蛛
		$this -> _initialize();
	}
	private function setBaseUrl($url) {
		if ($this -> rule['set']['basetype'] > 1) return false;
		if ($this -> rule['set']['basetype'] == 0) {
			$tmp = parse_url($url);
			if ($tmp) {
				$this -> baseUrl = $tmp['scheme'] . '://' . $tmp['host'];
			}
		} else $this -> baseUrl = dirname($url);
	}
	// 获取匹配字段内容
	private function getField($begin, $end, $content) {
		$begin = $this -> pregQuote($begin);
		$end = $this -> pregQuote($end);
		if (@preg_match("/{$begin}(.*?){$end}/is", $content, $rs)) $rs = $rs[1];
		else $rs = '';
		return $rs;
	}
	// 检查字符编码方式
	private function checkCharset($c) {
		if ($c) {
			$c = stripslashes($c);
			if ($this -> rule['set']['c_charset']) {
				$charset = $this -> rule['set']['c_charset'];
			} else {
				preg_match("/<meta.+?charset=([-\w]+)/i", $c, $rs);
				$charset = $rs[1];
			}
			if (!empty($charset)) {
				$charset = strtolower($charset);
				if ($charset == 'gb2312')$charset = 'gbk';
				if ($this -> charset != $charset) {
					$c = $this -> charset == 'utf-8'?iconv($charset, $this -> charset, $c):iconv($charset, $this -> charset . '//IGNORE', $c);
				}
			}
			$c = preg_replace("/[\r\n]+/", '', $c);
		}
		return $c;
	}
	// 获取内容
	public function getContent($url) {
		$c = $this -> rule['set']['sim_browser']?@FileUtil :: fsockGetContents($url):@file_get_contents($url);
		return $this -> checkCharset($c);
	}
	/**
	 * 根据二维数组(一般通过SQL查询获得)网址批量获取内容
	 *
	 * @param  $urls 二维数组array(array('url'=>url1), array('url'=>url2), ...)
	 * @access public
	 */
	public function getMoreContent2($data) {
		$rs = '';
		if (is_array($data)) {
			$urls = array();
			foreach($data as $k => $v) {
				$urls[$k] = $v['url'];
			}
			$rs = $this -> getMoreContent($urls);
		}
		return $rs;
	}
	/**
	 * 批量获取内容
	 *
	 * @param  $urls 一维数组array(url1, url2, ...)
	 * @access public
	 */
	public function getMoreContent($urls) {
		$rs = array();
		$sockets = array();
		$userAgent = $_SERVER['HTTP_USER_AGENT'];
		foreach($urls as $id => $url) {
			$tmp = parse_url($url);
			$host = $tmp['host'];
			$path = isset($tmp['path'])?$tmp['path']:'/';
			empty($tmp['query']) or $path .= '?' . $tmp['query'];
			if (empty($tmp['port'])) {
				$port = $tmp['scheme'] == 'https'?443:80;
			} else $port = $tmp['port'];
			$fp = stream_socket_client("$host:$port", $errno, $errstr, 20);
			if ($fp) {
				$rs[$id] = '';
				$sockets[$id] = $fp;
				fwrite($fp, "GET $path HTTP/1.1\r\nHost: $host\r\nUser-Agent: $userAgent\r\nConnection: Close\r\n\r\n");
			}
		}
		// Now, wait for the results to come back in
		while (count($sockets)) {
			$read = $sockets;
			// This is the magic function - explained below
			if (stream_select($read, $write = null, $e = null, 20)) {
				// readable sockets either have data for us, or are failed connection attempts
				foreach ($read as $r) {
					$id = array_search($r, $sockets);
					if (!feof($r)) $rs[$id] .= @stream_get_line($r, 4096);
					else {
						fclose($r);
						$tmp = explode("\r\n\r\n", $rs[$id], 2);
						$rs[$id] = strpos($tmp[0], '200')?$this -> checkCharset($tmp[1]):'';
						unset($sockets[$id]);
					}
				}
			}
		}
		return $rs;
	}
	// 采集字段内容,参数:当前url,字段数组,替换规则数组,分页规则,保护规则,内容
	public function getData($url, $af, $arp, $pageRule = '', $protect = '', $c = '') {
		if (!($url && is_array($af))) return false;
		$this -> setBaseUrl($url);
		$c or $c = $this -> getContent($url);
		$pageUrls = array($url); //分页地址
		// 获取分页地址start
		if (is_array($pageRule)) {
			$tmp = $this -> getField($pageRule['begin'], $pageRule['end'], $c);
			preg_match_all("/href\s*=\s*([\"']?)\s*([^ \"'>]+)\\1/i", $tmp, $link);
			if (isset($link[2])) {
				$tmp = $this -> formatUrl($link[2][0], $this -> baseUrl);
				$tmp = pathinfo($tmp);
				$tmp1 = pathinfo($url);
				$baseUrl = $tmp['dirname'] == $tmp1['dirname']?$this -> baseUrl:$tmp1['dirname'];
				foreach($link[2] as $tmp) {
					$tmp && $pageUrls[] = $this -> formatUrl($tmp, $baseUrl);
				}
				$pageMerge = $pageRule['ismerge']?'':'[nextpage]';
				$pageUrls = array_unique($pageUrls);
			}
		}
		// 获取分页地址end
		$rs = array();
		foreach($af as $k => $v) {
			if (!(empty($v['begin']) or empty($v['end']))) {
				$rs[$k] = $this -> getField($v['begin'], $v['end'], $c);
				if ($k == 'content' && count($pageUrls) > 1) {
					unset($pageUrls[0]);
					$c = $this -> getMoreContent($pageUrls);
					foreach($c as $tmp) {
						$rs[$k] .= $pageMerge . $this -> getField($v['begin'], $v['end'], $tmp);
					}
				}
				if ($rs[$k]) {
					if (is_array($protect)) {
						foreach($protect as $tmp) {
							$tmprep = substr($tmp, 0, 1) . '-=-j-x-c-m-s-=-' . substr($tmp, 1);
							$rs[$k] = str_replace($tmp, $tmprep, $rs[$k]);
						}
					}
					if (is_array($arp[$k])) {
						$old = $this -> replaceRule($arp[$k]['old'], 'old');
						$new = $this -> replaceRule($arp[$k]['new'], 'new');
						foreach($old as $tk => $pattern) {
							$tmp = isset($new[$tk]) ? $new[$tk] : '';
							$rs[$k] = preg_replace("/{$pattern}/is", $tmp, $rs[$k]);
						}
					}
					if ($k == 'content') {
						$rs[$k] = preg_replace_callback('/((href|src)\s*=\s*["\']?\s*)([^ "\'>]+)(\s*["\']?\s*(.*?)>)/i', array(&$this, '_getDataCallback'), $rs[$k]);
					}
					if (is_array($protect)) {
						$rs[$k] = str_replace('-=-j-x-c-m-s-=-', '', $rs[$k]);
					}
				}
				$tmp = substr($rs[$k], strrpos($rs[$k], '</'));
				$tmp = substr($tmp, strpos($tmp, '>') + 1);
				$tmp2 = str_replace('&nbsp;', '', $tmp);
				$tmp2 = preg_replace("/[\s]+/", '', $tmp2);
				if ($tmp && trim(strip_tags($tmp2)) == '') {
					$rs[$k] = substr($rs[$k], 0, - strlen($tmp));
				}
			} elseif (!empty($v['begin'])) {
				$rs[$k] = str_replace(array('{TIME}', '{USERNAME}'), array(date('Y-m-d H:i:s', TIME), $this -> username), $v['begin']);
			}
		}
		return $rs;
	}
	private function _getDataCallback($matches) {
		return $matches[1] . $this -> formatUrl($matches[3], $this -> baseUrl) . $matches[4];
	}

	public function getLinks($url, $begin = '', $end = '', $in = '', $out = '') {
		$this -> setBaseUrl($url);
		$c = $this -> getContent($url);
		if ($begin && $end) $c = $this -> getField($begin, $end, $c);
		preg_match_all('/<a(.*?)href\s*=\s*["\']?\s*([^ "\'>]+)\s*["\']?\s*(.*?)>(.*?)<\/a>/i', $c, $link);
		$array = $arrayUrl = array();
		if (isset($link[2])) {
			foreach($link[2] as $k => $v) {
				if ($in && $v && !strstr($v, $in)) $v = '';
				if ($out && $v && strstr($v, $out)) $v = '';
				if ($v) {
					$tmpurl = $this -> formatUrl($v, $this -> baseUrl);
					$key = array_search($tmpurl, $arrayUrl);
					if ($key === false) {
						$arrayUrl[] = $tmpurl;
						$img = @preg_match("/src\s*=\s*[\"']?([^\"'>]+\.(gif|jpg|jpeg|bmp|png))/i", $link[4][$k], $m)?$m[1]:'';
						if ($img) $img = $this -> formatUrl($img, $this -> baseUrl);
						$array[] = array('url' => $tmpurl, 'title' => trim(strip_tags($link[4][$k])), 'img' => $img);
					} else {
						if (empty($array[$key]['title'])) $array[$key]['title'] = trim(strip_tags($link[4][$k]));
						if (empty($array[$key]['img'])) {
							$img = @preg_match("/src\s*=\s*[\"']?([^\"'>]+\.(gif|jpg|jpeg|bmp|png))/i", $link[4][$k], $m)?$m[1]:'';
							if ($img) $img = $this -> formatUrl($img, $this -> baseUrl);
							$array[$key]['img'] = $img;
						}
					}
				}
			}
		}
		return $array;
	}
	// 发布
	public function post($url, $moduleObj) {
		$info = Cache :: read($url['urlid'], 'data', 0, 'spider');
		if (!empty($info['content'])) {
			if (!empty($info['addtime'])) $info['addtime'] = Func :: timeStamp($info['addtime']);
			if ($this -> rule['set']['remotetype']) {
				$this -> param['remotetype'] = $this -> rule['set']['remotetype'];
				$this -> param['saveremote'] = 1;
			}
			if ($this -> rule['set']['pic1_default']) $this -> param['pic1_default'] = 1;
			$info['catid'] = $this -> rule['set']['catid'];
			$info['hits'] = mt_rand($this -> rule['set']['hitsmin'], $this -> rule['set']['hitsmax']);
			$info['passed'] = $this -> rule['set']['passed']?1:0;
			if ($url['img']) $info['defaultpicurl'] = $url['img'];
			$this -> info = array_merge($info, $this -> addFields);
			$this -> _beforePost();
			if ($moduleObj -> save($this -> info, $this -> param)) {
				Cache :: del($url['urlid'], 'data', 'spider');
				return true;
			} else $this -> error = $moduleObj -> error;
		}
		return false;
	}
	// 正则转义
	private function pregQuote($str) {
		$str = preg_quote($str); //php自带正则转义
		$str = str_replace('\(\*\)', '.*?', $str); //转义自定义通配符
		$str = str_replace('/', '\/', $str); //转义斜杠
		$str = str_replace('"', '\"', $str); //转义双引号
		$str = preg_replace("/[\r\n]+/", '', $str); //转义回车换行
		return $str;
	}
	// 替换规则参数处理，参数$type 为 old 或 new
	private function replaceRule($str, $type = 'old') {
		$str = explode('(|)', $str);
		$array = array();
		foreach($str as $tmp) {
			$array[] = $type == 'old'?$this -> pregQuote($tmp):preg_replace('/\(\$([0-9]+)\)/', '\\\\\\1', $tmp);
		}
		return $array;
	}
	// 格式化路径 参数:当前路径,基础url;输出完整路径
	private function formatUrl($path, $baseUrl = '') {
		$tmp = parse_url($path);
		if (empty($tmp['scheme'])) {
			if (substr($baseUrl, -1) == '/') $baseUrl = substr($baseUrl, 0, -1);
			$tmp = parse_url($baseUrl);
			if (empty($tmp['scheme'])) return false;
			if (substr($path, 0, 1) == '/' && !empty($tmp['path'])) {
				$baseUrlArray = explode('://', $baseUrl);
				$tmp = explode('/', $baseUrlArray[1]);
				$url = $baseUrlArray[0] . '://' . $tmp[0] . $path;
			} else $url = $baseUrl . '/' . $path;
		} else $url = $path;
		$url = explode('://', $url);
		if (!isset($url[1])) return false;
		$path = $url[1];
		$path = preg_replace('{/+}', '/', $path);
		$path = explode('/', $path);
		$cnt = count($path);
		for($i = 0; $i < $cnt; $i++) {
			if ($path[$i] == '..') {
				unset($path[$i]);
				if ($i > 1) unset($path[$i-1]);
			}
		}
		return $url[0] . '://' . implode('/', $path);
	}
	// 回调方法 初始化蜘蛛
	protected function _initialize() {
	}
	// 回调方法 发布信息前处理
	protected function _beforePost() {
	}
}
