NokitaKaze icon

Парсер и сабмиттер rss для booru

NokitaKaze | PRO | 07/05/13 05:46:37 PM UTC | 0 ⭐ | 340 👁️ | Never ⏰ | []
PHP |

6.72 KB

|

None

|

0 👍

/

0 👎

<?php
/*
 http://danbooru.donmai.us/post/atom?tags=
 http://chan.sankakucomplex.com/post/atom?tags=
 http://gelbooru.com/index.php?page=atom&tags=
*/
 $id      =(int)($_GET['id']);
 $sad_hide=true;
 
 $q2_c=mysql_fetch_assoc(mysql_query('select * from `'.$sad_prefix.'booru_feed` where `id`='.$id));
 if ((int)$q2_c['id']<1){
  header('http/1.0 404 not found');
  echo 'not found';
  return;
 }
 
 if ((int)($q2_c['ttl'])==0){$q2_c['ttl']=3*3600;}
 if (($q2_c['time']+$q2_c['ttl']>=time() and strlen($q2_c['rss'])>0) or
     (!iam() and ($_SERVER['REMOTE_ADDR']!==my_ip())) ){
  $sad_contype='text/xml';
  echo '<'.'?xml version="1.0" encoding="utf-8"?'.'>';
  echo $q2_c['rss'];
  echo '<!-- cached: '.date('Y-m-d H:i:sO',$q2_c['time']).' -->';
  return;
 }
 
 if (!iam() and ($_SERVER['REMOTE_ADDR']!==my_ip()) and (int)sad_option('dis_booru')==1){header('http/1.0 503');return;}
 
 function numnum($img){
  if (!preg_match('|https?://([0-9a-z.-]+)|',$img,$a)){return 0;}
  if ($a[1]=='danbooru.donmai.us'){return 11;}
  if ($a[1]=='kanaria.ru'){return 10;}
  if ($a[1]=='gelbooru.com'){return 9;}
  if ($a[1]=='chan.sankakucomplex.com'){return 8;}
  
  
  
  return 1;
 }
 
 
 $feed=array();set_time_limit(20*60);$debug=true;
 
 if ($debug){$fo=fopen($sad_prefix.'/debug','a');
  fwrite($fo,date('Y-m-d H:i:s').': start = '.$id."\r\n");fclose($fo);}
 
// Перебираем все дочерные фиды
 $q   =mysql_query('select * from `'.$sad_prefix.'booru_rss` where `parent`='.$id);
 while ($q_c=mysql_fetch_assoc($q)){
  if ($debug){
   $fo=fopen($sad_prefix.'/debug','a');
   fwrite($fo, date('Y-m-d H:i:s').': q_c id = '.$q_c['id'].'; url = "'.
               $q_c['url'].'"; time = '.date('Y-m-d H:iO',$q_c['time'])."\r\n");
   fclose($fo);
  }
  
  //pull a feed
  if ($q_c['time']+6*3600<time() or strlen($q_c['rss'])<1){
   $ch=curl_init($q_c['url']);
   curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
   curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
   curl_setopt($ch, CURLOPT_MAXREDIRS, 5);
   $q_c['rss']=curl_exec($ch);
   curl_close($ch);
   mysql_query('update `'.$sad_prefix.'booru_rss` set `time`=UNIX_TIMESTAMP(), `rss`="'.
               sad_safe_mysql($q_c['rss']).'" where `id`='.$q_c['id']);
   
  }
  
  //danbooru pool
  if (preg_match('|^https?://'.$sad_domain.'/booru/danpool\\.php\\?id=([0-9]+)|',$q_c['url'],$a)){
   // Делаем финт ушами и читаем не из rss, а из своей же базы данных
   $q3=mysql_query('select * from `'.$sad_prefix.'booru_pool_e` where `parent`='.$a[1]);
   
   if ($debug){$fo=fopen($sad_prefix.'/debug','a');
    fwrite($fo,date('Y-m-d H:i:s').': q_c id = '.$q_c['id'].'; danbooru pool = '.(int)$a[1].'; count = '.mysql_num_rows($q2)."\r\n");fclose($fo);}
   
   while ($q3_c=mysql_fetch_assoc($q3)){
    $feed[$q3_c['md5']]=array(
     'title'=>$q3_c['title'],
     'links'=>array('http://danbooru.donmai.us/posts/'.$q3_c['dan_id'].'/'),
     'href' =>'http://danbooru.donmai.us/posts/'.$q3_c['dan_id'].'/',
     'img'  =>'http://danbooru.donmai.us'.$q3_c['src'],
     'time' =>0
    );
   }
   
   continue;
  }
  
  //other places
  if ($q_c['time']+6*3600<time() or strlen($q_c['rss'])<1){
   $ch=curl_init($q_c['url']);
   curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
   curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
   curl_setopt($ch, CURLOPT_MAXREDIRS, 5);
   $q_c['rss']=curl_exec($ch);
   curl_close($ch);
   mysql_query('update `'.$sad_prefix.'booru_rss` set `time`=UNIX_TIMESTAMP(), `rss`="'.sad_safe_mysql($q_c['rss']).
               '" where `id`='.$q_c['id']);
   
  }
  
  preg_match('|^https?://([a-z0-9.-]+)|',$q_c['url'],$a);$domain=$a[1];
  $xml=sad_xml_rawtext($q_c['rss']);
  for ($i=1;true;$i++){
   $ent=sad_xml_getnode($xml,'entry|'.$i);
   if (gettype($ent)!=='object'){break;}
 
   $tmp  = new DOMDocument('1.0');
   $tmp->appendChild($tmp->importNode(sad_xml_getnode($ent,'content'),true));
   $buf = $tmp->saveHTML();
   
   preg_match('|([a-f0-9]{32,32})|',$buf,$a); $md5=$a[1];
   if (!isset($feed[$md5])){$feed[$md5]=array('time'=>0,'links'=>array());}
   preg_match('|src="(.+?)"|i',$buf,$a);$img=$a[1];
 
   if (preg_match('|^/|',$img)){$img='http://'.$domain.$img;}
   if (numnum($feed[$md5]['img'])<numnum($img)){$feed[$md5]['img']=$img;}
 
   preg_match('|([0-9]{4,4})\-([0-9]{1,2})\-([0-9]{1,2})T([0-9]{1,2})\:([0-9]{1,2})\:([0-9]{1,2})Z|',sad_xml_getnode($ent,'updated#'),$a);//updated>2011-09-13T04:12:12Z
   $time=gmmktime($a[4],$a[5],$a[6], $a[2],$a[3],$a[1]);
   if ($feed[$md5]['time']>$time or $feed[$md5]['time']<1){$feed[$md5]['time']=$time;}
 
   if (strlen($feed[$md5]['title'])<1){$feed[$md5]['title']=sad_xml_getnode($ent,'title#'); }
 
   preg_match('|href="(.+?)"|i',$buf,$a);$href=$a[1];
   if (numnum($feed[$md5]['href'])<numnum($href)){$feed[$md5]['href']=$href;}
   
   $xml=sad_xml_rawtext($q_c['rss']);
   for ($j=1;true;$j++){
    $link=sad_xml_getnode($ent,'link|'.$j);
    if (gettype($link)!=='object'){break;}
    array_push($feed[$md5]['links'],sad_xml_attribute($link,'href'));
   }
   
  }
  if ($debug){$fo=fopen($sad_prefix.'/debug','a');
   fwrite($fo,date('Y-m-d H:i:s').': q_c id = '.$q_c['id'].'; i = '.$i."\r\n");fclose($fo);}
 } // while $q_c
 
 if ($debug){$fo=fopen($sad_prefix.'/debug','a');
  fwrite($fo,date('Y-m-d H:i:s').': count of feed = '.count($feed)."\r\n");fclose($fo);}
// echo '<pre>';print_r($feed);
 
 $buf='';unset($md5);$time=0;
 foreach ($feed as $md5 => $e){
  if ($md5==''){continue;}
  $descr='<a href="'.$e['href'].'" target="_blank"><img src="'.$e['img'].'"></a><br>';
  foreach (array_unique($e['links']) as $link){
   if (strlen($link)>0){$descr.='<br><a href="'.$link.'" target="_blank">'.$link.'</a>';}
  }
  $buf.='<item>
      <title>'.sad_safe_xml($e['title']).'</title>
      <link>'.sad_safe_xml($e['href']).'</link>
      <description>'.sad_safe_xml($descr).'</description>
      <pubDate>'.gmdate("D, d M Y H:i:s \G\M\T",$e['time']).'</pubDate>
      <guid>booru-'.$md5.'</guid>
    </item>';
  
  $time=max($time,$e['time']);
 }
 
 if ($debug){$fo=fopen($sad_prefix.'/debug','a');
  fwrite($fo,date('Y-m-d H:i:s').': strlen($buf) = '.strlen($buf)."\r\n");fclose($fo);}
 $buf='<rss version="2.0"><channel><title>'.sad_safe_xml($q2_c['title'].' / Meta Booru').'</title><pubDate>'.
       gmdate("D, d M Y H:i:s \G\M\T",$time).'</pubDate><ttl>'.round($q2_c['ttl']/60).'</ttl>'.$buf.'</channel></rss>';
 
 $sad_contype='text/xml';
 echo '<'.'?xml version="1.0" encoding="utf-8"?'.'>';
 echo $buf;
 echo '<!-- generated now: '.date('Y-m-d H:i:sO').' -->';
 mysql_query('update `'.$sad_prefix.'booru_feed` set `rss`="'.sad_safe_mysql($buf).'", `time`=UNIX_TIMESTAMP() where `id`='.$id);
?>

Comments