<?php
/**
 * Generates a list of robot useragent deffinitions for use with
 * $_SERVER['HTTP_USER_AGENT'] to identify robots
 *
 * Created by: Muddy_Funster @ forums.phpfreaks.com - 09/2013
 * Updated and modified by: Jessica Brown for use on DIABLOS - 07/2017
 *
 * A Huge Thank You to Psycho, Kicken and Thorpe @ forums.phpfreaks.com
 * for their help and advice.
 *
 * This links into the robotstext.org site to access thier current
 * robot list.  It then produces an array of these useragents that
 * can be used to check if a visitor is a robot or not.
 * Call: $yourVar = new getRobots();
 * Setter : $yourVar->setExclude(mixed $mixed)
 * Getter : $robotArray = $yourVar->makeBots;
 * $yourVar->exclude(mixed $mixed); - send values to be excluded.
 *         Accepts either an array of values or a single string vlaue
 * JSON output (if you want to pass to javascript): echo $yourVar;
 *
 * --------------------------------------------------------------
 * @example 1 : PHP BOT Check
 *
 * $bots = new getRobots;
 * $bots->setExclude(array("", "none", "no", "yes"));
 * $bots->makeBotList();
 * $botArray = $bots->robots;
 *
 * if(!in_array($_SERVER('HTTP_USER_AGENT'), $botArray){
 *        import_request_variables("g", "user_"); //example of something to do
 *         ...
 *         ...
 * }
 * else{
 *        echo "Bot Safe Site Visited"; //example of something to do
 *         ...
 *         ...
 * }
 * -------------------------------------------------------------
 * @example 2 : output to JSON
 *
 * $bots = new getRobots;
 * $bots->setExclude("");
 * $bots->setExclude("none");
 * $bots->setExclude("???");
 * $bots->setExclude("no");
 * $bots->setExclude("yes");
 * $bots->makeBotList();
 *
 * header("Content-type: application/json");
 * echo $bots;
 * exit;
 * -----------------------------------------------------------
 *
 * @param array $robots the array list of useragents
 * @param array $excludes array of exlusions from the bot list
 * @param string $url static url value for linking to the
 * @param string $lfPath path to generate subfolder to store cache files in
 * @param string $masterFile path to master cache file of robotstxt.org data
 * @param string $botFile path to cached bot file for qicker repeat array building
 * @param string $mdCheckFile path to md5Checksum cache to establish if cached bot file can be used
 * @param array $hashVals generated md5 values from current call
 * @param array $hashFileVals values from md5 checksum cache file use for comparison
 * @param string $output contents retrieved from robotstxt.org site
 * @return array getBots() returns array of robot user aganents
 * @return string __toString() Returns JSON string of Object{"robots":array[{"numericalKey":"useragentText"}]
 */
class getRobots{
    public $robots;
    public $excludes;
    private $url;
    private $lfPath;
    private $masterFile;
    private $botFile;
    private $mdCheckFile;
    private $hashVals;
    private $hashFileVals;
    private $output;

    public function __construct(){
        $this->url = "http://www.robotstxt.org/db/all.txt";
        $this->lfPath= substr(__FILE__,0,strripos(__FILE__,'\\')+1).'robots';
        $this->masterFile= $this->lfPath.'\\rbtList.txt';
        $this->botFile = $this->lfPath."\\allBots.txt";
        $this->mdCheckFile = $this->lfPath."\\mdHashFile.txt";
        $this->excludes[] = "Due to a deficiency in Java it's not currently possible to set the User-Agent.";
        $this->excludes[] = "Due to a deficiency in Java it's not currently possible";
        if(!is_dir($this->lfPath)){
            if(!mkdir($this->lfPath)){
                throw new RuntimeException("error creating directory! PHP must have write permissions for this folder -- $lfPath");
            }
        }
    }
    public function setExclude($mixed){
    $mixed = (array)$mixed;
    $this->excludes = array_merge($this->excludes, $mixed);
    $this->excludes = array_unique($this->excludes);
    sort($this->excludes);
    }
    public function makeBots(){
        $this->checkFile();
        $this->checkBotList();
    }
    private function checkFile(){
        if (file_exists($this->masterFile)){
            $mtime = filemtime($this->masterFile);
            $ctx = stream_context_create(array(
                'http' => array(
                    'header' => "If-modified-since: ".gmdate(DATE_RFC1123, $mtime)
                )
            ));
        }
        else {
            $ctx = stream_context_create();
        }
        $fp = fopen($this->url, 'rb', false, $ctx);
        $this->output = stream_get_contents($fp);
        $meta = stream_get_meta_data($fp);
        if (strpos($meta['wrapper_data'][0], ' 200 ') !== false){
            file_put_contents($this->masterFile, $this->output);
        }
        fclose($fp);
    }
    private function checkBotList(){
        $robots = array();
        $this->hashVals[0] = md5(implode("|",$this->excludes));
        if(!file_exists($this->mdCheckFile)){
            $fileVals = explode("\n",$this->output);
        }
        else{
            $this->hashFileVals = file($this->mdCheckFile);
            if(trim($this->hashVals[0]) == trim($this->hashFileVals[0])){
                $this->robots = file($this->botFile);

            }
            else{
                $fileVals = file($this->masterFile);
            }

        }
        if(isset($fileVals)){
            foreach ($fileVals as $line=>$text){
                if (strpos($text, "robot-useragent:") !== FALSE){
                    $robots[] = trim(substr($text,16));
                }
            }
            $filterRobs = array_diff($robots, $this->excludes);
            $filterRobs = array_unique($filterRobs);
            $this->robots = $filterRobs;
            $botOut = implode("\n", $filterRobs);
            $botHandle = fopen($this->botFile, 'w');
            fwrite($botHandle, $botOut);
            fclose($botHandle);
            $this->hashVals[1] = md5(implode("|", $filterRobs));
            $difCheck = array_diff($this->hashVals, (array)$this->hashFileVals);
            if(count($difCheck) >= 1){
                $writeback = implode("\n", $this->hashVals);
                $mdHandle = fopen($this->mdCheckFile, 'w');
                fwrite($mdHandle, $writeback);
            }
        }
    }
    public function __toString(){
        return json_encode(array('robots' => $this->robots));
    }
}