path: root/refresh.php

<?php
chdir(dirname($_SERVER["SCRIPT_FILENAME"]));

if (isset($_SERVER["SERVER_NAME"])) {
	echo "<pre>";
	ini_set("max_execution_time", "0");
	ini_set("memory_limit", "-1");

	die("This script is launched twice a day, be patient");
}
require "config.inc.php";

function addUpstream ($db, $type, $channel, $name, $state, $ver, $statename="") {

	$sql1 = "INSERT INTO `upstream` SET type='$type', name='$name'";
	$db->exec($sql1);

	$sql2 = "UPDATE `upstream` SET channel='$channel', $state='$ver'";
	if ($statename) $sql2 .= ", state='$statename'";
	$sql2 .= " WHERE type='$type' AND name='$name'";
	$nb = $db->exec($sql2);
	if ($nb===false) echo date("r : ") . "SQL ERROR($sql2) = " . implode(" ", $db->errorInfo()) . "\n";

	return $nb;
}
if ($_SERVER['argc']>1 && in_array('help', $_SERVER['argv'])) {
	echo "Options in: repo owner R pear pecl old\n";
	echo "Defaults:   repo owner R pear pecl\n";
	die("\n");
}

try {
	$db = new PDO ("mysql:dbname=" . MYBASE . ";host=" . MYHOST, MYUSER, MYPASS);

	// -------------------------------------------------------------------
	//	Fedora Packages
	// -------------------------------------------------------------------
	echo date("r : ") . "Refreshing " . MYBASE . " database\n";

	if ($_SERVER['argc']==1 || in_array('repo', $_SERVER['argv'])) {
	if (in_array('empty', $_SERVER['argv'])) {
		$res=$db->query("SELECT * FROM repo WHERE stamp IS NULL");
	} else if (in_array('old', $_SERVER['argv'])) {
		$res=$db->query("SELECT * FROM repo");
	} else {
		$res=$db->query("SELECT * FROM repo WHERE active=1");
	}
	if (!$res) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
	else while ($row = $res->fetchObject()) {
		echo date("r : ") . "REPOSITORY " . $row->main . " " . $row->sub . "\n";
		$TimRemote = 0;
		$repomd = simplexml_load_file($row->url . "repodata/repomd.xml");
		if ($repomd) foreach($repomd->data as $data) if ($data->attributes()=="primary") {
			$TimRemote = $data->timestamp;
			$UrlRemote = $row->url . $data->location->attributes();
		}
		if ($TimRemote > $row->stamp) {
			echo date("r : ") . "Loading $UrlRemote\n";

			//$fic=gzopen("primary.xml.gz", "r");
			$fic=gzopen($UrlRemote, "r");
			if ($fic) {
				$txt="";
				while ($buf=gzread($fic,8196))
					$txt .= $buf;
				echo date("r : ") . "Read " . strlen($txt) . " bytes\n";
				gzclose($fic);

				$primary = simplexml_load_string($txt);
				echo date("r : ") . "Read " . $primary->attributes() . " packages\n";
				unset($txt);

				$sql = sprintf("DELETE FROM rpm WHERE repo_main='%s' AND repo_sub='%s'",
					$row->main, $row->sub);
				$nb=$db->exec($sql);
				if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
				else  echo date("r : ") . "Delete $nb packages\n";

				$tot=0;
				foreach ($primary->package as $package) if ($package->attributes()=='rpm'){
					$ver = $package->version->attributes();
					$sql = sprintf("INSERT INTO rpm SET repo_main='%s', repo_sub='%s', name='%s', epoch='%s', ver='%s', rel='%s', summary='%s', url='%s'",
						$row->main, $row->sub, $package->name, $ver['epoch'], $ver['ver'], $ver['rel'], addslashes($package->summary), $package->url);
					$nb=$db->exec($sql);
					if ($nb) $tot+=$nb;
					else  echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
				}
				echo date("r : ") . "Write $tot packages\n";

				$sql=sprintf("UPDATE repo SET stamp=%d WHERE ID=%d", $TimRemote, $row->ID);
				$nb=$db->exec($sql);
				if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";

				unset($primary);
			} else {
				echo date("r : ") . "ERROR : can't read $UrlRemote\n";
			}
		} else {
			echo date("r : ") . "no update needed : $TimRemote / " . $row->stamp . "\n";
		}
	}
	} // If ask

	// -------------------------------------------------------------------
	//	Upstream Version
	// -------------------------------------------------------------------
	$sql="CREATE TABLE IF NOT EXISTS `upstream` (
  		`name` varchar(100) NOT NULL,
  		`type` varchar(20) NOT NULL,
  		`channel` varchar(20) NOT NULL,
  		`stable` varchar(20) DEFAULT NULL,
  		`unstable` varchar(20) DEFAULT NULL,
		`state` varchar(20) DEFAULT NULL,
		PRIMARY KEY (`name`,`type`),
  		KEY `name` (`name`),
  		KEY `type` (`type`),
  		KEY `channel` (`channel`))";

	if ($db->exec($sql)!==false)
	  	echo date("r : ") . "Check table 'upstream' ok\n";
	else  	echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";

	// -------------------------------------------------------------------
	if ($_SERVER['argc']==1 || in_array('pecl', $_SERVER['argv'])) {
	echo date("r : ") . "PECL listLatestReleases - stable\n";

	$request = xmlrpc_encode_request("package.listLatestReleases", "stable");
	$context = stream_context_create(array('http' => array(
	    'method' => "POST",
	    'header' => "Content-Type: text/xml",
	    'content' => $request
	)));
	$file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)");
	$stable = xmlrpc_decode($file);
	if (xmlrpc_is_fault($stable)) {
	 	echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])";
	} else {
		$sql = sprintf("DELETE FROM upstream WHERE type='pecl' AND channel='pecl'");
		$nb=$db->exec($sql);
		if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
		else  echo date("r : ") . "Delete $nb packages\n";

		$nb=0;
		foreach ($stable as $name => $info) {
			$rpmname="php-pecl-".str_replace("_","-",$name);

			$nb += addUpstream($db, "pecl", "pecl", $rpmname, "stable", $info["version"]);
		}
		echo date("r : ") . "Write $nb packages\n";
	 	// echo date("r : ") . "saved " . file_put_contents("cache/pecl-stable", serialize($stable)) . " bytes\n";
	}

	// -------------------------------------------------------------------
	echo date("r : ") . "PECL listLatestReleases - unstable\n";

	$request = xmlrpc_encode_request("package.listLatestReleases", array());
	$context = stream_context_create(array('http' => array(
	    'method' => "POST",
	    'header' => "Content-Type: text/xml",
	    'content' => $request
	)));
	$file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)");
	$unstable = xmlrpc_decode($file);
	if (xmlrpc_is_fault($unstable)) {
	    echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])";
	} else {
		$nb=0;
		foreach ($unstable as $name => $info) {
			$rpmname="php-pecl-".str_replace("_","-",$name);

			$nb += addUpstream($db, "pecl", "pecl", $rpmname, "unstable", $info["version"], $info["state"]);
		}
		echo date("r : ") . "Write $nb packages\n";
	 	//echo date("r : ") . "saved " . file_put_contents("cache/pecl-unstable", serialize($unstable)) . " bytes\n";
	}
	} // if in options

	// -------------------------------------------------------------------
	if ($_SERVER['argc']==1 || in_array('pear', $_SERVER['argv'])) {
 	echo date("r : ") . "PEAR reading channels\n";
	$channels=array(
		"pear"	    => "pear.php.net"
		,"doctrine" => "pear.phpdoctrine.org"
		,"ezc"      => "components.ez.no"
		,"pdepend"  => "pear.pdepend.org"
		,"phing"    => "pear.phing.info"
		,"phpdb"    => "pear.phpdb.org"
		,"phpmd"    => "pear.phpmd.org"
		,"phpunit"  => "pear.phpunit.de"
		,"swift"    => "pear.swiftmailer.org"
		,"symphony" => "pear.symfony-project.com"
		);
	try {
		$nbtot=0;

		foreach ($channels as $channelname => $channelurl) {

			$channel = @simplexml_load_file("http://$channelurl/channel.xml");
			if (!$channel) throw new Exception("can't read PEAR site (channel of $channelname)");

			$rest = $channel->servers->primary->rest->baseurl[0];
			echo date("r : ") . "PEAR reading channel=$channelname, baseurl = $rest\n";

			$categories = @simplexml_load_file($rest."c/categories.xml");
			if (!$categories) throw new Exception("can't read PEAR site (categories)");

			$sql = sprintf("DELETE FROM upstream WHERE type='pear' AND channel='$channelname'");
			$nb=$db->exec($sql);
			if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
			else  echo date("r : ") . "Delete $nb packages\n";

			$nb=0;
			if (!isset($categories->c[0])) {
				echo date("r : ") . "Reading ALL\n"; // ezc only
				$pitxt = @file_get_contents($rest."p/packages.xml");
				if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."p/packagesinfo.xml)");
				$allpi = @simplexml_load_string($pitxt);
				foreach ($allpi->p as $name) {
					$pitxt = @file_get_contents($rest."r/".strtolower($name)."/allreleases.xml");
					if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."r/".strtolower($name)."/allreleases.xml");
					$pi = @simplexml_load_string($pitxt);

					$rpmname1="php-".$channelname."-".str_replace("_","-",$name);
					$rpmname2="php-".$channelname."-".$name;

					//echo $rpmname ."/".(string)$pi->r[0]->v."/". (string)$pi->r[0]->s."\n";
					addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s);
					addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s);
					foreach($pi->r as $rev) if ($rev->s=='stable') {
						//echo $rpmname ."/".(string)$rev->v."/". (string)$rev->s."\n";
						addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v);
						addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v);
						break;
					}
					$nb++;
				}

			} else foreach ($categories->c as $cat) {
				echo date("r : ") . "Reading $cat\n";

				$pitxt = @file_get_contents($rest."c/".urlencode($cat)."/packagesinfo.xml");
				if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."c/".urlencode($cat)."/packagesinfo.xml)");
				$pitxt = "<?" . preg_replace("/<\?xml.*?>/U", "", str_replace("\r\n", "\n", substr($pitxt, 2)));

				$pi = @simplexml_load_string($pitxt);
				if (!$pi) throw new Exception("can't read response ($cat)");
				foreach ($pi->pi as $ps) if (isset($ps->p->n) && isset($ps->a->r)) {
					$name=(string)$ps->p->n;

					if ($channelname=='phing' && $name=='phing') {
						$rpmname1="php-pear-phing";
					} else if ($channelname=='phpunit' && $name=='PHPUnit') {
						$rpmname1="php-pear-PHPUnit";
					} else {
						$rpmname1="php-".$channelname."-".str_replace("_","-",$name);
					}
					$rpmname2="php-".$channelname."-".$name;

					addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s);
					addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s);
					foreach($ps->a->r as $rev) if ($rev->s=='stable') {
						addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v);
						addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v);
						break;
					}
					$nb++;
					// echo $ps->p->n."\n";
				}
			}
		 	echo date("r : ") . "read $nb packages in $channelname\n";
			$nbtot += $nb;
		}
	 	echo date("r : ") . "read $nbtot packages in all channels\n";

	} catch (Exception $e) {
		echo date("r : ") . 'ERROR ' .  $e->getMessage(), "\n";
	}
	} // if in options
	// -------------------------------------------------------------------
	if ($_SERVER['argc']==1 || in_array('R', $_SERVER['argv'])) {

      $repos = array(
         array("name"=>"R",      "state"=>"stable",   "url" => "http://cran.at.r-project.org/src/contrib/PACKAGES"),
       //array("name"=>"R-forge","state"=>"unstable", "url" => "http://r-forge.r-project.org/src/contrib/PACKAGES"),
         array("name"=>"biocp",  "state"=>"stable",   "url" => "http://www.bioconductor.org/packages/release/bioc/src/contrib/PACKAGES"),
         array("name"=>"biocp",  "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/bioc/src/contrib/PACKAGES"),
         array("name"=>"bioca",  "state"=>"stable",   "url" => "http://www.bioconductor.org/packages/release/data/annotation/src/contrib/PACKAGES"),
         array("name"=>"bioca",  "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/annotation/src/contrib/PACKAGES"),
         array("name"=>"bioce",  "state"=>"stable",   "url" => "http://www.bioconductor.org/packages/release/data/experiment/src/contrib/PACKAGES"),
         array("name"=>"bioce",  "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/experiment/src/contrib/PACKAGES")
         );
      $tottot=0;
      foreach ($repos as $repo) {
         echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n";
         $index = @file_get_contents($repo["url"]);
         if (!$index) {
            echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n";
            continue;
         }
         if ($repo["state"]=="stable") {
            $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]);
            $nb=$db->exec($sql);
            if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
            else  echo date("r : ") . "Delete $nb packages\n";
         }
         $results=array();
         if (preg_match_all('/Package: *(.*)\nVersion: *(.*)\n/i',$index,$results,PREG_SET_ORDER)) {
            $tot=0;
            foreach ($results as $result) {
               //echo $result[1]." = ".$result[2]."\n";
               $rpmname = "R-".$result[1];
               $ver = str_replace('-', '.', $result[2]);
               if (addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"))) {
                  $tot++;
               }
            }
            echo date("r : ") . "Write $tot packages in this channel\n";
            $tottot += $tot;
         } else {
            echo date("r : ") . "No package in this channel\n";
         }
      }
      echo date("r : ") . "Write $tottot packages in all channels\n";
/*
 * OLD VERSION parsing the HTML pages
	$repos = array(
		array("name"=>"R", 	"state"=>"stable", 	"url"	=> "http://cran.r-project.org/web/packages/"),
		array("name"=>"biocp", 	"state"=>"stable", 	"url"	=> "http://bioconductor.org/packages/release/bioc/"),
		array("name"=>"biocp", 	"state"=>"unstable", 	"url"	=> "http://bioconductor.org/packages/devel/bioc/"),
		array("name"=>"bioca", 	"state"=>"stable", 	"url"	=> "http://bioconductor.org/packages/release/data/annotation/"),
		array("name"=>"bioca", 	"state"=>"unstable", 	"url"	=> "http://bioconductor.org/packages/devel/data/annotation/"),
		array("name"=>"bioce", 	"state"=>"stable", 	"url"	=> "http://bioconductor.org/packages/release/data/experiment/"),
		array("name"=>"bioce", 	"state"=>"unstable", 	"url"	=> "http://bioconductor.org/packages/devel/data/experiment/")
		);
	$tottot=0;
	foreach ($repos as $repo) {
		echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n";
		$index = @file_get_contents($repo["url"]);
		if (!$index) {
			echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n";
			continue;
		}
		if ($repo["state"]=="stable") {
			$sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]);
			$nb=$db->exec($sql);
			if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
			else  echo date("r : ") . "Delete $nb packages\n";
		}
		if ($repo["name"] == "R") {
			$pos = strpos($index, 'ID="available-packages');
			$pat1 = '/<A HREF="(\.\.\/\.\.\/web\/.*)">(.*)<\/A>/iU';
		} else {
			$pos = 0;
			$pat1 = '/<a href="(.*)">(.*)<\/A>/iu';
		}
		$results=array();
		if (preg_match_all($pat1,$index,$results,PREG_SET_ORDER,$pos)) {
			$tot=0;
			foreach ($results as $result) {
				$page = @file_get_contents($repo["url"].$result[1]);
				$name = $result[2];
				$rpmname = "R-".$result[2];
				$ver=array();
				if (!$page) {
					echo date("r : ") . "Can't read [" . $repo["url"].$result[1] . "], skip this package\n";

				} else if (preg_match("/".$name."_((\d|-|\.)*)\.tar.gz/", $page, $ver)) {

					$ver = str_replace('-', '.', $ver[1]);
					$nb = addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"));

				}

				if ($nb) $tot+=$nb;

			}
			echo date("r : ") . "Write $tot packages in this channel\n";
			$tottot += $tot;
		} // Each package
	} // Each repo
	echo date("r : ") . "Write $tottot packages in all channels\n";
   */
	} // If R in options

	// -------------------------------------------------------------------
	//	Package Owners from pkgdb (thanks Smootherfrog)
	// -------------------------------------------------------------------
	if ($_SERVER['argc']==1 || in_array('owner', $_SERVER['argv'])) {

	$sql="CREATE TABLE IF NOT EXISTS `acls` (
  		`ID` int(11) NOT NULL AUTO_INCREMENT,
  		`collection` varchar(100) NOT NULL,
  		`name` varchar(100) NOT NULL,
  		`summary` varchar(200) NOT NULL,
  		`owner` varchar(50) DEFAULT NULL,
  		`qa` varchar(50) DEFAULT NULL,
  		`cc` varchar(100) DEFAULT NULL,
  		PRIMARY KEY (`ID`),
  		KEY `name` (`name`),
  		KEY `collection` (`collection`),
  		KEY `owner` (`owner`)) ENGINE=MyISAM";

	if ($db->exec($sql)!==false)
	  	echo date("r : ") . "Check table 'acls' ok\n";
	else  	echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";

	$fic=fopen("https://admin.fedoraproject.org/pkgdb/lists/bugzilla?tg_format=plain", "r");
	if (!$fic) {
		echo date("r : ") . "ERROR reading pkgdb\n";
	} else {
		$sql="TRUNCATE `acls`";
		$nb=$db->exec($sql);
		if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
		else  echo date("r : ") . "Delete $nb packages\n";

		for ($tot=0 ; $line=fgetcsv($fic,1024,'|') ; ) if (count($line)>5 && substr($line[0],0,1)!='#') {
			for ($i=0; $i<6; $i++) $line[$i]=trim($line[$i]);
			$sql=sprintf("INSERT INTO `acls` SET collection='%s', name='%s', summary='%s'", $line[0], $line[1], addslashes($line[2]));
			if (!empty($line[3])) $sql .= sprintf(", owner='%s'", $line[3]);
			if (!empty($line[4])) $sql .= sprintf(", qa='%s'", $line[4]);
			if (!empty($line[5])) $sql .= sprintf(", cc='%s'", $line[5]);
			$nb=$db->exec($sql);
			if ($nb) $tot+=$nb;
			else  echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n";
		}
		fclose($fic);
	 	echo date("r : ") . "wrote $tot package's owner\n";
	}
	} // If in options

}
catch(PDOException $e) {
	printf("%s ERREUR : %s\n", date("r"),  $e->getMessage());
}

if (isset($_SERVER["SERVER_NAME"])) echo "</pre>";
?>