diff options
| -rw-r--r-- | refresh.php | 1094 |
1 files changed, 676 insertions, 418 deletions
diff --git a/refresh.php b/refresh.php index eb612de..3be354f 100644 --- a/refresh.php +++ b/refresh.php @@ -1,437 +1,695 @@ <?php + +/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */ + +/** + * Refresh datas from server + * + * PHP version 5 + * + * Copyright © 2010 Remi Collet + * + * This file is part of rpmphp. + * + * rpmphp is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * rpmphp is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with rpmphp. If not, see <http://www.gnu.org/licenses/>. + * + * @category Main + * @package RPMPHP + * + * @author Remi Collet <unknown@unknwown.com> + * @author Johan Cwiklinski <johan@x-tnd.be> + * @copyright 2010 Remi Collet + * @license http://www.gnu.org/licenses/agpl-3.0-standalone.html AGPL License 3.0 or (at your option) any later version + * @link http://github.com/remicollet/rpmphp/ + * @since The begining of times. +*/ chdir(dirname($_SERVER["SCRIPT_FILENAME"])); if (isset($_SERVER["SERVER_NAME"])) { - echo "<pre>"; - ini_set("max_execution_time", "0"); - ini_set("memory_limit", "-1"); + echo "<pre>"; + ini_set("max_execution_time", "0"); + ini_set("memory_limit", "-1"); - die("This script is launched twice a day, be patient"); + die("This script is launched twice a day, be patient"); } require "config.inc.php"; -function addUpstream ($db, $type, $channel, $name, $state, $ver, $statename="") { - - $sql1 = "INSERT INTO `upstream` SET type='$type', name='$name'"; - $db->exec($sql1); - - $sql2 = "UPDATE `upstream` SET channel='$channel', $state='$ver'"; - if ($statename) $sql2 .= ", state='$statename'"; - $sql2 .= " WHERE type='$type' AND name='$name'"; - $nb = $db->exec($sql2); - if ($nb===false) echo date("r : ") . "SQL ERROR($sql2) = " . implode(" ", $db->errorInfo()) . "\n"; - - return $nb; +/** +* To document +* +* @param Object $db Database reference +* @param string $type type +* @param string $channel channel +* @param string $name name +* @param string $state state +* @param string $ver version +* @param string $statename state name +* +* @return rows updated +*/ +function addUpstream ($db, $type, $channel, $name, $state, $ver, $statename="") +{ + + $sql1 = "INSERT INTO `upstream` SET type='$type', name='$name'"; + $db->exec($sql1); + + $sql2 = "UPDATE `upstream` SET channel='$channel', $state='$ver'"; + if ($statename) { + $sql2 .= ", state='$statename'"; + } + $sql2 .= " WHERE type='$type' AND name='$name'"; + $nb = $db->exec($sql2); + if ($nb===false) { + echo date("r : ") . "SQL ERROR($sql2) = " . + implode(" ", $db->errorInfo()) . "\n"; + } + + return $nb; } + if ($_SERVER['argc']>1 && in_array('help', $_SERVER['argv'])) { - echo "Options in: repo owner R pear pecl old\n"; - echo "Defaults: repo owner R pear pecl\n"; - die("\n"); + echo "Options in: repo owner R pear pecl old\n"; + echo "Defaults: repo owner R pear pecl\n"; + die("\n"); } try { - $db = new PDO ("mysql:dbname=" . MYBASE . ";host=" . MYHOST, MYUSER, MYPASS); - - // ------------------------------------------------------------------- - // Fedora Packages - // ------------------------------------------------------------------- - echo date("r : ") . "Refreshing " . MYBASE . " database\n"; - - if ($_SERVER['argc']==1 || in_array('repo', $_SERVER['argv'])) { - if (in_array('empty', $_SERVER['argv'])) { - $res=$db->query("SELECT * FROM repo WHERE stamp IS NULL"); - } else if (in_array('old', $_SERVER['argv'])) { - $res=$db->query("SELECT * FROM repo"); - } else { - $res=$db->query("SELECT * FROM repo WHERE active=1"); - } - if (!$res) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else while ($row = $res->fetchObject()) { - echo date("r : ") . "REPOSITORY " . $row->main . " " . $row->sub . "\n"; - $TimRemote = 0; - $repomd = simplexml_load_file($row->url . "repodata/repomd.xml"); - if ($repomd) foreach($repomd->data as $data) if ($data->attributes()=="primary") { - $TimRemote = $data->timestamp; - $UrlRemote = $row->url . $data->location->attributes(); - } - if ($TimRemote > $row->stamp) { - echo date("r : ") . "Loading $UrlRemote\n"; - - //$fic=gzopen("primary.xml.gz", "r"); - $fic=gzopen($UrlRemote, "r"); - if ($fic) { - $txt=""; - while ($buf=gzread($fic,8196)) - $txt .= $buf; - echo date("r : ") . "Read " . strlen($txt) . " bytes\n"; - gzclose($fic); - - $primary = simplexml_load_string($txt); - echo date("r : ") . "Read " . $primary->attributes() . " packages\n"; - unset($txt); - - $sql = sprintf("DELETE FROM rpm WHERE repo_main='%s' AND repo_sub='%s'", - $row->main, $row->sub); - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - - $tot=0; - foreach ($primary->package as $package) if ($package->attributes()=='rpm'){ - $ver = $package->version->attributes(); - $sql = sprintf("INSERT INTO rpm SET repo_main='%s', repo_sub='%s', name='%s', epoch='%s', ver='%s', rel='%s', summary='%s', url='%s'", - $row->main, $row->sub, $package->name, $ver['epoch'], $ver['ver'], $ver['rel'], addslashes($package->summary), $package->url); - $nb=$db->exec($sql); - if ($nb) $tot+=$nb; - else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - } - echo date("r : ") . "Write $tot packages\n"; - - $sql=sprintf("UPDATE repo SET stamp=%d WHERE ID=%d", $TimRemote, $row->ID); - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - - unset($primary); - } else { - echo date("r : ") . "ERROR : can't read $UrlRemote\n"; - } - } else { - echo date("r : ") . "no update needed : $TimRemote / " . $row->stamp . "\n"; - } - } - } // If ask - - // ------------------------------------------------------------------- - // Upstream Version - // ------------------------------------------------------------------- - $sql="CREATE TABLE IF NOT EXISTS `upstream` ( - `name` varchar(100) NOT NULL, - `type` varchar(20) NOT NULL, - `channel` varchar(20) NOT NULL, - `stable` varchar(20) DEFAULT NULL, - `unstable` varchar(20) DEFAULT NULL, - `state` varchar(20) DEFAULT NULL, - PRIMARY KEY (`name`,`type`), - KEY `name` (`name`), - KEY `type` (`type`), - KEY `channel` (`channel`))"; - - if ($db->exec($sql)!==false) - echo date("r : ") . "Check table 'upstream' ok\n"; - else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - - // ------------------------------------------------------------------- - if ($_SERVER['argc']==1 || in_array('pecl', $_SERVER['argv'])) { - echo date("r : ") . "PECL listLatestReleases - stable\n"; - - $request = xmlrpc_encode_request("package.listLatestReleases", "stable"); - $context = stream_context_create(array('http' => array( - 'method' => "POST", - 'header' => "Content-Type: text/xml", - 'content' => $request - ))); - $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); - $stable = xmlrpc_decode($file); - if (xmlrpc_is_fault($stable)) { - echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])"; - } else { - $sql = sprintf("DELETE FROM upstream WHERE type='pecl' AND channel='pecl'"); - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - - $nb=0; - foreach ($stable as $name => $info) { - $rpmname="php-pecl-".str_replace("_","-",$name); - - $nb += addUpstream($db, "pecl", "pecl", $rpmname, "stable", $info["version"]); - } - echo date("r : ") . "Write $nb packages\n"; - // echo date("r : ") . "saved " . file_put_contents("cache/pecl-stable", serialize($stable)) . " bytes\n"; - } - - // ------------------------------------------------------------------- - echo date("r : ") . "PECL listLatestReleases - unstable\n"; - - $request = xmlrpc_encode_request("package.listLatestReleases", array()); - $context = stream_context_create(array('http' => array( - 'method' => "POST", - 'header' => "Content-Type: text/xml", - 'content' => $request - ))); - $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); - $unstable = xmlrpc_decode($file); - if (xmlrpc_is_fault($unstable)) { - echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])"; - } else { - $nb=0; - foreach ($unstable as $name => $info) { - $rpmname="php-pecl-".str_replace("_","-",$name); - - $nb += addUpstream($db, "pecl", "pecl", $rpmname, "unstable", $info["version"], $info["state"]); - } - echo date("r : ") . "Write $nb packages\n"; - //echo date("r : ") . "saved " . file_put_contents("cache/pecl-unstable", serialize($unstable)) . " bytes\n"; - } - } // if in options - - // ------------------------------------------------------------------- - if ($_SERVER['argc']==1 || in_array('pear', $_SERVER['argv'])) { - echo date("r : ") . "PEAR reading channels\n"; - $channels=array( - "pear" => "pear.php.net" - ,"doctrine" => "pear.phpdoctrine.org" - ,"ezc" => "components.ez.no" - ,"pdepend" => "pear.pdepend.org" - ,"phing" => "pear.phing.info" - ,"phpdb" => "pear.phpdb.org" - ,"phpmd" => "pear.phpmd.org" - ,"phpunit" => "pear.phpunit.de" - ,"swift" => "pear.swiftmailer.org" - ,"symphony" => "pear.symfony-project.com" - ); - try { - $nbtot=0; - - foreach ($channels as $channelname => $channelurl) { - - $channel = @simplexml_load_file("http://$channelurl/channel.xml"); - if (!$channel) throw new Exception("can't read PEAR site (channel of $channelname)"); - - $rest = $channel->servers->primary->rest->baseurl[0]; - echo date("r : ") . "PEAR reading channel=$channelname, baseurl = $rest\n"; - - $categories = @simplexml_load_file($rest."c/categories.xml"); - if (!$categories) throw new Exception("can't read PEAR site (categories)"); - - $sql = sprintf("DELETE FROM upstream WHERE type='pear' AND channel='$channelname'"); - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - - $nb=0; - if (!isset($categories->c[0])) { - echo date("r : ") . "Reading ALL\n"; // ezc only - $pitxt = @file_get_contents($rest."p/packages.xml"); - if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."p/packagesinfo.xml)"); - $allpi = @simplexml_load_string($pitxt); - foreach ($allpi->p as $name) { - $pitxt = @file_get_contents($rest."r/".strtolower($name)."/allreleases.xml"); - if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."r/".strtolower($name)."/allreleases.xml"); - $pi = @simplexml_load_string($pitxt); - - $rpmname1="php-".$channelname."-".str_replace("_","-",$name); - $rpmname2="php-".$channelname."-".$name; - - //echo $rpmname ."/".(string)$pi->r[0]->v."/". (string)$pi->r[0]->s."\n"; - addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s); - addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$pi->r[0]->v, (string)$pi->r[0]->s); - foreach($pi->r as $rev) if ($rev->s=='stable') { - //echo $rpmname ."/".(string)$rev->v."/". (string)$rev->s."\n"; - addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v); - addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v); - break; - } - $nb++; - } - - } else foreach ($categories->c as $cat) { - echo date("r : ") . "Reading $cat\n"; - - $pitxt = @file_get_contents($rest."c/".urlencode($cat)."/packagesinfo.xml"); - if (!$pitxt) throw new Exception("can't read PEAR site (".$rest."c/".urlencode($cat)."/packagesinfo.xml)"); - $pitxt = "<?" . preg_replace("/<\?xml.*?>/U", "", str_replace("\r\n", "\n", substr($pitxt, 2))); - - $pi = @simplexml_load_string($pitxt); - if (!$pi) throw new Exception("can't read response ($cat)"); - foreach ($pi->pi as $ps) if (isset($ps->p->n) && isset($ps->a->r)) { - $name=(string)$ps->p->n; - - if ($channelname=='phing' && $name=='phing') { - $rpmname1="php-pear-phing"; - } else if ($channelname=='phpunit' && $name=='PHPUnit') { - $rpmname1="php-pear-PHPUnit"; - } else { - $rpmname1="php-".$channelname."-".str_replace("_","-",$name); - } - $rpmname2="php-".$channelname."-".$name; - - addUpstream($db, "pear", $channelname, $rpmname1, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s); - addUpstream($db, "pear", $channelname, $rpmname2, "unstable", (string)$ps->a->r[0]->v, (string)$ps->a->r[0]->s); - foreach($ps->a->r as $rev) if ($rev->s=='stable') { - addUpstream($db, "pear", $channelname, $rpmname1, "stable", (string)$rev->v); - addUpstream($db, "pear", $channelname, $rpmname2, "stable", (string)$rev->v); - break; - } - $nb++; - // echo $ps->p->n."\n"; - } - } - echo date("r : ") . "read $nb packages in $channelname\n"; - $nbtot += $nb; - } - echo date("r : ") . "read $nbtot packages in all channels\n"; - - } catch (Exception $e) { - echo date("r : ") . 'ERROR ' . $e->getMessage(), "\n"; - } - } // if in options - // ------------------------------------------------------------------- - if ($_SERVER['argc']==1 || in_array('R', $_SERVER['argv'])) { - - $repos = array( - array("name"=>"R", "state"=>"stable", "url" => "http://cran.at.r-project.org/src/contrib/PACKAGES"), - //array("name"=>"R-forge","state"=>"unstable", "url" => "http://r-forge.r-project.org/src/contrib/PACKAGES"), - array("name"=>"biocp", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/bioc/src/contrib/PACKAGES"), - array("name"=>"biocp", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/bioc/src/contrib/PACKAGES"), - array("name"=>"bioca", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/annotation/src/contrib/PACKAGES"), - array("name"=>"bioca", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/annotation/src/contrib/PACKAGES"), - array("name"=>"bioce", "state"=>"stable", "url" => "http://www.bioconductor.org/packages/release/data/experiment/src/contrib/PACKAGES"), - array("name"=>"bioce", "state"=>"unstable", "url" => "http://www.bioconductor.org/packages/devel/data/experiment/src/contrib/PACKAGES") - ); - $tottot=0; - foreach ($repos as $repo) { - echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n"; - $index = @file_get_contents($repo["url"]); - if (!$index) { - echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n"; - continue; - } - if ($repo["state"]=="stable") { - $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]); + $db = new PDO ("mysql:dbname=" . MYBASE . ";host=" . MYHOST, MYUSER, MYPASS); + + // ------------------------------------------------------------------- + // Fedora Packages + // ------------------------------------------------------------------- + echo date("r : ") . "Refreshing " . MYBASE . " database\n"; + + if ($_SERVER['argc']==1 || in_array('repo', $_SERVER['argv'])) { + if (in_array('empty', $_SERVER['argv'])) { + $res=$db->query("SELECT * FROM repo WHERE stamp IS NULL"); + } else if (in_array('old', $_SERVER['argv'])) { + $res=$db->query("SELECT * FROM repo"); + } else { + $res=$db->query("SELECT * FROM repo WHERE active=1"); + } + if (!$res) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } else { + while ($row = $res->fetchObject()) { + echo date("r : ") . "REPOSITORY " . $row->main . " " . + $row->sub . "\n"; + $TimRemote = 0; + $repomd = simplexml_load_file($row->url . "repodata/repomd.xml"); + if ($repomd) { + foreach ($repomd->data as $data) { + if ($data->attributes()=="primary") { + $TimRemote = $data->timestamp; + $UrlRemote = $row->url . $data->location->attributes(); + } + } + } + if ($TimRemote > $row->stamp) { + echo date("r : ") . "Loading $UrlRemote\n"; + + //$fic=gzopen("primary.xml.gz", "r"); + $fic=gzopen($UrlRemote, "r"); + if ($fic) { + $txt=""; + while ($buf=gzread($fic, 8196)) { + $txt .= $buf; + } + echo date("r : ") . "Read " . strlen($txt) . " bytes\n"; + gzclose($fic); + + $primary = simplexml_load_string($txt); + echo date("r : ") . "Read " . $primary->attributes() . + " packages\n"; + unset($txt); + + $sql = sprintf( + "DELETE FROM rpm WHERE repo_main='%s' AND repo_sub='%s'", + $row->main, + $row->sub + ); + $nb=$db->exec($sql); + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } else { + echo date("r : ") . "Delete $nb packages\n"; + } + + $tot=0; + foreach ($primary->package as $package) { + if ($package->attributes()=='rpm') { + $ver = $package->version->attributes(); + $sql = sprintf( + "INSERT INTO rpm SET repo_main='%s', repo_sub='%s', name='%s', epoch='%s', ver='%s', rel='%s', summary='%s', url='%s'", + $row->main, + $row->sub, + $package->name, + $ver['epoch'], + $ver['ver'], + $ver['rel'], + addslashes($package->summary), + $package->url + ); + $nb=$db->exec($sql); + if ($nb) { + $tot+=$nb; + } else { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } + } + } + echo date("r : ") . "Write $tot packages\n"; + + $sql=sprintf( + "UPDATE repo SET stamp=%d WHERE ID=%d", + $TimRemote, + $row->ID + ); + $nb=$db->exec($sql); + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } + + unset($primary); + } else { + echo date("r : ") . "ERROR : can't read $UrlRemote\n"; + } + } else { + echo date("r : ") . "no update needed : $TimRemote / " . + $row->stamp . "\n"; + } + } + } + } // If ask + + // ------------------------------------------------------------------- + // Upstream Version + // ------------------------------------------------------------------- + $sql="CREATE TABLE IF NOT EXISTS `upstream` ( + `name` varchar(100) NOT NULL, + `type` varchar(20) NOT NULL, + `channel` varchar(20) NOT NULL, + `stable` varchar(20) DEFAULT NULL, + `unstable` varchar(20) DEFAULT NULL, + `state` varchar(20) DEFAULT NULL, + PRIMARY KEY (`name`,`type`), + KEY `name` (`name`), + KEY `type` (`type`), + KEY `channel` (`channel`))"; + + if ($db->exec($sql)!==false) { + echo date("r : ") . "Check table 'upstream' ok\n"; + } else { + echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; + } + + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('pecl', $_SERVER['argv'])) { + echo date("r : ") . "PECL listLatestReleases - stable\n"; + + $request = xmlrpc_encode_request("package.listLatestReleases", "stable"); + $context = stream_context_create( + array( + 'http' => array( + 'method' => "POST", + 'header' => "Content-Type: text/xml", + 'content' => $request + ) + ) + ); + $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); + $stable = xmlrpc_decode($file); + if (xmlrpc_is_fault($stable)) { + echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ($stable[faultCode])"; + } else { + $sql = sprintf("DELETE FROM upstream WHERE type='pecl' AND channel='pecl'"); $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - } - $results=array(); - if (preg_match_all('/Package: *(.*)\nVersion: *(.*)\n/i',$index,$results,PREG_SET_ORDER)) { - $tot=0; - foreach ($results as $result) { - //echo $result[1]." = ".$result[2]."\n"; - $rpmname = "R-".$result[1]; - $ver = str_replace('-', '.', $result[2]); - if (addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel"))) { - $tot++; - } + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } else { + echo date("r : ") . "Delete $nb packages\n"; } - echo date("r : ") . "Write $tot packages in this channel\n"; - $tottot += $tot; - } else { - echo date("r : ") . "No package in this channel\n"; - } - } - echo date("r : ") . "Write $tottot packages in all channels\n"; -/* - * OLD VERSION parsing the HTML pages - $repos = array( - array("name"=>"R", "state"=>"stable", "url" => "http://cran.r-project.org/web/packages/"), - array("name"=>"biocp", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/bioc/"), - array("name"=>"biocp", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/bioc/"), - array("name"=>"bioca", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/annotation/"), - array("name"=>"bioca", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/annotation/"), - array("name"=>"bioce", "state"=>"stable", "url" => "http://bioconductor.org/packages/release/data/experiment/"), - array("name"=>"bioce", "state"=>"unstable", "url" => "http://bioconductor.org/packages/devel/data/experiment/") - ); - $tottot=0; - foreach ($repos as $repo) { - echo date("r : ") . "Reading " . $repo["name"] . " (" . $repo["state"] . ")\n"; - $index = @file_get_contents($repo["url"]); - if (!$index) { - echo date("r : ") . "Can't read [" . $repo["url"] . "], skip this channel\n"; - continue; - } - if ($repo["state"]=="stable") { - $sql = sprintf("DELETE FROM upstream WHERE type='R' AND channel='%s'", $repo["name"]); - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - } - if ($repo["name"] == "R") { - $pos = strpos($index, 'ID="available-packages'); - $pat1 = '/<A HREF="(\.\.\/\.\.\/web\/.*)">(.*)<\/A>/iU'; - } else { - $pos = 0; - $pat1 = '/<a href="(.*)">(.*)<\/A>/iu'; - } - $results=array(); - if (preg_match_all($pat1,$index,$results,PREG_SET_ORDER,$pos)) { - $tot=0; - foreach ($results as $result) { - $page = @file_get_contents($repo["url"].$result[1]); - $name = $result[2]; - $rpmname = "R-".$result[2]; - $ver=array(); - if (!$page) { - echo date("r : ") . "Can't read [" . $repo["url"].$result[1] . "], skip this package\n"; - - } else if (preg_match("/".$name."_((\d|-|\.)*)\.tar.gz/", $page, $ver)) { - - $ver = str_replace('-', '.', $ver[1]); - $nb = addUpstream($db, "R", $repo["name"], $rpmname, $repo["state"], $ver, ($repo["state"]=="stable"?"":"devel")); - - } - - if ($nb) $tot+=$nb; - - } - echo date("r : ") . "Write $tot packages in this channel\n"; - $tottot += $tot; - } // Each package - } // Each repo - echo date("r : ") . "Write $tottot packages in all channels\n"; - */ - } // If R in options - - // ------------------------------------------------------------------- - // Package Owners from pkgdb (thanks Smootherfrog) - // ------------------------------------------------------------------- - if ($_SERVER['argc']==1 || in_array('owner', $_SERVER['argv'])) { - - $sql="CREATE TABLE IF NOT EXISTS `acls` ( - `ID` int(11) NOT NULL AUTO_INCREMENT, - `collection` varchar(100) NOT NULL, - `name` varchar(100) NOT NULL, - `summary` varchar(200) NOT NULL, - `owner` varchar(50) DEFAULT NULL, - `qa` varchar(50) DEFAULT NULL, - `cc` varchar(100) DEFAULT NULL, - PRIMARY KEY (`ID`), - KEY `name` (`name`), - KEY `collection` (`collection`), - KEY `owner` (`owner`)) ENGINE=MyISAM"; - - if ($db->exec($sql)!==false) - echo date("r : ") . "Check table 'acls' ok\n"; - else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - - $fic=fopen("https://admin.fedoraproject.org/pkgdb/lists/bugzilla?tg_format=plain", "r"); - if (!$fic) { - echo date("r : ") . "ERROR reading pkgdb\n"; - } else { - $sql="TRUNCATE `acls`"; - $nb=$db->exec($sql); - if ($nb===false) echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - else echo date("r : ") . "Delete $nb packages\n"; - - for ($tot=0 ; $line=fgetcsv($fic,1024,'|') ; ) if (count($line)>5 && substr($line[0],0,1)!='#') { - for ($i=0; $i<6; $i++) $line[$i]=trim($line[$i]); - $sql=sprintf("INSERT INTO `acls` SET collection='%s', name='%s', summary='%s'", $line[0], $line[1], addslashes($line[2])); - if (!empty($line[3])) $sql .= sprintf(", owner='%s'", $line[3]); - if (!empty($line[4])) $sql .= sprintf(", qa='%s'", $line[4]); - if (!empty($line[5])) $sql .= sprintf(", cc='%s'", $line[5]); - $nb=$db->exec($sql); - if ($nb) $tot+=$nb; - else echo date("r : ") . "SQL ERROR = " . implode(" ", $db->errorInfo()) . "\n"; - } - fclose($fic); - echo date("r : ") . "wrote $tot package's owner\n"; - } - } // If in options -} -catch(PDOException $e) { - printf("%s ERREUR : %s\n", date("r"), $e->getMessage()); + $nb=0; + foreach ($stable as $name => $info) { + $rpmname="php-pecl-".str_replace("_", "-", $name); + + $nb += addUpstream( + $db, + "pecl", + "pecl", + $rpmname, + "stable", + $info["version"] + ); + } + echo date("r : ") . "Write $nb packages\n"; + // echo date("r : ") . "saved " . file_put_contents("cache/pecl-stable", serialize($stable)) . " bytes\n"; + } + + // ------------------------------------------------------------------- + echo date("r : ") . "PECL listLatestReleases - unstable\n"; + + $request = xmlrpc_encode_request("package.listLatestReleases", array()); + $context = stream_context_create( + array( + 'http' => array( + 'method' => "POST", + 'header' => "Content-Type: text/xml", + 'content' => $request + ) + ) + ); + $file = file_get_contents("http://pecl.php.net/xmlrpc.php", false, $context) or die ("Can't file_get_contents(http://pecl.php.net/xmlrpc.php)"); + $unstable = xmlrpc_decode($file); + if (xmlrpc_is_fault($unstable)) { + echo date("r : ") . "ERROR xmlrpc: $stable[faultString] ". + "($stable[faultCode])"; + } else { + $nb=0; + foreach ($unstable as $name => $info) { + $rpmname="php-pecl-".str_replace("_", "-", $name); + + $nb += addUpstream( + $db, + "pecl", + "pecl", + $rpmname, + "unstable", + $info["version"], + $info["state"] + ); + } + echo date("r : ") . "Write $nb packages\n"; + //echo date("r : ") . "saved " . + // file_put_contents("cache/pecl-unstable", serialize($unstable)) . + // " bytes\n"; + } + } // if in options + + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('pear', $_SERVER['argv'])) { + echo date("r : ") . "PEAR reading channels\n"; + $channels=array( + "pear" => "pear.php.net" + ,"doctrine" => "pear.phpdoctrine.org" + ,"ezc" => "components.ez.no" + ,"pdepend" => "pear.pdepend.org" + ,"phing" => "pear.phing.info" + ,"phpdb" => "pear.phpdb.org" + ,"phpmd" => "pear.phpmd.org" + ,"phpunit" => "pear.phpunit.de" + ,"swift" => "pear.swiftmailer.org" + ,"symphony" => "pear.symfony-project.com" + ); + try { + $nbtot=0; + + foreach ($channels as $channelname => $channelurl) { + $channel = @simplexml_load_file("http://$channelurl/channel.xml"); + if (!$channel) { + throw new Exception( + "can't read PEAR site (channel of $channelname)" + ); + } + + $rest = $channel->servers->primary->rest->baseurl[0]; + echo date("r : ") . "PEAR reading channel=$channelname" . + ", baseurl = $rest\n"; + + $categories = @simplexml_load_file($rest."c/categories.xml"); + if (!$categories) { + throw new Exception("can't read PEAR site (categories)"); + } + + $sql = "DELETE FROM upstream WHERE type='pear' AND channel='". + $channelname."'"; + $nb=$db->exec($sql); + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . implode( + " ", + $db->errorInfo() + ) . "\n"; + } else { + echo date("r : ") . "Delete $nb packages\n"; + } + + $nb=0; + if (!isset($categories->c[0])) { + echo date("r : ") . "Reading ALL\n"; // ezc only + $pitxt = @file_get_contents($rest."p/packages.xml"); + if (!$pitxt) { + throw new Exception( + "can't read PEAR site (".$rest."p/packagesinfo.xml)" + ); + } + $allpi = @simplexml_load_string($pitxt); + foreach ($allpi->p as $name) { + $pitxt = @file_get_contents( + $rest."r/".strtolower($name)."/allreleases.xml" + ); + if (!$pitxt) { + throw new Exception( + "can't read PEAR site (".$rest."r/". + strtolower($name)."/allreleases.xml" + ); + } + $pi = @simplexml_load_string($pitxt); + + $rpmname1="php-".$channelname."-". + str_replace("_", "-", $name); + $rpmname2="php-".$channelname."-".$name; + + //echo $rpmname ."/".(string)$pi->r[0]->v."/". + // (string)$pi->r[0]->s."\n"; + addUpstream( + $db, + "pear", + $channelname, + $rpmname1, + "unstable", + (string)$pi->r[0]->v, + (string)$pi->r[0]->s + ); + addUpstream( + $db, + "pear", + $channelname, + $rpmname2, + "unstable", + (string)$pi->r[0]->v, + (string)$pi->r[0]->s + ); + foreach ($pi->r as $rev) { + if ($rev->s=='stable') { + //echo $rpmname ."/".(string)$rev->v."/". + // (string)$rev->s."\n"; + addUpstream( + $db, + "pear", + $channelname, + $rpmname1, + "stable", + (string)$rev->v + ); + addUpstream( + $db, + "pear", + $channelname, + $rpmname2, + "stable", + (string)$rev->v + ); + break; + } + } + $nb++; + } + + } else { + foreach ($categories->c as $cat) { + echo date("r : ") . "Reading $cat\n"; + + $pitxt = @file_get_contents( + $rest."c/".urlencode($cat)."/packagesinfo.xml" + ); + if (!$pitxt) { + throw new Exception( + "can't read PEAR site (".$rest."c/". + urlencode($cat)."/packagesinfo.xml)" + ); + } + $pitxt = "<?" . preg_replace( + "/<\?xml.*?>/U", + "", + str_replace("\r\n", "\n", substr($pitxt, 2)) + ); + + $pi = @simplexml_load_string($pitxt); + if (!$pi) { + throw new Exception("can't read response ($cat)"); + } + foreach ($pi->pi as $ps) { + if (isset($ps->p->n) && isset($ps->a->r)) { + $name=(string)$ps->p->n; + + if ($channelname=='phing' && $name=='phing') { + $rpmname1="php-pear-phing"; + } else if ($channelname=='phpunit' + && $name=='PHPUnit' + ) { + $rpmname1="php-pear-PHPUnit"; + } else { + $rpmname1="php-".$channelname."-". + str_replace("_", "-", $name); + } + $rpmname2="php-".$channelname."-".$name; + + addUpstream( + $db, + "pear", + $channelname, + $rpmname1, + "unstable", + (string)$ps->a->r[0]->v, + (string)$ps->a->r[0]->s + ); + addUpstream( + $db, + "pear", + $channelname, + $rpmname2, + "unstable", + (string)$ps->a->r[0]->v, + (string)$ps->a->r[0]->s + ); + foreach ($ps->a->r as $rev) { + if ($rev->s=='stable') { + addUpstream( + $db, "pear", + $channelname, + $rpmname1, + "stable", + (string)$rev->v + ); + addUpstream( + $db, + "pear", + $channelname, + $rpmname2, + "stable", + (string)$rev->v + ); + break; + } + } + $nb++; + // echo $ps->p->n."\n"; + } + } + } + } + echo date("r : ") . "read $nb packages in $channelname\n"; + $nbtot += $nb; + } + echo date("r : ") . "read $nbtot packages in all channels\n"; + + } catch (Exception $e) { + echo date("r : ") . 'ERROR ' . $e->getMessage(), "\n"; + } + } // if in options + + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('R', $_SERVER['argv'])) { + + $repos = array( + array( + "name" => "R", + "state" => "stable", + "url" => "http://cran.at.r-project.org/src/contrib/PACKAGES" + ), + //array("name"=>"R-forge","state"=>"unstable", "url" => "http://r-forge.r-project.org/src/contrib/PACKAGES"), + array( + "name" => "biocp", + "state" => "stable", + "url" => "http://www.bioconductor.org/packages/release/bioc/src/contrib/PACKAGES" + ), + array( + "name" => "biocp", + "state" => "unstable", + "url" => "http://www.bioconductor.org/packages/devel/bioc/src/contrib/PACKAGES" + ), + array( + "name" => "bioca", + "state" => "stable", + "url" => "http://www.bioconductor.org/packages/release/data/annotation/src/contrib/PACKAGES" + ), + array( + "name" => "bioca", + "state" => "unstable", + "url" => "http://www.bioconductor.org/packages/devel/data/annotation/src/contrib/PACKAGES" + ), + array( + "name" => "bioce", + "state" => "stable", + "url" => "http://www.bioconductor.org/packages/release/data/experiment/src/contrib/PACKAGES" + ), + array( + "name" => "bioce", + "state" => "unstable", + "url" => "http://www.bioconductor.org/packages/devel/data/experiment/src/contrib/PACKAGES" + ) + ); + $tottot=0; + foreach ($repos as $repo) { + echo date("r : ") . "Reading " . $repo["name"] . " (" . + $repo["state"] . ")\n"; + $index = @file_get_contents($repo["url"]); + if (!$index) { + echo date("r : ") . "Can't read [" . $repo["url"] . + "], skip this channel\n"; + continue; + } + if ($repo["state"]=="stable") { + $sql = sprintf( + "DELETE FROM upstream WHERE type='R' AND channel='%s'", + $repo["name"] + ); + $nb=$db->exec($sql); + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } else { + echo date("r : ") . "Delete $nb packages\n"; + } + } + $results=array(); + if (preg_match_all('/Package: *(.*)\nVersion: *(.*)\n/i', $index, $results, PREG_SET_ORDER)) { + $tot=0; + foreach ($results as $result) { + //echo $result[1]." = ".$result[2]."\n"; + $rpmname = "R-".$result[1]; + $ver = str_replace('-', '.', $result[2]); + $add_upstream = addUpstream( + $db, + "R", + $repo["name"], + $rpmname, + $repo["state"], + $ver, + ($repo["state"]=="stable"?"":"devel") + ); + + if ($add_upstream) { + $tot++; + } + } + echo date("r : ") . "Write $tot packages in this channel\n"; + $tottot += $tot; + } else { + echo date("r : ") . "No package in this channel\n"; + } + } + echo date("r : ") . "Write $tottot packages in all channels\n"; + + } // If R in options + + // ------------------------------------------------------------------- + // Package Owners from pkgdb (thanks Smootherfrog) + // ------------------------------------------------------------------- + if ($_SERVER['argc']==1 || in_array('owner', $_SERVER['argv'])) { + + $sql="CREATE TABLE IF NOT EXISTS `acls` ( + `ID` int(11) NOT NULL AUTO_INCREMENT, + `collection` varchar(100) NOT NULL, + `name` varchar(100) NOT NULL, + `summary` varchar(200) NOT NULL, + `owner` varchar(50) DEFAULT NULL, + `qa` varchar(50) DEFAULT NULL, + `cc` varchar(100) DEFAULT NULL, + PRIMARY KEY (`ID`), + KEY `name` (`name`), + KEY `collection` (`collection`), + KEY `owner` (`owner`)) ENGINE=MyISAM"; + + if ($db->exec($sql)!==false) { + echo date("r : ") . "Check table 'acls' ok\n"; + } else { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } + + $fic=fopen( + "https://admin.fedoraproject.org/pkgdb/lists/bugzilla?tg_format=plain", + "r" + ); + if (!$fic) { + echo date("r : ") . "ERROR reading pkgdb\n"; + } else { + $sql="TRUNCATE `acls`"; + $nb=$db->exec($sql); + if ($nb===false) { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } else { + echo date("r : ") . "Delete $nb packages\n"; + } + + for ($tot=0 ; $line=fgetcsv($fic, 1024, '|'); ) { + if (count($line)>5 && substr($line[0], 0, 1)!='#') { + for ($i=0; $i<6; $i++) { + $line[$i]=trim($line[$i]); + } + $sql=sprintf( + "INSERT INTO `acls` SET collection='%s', name='%s', summary='%s'", + $line[0], + $line[1], + addslashes($line[2]) + ); + if (!empty($line[3])) { + $sql .= sprintf(", owner='%s'", $line[3]); + } + if (!empty($line[4])) { + $sql .= sprintf(", qa='%s'", $line[4]); + } + if (!empty($line[5])) { + $sql .= sprintf(", cc='%s'", $line[5]); + } + $nb=$db->exec($sql); + if ($nb) { + $tot+=$nb; + } else { + echo date("r : ") . "SQL ERROR = " . + implode(" ", $db->errorInfo()) . "\n"; + } + } + } + fclose($fic); + echo date("r : ") . "wrote $tot package's owner\n"; + } + } // If in options + +} catch(PDOException $e) { + printf("%s ERREUR : %s\n", date("r"), $e->getMessage()); } -if (isset($_SERVER["SERVER_NAME"])) echo "</pre>"; +if (isset($_SERVER["SERVER_NAME"])) { + echo "</pre>"; +} ?> |