From 66e2a68f902cfa132a22b02892871d0be375d1c2 Mon Sep 17 00:00:00 2001 From: Christian Weiske Date: Thu, 13 Jun 2013 00:08:28 +0200 Subject: [PATCH] first work on feed updater + pinger --- .gitignore | 1 + bin/stapibas | 12 +++ data/config.php.dist | 6 ++ src/stapibas/Feed/UpdateFeeds.php | 128 ++++++++++++++++++++++++++++++ src/stapibas/Logger.php | 17 ++++ src/stapibas/PDO.php | 28 +++++++ src/stapibas/autoloader.php | 20 +++++ www/xmlrpc.php | 16 +--- 8 files changed, 213 insertions(+), 15 deletions(-) create mode 100755 bin/stapibas create mode 100644 src/stapibas/Feed/UpdateFeeds.php create mode 100644 src/stapibas/Logger.php create mode 100644 src/stapibas/PDO.php create mode 100644 src/stapibas/autoloader.php diff --git a/.gitignore b/.gitignore index f2b734f..a09bfc2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ data/config.php www/test.htm +lib/simplepie/* diff --git a/bin/stapibas b/bin/stapibas new file mode 100755 index 0000000..77763ce --- /dev/null +++ b/bin/stapibas @@ -0,0 +1,12 @@ +#!/usr/bin/env php +db = new PDO($dbdsn, $dbuser, $dbpass); +$uf->log = new Logger(); +$uf->updateAll(); + +?> diff --git a/data/config.php.dist b/data/config.php.dist index 4b9a97c..034a469 100644 --- a/data/config.php.dist +++ b/data/config.php.dist @@ -2,4 +2,10 @@ $dbdsn = 'mysql:dbname=stapibas;host=127.0.0.1'; $dbuser = 'stapibas'; $dbpass = 'password'; + +set_include_path( + __DIR__ . '/../src/' . PATH_SEPARATOR + . get_include_path() +); +$GLOBALS['stapibas_libdir'] = __DIR__ . '/../lib'; ?> diff --git a/src/stapibas/Feed/UpdateFeeds.php b/src/stapibas/Feed/UpdateFeeds.php new file mode 100644 index 0000000..642f29e --- /dev/null +++ b/src/stapibas/Feed/UpdateFeeds.php @@ -0,0 +1,128 @@ +db->query( + 'SELECT * FROM feeds' + . ' WHERE f_needs_update = 1 OR f_updated = "0000-00-00"' + ); + while ($feedRow = $res->fetch(\PDO::FETCH_OBJ)) { + $this->log->info( + sprintf('Updating feed #%d: %s', $feedRow->f_id, $feedRow->f_url) + ); + $this->updateFeed($feedRow); + } + } + + protected function updateFeed($feedRow) + { + $req = new \HTTP_Request2($feedRow->f_url); + $req->setHeader('User-Agent', 'stapibas'); + + if ($feedRow->f_updated != '0000-00-00 00:00:00') { + $req->setHeader( + 'If-Modified-Since', + gmdate('r', strtotime($feedRow->f_updated)) + ); + } + + $res = $req->send(); + if ($res->getStatus() == 304) { + //not modified + $this->setNoUpdate($feedRow); + $this->log->info('Not modified'); + return; + } + + if (intval($res->getStatus() / 100) != 2) { + //no 2xx is an error for us + $this->log->info('Error fetching feed'); + return; + } + + $this->updateEntries($feedRow, $res); + } + + protected function updateEntries($feedRow, \HTTP_Request2_Response $res) + { + require_once $GLOBALS['stapibas_libdir'] . '/simplepie/autoloader.php'; + $sp = new \SimplePie(); + $sp->set_raw_data($res->getBody()); + $sp->init(); + + $new = $updated = $items = 0; + foreach ($sp->get_items() as $item) { + ++$items; + $url = $item->get_permalink(); + $entryRow = $this->db->query( + 'SELECT fe_id, fe_updated, fe_needs_update FROM feedentries' + . ' WHERE fe_url = ' . $this->db->quote($url) + . ' AND fe_f_id = ' . $this->db->quote($feedRow->f_id) + )->fetch(\PDO::FETCH_OBJ); + + if ($entryRow === false) { + //new item! + $this->db->exec( + 'INSERT INTO feedentries SET' + . ' fe_f_id = ' . $this->db->quote($feedRow->f_id) + . ', fe_url = ' . $this->db->quote($url) + . ', fe_needs_update = 1' + ); + ++$new; + continue; + } + if ($entryRow->fe_needs_update == 0 + && $item->get_updated_gmdate('U') > strtotime($entryRow->fe_updated) + ) { + //updated + $this->db->exec( + 'UPDATE feedentries SET' + . ' fe_url = ' . $this->db->quote($url) + . ', fe_needs_update = 1' + . ' WHERE fe_id = ' . $this->db->quote($entryRow->fe_id) + ); + ++$updated; + continue; + } + } + $this->log->info( + sprintf( + 'Feed #%d: %d new, %d updated of %d entries', + $feedRow->f_id, $new, $updated, $items + ) + ); + $this->setUpdated($feedRow, $res); + } + + protected function setNoUpdate($feedRow) + { + $this->db->exec( + 'UPDATE feeds SET f_needs_update = 0' + . ' WHERE f_id = ' . $this->db->quote($feedRow->f_id) + ); + } + + protected function setUpdated($feedRow, \HTTP_Request2_Response $res) + { + $this->db->exec( + 'UPDATE feeds' + . ' SET f_needs_update = 0' + . ', f_updated = ' . $this->db->quote( + gmdate('Y-m-d H:i:s', strtotime($res->getHeader('last-modified'))) + ) + . ' WHERE f_id = ' . $this->db->quote($feedRow->f_id) + ); + } + +} + +?> \ No newline at end of file diff --git a/src/stapibas/Logger.php b/src/stapibas/Logger.php new file mode 100644 index 0000000..9068609 --- /dev/null +++ b/src/stapibas/Logger.php @@ -0,0 +1,17 @@ +log($msg); + } + + public function log($msg) + { + echo $msg . "\n"; + } +} + +?> diff --git a/src/stapibas/PDO.php b/src/stapibas/PDO.php new file mode 100644 index 0000000..a85008b --- /dev/null +++ b/src/stapibas/PDO.php @@ -0,0 +1,28 @@ +handleError(); + } + + protected function handleError() + { + echo "SQL error\n"; + echo " " . $this->errorCode() . "\n"; + echo " " . implode(' - ', $this->errorInfo()) . "\n"; + } +} + +?> \ No newline at end of file diff --git a/src/stapibas/autoloader.php b/src/stapibas/autoloader.php new file mode 100644 index 0000000..6ff3f1b --- /dev/null +++ b/src/stapibas/autoloader.php @@ -0,0 +1,20 @@ + diff --git a/www/xmlrpc.php b/www/xmlrpc.php index 52bdf6e..9221e2b 100644 --- a/www/xmlrpc.php +++ b/www/xmlrpc.php @@ -3,21 +3,7 @@ * Simply stores all pingbacks in the database. */ require_once __DIR__ . '/../data/config.php'; - -function __autoload($className) -{ - $className = ltrim($className, '\\'); - $fileName = ''; - $namespace = ''; - if ($lastNsPos = strripos($className, '\\')) { - $namespace = substr($className, 0, $lastNsPos); - $className = substr($className, $lastNsPos + 1); - $fileName = str_replace('\\', '/', $namespace) . '/'; - } - $fileName .= str_replace('_', '/', $className) . '.php'; - - require $fileName; -} +require_once 'stapibas/autoloader.php'; $db = new PDO($dbdsn, $dbuser, $dbpass); -- 2.30.2