Merge lp:~tristan-rivoallan/vanilla-miner/vm-598020 into lp:vanilla-miner

[Tristan Rivoallan]

[Tristan Rivoallan (tristan-rivoallan)]

review: Approve

=== modified file 'config/doctrine/schema.yml'
--- config/doctrine/schema.yml	2010-06-23 20:45:16 +0000
+++ config/doctrine/schema.yml	2010-06-24 16:15:42 +0000
@@ -48,4 +48,24 @@
     idx_availability:
       fields: 
         availability:
-          length: 11
\ No newline at end of file
+          length: 11
+
+ExtractionLog:
+  actAs:
+    Timestampable:
+  columns:
+    id: 
+      type: integer
+      autoincrement: true
+      primary: true
+      notnull: true
+    extraction_driver:
+      type: string
+    started_on:
+      type: timestamp
+    finished_on:
+      type: timestamp
+    resources_parsed:
+      type: int
+    urls_extracted:
+      type: int
\ No newline at end of file

=== added file 'data/sql/init_extraction_log.sql'
--- data/sql/init_extraction_log.sql	1970-01-01 00:00:00 +0000
+++ data/sql/init_extraction_log.sql	2010-06-24 16:15:42 +0000
@@ -0,0 +1,57 @@
+-- MySQL dump 10.13  Distrib 5.1.41, for debian-linux-gnu (i486)
+--
+-- Host: localhost    Database: miner
+-- ------------------------------------------------------
+-- Server version	5.1.41-3ubuntu12.3
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `extraction_log`
+--
+
+DROP TABLE IF EXISTS `extraction_log`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `extraction_log` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `extraction_driver` text,
+  `created_at` datetime NOT NULL,
+  `updated_at` datetime NOT NULL,
+  `started_on` datetime DEFAULT NULL,
+  `finished_on` datetime DEFAULT NULL,
+  `resources_parsed` int(11) DEFAULT NULL,
+  `urls_extracted` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `extraction_log`
+--
+
+LOCK TABLES `extraction_log` WRITE;
+/*!40000 ALTER TABLE `extraction_log` DISABLE KEYS */;
+INSERT INTO `extraction_log` VALUES (1,'CI_Extractor_LussumoVanilla1','2010-06-23 12:00:00','2010-06-23 12:00:00','2010-06-23 12:00:00',NULL,32176,16535);
+/*!40000 ALTER TABLE `extraction_log` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2010-06-24 17:40:00

=== added file 'lib/filter/doctrine/ExtractionLogFormFilter.class.php'
--- lib/filter/doctrine/ExtractionLogFormFilter.class.php	1970-01-01 00:00:00 +0000
+++ lib/filter/doctrine/ExtractionLogFormFilter.class.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,16 @@
+<?php
+
+/**
+ * ExtractionLog filter form.
+ *
+ * @package    vanilla-miner
+ * @subpackage filter
+ * @author     Constructions Incongrues
+ * @version    SVN: $Id: sfDoctrineFormFilterTemplate.php 23810 2009-11-12 11:07:44Z Kris.Wallsmith $
+ */
+class ExtractionLogFormFilter extends BaseExtractionLogFormFilter
+{
+  public function configure()
+  {
+  }
+}

=== added file 'lib/form/doctrine/ExtractionLogForm.class.php'
--- lib/form/doctrine/ExtractionLogForm.class.php	1970-01-01 00:00:00 +0000
+++ lib/form/doctrine/ExtractionLogForm.class.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,16 @@
+<?php
+
+/**
+ * ExtractionLog form.
+ *
+ * @package    vanilla-miner
+ * @subpackage form
+ * @author     Constructions Incongrues
+ * @version    SVN: $Id: sfDoctrineFormTemplate.php 23810 2009-11-12 11:07:44Z Kris.Wallsmith $
+ */
+class ExtractionLogForm extends BaseExtractionLogForm
+{
+  public function configure()
+  {
+  }
+}

=== added file 'lib/migration/doctrine/1277385385_version2.php'
--- lib/migration/doctrine/1277385385_version2.php	1970-01-01 00:00:00 +0000
+++ lib/migration/doctrine/1277385385_version2.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,51 @@
+<?php
+/**
+ * This class has been auto-generated by the Doctrine ORM Framework
+ */
+class Version2 extends Doctrine_Migration_Base
+{
+    public function up()
+    {
+        $this->createTable('extraction_log', array(
+             'id' => 
+             array(
+              'type' => 'integer',
+              'autoincrement' => '1',
+              'primary' => '1',
+              'length' => '8',
+             ),
+             'extraction_driver' => 
+             array(
+              'type' => 'string',
+              'length' => '',
+             ),
+             'duration' => 
+             array(
+              'type' => 'int',
+              'length' => '',
+             ),
+             'created_at' => 
+             array(
+              'notnull' => '1',
+              'type' => 'timestamp',
+              'length' => '25',
+             ),
+             'updated_at' => 
+             array(
+              'notnull' => '1',
+              'type' => 'timestamp',
+              'length' => '25',
+             ),
+             ), array(
+             'primary' => 
+             array(
+              0 => 'id',
+             ),
+             ));
+    }
+
+    public function down()
+    {
+        $this->dropTable('extraction_log');
+    }
+}
\ No newline at end of file

=== added file 'lib/migration/doctrine/1277387331_version3.php'
--- lib/migration/doctrine/1277387331_version3.php	1970-01-01 00:00:00 +0000
+++ lib/migration/doctrine/1277387331_version3.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,23 @@
+<?php
+/**
+ * This class has been auto-generated by the Doctrine ORM Framework
+ */
+class Version3 extends Doctrine_Migration_Base
+{
+    public function up()
+    {
+        $this->removeColumn('extraction_log', 'duration');
+        $this->addColumn('extraction_log', 'started_on', 'timestamp', '25', array(
+             ));
+        $this->addColumn('extraction_log', 'finished_on', 'timestamp', '25', array(
+             ));
+    }
+
+    public function down()
+    {
+        $this->addColumn('extraction_log', 'duration', 'int', '', array(
+             ));
+        $this->removeColumn('extraction_log', 'started_on');
+        $this->removeColumn('extraction_log', 'finished_on');
+    }
+}
\ No newline at end of file

=== added file 'lib/migration/doctrine/1277393030_version4.php'
--- lib/migration/doctrine/1277393030_version4.php	1970-01-01 00:00:00 +0000
+++ lib/migration/doctrine/1277393030_version4.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,20 @@
+<?php
+/**
+ * This class has been auto-generated by the Doctrine ORM Framework
+ */
+class Version4 extends Doctrine_Migration_Base
+{
+    public function up()
+    {
+        $this->addColumn('extraction_log', 'resources_parsed', 'int', '', array(
+             ));
+        $this->addColumn('extraction_log', 'urls_extracted', 'int', '', array(
+             ));
+    }
+
+    public function down()
+    {
+        $this->removeColumn('extraction_log', 'resources_parsed');
+        $this->removeColumn('extraction_log', 'urls_extracted');
+    }
+}
\ No newline at end of file

=== added file 'lib/model/doctrine/ExtractionLog.class.php'
--- lib/model/doctrine/ExtractionLog.class.php	1970-01-01 00:00:00 +0000
+++ lib/model/doctrine/ExtractionLog.class.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,15 @@
+<?php
+
+/**
+ * ExtractionLog
+ * 
+ * This class has been auto-generated by the Doctrine ORM Framework
+ * 
+ * @package    vanilla-miner
+ * @subpackage model
+ * @author     Constructions Incongrues
+ * @version    SVN: $Id: Builder.php 7490 2010-03-29 19:53:27Z jwage $
+ */
+class ExtractionLog extends BaseExtractionLog
+{
+}

=== added file 'lib/model/doctrine/ExtractionLogTable.class.php'
--- lib/model/doctrine/ExtractionLogTable.class.php	1970-01-01 00:00:00 +0000
+++ lib/model/doctrine/ExtractionLogTable.class.php	2010-06-24 16:15:42 +0000
@@ -0,0 +1,11 @@
+<?php
+
+
+class ExtractionLogTable extends Doctrine_Table
+{
+    
+    public static function getInstance()
+    {
+        return Doctrine_Core::getTable('ExtractionLog');
+    }
+}
\ No newline at end of file

=== modified file 'lib/task/minerExtractlinksTask.class.php'
--- lib/task/minerExtractlinksTask.class.php	2010-06-23 16:09:40 +0000
+++ lib/task/minerExtractlinksTask.class.php	2010-06-24 16:15:42 +0000
@@ -20,6 +20,7 @@
       new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'dev'),
       new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'doctrine'),
       new sfCommandOption('extraction-driver', null, sfCommandOption::PARAMETER_REQUIRED, 'Extraction driver class name', 'CI_Extractor_LussumoVanilla1'),
+      new sfCommandOption('incremental', null, sfCommandOption::PARAMETER_REQUIRED, 'If true, only extracts URLs from new and updated resources since last extraction', true),
       new sfCommandOption('progress', null, sfCommandOption::PARAMETER_NONE, 'Displays a progress bar'),
     ));
 
@@ -55,8 +56,38 @@
       throw new InvalidArgumentException(sprintf('Class "%s" does not exist', $options['extraction-driver']));
     }
 
+    if ($options['incremental'] === 'false')
+    {
+      $options['incremental'] = false;
+    }
+
+    // Instanciate database connection
+    $databaseManager = new sfDatabaseManager($this->configuration);
+    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();
+
+    // If extraction is incremental, retrieve last extraction date
+    $since = null;
+    if ($options['incremental'])
+    {
+      $since = $this->getLastExtractionDate($connection);
+      if ($since)
+      {
+        $this->logSection('extract', sprintf('Incrementally extracting URLs from resources created or updated since "%s"', $since));
+      }
+      else
+      {
+        $this->logSection('extract', 'Found no entries in extraction log. Extracting URLs from all resources.');
+      }
+    }
+
+    // Create new extraction log entry
+    $log_entry = new ExtractionLog();
+    $log_entry->extraction_driver = $options['extraction-driver'];
+    $log_entry->started_on = date('Y-m-d H:i:s');
+    $log_entry->save();
+
     // Instanciate and configure extraction engine
-    $extractor = new $options['extraction-driver']($this->dispatcher, $this->configuration);
+    $extractor = new $options['extraction-driver']($this->dispatcher, $this->configuration, $since);
 
     // Extraction statistics
     $urls_found_count = 0;
@@ -76,11 +107,16 @@
       }
 
       // Extract resources from source and insert them in Links database
-      while ($resource_extraction_info = $extractor->extract($arguments['dsn'], $options['connection']))
+      while ($resource_extraction_info = $extractor->extract($arguments['dsn'], $options['connection'], $since))
       {
         // Update extraction statistics
         $urls_found_count += $resource_extraction_info['urls_found_count'];
 
+        // Update extraction log
+        $log_entry->resources_parsed = $resource_extraction_info['resources_parsed_count'];
+        $log_entry->urls_extracted = $urls_found_count;
+        $log_entry->save();
+
         // Update progress bar
         if ($options['progress'])
         {
@@ -95,6 +131,33 @@
     {
       $this->logSection('extract', 'No resources to extract. Exiting.');
     }
+
+    // Record finish time
+    $log_entry->finished_on = date('Y-m-d H:i:s');
+    $log_entry->save();
+  }
+
+  /**
+   * Returns date of most recent extraction.
+   *
+   * @param string $doctrine_connection
+   */
+  private function getLastExtractionDate()
+  {
+    // Retrieve last extraction date
+    $last_extraction_date = Doctrine_Query::create()
+      ->select('l.started_on')
+      ->from('ExtractionLog l')
+      ->orderBy('l.started_on desc')
+      ->limit(1)
+      ->execute(null, Doctrine_Core::HYDRATE_SINGLE_SCALAR);
+
+    if (!$last_extraction_date)
+    {
+      $last_extraction_date = null;
+    }
+
+    return $last_extraction_date;
   }
 
   /**

=== modified file 'lib/vendor/CI/Extractor.php'
--- lib/vendor/CI/Extractor.php	2010-06-23 09:28:02 +0000
+++ lib/vendor/CI/Extractor.php	2010-06-24 16:15:42 +0000
@@ -13,7 +13,7 @@
    *
    * @return array
    */
-  abstract protected function getResources($dsn_source);
+  abstract protected function getResources($dsn_source, $sources = null);
 
   /**
    * Returns text from which URLs will be extracted.
@@ -39,7 +39,7 @@
    *
    * @return int
    */
-  abstract protected function countResources($dsn_source);
+  abstract protected function countResources($dsn_source, $since = null);
 
   /**
    * @var array
@@ -69,7 +69,7 @@
   /**
    * Instanciates and configures extractor.
    *
-   * @param sfEventDispatcher $event_dispatcher
+   * @param sfEventDispatcher      $event_dispatcher
    * @param sfProjectConfiguration $configuration
    */
   public function __construct(sfEventDispatcher $event_dispatcher, sfProjectConfiguration $configuration)
@@ -115,15 +115,16 @@
   /**
    * Extracts links from source database and inserts them into links collection.
    *
-   * @param string               $dsn_source
-   * @param sfDoctrineConnection $connection_dest
+   * @param string $dsn_source
+   * @param string $connection_dest
+   * @param string $since           Extract urls from resources updated or create since this date (Y-m-d H:i:s)
    */
-  public function extract($dsn_source, $connection_dest)
+  public function extract($dsn_source, $connection_dest, $since = null)
   {
     if (!$this->resources)
     {
       // Retrieve comments from database
-      $resources = $this->getResources($dsn_source);
+      $resources = $this->getResources($dsn_source, $since);
       $this->log(sprintf('Extracting URLs from %d resources using extractor "%s"', count($resources), get_class($this)));
       $this->resources = $resources;
     }

=== modified file 'lib/vendor/CI/Extractor/LussumoVanilla1.php'
--- lib/vendor/CI/Extractor/LussumoVanilla1.php	2010-06-23 10:13:43 +0000
+++ lib/vendor/CI/Extractor/LussumoVanilla1.php	2010-06-24 16:15:42 +0000
@@ -15,13 +15,18 @@
    *
    * NEXT : make table prefix configurable
    */
-  protected function getResources($dsn_source)
+  protected function getResources($dsn_source, $since = null)
   {
     $q = 'select c.CommentID, c.Body, c.DateCreated, c.AuthUserID, c.DiscussionID, d.Name as DiscussionName, u.Name
     	from LUM_Comment c
     	inner join LUM_User u on c.AuthUserID = u.UserID
     	inner join LUM_Discussion d on c.DiscussionID = d.DiscussionID
     	where c.Deleted != 1 and c.WhisperUserID = 0';
+    if ($since)
+    {
+      $q .= sprintf(' and c.DateCreated > "%s"', $since);
+    }
+
 
     return $this->getConnection($dsn_source)->fetchAssoc($q);
   }
@@ -74,9 +79,13 @@
    *
    * @return int
    */
-  public function countResources($dsn)
+  public function countResources($dsn, $since = null)
   {
     $q = 'select count(c.CommentID) from LUM_Comment c where c.Deleted != 1 and c.WhisperUserID = 0';
+    if ($since)
+    {
+      $q .= sprintf(' where c.DateCreated > "%s"', $since);
+    }
 
     return (int)$this->getConnection($dsn)->fetchOne($q);
   }