Merge lp:~tristan-rivoallan/vanilla-miner/vm-594599 into lp:vanilla-miner

[Tristan Rivoallan]

[Tristan Rivoallan (tristan-rivoallan)]

review: Approve

=== modified file 'apps/frontend/modules/resource/templates/_documentation/link/schema.php'
--- apps/frontend/modules/resource/templates/_documentation/link/schema.php	2010-06-03 16:39:07 +0000
+++ apps/frontend/modules/resource/templates/_documentation/link/schema.php	2010-06-23 20:53:23 +0000
@@ -1,40 +1,32 @@
 <p>Cette collection expose les attributs suivants :</p>
 
-<h4 id="schema-url">url</h4>
-<p>C'est l'URL vers la ressource. Par exemple : </p>
-<pre>http://www.glafouk.com/dlz/radioclash_astrotease.mp3</pre>
-
-<h4 id="schema-domain_fqdn">domain_fqdn</h4>
-<p>C'est le nom de domaine complet de l'URL vers la ressource. Par exemple :</p>
-<pre>data.musiques-incongrues.net</pre>
-
-<h4 id="schema-domain_parent">domain_parent</h4>
-<p>C'est le domaine parent de l'URL vers la ressource. Par exemple :</p>
-<pre>musiques-incongrues.net</pre>
-<p>Les deux URLs http://www.musiques-incongrues.net et http://data.musiques-incongrues.net ont un domaine parent identique.</p>
-
-<h4 id="schema-mime_type">mime_type</h4>
-<p>C'est le type MIME de la ressource. Cet attribut n'est pas toujours définit. Il l'est toujours pour les fichiers binaires (mp3, image, etc). Par exemple :</p>
-<pre>audio/mpeg</pre>
+<h4 id="schema-availability">availability</h4>
+<p>Ce paramètre correspond à la disponibilité du lien.</p>
+<dl>
+  <dt>Valeurs possibles</dt>
+  <dd><code>unknown</code> : On ne sait pas si l'URL est accessible ou non</dd>
+  <dd><code>available</code> : L'URL est accessible</dd>
+  <dd><code>unavailable</code> : L'URL n'est pas accessible</dd>
+</dl>
+<p>Par défaut, les liens avec une URL non accessible ne sont pas retournés.</p>
+
+<h4 id="schema-comment_id">comment_id</h4>
+<p>C'est l'identifiant du commentaire sur le forum dans lequel à été contribué le lien. Par exemple :</p>
+<pre>15336</pre>
 
 <h4 id="schema-contributed_at">contributed_at</h4>
 <p>C'est la date à laquelle a été contribué le lien. Par exemple :</p>
 <pre>2007-05-09T21:22:05Z</pre>
 
+<h4 id="schema-contributor_name">contributor_name</h4>
+<p>C'est le nom sur le forum de l'utilisateur ayant contribué le lien. Par exemple :</p>
+<pre>mbertier</pre>
 
 <h4 id="schema-contributor_id">contributor_id</h4>
 <p>C'est l'identifiant sur le forum de l'utilisateur ayant contribué le lien. Par exemple :</p>
 <pre>34</pre>
 <p>Les URL pour accéder au profil d'un utilisateur sur Musiques Incongrues ont la forme http://www.musiques-incongrues.net/forum/account/<strong>contributor_id</strong>/</p>
 
-<h4 id="schema-contributor_name">contributor_name</h4>
-<p>C'est le nom sur le forum de l'utilisateur ayant contribué le lien. Par exemple :</p>
-<pre>mbertier</pre>
-
-<h4 id="schema-comment_id">comment_id</h4>
-<p>C'est l'identifiant du commentaire sur le forum dans lequel à été contribué le lien. Par exemple :</p>
-<pre>15336</pre>
-
 <h4 id="schema-discussion_id">discussion_id</h4>
 <p>C'est l'identifiant de la discussion dans laquelle a été contribué le lien. Par exemple :</p>
 <pre>5455</pre>
@@ -42,4 +34,20 @@
 
 <h4 id="schema-discussion_name">discussion_name</h4>
 <p>C'est le titre de la discussion dans laquelle a été contribué le lien. Par exemple :</p>
-<pre>Des clips, des clips, rien que des clips</pre>
\ No newline at end of file
+<pre>Des clips, des clips, rien que des clips</pre>
+<h4 id="schema-domain_fqdn">domain_fqdn</h4>
+<p>C'est le nom de domaine complet de l'URL vers la ressource. Par exemple :</p>
+<pre>data.musiques-incongrues.net</pre>
+
+<h4 id="schema-domain_parent">domain_parent</h4>
+<p>C'est le domaine parent de l'URL vers la ressource. Par exemple :</p>
+<pre>musiques-incongrues.net</pre>
+<p>Les deux URLs http://www.musiques-incongrues.net et http://data.musiques-incongrues.net ont un domaine parent identique.</p>
+
+<h4 id="schema-mime_type">mime_type</h4>
+<p>C'est le type MIME de la ressource. Cet attribut n'est pas toujours définit. Il l'est toujours pour les fichiers binaires (mp3, image, etc). Par exemple :</p>
+<pre>audio/mpeg</pre>
+
+<h4 id="schema-url">url</h4>
+<p>C'est l'URL vers la ressource. Par exemple : </p>
+<pre>http://www.glafouk.com/dlz/radioclash_astrotease.mp3</pre>
\ No newline at end of file

=== modified file 'config/doctrine/schema.yml'
--- config/doctrine/schema.yml	2010-06-15 13:22:18 +0000
+++ config/doctrine/schema.yml	2010-06-23 20:53:23 +0000
@@ -31,9 +31,21 @@
       type:    integer
     discussion_name:
       type:    string
+    # available, unavailable, unknown
+    availability:
+      type:    string
+      default: 'unknown'
+    expanded_at:
+      type:    timestamp
   indexes:
-    url_index:
+    idx_url:
       fields:
         url:
           length: 512
-      type:   unique
\ No newline at end of file
+      type:   unique
+    idx_expanded_at:
+      fields: [expanded_at]
+    idx_availability:
+      fields: 
+        availability:
+          length: 11
\ No newline at end of file

=== modified file 'config/search.yml'
--- config/search.yml	2010-06-02 16:53:29 +0000
+++ config/search.yml	2010-06-23 20:53:23 +0000
@@ -26,7 +26,9 @@
           type:      int
         discussion_name:
           stored:    true
-          
+        availability:
+          stored:    true
+
   index:
     encoding: UTF-8
     cultures: [fr]

=== modified file 'config/solr/IndexA_fr/conf/schema.xml'
--- config/solr/IndexA_fr/conf/schema.xml	2010-06-03 13:14:51 +0000
+++ config/solr/IndexA_fr/conf/schema.xml	2010-06-23 20:53:23 +0000
@@ -258,6 +258,7 @@
   <field name='comment_id' type='int' stored='true' multiValued='false' required='false'  />
   <field name='discussion_id' type='int' stored='true' multiValued='false' required='false'  />
   <field name='discussion_name' type='text' stored='true' multiValued='false' required='false'  />
+  <field name='availability' type='text' stored='true' multiValued='false' required='false'  />
  </fields>
 
  <!-- field to use to determine and enforce document uniqueness. -->
@@ -283,5 +284,6 @@
   <copyField source='comment_id' dest='sfl_all' />
   <copyField source='discussion_id' dest='sfl_all' />
   <copyField source='discussion_name' dest='sfl_all' />
+  <copyField source='availability' dest='sfl_all' />
   
 </schema>

=== added directory 'data/utils'
=== added file 'data/utils/proxy.php'
--- data/utils/proxy.php	1970-01-01 00:00:00 +0000
+++ data/utils/proxy.php	2010-06-23 20:53:23 +0000
@@ -0,0 +1,13 @@
+<?php
+// Sanity checks
+if (is_null($_SERVER['PATH_INFO']))
+{
+  throw new InvalidArgumentException('Please specify path info.');
+}
+
+// Call service
+$curl = curl_init(sprintf('http://data.musiques-incongrues.net/%s?%s', $_SERVER['PATH_INFO'], $_SERVER['QUERY_STRING']));
+curl_exec($curl);
+
+// Clean up
+curl_close($curl);
\ No newline at end of file

=== removed file 'lib/filter/doctrine/ResourceTypeFormFilter.class.php'
--- lib/filter/doctrine/ResourceTypeFormFilter.class.php	2010-06-02 10:18:41 +0000
+++ lib/filter/doctrine/ResourceTypeFormFilter.class.php	1970-01-01 00:00:00 +0000
@@ -1,16 +0,0 @@
-<?php
-
-/**
- * ResourceType filter form.
- *
- * @package    vanilla-miner
- * @subpackage filter
- * @author     Your name here
- * @version    SVN: $Id: sfDoctrineFormFilterTemplate.php 23810 2009-11-12 11:07:44Z Kris.Wallsmith $
- */
-class ResourceTypeFormFilter extends BaseResourceTypeFormFilter
-{
-  public function configure()
-  {
-  }
-}

=== removed file 'lib/form/doctrine/ResourceTypeForm.class.php'
--- lib/form/doctrine/ResourceTypeForm.class.php	2010-06-02 10:18:41 +0000
+++ lib/form/doctrine/ResourceTypeForm.class.php	1970-01-01 00:00:00 +0000
@@ -1,16 +0,0 @@
-<?php
-
-/**
- * ResourceType form.
- *
- * @package    vanilla-miner
- * @subpackage form
- * @author     Your name here
- * @version    SVN: $Id: sfDoctrineFormTemplate.php 23810 2009-11-12 11:07:44Z Kris.Wallsmith $
- */
-class ResourceTypeForm extends BaseResourceTypeForm
-{
-  public function configure()
-  {
-  }
-}

=== removed file 'lib/model/doctrine/ResourceType.class.php'
--- lib/model/doctrine/ResourceType.class.php	2010-06-02 10:18:41 +0000
+++ lib/model/doctrine/ResourceType.class.php	1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
-<?php
-
-/**
- * ResourceType
- * 
- * This class has been auto-generated by the Doctrine ORM Framework
- * 
- * @package    vanilla-miner
- * @subpackage model
- * @author     Your name here
- * @version    SVN: $Id: Builder.php 7490 2010-03-29 19:53:27Z jwage $
- */
-class ResourceType extends BaseResourceType
-{
-}

=== removed file 'lib/model/doctrine/ResourceTypeTable.class.php'
--- lib/model/doctrine/ResourceTypeTable.class.php	2010-06-02 10:18:41 +0000
+++ lib/model/doctrine/ResourceTypeTable.class.php	1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
-<?php
-
-
-class ResourceTypeTable extends Doctrine_Table
-{
-    
-    public static function getInstance()
-    {
-        return Doctrine_Core::getTable('ResourceType');
-    }
-}
\ No newline at end of file

=== added file 'lib/task/minerExpandLinksTask.class.php'
--- lib/task/minerExpandLinksTask.class.php	1970-01-01 00:00:00 +0000
+++ lib/task/minerExpandLinksTask.class.php	2010-06-23 20:53:23 +0000
@@ -0,0 +1,178 @@
+<?php
+
+class minerExpandLinksTask extends sfBaseTask
+{
+  protected function configure()
+  {
+    $this->addOptions(array(
+      new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'dev'),
+      new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'doctrine'),
+      new sfCommandOption('progress', null, sfCommandOption::PARAMETER_NONE, 'Display a progress bar'),
+      new sfCommandOption('verbose', null, sfCommandOption::PARAMETER_NONE, 'Display more informations about extraction process'),
+      new sfCommandOption('all', null, sfCommandOption::PARAMETER_NONE, 'Expand all links in database. By default, only new links are expanded'),
+      new sfCommandOption('with-unavailable', null, sfCommandOption::PARAMETER_NONE, 'When expanding all links (--all), also include links previously marked as unavailable'),
+      // TODO : add --older-than option
+    ));
+
+    $this->namespace        = 'miner';
+    $this->name             = 'expand-links';
+    // TODO : write descriptions
+    $this->briefDescription = 'Expands informations about links by crawling their URLs';
+    $this->detailedDescription = <<<EOF
+
+Use cases :
+ * Expand new urls : [php symfony miner:expand-links|INFO]
+ * Expand all urls (a word about --with-unavailable) : [php symfony miner:expand-links --all|INFO]
+ * Expand all urls, including those previously marked as unavailable : [php symfony miner:expand-links --all --with-unavailable|INFO]
+EOF;
+  }
+
+  protected function execute($arguments = array(), $options = array())
+  {
+    // Open database connection
+    $databaseManager = new sfDatabaseManager($this->configuration);
+    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();
+
+    // Build query for fetching links from database
+    $q = Doctrine_Query::create()
+      ->select('l.url')
+      ->from('Link l');
+    if (!$options['all'])
+    {
+      $q->where('l.expanded_at is null');
+    }
+    if (!$options['with-unavailable'])
+    {
+      $q->andWhere('l.availability != "unavailable"');
+    }
+
+    // Fetch links from database
+    $links_count = $q->count();
+    $links = $q->execute(null, Doctrine_Core::HYDRATE_ON_DEMAND);
+    $q->free();
+    $this->logSection('info', sprintf('Expanding %s links', $links_count));
+
+    // Instanciate progress bar, if user requested so
+    $links_expanded = 0;
+    if ($options['progress'])
+    {
+      include 'Console/ProgressBar.php';
+      $progress_bar = new Console_ProgressBar(
+      '** Links %fraction% comments [%bar%] %percent% | ',
+      '=>', '-', 80, $links_count, array('ansi_terminal' => true)
+      );
+      $progress_bar->update($links_expanded);
+    }
+
+    // Launch a HEAD request on each link, and use data in response headers to update informations about link in database
+    // TODO : move crawling code to dedicated class. and then create miner:crawl-url task
+    require 'HTTP/Request2.php';
+    $request = new HTTP_Request2(null, HTTP_Request2::METHOD_HEAD, array('follow_redirects' => true));
+    $request->setHeader('user-agent', 'vanilla-miner/1.1 (https://launchpad.net/vanilla-miner)');
+
+    foreach ($links as $link)
+    {
+      $link->expanded_at = time();
+      try
+      {
+        $request->setUrl($link->url);
+        $response = $request->send();
+        if (200 == $response->getStatus())
+        {
+          if ($options['progress'])
+          {
+            $this->log(sprintf('[%d] %s', $response->getStatus(), $link->url));
+          }
+          else
+          {
+            $this->logSection('info', sprintf('[%d] %s - Updating metadata, marking as available', $response->getStatus(), $link->url));
+          }
+
+          // Extract meaningful informations from server response
+          $header = $response->getHeader();
+          $header = $this->normalizeHeader($header);
+          $link->mime_type = $this->getMimeType($header);
+
+          // Mark link as available
+          $link->availability = 'available';
+
+          // Save link to database
+          $link->replace();
+        }
+        else
+        {
+          if ($options['progress'])
+          {
+            $this->log(sprintf('[%d] %s', $response->getStatus(), $link->url));
+          }
+          else
+          {
+            $this->logSection('notice', sprintf(
+                '[%d] %s (%d %s) - Marking as unavailable',
+                $response->getStatus(),
+                $link->url,
+                $response->getStatus(),
+                $response->getReasonPhrase()
+              )
+            );
+          }
+          $link->availability = 'unavailable';
+          $link->replace();
+        }
+      }
+      catch (HTTP_Request2_Exception $e)
+      {
+        if ($options['progress'])
+        {
+          $this->log(sprintf('[ERR] %s', $link->url));
+        }
+        else
+        {
+          $this->logSection('error', sprintf('[ERR] Received exception with message "%s" for link "%s" - Marking as unavailable.', $e->getMessage(), $link->url));
+        }
+        $link->availability = 'unavailable';
+        $link->replace();
+      }
+
+      // Update progress bar
+      if ($options['progress'])
+      {
+        $progress_bar->update(++$links_expanded);
+      }
+
+    }
+  }
+
+  private function normalizeHeader(array $header)
+  {
+    // Make all header names lower case
+    $header_rev = array_flip($header);
+    array_walk($header_rev, create_function('&$item, $key', 'strtolower($item);'));
+    $header = array_flip($header_rev);
+
+    return $header;
+  }
+
+  private function getMimeType(array $header)
+  {
+    $mime_type = null;
+
+    if (isset($header['content-type']))
+    {
+      $mime_type = $header['content-type'];
+
+      // Extract mime type from content-type header
+      // TODO : use a regular expression instead of this crappy flow
+      $matches = array();
+      if (strpos($header['content-type'], 'charset') !== false)
+      {
+        if (preg_match('/(.+); ?charset=.+/i', $header['content-type'], $matches))
+        {
+          $mime_type = $matches[1];
+        }
+      }
+    }
+
+    return $mime_type;
+  }
+}
\ No newline at end of file

=== modified file 'lib/task/minerExtractlinksTask.class.php'
--- lib/task/minerExtractlinksTask.class.php	2010-06-23 13:13:30 +0000
+++ lib/task/minerExtractlinksTask.class.php	2010-06-23 20:53:23 +0000
@@ -63,31 +63,38 @@
     $resources_parsed = 0;
     $resources_total = $extractor->countResources($arguments['dsn']);
 
-    // Instanciate an configure progress bar
-    if ($options['progress'])
-    {
-      include 'Console/ProgressBar.php';
-      $progress_bar = new Console_ProgressBar(
-        '** '.$arguments['dsn'].' %fraction% resources [%bar%] %percent% | ',
-        '=>', '-', 80, $resources_total, array('ansi_terminal' => true)
-      );
-    }
-
-    // Extract resources from source and insert them in Links database
-    while ($resource_extraction_info = $extractor->extract($arguments['dsn'], $options['connection']))
-    {
-      // Update extraction statistics
-      $urls_found_count += $resource_extraction_info['urls_found_count'];
-
-      // Update progress bar
+    if ($resources_total > 0)
+    {
+      // Instanciate an configure progress bar
       if ($options['progress'])
       {
-        $progress_bar->update($resource_extraction_info['resources_parsed_count']);
-      }
-    }
-
-    // Log
-    $this->logSection('extract', sprintf('%d URLs where extracted from %d resources', $urls_found_count, $resources_total));
+        include 'Console/ProgressBar.php';
+        $progress_bar = new Console_ProgressBar(
+          '** '.$arguments['dsn'].' %fraction% resources [%bar%] %percent% | ',
+          '=>', '-', 80, $resources_total, array('ansi_terminal' => true)
+        );
+      }
+
+      // Extract resources from source and insert them in Links database
+      while ($resource_extraction_info = $extractor->extract($arguments['dsn'], $options['connection']))
+      {
+        // Update extraction statistics
+        $urls_found_count += $resource_extraction_info['urls_found_count'];
+
+        // Update progress bar
+        if ($options['progress'])
+        {
+          $progress_bar->update($resource_extraction_info['resources_parsed_count']);
+        }
+      }
+
+      // Log
+      $this->logSection('extract', sprintf('%d URLs where extracted from %d resources', $urls_found_count, $resources_total));
+    }
+    else
+    {
+      $this->logSection('extract', 'No resources to extract. Exiting.');
+    }
   }
 
   /**

=== modified file 'lib/vendor/CI/Search/Link/Segment.php'
--- lib/vendor/CI/Search/Link/Segment.php	2010-06-15 13:21:38 +0000
+++ lib/vendor/CI/Search/Link/Segment.php	2010-06-23 20:53:23 +0000
@@ -59,6 +59,12 @@
     }
     $c->setLimit($limit);
 
+    // default : return links with availability being marked as "available" or "unknown"
+    if ($parameters->get('availability', null) === null)
+    {
+        $c->addField('-availability', 'unavailable');
+    }
+
     // Define sorting
     $sorting_direction = $parameters->get('sort_direction', 'asc');
     if ($sorting_direction == 'desc')
@@ -108,5 +114,4 @@
 
     return array_keys($schema_fields);
   }
-
 }
\ No newline at end of file