datamining.html

<!DOCTYPE html>
<!--
  Copyright 2010 Google Inc.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

  Original slides: Marcin Wichary (mwichary@google.com)
  Modifications: Ernest Delgado (ernestd@google.com)
                 Alex Russell (slightlyoff@chromium.org)

  landslide modifications: Adam Zapletal (adamzap@gmail.com)
                           Nicolas Perriault (nperriault@gmail.com)
-->
<html>
<head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="chrome=1">
    <title>«Web Scraping 101»</title>
    <!-- Styles -->
    
    <link rel="stylesheet" media="print" href="avalanche/css/print.css">
    <link rel="stylesheet" media="screen, projection" href="avalanche/css/screen.css">
    
    
      
      <link rel="stylesheet" href="avalanche/custom_css/pygments/monokai.css">
      
    
      
      <link rel="stylesheet" href="avalanche/css/screen.css">
      
    
    <!-- /Styles -->
    <!-- Javascripts -->
    
    <script type="text/javascript" src="avalanche/js/slides.js"></script>
    
    
    <!-- /Javascripts -->
</head>
<body>
  <div id="blank"></div>
  <div class="presentation">
    <div id="time">
      <span id="hours">00</span>:<span id="minutes">00</span>:<span id="seconds">00</span>
    </div>
    <div id="current_presenter_notes">
      <div id="presenter_note"></div>
    </div>
    <div class="slides">
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>«Web Scraping 101»</h1></header>
            
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              1/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h1>Présentation</h1></header>
            
            
            <section><div class="highlight"><pre><span class="kn">from</span> <span class="nn">Quebec</span> <span class="kn">import</span> <span class="n">Akim</span> <span class="n">Couture</span>
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              2/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Définir «Web Scraping» ?</h1></header>
            
            
            <section><blockquote>
<p>Le Web scraping (parfois appelé Harvesting) est une technique d'extraction du contenu de sites Web, via un script ou un programme, dans le but de le transformer pour permettre son utilisation dans un autre contexte.</p>
</blockquote></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              3/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            
            <section><p><img alt="Scrapy" src="Scrapy_logo.jpg" /></p></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              4/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h3>Pourquoi l'utiliser ?</h3></header>
            
            
            <section><ul>
<li>100% Python !</li>
<li>Un des plus populaires donc grosse communauté</li>
<li>Un environnement pour crée plusieurs «spiders»</li>
</ul></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              5/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Installation</h1></header>
            
            
            <section><h2>Dépendences :</h2>
<ul>
<li>Python 2.7</li>
<li>lxml</li>
<li>OpenSSL</li>
<li>pip or easy_install</li>
</ul></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              6/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            
            <section><div class="highlight"><pre>mkdir scraping
virtualenv venv
<span class="nb">source </span>venv/bin/activate
pip install Scrapy
</pre></div>

<p>Enjoy !</p>
<p>Windows : scrapy docs (vous devez installer toutes les dépences manuellement)</p>
<p>Ubuntu : N'utilisez pas python-scrapy trop vieux !</p></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              7/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Comment ça marche ?</h1></header>
            
            
            <section><p><img alt="" src="scrapy_architecture.png" /></p></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              8/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h1>HN Scraping</h1></header>
            
            
            <section><div class="highlight"><pre>scrapy startproject hn
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              9/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            
            <section><div class="highlight"><pre>hn/

    scrapy.cfg <span class="c"># le fichier de configuration du projet</span>

    hn/ <span class="c"># le module du projet</span>

        __init__.py

        items.py <span class="c"># fichier «items»</span>

        pipelines.py <span class="c"># fichier «pipelines»</span>

        settings.py <span class="c">#</span>

    spiders/ <span class="c"># nos araignées !</span>
        __init__.py
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              10/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h2>Items.py</h2></header>
            
            
            <section><div class="highlight"><pre><span class="kn">from</span> <span class="nn">scrapy.item</span> <span class="kn">import</span> <span class="n">Item</span><span class="p">,</span> <span class="n">Field</span>

<span class="k">class</span> <span class="nc">HnItem</span><span class="p">(</span><span class="n">Item</span><span class="p">):</span>
        <span class="n">title</span> <span class="o">=</span> <span class="n">Field</span><span class="p">()</span>
        <span class="n">link</span> <span class="o">=</span> <span class="n">Field</span><span class="p">()</span>
</pre></div>

<p>Cela permet de ne pas prendre des champs par erreur.</p>
<p>La classe Item est très similaire à un dictionnaire Python.</p></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              11/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h2>Les araignées (spider)</h2></header>
            
            
            <section><blockquote>
<p>Le code qui définit qu'est-ce que notre «crawler» va parser.</p>
</blockquote>
<h2>Nous allons donner de la logique à nos araignées !</h2>
<ul>
<li>nom : pour identifier l'araignée</li>
<li>start_urls : le début d' une liste de liens internet</li>
<li>parse() : La  méthode qui nous envoye une réponse lorsque l'url est téléchargé</li>
</ul></section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              12/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h3>hn_spider.py</h3></header>
            
            
            <section><div class="highlight"><pre><span class="kn">from</span> <span class="nn">scrapy.spider</span> <span class="kn">import</span> <span class="n">BaseSpider</span>
<span class="kn">from</span> <span class="nn">scrapy.selector</span> <span class="kn">import</span> <span class="n">Selector</span>

<span class="k">class</span> <span class="nc">HnSpider</span><span class="p">(</span><span class="n">BaseSpider</span><span class="p">):</span>
    <span class="n">name</span> <span class="o">=</span> <span class="s">&#39;hn&#39;</span> <span class="c">#nom</span>
    <span class="n">allowed_domains</span> <span class="o">=</span> <span class="p">[]</span> <span class="c">#</span>
    <span class="n">start_urls</span> <span class="o">=</span> <span class="p">[</span><span class="s">&#39;http://news.ycombinator.com&#39;</span><span class="p">]</span>

<span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">):</span>
    <span class="n">sel</span> <span class="o">=</span> <span class="n">Selector</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
    <span class="n">sites</span> <span class="o">=</span> <span class="n">sel</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s">&#39;//td[@class=&quot;title&quot;]&#39;</span><span class="p">)</span>
    <span class="k">for</span> <span class="n">site</span> <span class="ow">in</span> <span class="n">sites</span><span class="p">:</span>
        <span class="n">title</span> <span class="o">=</span> <span class="n">site</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s">&#39;a/text()&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">extract</span><span class="p">()</span>
        <span class="n">link</span> <span class="o">=</span> <span class="n">site</span><span class="o">.</span><span class="n">xpath</span><span class="p">(</span><span class="s">&#39;a/@href&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">extract</span><span class="p">()</span>

        <span class="k">print</span> <span class="n">title</span><span class="p">,</span> <span class="n">link</span>
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              13/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Beautiful Soup</h1></header>
            
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              14/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h1>Crawl !</h1></header>
            
            
            <section><div class="highlight"><pre>scrapy crawl hn
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              15/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide has_code">
          <div class="inner">
            
            <header><h1>JSON</h1></header>
            
            
            <section><div class="highlight"><pre>scrapy crawl hn -o items.json -t json
</pre></div>
</section>
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              16/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Next ? Atelier ? web-scraping 102 ?</h1></header>
            
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              17/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Remerciement</h1></header>
            
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              18/19
            </aside>
          </footer>
        </div>
      </div>
      
      <!-- slide source: datamining.md -->
      <div class="slide-wrapper">
        <div class="slide">
          <div class="inner">
            
            <header><h1>Questions ?</h1></header>
            
            
          </div>
          <div class="presenter_notes">
            <header><h1>Notes</h1></header>
            <section>
            
            </section>
          </div>
          <footer>
            
            <aside class="source">
              Source: <a href="datamining.md">datamining.md</a>
            </aside>
            
            <aside class="page_number">
              19/19
            </aside>
          </footer>
        </div>
      </div>
      
    </div>
  </div>

  
  <div id="toc" class="sidebar hidden">
    <h2>Table of Contents</h2>
    <table>
      <caption>Table of Contents</caption>
      
      <tr id="toc-row-1">
        <th><a href="#slide1">«Web Scraping 101»</a></th>
        <td><a href="#slide1">1</a></td>
      </tr>
      
      
      <tr id="toc-row-2">
        <th><a href="#slide2">Présentation</a></th>
        <td><a href="#slide2">2</a></td>
      </tr>
      
      
      <tr id="toc-row-3">
        <th><a href="#slide3">Définir «Web Scraping» ?</a></th>
        <td><a href="#slide3">3</a></td>
      </tr>
      
      
      <tr id="toc-row-4">
        <th><a href="#slide4">-</a></th>
        <td><a href="#slide4">4</a></td>
      </tr>
      
      
      <tr id="toc-row-5">
        <th><a href="#slide5">-</a></th>
        <td><a href="#slide5">5</a></td>
      </tr>
      
      
      <tr id="toc-row-6">
        <th><a href="#slide6">Installation</a></th>
        <td><a href="#slide6">6</a></td>
      </tr>
      
      
      <tr id="toc-row-7">
        <th><a href="#slide7">-</a></th>
        <td><a href="#slide7">7</a></td>
      </tr>
      
      
      <tr id="toc-row-8">
        <th><a href="#slide8">Comment ça marche ?</a></th>
        <td><a href="#slide8">8</a></td>
      </tr>
      
      
      <tr id="toc-row-9">
        <th><a href="#slide9">HN Scraping</a></th>
        <td><a href="#slide9">9</a></td>
      </tr>
      
      
      <tr id="toc-row-10">
        <th><a href="#slide10">-</a></th>
        <td><a href="#slide10">10</a></td>
      </tr>
      
        
        <tr id="toc-row-11" class="sub">
          <th><a href="#slide11">Items.py</a></th>
          <td><a href="#slide11">11</a></td>
        </tr>
        
        <tr id="toc-row-12" class="sub">
          <th><a href="#slide12">Les araignées (spider)</a></th>
          <td><a href="#slide12">12</a></td>
        </tr>
        
      
      
      <tr id="toc-row-13">
        <th><a href="#slide13">-</a></th>
        <td><a href="#slide13">13</a></td>
      </tr>
      
      
      <tr id="toc-row-14">
        <th><a href="#slide14">Beautiful Soup</a></th>
        <td><a href="#slide14">14</a></td>
      </tr>
      
      
      <tr id="toc-row-15">
        <th><a href="#slide15">Crawl !</a></th>
        <td><a href="#slide15">15</a></td>
      </tr>
      
      
      <tr id="toc-row-16">
        <th><a href="#slide16">JSON</a></th>
        <td><a href="#slide16">16</a></td>
      </tr>
      
      
      <tr id="toc-row-17">
        <th><a href="#slide17">Next ? Atelier ? web-scraping 102 ?</a></th>
        <td><a href="#slide17">17</a></td>
      </tr>
      
      
      <tr id="toc-row-18">
        <th><a href="#slide18">Remerciement</a></th>
        <td><a href="#slide18">18</a></td>
      </tr>
      
      
      <tr id="toc-row-19">
        <th><a href="#slide19">Questions ?</a></th>
        <td><a href="#slide19">19</a></td>
      </tr>
      
      
    </table>
  </div>
  
  <div id="help" class="sidebar hidden">
    <h2>Help</h2>
    <table>
      <caption>Help</caption>
      <tr>
        <th>Table of contents</th>
        <td>t</td>
      </tr>
      <tr>
        <th>Exposé</th>
        <td>ESC</td>
      </tr>
      <tr>
        <th>Autoscale</th>
        <td>e</td>
      </tr>
      <tr>
        <th>Full screen slides</th>
        <td>f</td>
      </tr>
      <tr>
        <th>Presenter view</th>
        <td>p</td>
      </tr>
      <tr>
        <th>Source files</th>
        <td>s</td>
      </tr>
      <tr>
        <th>Slide numbers</th>
        <td>n</td>
      </tr>
      <tr>
        <th>Blank screen</th>
        <td>b</td>
      </tr>
      <tr>
        <th>Notes</th>
        <td>2</td>
      </tr>
      <tr>
        <th>Help</th>
        <td>h</td>
      </tr>
    </table>
  </div>
  <script>$().landslide();</script>
</body>
</html>