week9-data-wrangling.html

<!DOCTYPE html>

<html>

<head>

<meta charset="utf-8" />
<meta name="generator" content="pandoc" />
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />


<title>Week 9: Intro to piping and tidyverse data wrangling</title>

<script src="site_libs/header-attrs-2.9/header-attrs.js"></script>
<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="site_libs/bootstrap-3.3.5/css/readable.min.css" rel="stylesheet" />
<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<style>h1 {font-size: 34px;}
       h1.title {font-size: 38px;}
       h2 {font-size: 30px;}
       h3 {font-size: 24px;}
       h4 {font-size: 18px;}
       h5 {font-size: 16px;}
       h6 {font-size: 12px;}
       code {color: inherit; background-color: rgba(0, 0, 0, 0.04);}
       pre:not([class]) { background-color: white }</style>
<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>
<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />
<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>
<script src="site_libs/navigation-1.1/tabsets.js"></script>
<link href="site_libs/highlightjs-9.12.0/textmate.css" rel="stylesheet" />
<script src="site_libs/highlightjs-9.12.0/highlight.js"></script>
<link href="site_libs/font-awesome-5.1.0/css/all.css" rel="stylesheet" />
<link href="site_libs/font-awesome-5.1.0/css/v4-shims.css" rel="stylesheet" />
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">

<style type="text/css">
  code{white-space: pre-wrap;}
  span.smallcaps{font-variant: small-caps;}
  span.underline{text-decoration: underline;}
  div.column{display: inline-block; vertical-align: top; width: 50%;}
  div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
  ul.task-list{list-style: none;}
    </style>

<style type="text/css">code{white-space: pre;}</style>
<script type="text/javascript">
if (window.hljs) {
  hljs.configure({languages: []});
  hljs.initHighlightingOnLoad();
  if (document.readyState && document.readyState === "complete") {
    window.setTimeout(function() { hljs.initHighlighting(); }, 0);
  }
}
</script>


<link rel="stylesheet" href="styles.css" type="text/css" />


<style type = "text/css">
.main-container {
  max-width: 940px;
  margin-left: auto;
  margin-right: auto;
}
img {
  max-width:100%;
}
.tabbed-pane {
  padding-top: 12px;
}
.html-widget {
  margin-bottom: 20px;
}
button.code-folding-btn:focus {
  outline: none;
}
summary {
  display: list-item;
}
pre code {
  padding: 0;
}
</style>


<style type="text/css">
.dropdown-submenu {
  position: relative;
}
.dropdown-submenu>.dropdown-menu {
  top: 0;
  left: 100%;
  margin-top: -6px;
  margin-left: -1px;
  border-radius: 0 6px 6px 6px;
}
.dropdown-submenu:hover>.dropdown-menu {
  display: block;
}
.dropdown-submenu>a:after {
  display: block;
  content: " ";
  float: right;
  width: 0;
  height: 0;
  border-color: transparent;
  border-style: solid;
  border-width: 5px 0 5px 5px;
  border-left-color: #cccccc;
  margin-top: 5px;
  margin-right: -10px;
}
.dropdown-submenu:hover>a:after {
  border-left-color: #adb5bd;
}
.dropdown-submenu.pull-left {
  float: none;
}
.dropdown-submenu.pull-left>.dropdown-menu {
  left: -100%;
  margin-left: 10px;
  border-radius: 6px 0 6px 6px;
}
</style>

<script type="text/javascript">
// manage active state of menu based on current page
$(document).ready(function () {
  // active menu anchor
  href = window.location.pathname
  href = href.substr(href.lastIndexOf('/') + 1)
  if (href === "")
    href = "index.html";
  var menuAnchor = $('a[href="' + href + '"]');

  // mark it active
  menuAnchor.tab('show');

  // if it's got a parent navbar menu mark it active as well
  menuAnchor.closest('li.dropdown').addClass('active');

  // Navbar adjustments
  var navHeight = $(".navbar").first().height() + 15;
  var style = document.createElement('style');
  var pt = "padding-top: " + navHeight + "px; ";
  var mt = "margin-top: -" + navHeight + "px; ";
  var css = "";
  // offset scroll position for anchor links (for fixed navbar)
  for (var i = 1; i <= 6; i++) {
    css += ".section h" + i + "{ " + pt + mt + "}\n";
  }
  style.innerHTML = "body {" + pt + "padding-bottom: 40px; }\n" + css;
  document.head.appendChild(style);
});
</script>

<!-- tabsets -->

<style type="text/css">
.tabset-dropdown > .nav-tabs {
  display: inline-table;
  max-height: 500px;
  min-height: 44px;
  overflow-y: auto;
  border: 1px solid #ddd;
  border-radius: 4px;
}

.tabset-dropdown > .nav-tabs > li.active:before {
  content: "";
  font-family: 'Glyphicons Halflings';
  display: inline-block;
  padding: 10px;
  border-right: 1px solid #ddd;
}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li.active:before {
  content: "&#xe258;";
  border: none;
}

.tabset-dropdown > .nav-tabs.nav-tabs-open:before {
  content: "";
  font-family: 'Glyphicons Halflings';
  display: inline-block;
  padding: 10px;
  border-right: 1px solid #ddd;
}

.tabset-dropdown > .nav-tabs > li.active {
  display: block;
}

.tabset-dropdown > .nav-tabs > li > a,
.tabset-dropdown > .nav-tabs > li > a:focus,
.tabset-dropdown > .nav-tabs > li > a:hover {
  border: none;
  display: inline-block;
  border-radius: 4px;
  background-color: transparent;
}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li {
  display: block;
  float: none;
}

.tabset-dropdown > .nav-tabs > li {
  display: none;
}
</style>

<!-- code folding -->


<style type="text/css">

#TOC {
  margin: 25px 0px 20px 0px;
}
@media (max-width: 768px) {
#TOC {
  position: relative;
  width: 100%;
}
}

@media print {
.toc-content {
  /* see https://github.com/w3c/csswg-drafts/issues/4434 */
  float: right;
}
}

.toc-content {
  padding-left: 30px;
  padding-right: 40px;
}

div.main-container {
  max-width: 1200px;
}

div.tocify {
  width: 20%;
  max-width: 260px;
  max-height: 85%;
}

@media (min-width: 768px) and (max-width: 991px) {
  div.tocify {
    width: 25%;
  }
}

@media (max-width: 767px) {
  div.tocify {
    width: 100%;
    max-width: none;
  }
}

.tocify ul, .tocify li {
  line-height: 20px;
}

.tocify-subheader .tocify-item {
  font-size: 0.90em;
}

.tocify .list-group-item {
  border-radius: 0px;
}

.tocify-subheader {
  display: inline;
}
.tocify-subheader .tocify-item {
  font-size: 0.95em;
}

</style>


</head>

<body>


<div class="container-fluid main-container">


<!-- setup 3col/9col grid for toc_float and main content  -->
<div class="row">
<div class="col-xs-12 col-sm-4 col-md-3">
<div id="TOC" class="tocify">
</div>
</div>

<div class="toc-content col-xs-12 col-sm-8 col-md-9">


<div class="navbar navbar-default  navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="index.html">RWorkflow</a>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav">
        <li>
  <a href="set-up.html">
    <span class="fa fa-clone"></span>
     
    Set-up
  </a>
</li>
<li class="dropdown">
  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
    <span class="fa fa-calendar"></span>
     
    Workshop Weekly Notes
     
    <span class="caret"></span>
  </a>
  <ul class="dropdown-menu" role="menu">
    <li>
      <a href="week1.html">
        <span class="fa fa-cogs"></span>
         
        Week 1 RStudio + Version Control
      </a>
    </li>
    <li>
      <a href="week2.html">
        <span class="fa fa-gitalt"></span>
         
        Week 2 More Git/GitHub
      </a>
    </li>
    <li>
      <a href="week3.html">
        <span class="fab fa-github"></span>
         
        Week 3 GitHub
      </a>
    </li>
  </ul>
</li>
<li>
  <a href="https://rverse-tutorials.github.io/RWorkflow/">
    <span class="fa fa-folder"></span>
     
    Archive
  </a>
</li>
<li>
  <a href="https://rverse-tutorials.github.io/RWorkflow-NWFSC-2020">
    <span class="fa fa-clock"></span>
     
    2020 website
  </a>
</li>
<li>
  <a href="links.html">
    <span class="fa fa-info"></span>
     
    More resources
  </a>
</li>
      </ul>
      <ul class="nav navbar-nav navbar-right">
        <li>
  <a href="https://github.com/RVerse-Tutorials/RWorkflow-NWFSC-2021">
    <span class="fab fa-github"></span>
     
    Repo
  </a>
</li>
<li>
  <a href="https://github.com/orgs/RWorkflow-Workshop-2021">Org</a>
</li>
      </ul>
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

<div id="header">


<h1 class="title toc-ignore">Week 9: Intro to piping and tidyverse data wrangling</h1>

</div>


<p>This week I will give a brief introduction to piping and data wrangling with tidyr and tidyverse verbs. If you want to learn more, there are many resources on the web. I like <a href="https://dcl-wrangle.stanford.edu/">this course book</a> as a beginning, <a href="https://cengel.github.io/R-data-wrangling/">this course</a> goes into more depth, and <a href="https://r4ds.had.co.nz/">R for Data Science</a> has much more.</p>
<p>Today’s goal is to teach you 6 data “verbs” that are easy to use and that will cover many of your data-wrangling needs.</p>
<pre class="r"><code>library(dplyr)
library(tidyr)
library(ggplot2)</code></pre>
<div id="piping" class="section level1">
<h1>Piping</h1>
<p>You already use the concept of “piping”, but you do it with parentheses. Your pipe in this case goes from inside to outside.</p>
<pre><code>fun4(fun3(fun2(fun1(x))))</code></pre>
<p><code>x</code> is our initial object. We apply <code>fun1()</code> to that. Then that result goes into <code>fun2()</code>. The result of that goes into <code>fun3()</code>. etc. “Pipe” is referring to the flow of information, the “goes into” part.</p>
<p>This has many problems.</p>
<ul>
<li>You will lose track of your parentheses and waste much time sorting out where you lost one of your parentheses.</li>
<li>You are limited in how many levels you can add without your code being unreadable as it stretches quickly to multiple lines.</li>
<li>We don’t read inside to outside though from our math education, we are used to that (sort of).</li>
</ul>
<p>Piping is just a different way to write the same thing but left to right like we read (in English).</p>
<pre><code>x %&gt;% fun1() %&gt;% fun2() %&gt;% fun3() %&gt;% fun4()</code></pre>
<p>Example:</p>
<pre class="r"><code>1:10 %&gt;% sqrt() %&gt;% mean() %&gt;% round(digits=1) </code></pre>
<pre><code>## [1] 2.2</code></pre>
<p>That’s the same as</p>
<pre class="r"><code>round(mean(sqrt(1:10)),digits=1) </code></pre>
<pre><code>## [1] 2.2</code></pre>
<div id="upsides" class="section level2">
<h2>Upsides</h2>
<ul>
<li>It is used a lot in the tidyverse so you should be aware of how it works.</li>
<li>It works well in the tidyverse.</li>
<li>Can be more readable.</li>
<li>Makes it easy to avoid intermediate variables (<code>tmp</code> and <code>foo</code>).</li>
</ul>
</div>
<div id="downsides" class="section level2">
<h2>Downsides</h2>
<ul>
<li>It’s is easy to write code that is hard to understand. It tends to hide where the object on the left is going to in the function on the right.</li>
</ul>
<p>So you see code like this</p>
<pre class="r"><code>fun &lt;- function(a=1, b=10){c(max(a), min(b))}
a &lt;- 2:10
a %&gt;% fun(a^2)</code></pre>
<pre><code>## [1] 10  4</code></pre>
<p>and you have to go research <code>fun</code> to discover that this is <code>fun(a=a, b=a^2)</code>.</p>
<p>This kind of mistake is so easy to make. You want say, <code>rnorm(n=a)</code>, and accidentally write that in your pipe.</p>
<pre class="r"><code>a &lt;- 3
a %&gt;% rnorm(n=a)</code></pre>
<pre><code>## [1] 1.792934 3.277429 4.084441</code></pre>
<pre class="r"><code>a %&gt;% rnorm()</code></pre>
<pre><code>## [1] -2.3456977  0.4291247  0.5060559</code></pre>
<p>The left flows into the first function spot, UNLESS you specified that and then it flows to the second or third or whatever is “open”.</p>
<ul>
<li><p>You can spend hours trying to solve your piping issue when you could immediately write down code using intermediate values.</p></li>
<li><p>It’s really slow.</p></li>
</ul>
<pre class="r"><code>library(microbenchmark)
bm &lt;- microbenchmark(1:10 %&gt;% sqrt() %&gt;% mean() %&gt;% round(digits=1), round(mean(sqrt(1:10)),digits=1) )
ggplot2::autoplot(bm)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-6-1.png" width="672" /></p>
<ul>
<li>You must stay hyper-alert as to the names in the data frame you are working with. You might not have created this data frame. This may be a function you wrote to take any data frame from a user…. Otherwise, bad things like this can happen.</li>
</ul>
<pre class="r"><code>fun &lt;- function(x, cyl=1){
x %&gt;% transform(new = mpg^cyl) %&gt;% select(mpg, new)
}</code></pre>
<p>Works great!</p>
<pre class="r"><code>df &lt;- data.frame(mpg=runif(10, 10, 30))
fun(df, cyl=0.5)</code></pre>
<pre><code>##         mpg      new
## 1  16.89275 4.110079
## 2  17.38352 4.169355
## 3  18.58504 4.311037
## 4  28.37029 5.326377
## 5  25.68690 5.068224
## 6  24.75663 4.975604
## 7  15.61545 3.951639
## 8  19.13653 4.374532
## 9  15.75075 3.968722
## 10 23.92582 4.891403</code></pre>
<p>Why, why? It didn’t complain but this is wrong!</p>
<pre class="r"><code>fun(mtcars[1:5,], cyl=0.5)</code></pre>
<pre><code>##                    mpg          new
## Mazda RX4         21.0 8.576612e+07
## Mazda RX4 Wag     21.0 8.576612e+07
## Datsun 710        22.8 2.702336e+05
## Hornet 4 Drive    21.4 9.604674e+07
## Hornet Sportabout 18.7 1.495316e+10</code></pre>
</div>
</div>
<div id="tidyverse-verbs" class="section level1">
<h1>Tidyverse verbs</h1>
<p>Read up on tidyverse verbs <a href="https://dplyr.tidyverse.org/">here</a>.</p>
<p>NOTE:</p>
<ul>
<li>I am using piping because that is how you will see these verbs used in online material. They are just functions and you do not need to use them with piping.</li>
<li>They are very slow. For loops are not the place for these functions. <code>apply()</code> in base R is much faster.</li>
<li>Tidyverse is maturing and the semantics across functions is not quite the same. Careful.</li>
</ul>
<div id="workflow" class="section level2">
<h2>Workflow</h2>
<pre><code>data.frame %&gt;%
  select columns or filter rows %&gt;%
  summarize or mutate</code></pre>
<pre><code>data.frame %&gt;%
  group data %&gt;%
  summarize or mutate</code></pre>
<p>Verbs that we will learn:</p>
<ul>
<li><code>select()</code> select columns</li>
<li><code>filter()</code> select rows</li>
<li><code>group_by()</code> apply the function differently within groups.</li>
<li><code>summarize()</code> apply function over columns -&gt; fewer rows. Like <code>apply(df,2,function)</code></li>
<li><code>mutate()</code> apply function to columns and make new column. Like <code>apply(df,1,function)</code></li>
</ul>
<p>What we won’t learn: applying a function across a group of columns. See ?across to learn about that.</p>
</div>
<div id="selecting-and-filtering" class="section level2">
<h2>Selecting and filtering</h2>
<pre><code>data.frame %&gt;%
  select or filter</code></pre>
<div id="select" class="section level3">
<h3><code>select()</code></h3>
<p>Select columns.</p>
<pre class="r"><code>head(mtcars)</code></pre>
<pre><code>##                    mpg cyl disp  hp drat    wt  qsec vs am gear carb
## Mazda RX4         21.0   6  160 110 3.90 2.620 16.46  0  1    4    4
## Mazda RX4 Wag     21.0   6  160 110 3.90 2.875 17.02  0  1    4    4
## Datsun 710        22.8   4  108  93 3.85 2.320 18.61  1  1    4    1
## Hornet 4 Drive    21.4   6  258 110 3.08 3.215 19.44  1  0    3    1
## Hornet Sportabout 18.7   8  360 175 3.15 3.440 17.02  0  0    3    2
## Valiant           18.1   6  225 105 2.76 3.460 20.22  1  0    3    1</code></pre>
<pre class="r"><code>mtcars %&gt;% select(mpg)</code></pre>
<pre><code>##                      mpg
## Mazda RX4           21.0
## Mazda RX4 Wag       21.0
## Datsun 710          22.8
## Hornet 4 Drive      21.4
## Hornet Sportabout   18.7
## Valiant             18.1
## Duster 360          14.3
## Merc 240D           24.4
## Merc 230            22.8
## Merc 280            19.2
## Merc 280C           17.8
## Merc 450SE          16.4
## Merc 450SL          17.3
## Merc 450SLC         15.2
## Cadillac Fleetwood  10.4
## Lincoln Continental 10.4
## Chrysler Imperial   14.7
## Fiat 128            32.4
## Honda Civic         30.4
## Toyota Corolla      33.9
## Toyota Corona       21.5
## Dodge Challenger    15.5
## AMC Javelin         15.2
## Camaro Z28          13.3
## Pontiac Firebird    19.2
## Fiat X1-9           27.3
## Porsche 914-2       26.0
## Lotus Europa        30.4
## Ford Pantera L      15.8
## Ferrari Dino        19.7
## Maserati Bora       15.0
## Volvo 142E          21.4</code></pre>
<pre class="r"><code>mtcars %&gt;% select(-mpg, -qsec, -gear, -cyl)</code></pre>
<pre><code>##                      disp  hp drat    wt vs am carb
## Mazda RX4           160.0 110 3.90 2.620  0  1    4
## Mazda RX4 Wag       160.0 110 3.90 2.875  0  1    4
## Datsun 710          108.0  93 3.85 2.320  1  1    1
## Hornet 4 Drive      258.0 110 3.08 3.215  1  0    1
## Hornet Sportabout   360.0 175 3.15 3.440  0  0    2
## Valiant             225.0 105 2.76 3.460  1  0    1
## Duster 360          360.0 245 3.21 3.570  0  0    4
## Merc 240D           146.7  62 3.69 3.190  1  0    2
## Merc 230            140.8  95 3.92 3.150  1  0    2
## Merc 280            167.6 123 3.92 3.440  1  0    4
## Merc 280C           167.6 123 3.92 3.440  1  0    4
## Merc 450SE          275.8 180 3.07 4.070  0  0    3
## Merc 450SL          275.8 180 3.07 3.730  0  0    3
## Merc 450SLC         275.8 180 3.07 3.780  0  0    3
## Cadillac Fleetwood  472.0 205 2.93 5.250  0  0    4
## Lincoln Continental 460.0 215 3.00 5.424  0  0    4
## Chrysler Imperial   440.0 230 3.23 5.345  0  0    4
## Fiat 128             78.7  66 4.08 2.200  1  1    1
## Honda Civic          75.7  52 4.93 1.615  1  1    2
## Toyota Corolla       71.1  65 4.22 1.835  1  1    1
## Toyota Corona       120.1  97 3.70 2.465  1  0    1
## Dodge Challenger    318.0 150 2.76 3.520  0  0    2
## AMC Javelin         304.0 150 3.15 3.435  0  0    2
## Camaro Z28          350.0 245 3.73 3.840  0  0    4
## Pontiac Firebird    400.0 175 3.08 3.845  0  0    2
## Fiat X1-9            79.0  66 4.08 1.935  1  1    1
## Porsche 914-2       120.3  91 4.43 2.140  0  1    2
## Lotus Europa         95.1 113 3.77 1.513  1  1    2
## Ford Pantera L      351.0 264 4.22 3.170  0  1    4
## Ferrari Dino        145.0 175 3.62 2.770  0  1    6
## Maserati Bora       301.0 335 3.54 3.570  0  1    8
## Volvo 142E          121.0 109 4.11 2.780  1  1    2</code></pre>
<pre class="r"><code>mtcars %&gt;% select(disp:qsec)</code></pre>
<pre><code>##                      disp  hp drat    wt  qsec
## Mazda RX4           160.0 110 3.90 2.620 16.46
## Mazda RX4 Wag       160.0 110 3.90 2.875 17.02
## Datsun 710          108.0  93 3.85 2.320 18.61
## Hornet 4 Drive      258.0 110 3.08 3.215 19.44
## Hornet Sportabout   360.0 175 3.15 3.440 17.02
## Valiant             225.0 105 2.76 3.460 20.22
## Duster 360          360.0 245 3.21 3.570 15.84
## Merc 240D           146.7  62 3.69 3.190 20.00
## Merc 230            140.8  95 3.92 3.150 22.90
## Merc 280            167.6 123 3.92 3.440 18.30
## Merc 280C           167.6 123 3.92 3.440 18.90
## Merc 450SE          275.8 180 3.07 4.070 17.40
## Merc 450SL          275.8 180 3.07 3.730 17.60
## Merc 450SLC         275.8 180 3.07 3.780 18.00
## Cadillac Fleetwood  472.0 205 2.93 5.250 17.98
## Lincoln Continental 460.0 215 3.00 5.424 17.82
## Chrysler Imperial   440.0 230 3.23 5.345 17.42
## Fiat 128             78.7  66 4.08 2.200 19.47
## Honda Civic          75.7  52 4.93 1.615 18.52
## Toyota Corolla       71.1  65 4.22 1.835 19.90
## Toyota Corona       120.1  97 3.70 2.465 20.01
## Dodge Challenger    318.0 150 2.76 3.520 16.87
## AMC Javelin         304.0 150 3.15 3.435 17.30
## Camaro Z28          350.0 245 3.73 3.840 15.41
## Pontiac Firebird    400.0 175 3.08 3.845 17.05
## Fiat X1-9            79.0  66 4.08 1.935 18.90
## Porsche 914-2       120.3  91 4.43 2.140 16.70
## Lotus Europa         95.1 113 3.77 1.513 16.90
## Ford Pantera L      351.0 264 4.22 3.170 14.50
## Ferrari Dino        145.0 175 3.62 2.770 15.50
## Maserati Bora       301.0 335 3.54 3.570 14.60
## Volvo 142E          121.0 109 4.11 2.780 18.60</code></pre>
<pre class="r"><code>mtcars %&gt;% select(!disp:qsec)</code></pre>
<pre><code>##                      mpg cyl vs am gear carb
## Mazda RX4           21.0   6  0  1    4    4
## Mazda RX4 Wag       21.0   6  0  1    4    4
## Datsun 710          22.8   4  1  1    4    1
## Hornet 4 Drive      21.4   6  1  0    3    1
## Hornet Sportabout   18.7   8  0  0    3    2
## Valiant             18.1   6  1  0    3    1
## Duster 360          14.3   8  0  0    3    4
## Merc 240D           24.4   4  1  0    4    2
## Merc 230            22.8   4  1  0    4    2
## Merc 280            19.2   6  1  0    4    4
## Merc 280C           17.8   6  1  0    4    4
## Merc 450SE          16.4   8  0  0    3    3
## Merc 450SL          17.3   8  0  0    3    3
## Merc 450SLC         15.2   8  0  0    3    3
## Cadillac Fleetwood  10.4   8  0  0    3    4
## Lincoln Continental 10.4   8  0  0    3    4
## Chrysler Imperial   14.7   8  0  0    3    4
## Fiat 128            32.4   4  1  1    4    1
## Honda Civic         30.4   4  1  1    4    2
## Toyota Corolla      33.9   4  1  1    4    1
## Toyota Corona       21.5   4  1  0    3    1
## Dodge Challenger    15.5   8  0  0    3    2
## AMC Javelin         15.2   8  0  0    3    2
## Camaro Z28          13.3   8  0  0    3    4
## Pontiac Firebird    19.2   8  0  0    3    2
## Fiat X1-9           27.3   4  1  1    4    1
## Porsche 914-2       26.0   4  0  1    5    2
## Lotus Europa        30.4   4  1  1    5    2
## Ford Pantera L      15.8   8  0  1    5    4
## Ferrari Dino        19.7   6  0  1    5    6
## Maserati Bora       15.0   8  0  1    5    8
## Volvo 142E          21.4   4  1  1    4    2</code></pre>
<pre class="r"><code>df %&gt;% select(where(is.numeric))</code></pre>
<pre><code>##         mpg
## 1  16.89275
## 2  17.38352
## 3  18.58504
## 4  28.37029
## 5  25.68690
## 6  24.75663
## 7  15.61545
## 8  19.13653
## 9  15.75075
## 10 23.92582</code></pre>
</div>
<div id="filter" class="section level3">
<h3><code>filter()</code></h3>
<p>Select rows.</p>
<pre class="r"><code>mtcars %&gt;% filter(cyl==4)</code></pre>
<pre><code>##                 mpg cyl  disp  hp drat    wt  qsec vs am gear carb
## Datsun 710     22.8   4 108.0  93 3.85 2.320 18.61  1  1    4    1
## Merc 240D      24.4   4 146.7  62 3.69 3.190 20.00  1  0    4    2
## Merc 230       22.8   4 140.8  95 3.92 3.150 22.90  1  0    4    2
## Fiat 128       32.4   4  78.7  66 4.08 2.200 19.47  1  1    4    1
## Honda Civic    30.4   4  75.7  52 4.93 1.615 18.52  1  1    4    2
## Toyota Corolla 33.9   4  71.1  65 4.22 1.835 19.90  1  1    4    1
## Toyota Corona  21.5   4 120.1  97 3.70 2.465 20.01  1  0    3    1
## Fiat X1-9      27.3   4  79.0  66 4.08 1.935 18.90  1  1    4    1
## Porsche 914-2  26.0   4 120.3  91 4.43 2.140 16.70  0  1    5    2
## Lotus Europa   30.4   4  95.1 113 3.77 1.513 16.90  1  1    5    2
## Volvo 142E     21.4   4 121.0 109 4.11 2.780 18.60  1  1    4    2</code></pre>
<p>Same as <code>subset(df, cyl==4)</code> or <code>df[df$cyl==4,]</code></p>
<pre class="r"><code>mtcars %&gt;% filter(cyl==4, hp&gt;90)</code></pre>
<pre><code>##                mpg cyl  disp  hp drat    wt  qsec vs am gear carb
## Datsun 710    22.8   4 108.0  93 3.85 2.320 18.61  1  1    4    1
## Merc 230      22.8   4 140.8  95 3.92 3.150 22.90  1  0    4    2
## Toyota Corona 21.5   4 120.1  97 3.70 2.465 20.01  1  0    3    1
## Porsche 914-2 26.0   4 120.3  91 4.43 2.140 16.70  0  1    5    2
## Lotus Europa  30.4   4  95.1 113 3.77 1.513 16.90  1  1    5    2
## Volvo 142E    21.4   4 121.0 109 4.11 2.780 18.60  1  1    4    2</code></pre>
<p>Same as <code>subset(df, cyl==4 &amp; hp&gt;90)</code> or <code>df[df$cyl==4 &amp; df$hp&gt;90,]</code>.</p>
</div>
</div>
<div id="summarize" class="section level2">
<h2><code>summarize()</code></h2>
<p><code>summarize()</code> summarizes down columns and leads to fewer rows. <strong>You must tell it the columns to work on.</strong></p>
<pre><code>data.frame %&gt;%
  summarize</code></pre>
<p>Apply a function to columns.</p>
<pre class="r"><code>mtcars %&gt;% 
  summarize(mean.wt=mean(wt))</code></pre>
<pre><code>##   mean.wt
## 1 3.21725</code></pre>
<p>I can add on more functions.</p>
<pre class="r"><code>mtcars %&gt;% 
  summarize(mean.wt=mean(wt)) %&gt;%
  round(digits=2)</code></pre>
<pre><code>##   mean.wt
## 1    3.22</code></pre>
<p>Same as</p>
<pre class="r"><code>round(mean(mtcars$wt), digits=2)</code></pre>
<pre><code>## [1] 3.22</code></pre>
<p>You can specify multiple columns. You have to put in <code>c()</code> in this case.</p>
<pre class="r"><code>mtcars %&gt;% 
  summarize(mean.wt=mean(c(mpg, hp)))</code></pre>
<pre><code>##    mean.wt
## 1 83.38906</code></pre>
<p>Same as</p>
<pre class="r"><code>mean(c(mtcars$mpg, mtcars$hp))</code></pre>
<pre><code>## [1] 83.38906</code></pre>
<p>Sadly this does NOT work for specifying a range. It computes something but not what you’d think. <strong>This is the <code>apply across columns</code> case.</strong> Read up on that in <code>?across</code>.</p>
<pre class="r"><code>mtcars %&gt;% 
  summarize(mean.wt=mean(mpg:hp))</code></pre>
<pre><code>## Warning in mpg:hp: numerical expression has 32 elements: only the first used

## Warning in mpg:hp: numerical expression has 32 elements: only the first used</code></pre>
<pre><code>##   mean.wt
## 1    65.5</code></pre>
<p>This is computing the mean of this:</p>
<pre class="r"><code>mtcars[1,&quot;mpg&quot;]:mtcars[1,&quot;hp&quot;]</code></pre>
<pre><code>##  [1]  21  22  23  24  25  26  27  28  29  30  31  32  33  34  35  36  37  38  39
## [20]  40  41  42  43  44  45  46  47  48  49  50  51  52  53  54  55  56  57  58
## [39]  59  60  61  62  63  64  65  66  67  68  69  70  71  72  73  74  75  76  77
## [58]  78  79  80  81  82  83  84  85  86  87  88  89  90  91  92  93  94  95  96
## [77]  97  98  99 100 101 102 103 104 105 106 107 108 109 110</code></pre>
<p>which is</p>
<pre class="r"><code>mean(mtcars[1,&quot;mpg&quot;]:mtcars[1,&quot;hp&quot;])</code></pre>
<pre><code>## [1] 65.5</code></pre>
</div>
<div id="summarize-by-column" class="section level2">
<h2><code>summarize()</code> by column</h2>
<p><code>summarize_all()</code> and <code>summarize_at()</code></p>
<p>Apply to a range of columns or all columns. Note the <code>_at()</code> and <code>_all()</code> functions have been superseded with <code>across()</code> but I find <code>across()</code> really cumbersome. I personally would use <code>select()</code> and <code>summarize_all()</code>.</p>
<pre class="r"><code>mtcars %&gt;% summarize_at(vars(mpg:hp), mean)</code></pre>
<pre><code>##        mpg    cyl     disp       hp
## 1 20.09062 6.1875 230.7219 146.6875</code></pre>
<pre><code>data.frame %&gt;%
  select columns %&gt;%
  use summarize_all()</code></pre>
<pre class="r"><code>mtcars %&gt;% 
  select(mpg:hp) %&gt;% 
  summarize_all(mean)</code></pre>
<pre><code>##        mpg    cyl     disp       hp
## 1 20.09062 6.1875 230.7219 146.6875</code></pre>
<pre class="r"><code>apply(mtcars[,1:4],2,mean)</code></pre>
<pre><code>##       mpg       cyl      disp        hp 
##  20.09062   6.18750 230.72188 146.68750</code></pre>
<p><code>summarize_all()</code> is less picky than <code>apply()</code></p>
<pre class="r"><code>df &lt;- mtcars[,1:3]; df$label=&quot;test&quot;
df %&gt;% summarize_all(mean)</code></pre>
<pre><code>## Warning in mean.default(label): argument is not numeric or logical: returning NA</code></pre>
<pre><code>##        mpg    cyl     disp label
## 1 20.09062 6.1875 230.7219    NA</code></pre>
<p>while <code>apply()</code> gives all NA.</p>
<pre class="r"><code>apply(df,2,mean)</code></pre>
<pre><code>## Warning in mean.default(newX[, i], ...): argument is not numeric or logical:
## returning NA

## Warning in mean.default(newX[, i], ...): argument is not numeric or logical:
## returning NA

## Warning in mean.default(newX[, i], ...): argument is not numeric or logical:
## returning NA

## Warning in mean.default(newX[, i], ...): argument is not numeric or logical:
## returning NA</code></pre>
<pre><code>##   mpg   cyl  disp label 
##    NA    NA    NA    NA</code></pre>
<p>Another example.</p>
<pre><code>data.frame %&gt;%
  select rows %&gt;%
  select columns %&gt;%
  use summarize_all() verb to apply %&gt;%
  format with round()</code></pre>
<pre class="r"><code>mtcars %&gt;% 
  filter(cyl==4) %&gt;% 
  select(mpg:hp) %&gt;% 
  summarize_all(mean) %&gt;%
  round(digits=2) %&gt;%
  paste(collapse=&quot; -- &quot;)</code></pre>
<pre><code>## [1] &quot;26.66 -- 4 -- 105.14 -- 82.64&quot;</code></pre>
</div>
<div id="summarize-by-group" class="section level2">
<h2><code>summarize()</code> by group</h2>
<pre><code>data.frame %&gt;%
  define groups %&gt;%
  apply summarize within groups</code></pre>
<pre class="r"><code>df2 &lt;- data.frame(id=paste0(rep(&quot;s&quot;,8),1:2),
                  pop=paste0(rep(&quot;r&quot;,8),rep(1:2,each=4)),
                  Week1=rnorm(8), Week2=rnorm(8), Week3=rnorm(8), Week4=rnorm(8))
head(df2)</code></pre>
<pre><code>##   id pop      Week1      Week2      Week3      Week4
## 1 s1  r1  0.9182511 -0.1193895  1.0884661 -1.9132871
## 2 s2  r1 -0.2185388  2.3914560 -1.0225778 -0.2261837
## 3 s1  r1 -0.6963352  0.7349524 -0.3393304 -0.6063149
## 4 s2  r1  0.6997282  0.4736129 -0.8914761  2.0200296
## 5 s1  r2 -1.1981578 -0.2337429  1.4078441 -0.4946306
## 6 s2  r2 -0.4986149 -0.8535842  1.1866608  0.4332016</code></pre>
<p>The <code>group_by()</code> function allows you to do more complicated things. Take the mean by pop.</p>
<pre class="r"><code>df2 %&gt;% group_by(pop) %&gt;% summarize_all(mean)</code></pre>
<pre><code>## Warning in mean.default(id): argument is not numeric or logical: returning NA

## Warning in mean.default(id): argument is not numeric or logical: returning NA</code></pre>
<pre><code>## # A tibble: 2 x 6
##   pop      id  Week1   Week2  Week3  Week4
##   &lt;chr&gt; &lt;dbl&gt;  &lt;dbl&gt;   &lt;dbl&gt;  &lt;dbl&gt;  &lt;dbl&gt;
## 1 r1       NA 0.176   0.870  -0.291 -0.181
## 2 r2       NA 0.0591 -0.0867  0.361 -0.276</code></pre>
<pre class="r"><code>apply(df2[df2$pop==&quot;r1&quot;,3:6],2,mean)</code></pre>
<pre><code>##      Week1      Week2      Week3      Week4 
##  0.1757763  0.8701579 -0.2912295 -0.1814390</code></pre>
<p>Take the mean for all weeks by pop and id. Notice that I have to specify columns for <code>summarize()</code>.</p>
<pre class="r"><code>df2 %&gt;% 
  group_by(pop, id) %&gt;%
  summarize(mean=mean(c(Week1, Week2, Week3, Week4)))</code></pre>
<pre><code>## `summarise()` has grouped output by &#39;pop&#39;. You can override using the `.groups` argument.</code></pre>
<pre><code>## # A tibble: 4 x 3
## # Groups:   pop [2]
##   pop   id      mean
##   &lt;chr&gt; &lt;chr&gt;  &lt;dbl&gt;
## 1 r1    s1    -0.117
## 2 r1    s2     0.403
## 3 r2    s1    -0.154
## 4 r2    s2     0.183</code></pre>
</div>
<div id="combine-group-and-select" class="section level2">
<h2>Combine group and select</h2>
<p>Select based on properties. For some reason you have to wrap in <code>where()</code>.</p>
<pre class="r"><code>df2 %&gt;% group_by(pop) %&gt;% 
  select(where(is.numeric)) %&gt;% 
  summarize_all(mean)</code></pre>
<pre><code>## Adding missing grouping variables: `pop`</code></pre>
<pre><code>## # A tibble: 2 x 5
##   pop    Week1   Week2  Week3  Week4
##   &lt;chr&gt;  &lt;dbl&gt;   &lt;dbl&gt;  &lt;dbl&gt;  &lt;dbl&gt;
## 1 r1    0.176   0.870  -0.291 -0.181
## 2 r2    0.0591 -0.0867  0.361 -0.276</code></pre>
<p>Select based on name.</p>
<pre class="r"><code>df2 %&gt;% group_by(pop) %&gt;% 
  select(ends_with(&quot;2&quot;)) %&gt;% 
  summarize_all(mean)</code></pre>
<pre><code>## Adding missing grouping variables: `pop`</code></pre>
<pre><code>## # A tibble: 2 x 2
##   pop     Week2
##   &lt;chr&gt;   &lt;dbl&gt;
## 1 r1     0.870 
## 2 r2    -0.0867</code></pre>
</div>
<div id="mutate" class="section level2">
<h2><code>mutate()</code></h2>
<p>Apply a function across rows and return a new column. Like <code>apply(df,1,fun)</code>.</p>
<pre class="r"><code>mtcars %&gt;% 
   select(mpg:wt) %&gt;%
   mutate(wt.hp=wt/hp)</code></pre>
<pre><code>##                      mpg cyl  disp  hp drat    wt      wt.hp
## Mazda RX4           21.0   6 160.0 110 3.90 2.620 0.02381818
## Mazda RX4 Wag       21.0   6 160.0 110 3.90 2.875 0.02613636
## Datsun 710          22.8   4 108.0  93 3.85 2.320 0.02494624
## Hornet 4 Drive      21.4   6 258.0 110 3.08 3.215 0.02922727
## Hornet Sportabout   18.7   8 360.0 175 3.15 3.440 0.01965714
## Valiant             18.1   6 225.0 105 2.76 3.460 0.03295238
## Duster 360          14.3   8 360.0 245 3.21 3.570 0.01457143
## Merc 240D           24.4   4 146.7  62 3.69 3.190 0.05145161
## Merc 230            22.8   4 140.8  95 3.92 3.150 0.03315789
## Merc 280            19.2   6 167.6 123 3.92 3.440 0.02796748
## Merc 280C           17.8   6 167.6 123 3.92 3.440 0.02796748
## Merc 450SE          16.4   8 275.8 180 3.07 4.070 0.02261111
## Merc 450SL          17.3   8 275.8 180 3.07 3.730 0.02072222
## Merc 450SLC         15.2   8 275.8 180 3.07 3.780 0.02100000
## Cadillac Fleetwood  10.4   8 472.0 205 2.93 5.250 0.02560976
## Lincoln Continental 10.4   8 460.0 215 3.00 5.424 0.02522791
## Chrysler Imperial   14.7   8 440.0 230 3.23 5.345 0.02323913
## Fiat 128            32.4   4  78.7  66 4.08 2.200 0.03333333
## Honda Civic         30.4   4  75.7  52 4.93 1.615 0.03105769
## Toyota Corolla      33.9   4  71.1  65 4.22 1.835 0.02823077
## Toyota Corona       21.5   4 120.1  97 3.70 2.465 0.02541237
## Dodge Challenger    15.5   8 318.0 150 2.76 3.520 0.02346667
## AMC Javelin         15.2   8 304.0 150 3.15 3.435 0.02290000
## Camaro Z28          13.3   8 350.0 245 3.73 3.840 0.01567347
## Pontiac Firebird    19.2   8 400.0 175 3.08 3.845 0.02197143
## Fiat X1-9           27.3   4  79.0  66 4.08 1.935 0.02931818
## Porsche 914-2       26.0   4 120.3  91 4.43 2.140 0.02351648
## Lotus Europa        30.4   4  95.1 113 3.77 1.513 0.01338938
## Ford Pantera L      15.8   8 351.0 264 4.22 3.170 0.01200758
## Ferrari Dino        19.7   6 145.0 175 3.62 2.770 0.01582857
## Maserati Bora       15.0   8 301.0 335 3.54 3.570 0.01065672
## Volvo 142E          21.4   4 121.0 109 4.11 2.780 0.02550459</code></pre>
<p>You can write custom functions to apply to the groups.</p>
<pre class="r"><code>fun &lt;- function(x,y){fit &lt;- lm(y~x); coef(fit)[1]} #return interept
mtcars %&gt;% 
  group_by(carb) %&gt;% 
  summarize(lmm=fun(mpg,hp))</code></pre>
<pre><code>## # A tibble: 6 x 2
##    carb   lmm
##   &lt;dbl&gt; &lt;dbl&gt;
## 1     1  162.
## 2     2  252.
## 3     3  180 
## 4     4  379.
## 5     6  175 
## 6     8  335</code></pre>
<p>Use <code>mutate()</code> to add column. <strong>This will not change the number of rows.</strong></p>
<pre class="r"><code>mtcars %&gt;% 
  group_by(carb) %&gt;% 
  mutate(lmm=fun(mpg,hp)) %&gt;%
  select(mpg, hp, lmm)</code></pre>
<pre><code>## Adding missing grouping variables: `carb`</code></pre>
<pre><code>## # A tibble: 32 x 4
## # Groups:   carb [6]
##     carb   mpg    hp   lmm
##    &lt;dbl&gt; &lt;dbl&gt; &lt;dbl&gt; &lt;dbl&gt;
##  1     4  21     110  379.
##  2     4  21     110  379.
##  3     1  22.8    93  162.
##  4     1  21.4   110  162.
##  5     2  18.7   175  252.
##  6     1  18.1   105  162.
##  7     4  14.3   245  379.
##  8     2  24.4    62  252.
##  9     2  22.8    95  252.
## 10     4  19.2   123  379.
## # … with 22 more rows</code></pre>
<div id="careful-mutate-guesses" class="section level3">
<h3>Careful <code>mutate()</code> guesses</h3>
<p>That works.</p>
<pre class="r"><code>mtcars %&gt;% 
   select(mpg:wt) %&gt;%
   mutate(mean=sqrt(wt))</code></pre>
<pre><code>##                      mpg cyl  disp  hp drat    wt     mean
## Mazda RX4           21.0   6 160.0 110 3.90 2.620 1.618641
## Mazda RX4 Wag       21.0   6 160.0 110 3.90 2.875 1.695582
## Datsun 710          22.8   4 108.0  93 3.85 2.320 1.523155
## Hornet 4 Drive      21.4   6 258.0 110 3.08 3.215 1.793042
## Hornet Sportabout   18.7   8 360.0 175 3.15 3.440 1.854724
## Valiant             18.1   6 225.0 105 2.76 3.460 1.860108
## Duster 360          14.3   8 360.0 245 3.21 3.570 1.889444
## Merc 240D           24.4   4 146.7  62 3.69 3.190 1.786057
## Merc 230            22.8   4 140.8  95 3.92 3.150 1.774824
## Merc 280            19.2   6 167.6 123 3.92 3.440 1.854724
## Merc 280C           17.8   6 167.6 123 3.92 3.440 1.854724
## Merc 450SE          16.4   8 275.8 180 3.07 4.070 2.017424
## Merc 450SL          17.3   8 275.8 180 3.07 3.730 1.931321
## Merc 450SLC         15.2   8 275.8 180 3.07 3.780 1.944222
## Cadillac Fleetwood  10.4   8 472.0 205 2.93 5.250 2.291288
## Lincoln Continental 10.4   8 460.0 215 3.00 5.424 2.328948
## Chrysler Imperial   14.7   8 440.0 230 3.23 5.345 2.311926
## Fiat 128            32.4   4  78.7  66 4.08 2.200 1.483240
## Honda Civic         30.4   4  75.7  52 4.93 1.615 1.270827
## Toyota Corolla      33.9   4  71.1  65 4.22 1.835 1.354622
## Toyota Corona       21.5   4 120.1  97 3.70 2.465 1.570032
## Dodge Challenger    15.5   8 318.0 150 2.76 3.520 1.876166
## AMC Javelin         15.2   8 304.0 150 3.15 3.435 1.853375
## Camaro Z28          13.3   8 350.0 245 3.73 3.840 1.959592
## Pontiac Firebird    19.2   8 400.0 175 3.08 3.845 1.960867
## Fiat X1-9           27.3   4  79.0  66 4.08 1.935 1.391043
## Porsche 914-2       26.0   4 120.3  91 4.43 2.140 1.462874
## Lotus Europa        30.4   4  95.1 113 3.77 1.513 1.230041
## Ford Pantera L      15.8   8 351.0 264 4.22 3.170 1.780449
## Ferrari Dino        19.7   6 145.0 175 3.62 2.770 1.664332
## Maserati Bora       15.0   8 301.0 335 3.54 3.570 1.889444
## Volvo 142E          21.4   4 121.0 109 4.11 2.780 1.667333</code></pre>
<p>Yipes! It decided <em>you must want me to take the mean in the whole column</em>.</p>
<pre class="r"><code>mtcars %&gt;% 
   select(mpg:wt) %&gt;%
   mutate(mean=mean(wt))</code></pre>
<pre><code>##                      mpg cyl  disp  hp drat    wt    mean
## Mazda RX4           21.0   6 160.0 110 3.90 2.620 3.21725
## Mazda RX4 Wag       21.0   6 160.0 110 3.90 2.875 3.21725
## Datsun 710          22.8   4 108.0  93 3.85 2.320 3.21725
## Hornet 4 Drive      21.4   6 258.0 110 3.08 3.215 3.21725
## Hornet Sportabout   18.7   8 360.0 175 3.15 3.440 3.21725
## Valiant             18.1   6 225.0 105 2.76 3.460 3.21725
## Duster 360          14.3   8 360.0 245 3.21 3.570 3.21725
## Merc 240D           24.4   4 146.7  62 3.69 3.190 3.21725
## Merc 230            22.8   4 140.8  95 3.92 3.150 3.21725
## Merc 280            19.2   6 167.6 123 3.92 3.440 3.21725
## Merc 280C           17.8   6 167.6 123 3.92 3.440 3.21725
## Merc 450SE          16.4   8 275.8 180 3.07 4.070 3.21725
## Merc 450SL          17.3   8 275.8 180 3.07 3.730 3.21725
## Merc 450SLC         15.2   8 275.8 180 3.07 3.780 3.21725
## Cadillac Fleetwood  10.4   8 472.0 205 2.93 5.250 3.21725
## Lincoln Continental 10.4   8 460.0 215 3.00 5.424 3.21725
## Chrysler Imperial   14.7   8 440.0 230 3.23 5.345 3.21725
## Fiat 128            32.4   4  78.7  66 4.08 2.200 3.21725
## Honda Civic         30.4   4  75.7  52 4.93 1.615 3.21725
## Toyota Corolla      33.9   4  71.1  65 4.22 1.835 3.21725
## Toyota Corona       21.5   4 120.1  97 3.70 2.465 3.21725
## Dodge Challenger    15.5   8 318.0 150 2.76 3.520 3.21725
## AMC Javelin         15.2   8 304.0 150 3.15 3.435 3.21725
## Camaro Z28          13.3   8 350.0 245 3.73 3.840 3.21725
## Pontiac Firebird    19.2   8 400.0 175 3.08 3.845 3.21725
## Fiat X1-9           27.3   4  79.0  66 4.08 1.935 3.21725
## Porsche 914-2       26.0   4 120.3  91 4.43 2.140 3.21725
## Lotus Europa        30.4   4  95.1 113 3.77 1.513 3.21725
## Ford Pantera L      15.8   8 351.0 264 4.22 3.170 3.21725
## Ferrari Dino        19.7   6 145.0 175 3.62 2.770 3.21725
## Maserati Bora       15.0   8 301.0 335 3.54 3.570 3.21725
## Volvo 142E          21.4   4 121.0 109 4.11 2.780 3.21725</code></pre>
</div>
<div id="careful-regarding-across" class="section level3">
<h3>Careful regarding <em>across</em></h3>
<p>It is super easy to try to use this to apply functions across multiple columns. Doesn’t work. You need to use <code>?across</code> (or <code>group_by()</code> with a row factor).</p>
<pre class="r"><code>mtcars %&gt;% 
   select(mpg:wt) %&gt;%
   mutate(mean1 = (wt+hp)/2, mean2=mean(c(wt,hp)))</code></pre>
<pre><code>##                      mpg cyl  disp  hp drat    wt    mean1    mean2
## Mazda RX4           21.0   6 160.0 110 3.90 2.620  56.3100 74.95238
## Mazda RX4 Wag       21.0   6 160.0 110 3.90 2.875  56.4375 74.95238
## Datsun 710          22.8   4 108.0  93 3.85 2.320  47.6600 74.95238
## Hornet 4 Drive      21.4   6 258.0 110 3.08 3.215  56.6075 74.95238
## Hornet Sportabout   18.7   8 360.0 175 3.15 3.440  89.2200 74.95238
## Valiant             18.1   6 225.0 105 2.76 3.460  54.2300 74.95238
## Duster 360          14.3   8 360.0 245 3.21 3.570 124.2850 74.95238
## Merc 240D           24.4   4 146.7  62 3.69 3.190  32.5950 74.95238
## Merc 230            22.8   4 140.8  95 3.92 3.150  49.0750 74.95238
## Merc 280            19.2   6 167.6 123 3.92 3.440  63.2200 74.95238
## Merc 280C           17.8   6 167.6 123 3.92 3.440  63.2200 74.95238
## Merc 450SE          16.4   8 275.8 180 3.07 4.070  92.0350 74.95238
## Merc 450SL          17.3   8 275.8 180 3.07 3.730  91.8650 74.95238
## Merc 450SLC         15.2   8 275.8 180 3.07 3.780  91.8900 74.95238
## Cadillac Fleetwood  10.4   8 472.0 205 2.93 5.250 105.1250 74.95238
## Lincoln Continental 10.4   8 460.0 215 3.00 5.424 110.2120 74.95238
## Chrysler Imperial   14.7   8 440.0 230 3.23 5.345 117.6725 74.95238
## Fiat 128            32.4   4  78.7  66 4.08 2.200  34.1000 74.95238
## Honda Civic         30.4   4  75.7  52 4.93 1.615  26.8075 74.95238
## Toyota Corolla      33.9   4  71.1  65 4.22 1.835  33.4175 74.95238
## Toyota Corona       21.5   4 120.1  97 3.70 2.465  49.7325 74.95238
## Dodge Challenger    15.5   8 318.0 150 2.76 3.520  76.7600 74.95238
## AMC Javelin         15.2   8 304.0 150 3.15 3.435  76.7175 74.95238
## Camaro Z28          13.3   8 350.0 245 3.73 3.840 124.4200 74.95238
## Pontiac Firebird    19.2   8 400.0 175 3.08 3.845  89.4225 74.95238
## Fiat X1-9           27.3   4  79.0  66 4.08 1.935  33.9675 74.95238
## Porsche 914-2       26.0   4 120.3  91 4.43 2.140  46.5700 74.95238
## Lotus Europa        30.4   4  95.1 113 3.77 1.513  57.2565 74.95238
## Ford Pantera L      15.8   8 351.0 264 4.22 3.170 133.5850 74.95238
## Ferrari Dino        19.7   6 145.0 175 3.62 2.770  88.8850 74.95238
## Maserati Bora       15.0   8 301.0 335 3.54 3.570 169.2850 74.95238
## Volvo 142E          21.4   4 121.0 109 4.11 2.780  55.8900 74.95238</code></pre>
</div>
</div>
<div id="tidyverse-verbs-are-slow" class="section level2">
<h2>Tidyverse verbs are slow</h2>
<p>For loops are not the place for these verbs!</p>
<pre class="r"><code>bm &lt;- microbenchmark(
  apply(mtcars,2,sum), 
  summarize_all(mtcars, sum)
  )
ggplot2::autoplot(bm)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-44-1.png" width="672" /></p>
<pre class="r"><code>bm &lt;- microbenchmark(
   mtcars[mtcars$cyl==4,], 
   filter(mtcars, cyl==4),
    subset(mtcars, cyl==4)
   )
ggplot2::autoplot(bm)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-45-1.png" width="672" /></p>
</div>
</div>
<div id="long-and-wide-data-frames" class="section level1">
<h1>Long and wide data frames</h1>
<p>Wide is what you need for say <code>lm()</code>. You have your response variable in one column and then your predictor variables in other columns.</p>
<pre class="r"><code>df &lt;- subset(mtcars, select=c(mpg, wt, hp, carb))
df$model &lt;- rownames(mtcars)
head(df)</code></pre>
<pre><code>##                    mpg    wt  hp carb             model
## Mazda RX4         21.0 2.620 110    4         Mazda RX4
## Mazda RX4 Wag     21.0 2.875 110    4     Mazda RX4 Wag
## Datsun 710        22.8 2.320  93    1        Datsun 710
## Hornet 4 Drive    21.4 3.215 110    1    Hornet 4 Drive
## Hornet Sportabout 18.7 3.440 175    2 Hornet Sportabout
## Valiant           18.1 3.460 105    1           Valiant</code></pre>
<p>Fit with <code>lm()</code>.</p>
<pre class="r"><code>fit &lt;- lm(mpg ~ wt + hp, data=df)
summary(fit)</code></pre>
<pre><code>## 
## Call:
## lm(formula = mpg ~ wt + hp, data = df)
## 
## Residuals:
##    Min     1Q Median     3Q    Max 
## -3.941 -1.600 -0.182  1.050  5.854 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(&gt;|t|)    
## (Intercept) 37.22727    1.59879  23.285  &lt; 2e-16 ***
## wt          -3.87783    0.63273  -6.129 1.12e-06 ***
## hp          -0.03177    0.00903  -3.519  0.00145 ** 
## ---
## Signif. codes:  0 &#39;***&#39; 0.001 &#39;**&#39; 0.01 &#39;*&#39; 0.05 &#39;.&#39; 0.1 &#39; &#39; 1
## 
## Residual standard error: 2.593 on 29 degrees of freedom
## Multiple R-squared:  0.8268, Adjusted R-squared:  0.8148 
## F-statistic: 69.21 on 2 and 29 DF,  p-value: 9.109e-12</code></pre>
<p>Long form is what ggplot wants. Usually I have to ponder a bit before I decide how to make my long form data frame. There is not just “one” that you could make from your wide form data frame.</p>
<p>Let’s make a plot of the red, green and blue columns. We want the points to be red, green or blue.</p>
<pre class="r"><code>plot(df$mpg, col=&quot;red&quot;, ylim=c(0,200))
points(df$wt, col=&quot;green&quot;)
points(df$hp, col=&quot;blue&quot;)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-48-1.png" width="672" /></p>
<p>Same idea but let’s plot versus the carburator number. Let’s make a plot of the red, green and blue columns. We want the points to be red, green or blue.</p>
<pre class="r"><code>plot(df$carb, df$mpg, col=&quot;red&quot;, ylim=c(0,200))
points(df$carb, df$wt, col=&quot;green&quot;)
points(df$carb, df$hp, col=&quot;blue&quot;)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-49-1.png" width="672" /></p>
<p>Now let’s do this in ggplot. What values do we want in the <code>aes()</code> function? Those need to be columns. <code>aes(x=..., y=..., col=....)</code></p>
<pre><code>data.frame(
   col=name.of.thing.we.plot, 
   y=value.of.thing.we.plot, 
   x=x.value
   )</code></pre>
<p>So we’ll have a column of x (carb or model or 1:n), a column for our y (mpg,wt,hp) and a column for names (“mpg”,“wt”,“hb”).</p>
<pre class="r"><code>df.long &lt;- data.frame()
a &lt;- data.frame(model=df$model, val=df$mpg, name=&quot;mpg&quot;, carb=df$carb)
df.long &lt;- rbind(df.long, a)
a &lt;- data.frame(model=df$model, val=df$wt, name=&quot;wt&quot;, carb=df$carb)
df.long &lt;- rbind(df.long, a)
a &lt;- data.frame(model=df$model, val=df$hp, name=&quot;hp&quot;, carb=df$carb)
df.long &lt;- rbind(df.long, a)
df.long</code></pre>
<pre><code>##                  model     val name carb
## 1            Mazda RX4  21.000  mpg    4
## 2        Mazda RX4 Wag  21.000  mpg    4
## 3           Datsun 710  22.800  mpg    1
## 4       Hornet 4 Drive  21.400  mpg    1
## 5    Hornet Sportabout  18.700  mpg    2
## 6              Valiant  18.100  mpg    1
## 7           Duster 360  14.300  mpg    4
## 8            Merc 240D  24.400  mpg    2
## 9             Merc 230  22.800  mpg    2
## 10            Merc 280  19.200  mpg    4
## 11           Merc 280C  17.800  mpg    4
## 12          Merc 450SE  16.400  mpg    3
## 13          Merc 450SL  17.300  mpg    3
## 14         Merc 450SLC  15.200  mpg    3
## 15  Cadillac Fleetwood  10.400  mpg    4
## 16 Lincoln Continental  10.400  mpg    4
## 17   Chrysler Imperial  14.700  mpg    4
## 18            Fiat 128  32.400  mpg    1
## 19         Honda Civic  30.400  mpg    2
## 20      Toyota Corolla  33.900  mpg    1
## 21       Toyota Corona  21.500  mpg    1
## 22    Dodge Challenger  15.500  mpg    2
## 23         AMC Javelin  15.200  mpg    2
## 24          Camaro Z28  13.300  mpg    4
## 25    Pontiac Firebird  19.200  mpg    2
## 26           Fiat X1-9  27.300  mpg    1
## 27       Porsche 914-2  26.000  mpg    2
## 28        Lotus Europa  30.400  mpg    2
## 29      Ford Pantera L  15.800  mpg    4
## 30        Ferrari Dino  19.700  mpg    6
## 31       Maserati Bora  15.000  mpg    8
## 32          Volvo 142E  21.400  mpg    2
## 33           Mazda RX4   2.620   wt    4
## 34       Mazda RX4 Wag   2.875   wt    4
## 35          Datsun 710   2.320   wt    1
## 36      Hornet 4 Drive   3.215   wt    1
## 37   Hornet Sportabout   3.440   wt    2
## 38             Valiant   3.460   wt    1
## 39          Duster 360   3.570   wt    4
## 40           Merc 240D   3.190   wt    2
## 41            Merc 230   3.150   wt    2
## 42            Merc 280   3.440   wt    4
## 43           Merc 280C   3.440   wt    4
## 44          Merc 450SE   4.070   wt    3
## 45          Merc 450SL   3.730   wt    3
## 46         Merc 450SLC   3.780   wt    3
## 47  Cadillac Fleetwood   5.250   wt    4
## 48 Lincoln Continental   5.424   wt    4
## 49   Chrysler Imperial   5.345   wt    4
## 50            Fiat 128   2.200   wt    1
## 51         Honda Civic   1.615   wt    2
## 52      Toyota Corolla   1.835   wt    1
## 53       Toyota Corona   2.465   wt    1
## 54    Dodge Challenger   3.520   wt    2
## 55         AMC Javelin   3.435   wt    2
## 56          Camaro Z28   3.840   wt    4
## 57    Pontiac Firebird   3.845   wt    2
## 58           Fiat X1-9   1.935   wt    1
## 59       Porsche 914-2   2.140   wt    2
## 60        Lotus Europa   1.513   wt    2
## 61      Ford Pantera L   3.170   wt    4
## 62        Ferrari Dino   2.770   wt    6
## 63       Maserati Bora   3.570   wt    8
## 64          Volvo 142E   2.780   wt    2
## 65           Mazda RX4 110.000   hp    4
## 66       Mazda RX4 Wag 110.000   hp    4
## 67          Datsun 710  93.000   hp    1
## 68      Hornet 4 Drive 110.000   hp    1
## 69   Hornet Sportabout 175.000   hp    2
## 70             Valiant 105.000   hp    1
## 71          Duster 360 245.000   hp    4
## 72           Merc 240D  62.000   hp    2
## 73            Merc 230  95.000   hp    2
## 74            Merc 280 123.000   hp    4
## 75           Merc 280C 123.000   hp    4
## 76          Merc 450SE 180.000   hp    3
## 77          Merc 450SL 180.000   hp    3
## 78         Merc 450SLC 180.000   hp    3
## 79  Cadillac Fleetwood 205.000   hp    4
## 80 Lincoln Continental 215.000   hp    4
## 81   Chrysler Imperial 230.000   hp    4
## 82            Fiat 128  66.000   hp    1
## 83         Honda Civic  52.000   hp    2
## 84      Toyota Corolla  65.000   hp    1
## 85       Toyota Corona  97.000   hp    1
## 86    Dodge Challenger 150.000   hp    2
## 87         AMC Javelin 150.000   hp    2
## 88          Camaro Z28 245.000   hp    4
## 89    Pontiac Firebird 175.000   hp    2
## 90           Fiat X1-9  66.000   hp    1
## 91       Porsche 914-2  91.000   hp    2
## 92        Lotus Europa 113.000   hp    2
## 93      Ford Pantera L 264.000   hp    4
## 94        Ferrari Dino 175.000   hp    6
## 95       Maserati Bora 335.000   hp    8
## 96          Volvo 142E 109.000   hp    2</code></pre>
<p>Now we can plot with ggplot.</p>
<pre class="r"><code>ggplot(df.long, aes(x=model, y=val, col=name)) + geom_point()</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-51-1.png" width="672" /></p>
<p>And we can easily do panels.</p>
<pre class="r"><code>ggplot(df.long, aes(x=model, y=val, col=name)) + geom_point() +
  facet_wrap(~name)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-52-1.png" width="672" /></p>
<div id="pivoting" class="section level2">
<h2>Pivoting</h2>
<p>So creating your long data frames like that would get tidieous and inefficient. The tidyr package has <a href="https://dcl-wrangle.stanford.edu/pivot-basic.html">functions to make this easier</a>.</p>
<div id="pivot_longer" class="section level3">
<h3><code>pivot_longer</code></h3>
<pre><code>pivot_longer(
    data frame name,
    cols = what columns will you stack or pivot,
    names_to = colname for the column with the column names that your stacked,
    values_to = colname for the column with the values
  )</code></pre>
<pre class="r"><code>df.long2 &lt;- 
  pivot_longer(
    df,
    cols = c(&quot;mpg&quot;, &quot;wt&quot;, &quot;hp&quot;),
    names_to = &quot;name&quot;,
    values_to = &quot;val&quot;
  )</code></pre>
<pre class="r"><code>df.long2</code></pre>
<pre><code>## # A tibble: 96 x 4
##     carb model          name     val
##    &lt;dbl&gt; &lt;chr&gt;          &lt;chr&gt;  &lt;dbl&gt;
##  1     4 Mazda RX4      mpg    21   
##  2     4 Mazda RX4      wt      2.62
##  3     4 Mazda RX4      hp    110   
##  4     4 Mazda RX4 Wag  mpg    21   
##  5     4 Mazda RX4 Wag  wt      2.88
##  6     4 Mazda RX4 Wag  hp    110   
##  7     1 Datsun 710     mpg    22.8 
##  8     1 Datsun 710     wt      2.32
##  9     1 Datsun 710     hp     93   
## 10     1 Hornet 4 Drive mpg    21.4 
## # … with 86 more rows</code></pre>
<p>It changed the order our names but that’s ok. We can fix that if needed.</p>
<pre class="r"><code>ggplot(df.long, aes(x=model, y=val, col=name)) + geom_point() +
  facet_wrap(~name)</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-55-1.png" width="672" /></p>
<p>It changed the order our names but that’s ok. We can fix that if needed.</p>
<pre class="r"><code>df.long$name &lt;- factor(df.long$name, levels=c(&quot;mpg&quot;, &quot;wt&quot;, &quot;hp&quot;))
ggplot(df.long, aes(x=model, y=val, col=name)) + geom_point() +
  facet_wrap(~name) +
  theme(axis.text.x=element_blank())</code></pre>
<p><img src="week9-data-wrangling_files/figure-html/unnamed-chunk-56-1.png" width="672" /></p>
<p>Read <code>?tidyr::pivot_longer</code> to learn more options but this will be all you need most of the time.</p>
</div>
<div id="pivot_wider" class="section level3">
<h3><code>pivot_wider</code></h3>
<pre><code>pivot_wider( 
    data frame name,
    names_from = what column has the column names
    values_from = what column has the values
  )</code></pre>
<pre class="r"><code>pivot_wider( 
    df.long,
    names_from = &quot;name&quot;, 
    values_from = &quot;val&quot;
  )</code></pre>
<pre><code>## # A tibble: 32 x 5
##    model              carb   mpg    wt    hp
##    &lt;chr&gt;             &lt;dbl&gt; &lt;dbl&gt; &lt;dbl&gt; &lt;dbl&gt;
##  1 Mazda RX4             4  21    2.62   110
##  2 Mazda RX4 Wag         4  21    2.88   110
##  3 Datsun 710            1  22.8  2.32    93
##  4 Hornet 4 Drive        1  21.4  3.22   110
##  5 Hornet Sportabout     2  18.7  3.44   175
##  6 Valiant               1  18.1  3.46   105
##  7 Duster 360            4  14.3  3.57   245
##  8 Merc 240D             2  24.4  3.19    62
##  9 Merc 230              2  22.8  3.15    95
## 10 Merc 280              4  19.2  3.44   123
## # … with 22 more rows</code></pre>
<p>BTW, you will normally see the function called with piping.</p>
<pre class="r"><code>df.long %&gt;%
  pivot_wider( 
    names_from = &quot;name&quot;, 
    values_from = &quot;val&quot;
  )</code></pre>
</div>
</div>
</div>
<div id="comments" class="section level1">
<h1>Comments</h1>
<p>I rarely use piping. Why?</p>
<ul>
<li>Piping is fine but is super slow. Also I develop packages and the R package checks often through errors with piping or any type of implicit reference to data columns in data frames. So like <code>subset(df, x==1)</code> is implicit because it is implying that there is an <code>x</code> column in <code>df</code> but that code will actually work if <code>x</code> is not in <code>df</code>. If you are a package writer, the check will complain about that vagueness.</li>
<li>I have a hard time following long strings of pipes and lose track of what’s going on.</li>
</ul>
<p>And I rarely use dplyr, tidyr, or tidyverse verbs.</p>
<ul>
<li>tidyverse is “maturing” which means the functions keep changing…which means my code rots quickly.</li>
<li>I often have to do something that takes me a long time to figure out how to do with tidyverse.</li>
</ul>
<p>Whereas learning ggplot2 has made my plotting much, much faster, I can’t say the same with data-wrangling with tidyverse except for a few functions that I use to help out with ggplot.</p>
</div>

<hr>

<div style="text-align: center">
<i class="fas fa-cubes"></i> NWFSC Math Bio Program <i class="fas fa-cubes"></i>
</div>


</div>
</div>

</div>

<script>

// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
  $('tr.odd').parent('tbody').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
  bootstrapStylePandocTables();
});


</script>

<!-- tabsets -->

<script>
$(document).ready(function () {
  window.buildTabsets("TOC");
});

$(document).ready(function () {
  $('.tabset-dropdown > .nav-tabs > li').click(function () {
    $(this).parent().toggleClass('nav-tabs-open');
  });
});
</script>

<!-- code folding -->

<script>
$(document).ready(function ()  {

    // temporarily add toc-ignore selector to headers for the consistency with Pandoc
    $('.unlisted.unnumbered').addClass('toc-ignore')

    // move toc-ignore selectors from section div to header
    $('div.section.toc-ignore')
        .removeClass('toc-ignore')
        .children('h1,h2,h3,h4,h5').addClass('toc-ignore');

    // establish options
    var options = {
      selectors: "h1,h2",
      theme: "bootstrap3",
      context: '.toc-content',
      hashGenerator: function (text) {
        return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_');
      },
      ignoreSelector: ".toc-ignore",
      scrollTo: 0
    };
    options.showAndHide = false;
    options.smoothScroll = true;

    // tocify
    var toc = $("#TOC").tocify(options).data("toc-tocify");
});
</script>

<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>