costco rental car Thanksgiving week.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import urllib\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time as time\n",
    "import datetime as dt\n",
    "from lxml import html\n",
    "# import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#use selenium to key in the rental car parameters\n",
    "browser = webdriver.Firefox() #I only tested in firefox\n",
    "browser.get('http://costcotravel.com/Rental-Cars')\n",
    "browser.implicitly_wait(5)#wait for webpage download\n",
    "browser.find_element_by_xpath('//div/input[@id=\"pickupAirportTextWidget\"]').send_keys('PHX')\n",
    "browser.implicitly_wait(5) #wait for the airport suggestion box to show\n",
    "browser.find_element_by_xpath('//li[@class=\"sayt-result\"]').click() \n",
    "#click the  airport suggestion box \n",
    "\n",
    "browser.find_element_by_xpath('//input[@id=\"pickupDateWidget\"]').send_keys('11/21/2015')\n",
    "browser.find_element_by_xpath('//input[@id=\"dropoffDateWidget\"]').send_keys('11/29/2015',Keys.RETURN)\n",
    "\n",
    "browser.find_element_by_xpath('//select[@id=\"pickupTimeWidget\"]/option[@value=\"11:00 AM\"]').click()\n",
    "browser.find_element_by_xpath('//select[@id=\"dropoffTimeWidget\"]/option[@value=\"05:00 PM\"]').click()\n",
    "browser.implicitly_wait(5) #wait for the clicks to be completed\n",
    "browser.find_element_by_link_text('SEARCH').click()\n",
    "#click the search box\n",
    "\n",
    "time.sleep(8) #wait for firefox to download and render the page\n",
    "n = browser.page_source #grab the html source code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "soup = BeautifulSoup(n) #we use bs here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#grab all the tags of interest (from the main table, the cells that display the car class)\n",
    "\n",
    "#exclude the first row about vendor and put into a list\n",
    "all_tags = [i for i in soup.findAll('th',{\"class\":\"tar\"}) if i.text.startswith('Tax')==False]\n",
    "\n",
    "masterlist = []\n",
    "#go to the entir row, grab all the text of that rows, and clean it up (replace, split....) \n",
    "for i in range(len(all_tags)):\n",
    "    masterlist.append(str(all_tags[i].findParent().text).replace('Not Available','$NA').replace(',','').split('$'))\n",
    "#create dataframe from masterlist\n",
    "df = pd.DataFrame(masterlist)\n",
    "#clean up the name of the columns\n",
    "df.columns= [['car class', 'Alamo','Avis', 'Budget','Enterprise']]\n",
    "#calc the days away from first date of rental 11/21/2015 at 11AM\n",
    "df['days away']= abs((dt.datetime.now()-dt.datetime(2015,11,21,11)).days)\n",
    "#setup the index. Will do groupby later?\n",
    "df = df.set_index(['days away','car class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#save current df\n",
    "df.to_csv('data/%s.csv' %dt.datetime.now().strftime(\"%m-%d-%H-%M%p\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#open up master df, then append it with current df\n",
    "master= pd.read_csv('data/master.csv',index_col=[0,1])\n",
    "master = pd.concat([master, df])\n",
    "master.to_csv('data/master.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}