scrape-linkedin-selenium/scrape_linkedin/CompanyScraper.py at 43e86193cb1ea4925da696ee03ab24c1998efb83 · austinoboyle/scrape-linkedin-selenium · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import logging

from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from .Company import Company
from .Scraper import Scraper
from .utils import AnyEC

logger = logging.getLogger(__name__)


class CompanyScraper(Scraper):
    def scrape(self, company, overview=True, jobs=False, life=False, insights=False):
        # Get Overview
        self.load_initial(company)

        jobs_html = life_html = insights_html = overview_html = ''

        if overview:
            overview_html = self.get_overview()
        if life:
            life_html = self.get_life()
        if jobs:
            jobs_html = self.get_jobs()
        if insights:
            insights_html = self.get_insights()
        #print("JOBS", jobs_html, "\n\n\n\n\nLIFE", life_html)
        return Company(overview_html, jobs_html, life_html, insights_html)

    def load_initial(self, company):
        url = 'https://www.linkedin.com/company/{}'.format(company)

        self.driver.get(url)
        try:
            myElem = WebDriverWait(self.driver, self.timeout).until(AnyEC(
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.organization-outlet')),
                EC.presence_of_element_located(
                    (By.CSS_SELECTOR, '.error-container'))
            ))
        except TimeoutException as e:
            raise ValueError(
                """Took too long to load company.  Common problems/solutions:
                1. Invalid LI_AT value: ensure that yours is correct (they
                   update frequently)
                2. Slow Internet: increase the timeout parameter in the Scraper constructor""")
        try:
            self.driver.find_element_by_css_selector('.organization-outlet')
        except:
            raise ValueError(
                'Company Unavailable: Company link does not match any companies on LinkedIn')

    def get_overview(self):
        try:
            tab_link = self.driver.find_elements_by_css_selector(
                'li.org-page-navigation__item.m0')[1]
            tab_link.click()
            self.wait_for_el(
                'section.artdeco-card.p4.mb3')
            return self.driver.find_element_by_css_selector(
                'div.scaffold-layout__row.scaffold-layout__content').get_attribute('outerHTML')
        except:
            return ''

    def get_life(self):
        try:
            tab_link = self.driver.find_element_by_css_selector(
                'a[data-control-name="page_member_main_nav_life_tab"]')
            tab_link.click()
            self.wait_for_el(
                'a[data-control-name="page_member_main_nav_life_tab"].active')
            return self.driver.find_element_by_css_selector('.org-life').get_attribute('outerHTML')
        except:
            return ''

    def get_jobs(self):
        try:
            tab_link = self.driver.find_element_by_css_selector(
                'a[data-control-name="page_member_main_nav_jobs_tab"]')
            tab_link.click()
            self.wait_for_el(
                'a[data-control-name="page_member_main_nav_jobs_tab"].active')
            return self.driver.find_element_by_css_selector('.org-jobs-container').get_attribute('outerHTML')
        except:
            return ''

    def get_insights(self):
        try:
            tab_link = self.driver.find_element_by_css_selector(
                'a[data-control-name="page_member_main_nav_insights_tab"]')
            tab_link.click()
            self.wait_for_el(
                'a[data-control-name="page_member_main_nav_insights_tab"].active')
            return self.driver.find_element_by_css_selector('.org-premium-insights-module').get_attribute('outerHTML')
        except:
            return ''