dnd-beyond-scraper/scrapeMonster.js at master · leoric-crown/dnd-beyond-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
const puppeteer = require('puppeteer')
const fs = require('fs')
const { performance } = require('perf_hooks')
const statBlock = require('./config/statBlockSelectors.json')
const login = require('./login')
const parseMonsterAtUrl = async (page, url) => {
    return new Promise(async (resolve, reject) => {
        try {
            const t0 = performance.now()
            await page.goto(url, { waitUntil: 'networkidle0' })
            const monster = await parseMonster(page)
            const t1 = performance.now()
            console.log(`Parsing of monster: '${monster.header.name}' took ${(t1 - t0) / 1000} seconds.`)
            resolve(monster)
        } catch (err) {
            console.log('Error parsing monster at: ' + url)
            reject()
        }
    })
}

const parseMonster = async (page) => {
    return new Promise(async (resolve, reject) => {
        try {
            const monster = await page.evaluate(statBlock => {
                const scrape = {}
                scrape.header = {
                    name: document.querySelector(statBlock.header.name).innerText,
                    meta: document.querySelector(statBlock.header.meta).innerText
                }

                const attributes = Array.from(document.querySelectorAll(statBlock.attributes.selector))
                scrape.attributes = attributes.map(attribute => {
                    const label = attribute.querySelector(statBlock.attributes.data.label)
                    const value = attribute.querySelector(statBlock.attributes.data.value)
                    const extra = attribute.querySelector(statBlock.attributes.data.extra)
                    return {
                        label: label ? label.innerText : null,
                        value: value ? value.innerText : null,
                        extra: extra ? extra.innerText : null
                    }
                })

                const abilities = Array.from(document.querySelector(statBlock.abilities.selector).children)
                scrape.abilities = abilities.map(ability => {
                    const header = ability.querySelector(statBlock.abilities.data.header)
                    const score = ability.querySelector(statBlock.abilities.data.score)
                    const modifier = ability.querySelector(statBlock.abilities.data.modifier)
                    return {
                        header: header ? header.innerText : null,
                        score: score ? score.innerText : null,
                        modifier: modifier ? modifier.innerText : null
                    }
                })

                const tidbits = Array.from(document.querySelectorAll(statBlock.tidbits.selector))
                scrape.tidbits = tidbits.map(tidbit => {
                    let label = tidbit.querySelector(statBlock.tidbits.data.label)
                    label = label ? label.innerText : null
                    let data = tidbit.querySelector(statBlock.tidbits.data.data)
                    if (data) {
                        data = data.innerText.split(/[,;] /)
                        data = data.map(d => {
                            d = d.replace('and', '')
                            d = d.replace(' from Nonmagical Attacks', '')
                            return d
                        })
                        if (label === 'Challenge') data = data[0]
                    }
                    data = data || null
                    return {
                        label,
                        data
                    }
                })

                const descriptionBlocks = Array.from(document.querySelectorAll(statBlock.descriptionBlocks.selector))
                scrape.descriptionBlocks = descriptionBlocks.map(descriptionBlock => {
                    let header = descriptionBlock.querySelector(statBlock.descriptionBlocks.data.header)
                    header = header ? header.innerText : null
                    let content = descriptionBlock.querySelector(statBlock.descriptionBlocks.data.content)
                    content = Array.from(content.querySelectorAll('p'))
                    content = content.map(con => {
                        let title = con.querySelector('em > strong')
                        let description = con.innerText
                        if (title) {
                            title = title.innerText
                            description = description.replace(`${title} `, '')
                            if (title[title.length - 1] === '.') {
                                title = title.replace('.', '')
                            }
                        }
                        return {
                            title: title || null,
                            description
                        }
                    })

                    return {
                        header,
                        content
                    }
                })
                return scrape
            }, statBlock)
            resolve(monster)
        } catch (err) {
            console.log('Error parsing monster', err)
            reject()
        }
    })
}

const writeMonster = async (monster) => {
    if (!fs.existsSync('./monsters')) {
        fs.mkdirSync('./monsters')
    }
    fs.writeFileSync(`./monsters/${new Date().getTime()}.json`, JSON.stringify(monster))
}

async function main() {
    try {
        const browser = await puppeteer.launch({ headless: false })
        let page = await browser.newPage()
        await page.setViewport({ width: 1920, height: 1080 })

        let url = 'https://www.dndbeyond.com/monsters/adult-black-dragon'
        let monster = await parseMonsterAtUrl(page, url)
        writeMonster(monster)

        await login(page)
        url = 'https://www.dndbeyond.com/monsters/baba-lysaga'
        monster = await parseMonsterAtUrl(page, url)
        writeMonster(monster)

        url = 'https://www.dndbeyond.com/monsters/strahd-von-zarovich'
        monster = await parseMonsterAtUrl(page, url)
        writeMonster(monster)
    } catch (err) {
        console.log(err)
    }
}

main()