如何使用 Node.js 和 Puppeteer 抓取网站

1mkdir book-scraper
2cd book-scraper

1npm init

 1[secondary_label Output]
 2{
 3  "name": "sammy_scraper",
 4  "version": "1.0.0",
 5  "description": "a web scraper",
 6  "main": "index.js",
 7  "scripts": {
 8    "test": "echo \"Error: no test specified\" && exit 1"
 9  },
10  "keywords": [],
11  "author": "sammy the shark",
12  "license": "ISC"
13}
14
15Is this OK? (yes) yes

1npm install --save puppeteer

1ldd chrome | grep not

1nano package.json

 1[secondary_label Output]
 2{
 3  . . .
 4  "scripts": {
 5    "test": "echo \"Error: no test specified\" && exit 1",
 6    "start": "node index.js"
 7  },
 8  . . .
 9  "dependencies": {
10    "puppeteer": "^5.2.1"
11  }
12}

1nano browser.js

 1[label ./book-scraper/browser.js]
 2const puppeteer = require('puppeteer');
 3
 4async function startBrowser(){
 5    let browser;
 6    try {
 7        console.log("Opening the browser......");
 8        browser = await puppeteer.launch({
 9            headless: false,
10            args: ["--disable-setuid-sandbox"],
11            'ignoreHTTPSErrors': true
12        });
13    } catch (err) {
14        console.log("Could not create a browser instance => : ", err);
15    }
16    return browser;
17}
18
19module.exports = {
20    startBrowser
21};

1nano index.js

1[label ./book-scraper/index.js]
2const browserObject = require('./browser');
3const scraperController = require('./pageController');
4
5//Start the browser and create a browser instance
6let browserInstance = browserObject.startBrowser();
7
8// Pass the browser instance to the scraper controller
9scraperController(browserInstance)

1nano pageController.js

 1[label ./book-scraper/pageController.js]
 2const pageScraper = require('./pageScraper');
 3async function scrapeAll(browserInstance){
 4    let browser;
 5    try{
 6    	browser = await browserInstance;
 7    	await pageScraper.scraper(browser);	
 8    	
 9    }
10    catch(err){
11    	console.log("Could not resolve the browser instance => ", err);
12    }
13}
14
15module.exports = (browserInstance) => scrapeAll(browserInstance)

1nano pageScraper.js

 1[label ./book-scraper/pageScraper.js]
 2const scraperObject = {
 3    url: 'http://books.toscrape.com',
 4    async scraper(browser){
 5    	let page = await browser.newPage();
 6    	console.log(`Navigating to ${this.url}...`);
 7    	await page.goto(this.url);
 8    	
 9    }
10}
11
12module.exports = scraperObject;

1[secondary_label Output]
2.
3├── browser.js
4├── index.js
5├── node_modules
6├── package-lock.json
7├── package.json
8├── pageController.js
9└── pageScraper.js

1npm run start

1nano pageScraper.js

 1[label ./book-scraper/pageScraper.js]
 2
 3const scraperObject = {
 4    url: 'http://books.toscrape.com',
 5    async scraper(browser){
 6        let page = await browser.newPage();
 7    	console.log(`Navigating to ${this.url}...`);
 8    	// Navigate to the selected page
 9    	await page.goto(this.url);
10    	// Wait for the required DOM to be rendered
11    	await page.waitForSelector('.page_inner');
12    	// Get the link to all the required books
13    	let urls = await page.$$eval('section ol > li', links => {
14    		// Make sure the book to be scraped is in stock
15    		links = links.filter(link => link.querySelector('.instock.availability > i').textContent !== "In stock")
16    		// Extract the links from the data
17    		links = links.map(el => el.querySelector('h3 > a').href)
18    		return links;
19    	});
20    	console.log(urls);
21    }
22}
23
24module.exports = scraperObject;

1npm run start

 1[secondary_label Output]
 2> [email protected] start /Users/sammy/book-scraper
 3> node index.js
 4
 5Opening the browser......
 6Navigating to http://books.toscrape.com...
 7[
 8  'http://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
 9  'http://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html',
10  'http://books.toscrape.com/catalogue/soumission_998/index.html',
11  'http://books.toscrape.com/catalogue/sharp-objects_997/index.html',
12  'http://books.toscrape.com/catalogue/sapiens-a-brief-history-of-humankind_996/index.html',
13  'http://books.toscrape.com/catalogue/the-requiem-red_995/index.html',
14  'http://books.toscrape.com/catalogue/the-dirty-little-secrets-of-getting-your-dream-job_994/index.html',
15  'http://books.toscrape.com/catalogue/the-coming-woman-a-novel-based-on-the-life-of-the-infamous-feminist-victoria-woodhull_993/index.html',
16  'http://books.toscrape.com/catalogue/the-boys-in-the-boat-nine-americans-and-their-epic-quest-for-gold-at-the-1936-berlin-olympics_992/index.html',
17  'http://books.toscrape.com/catalogue/the-black-maria_991/index.html',
18  'http://books.toscrape.com/catalogue/starving-hearts-triangular-trade-trilogy-1_990/index.html',
19  'http://books.toscrape.com/catalogue/shakespeares-sonnets_989/index.html',
20  'http://books.toscrape.com/catalogue/set-me-free_988/index.html',
21  'http://books.toscrape.com/catalogue/scott-pilgrims-precious-little-life-scott-pilgrim-1_987/index.html',
22  'http://books.toscrape.com/catalogue/rip-it-up-and-start-again_986/index.html',
23  'http://books.toscrape.com/catalogue/our-band-could-be-your-life-scenes-from-the-american-indie-underground-1981-1991_985/index.html',
24  'http://books.toscrape.com/catalogue/olio_984/index.html',
25  'http://books.toscrape.com/catalogue/mesaerion-the-best-science-fiction-stories-1800-1849_983/index.html',
26  'http://books.toscrape.com/catalogue/libertarianism-for-beginners_982/index.html',
27  'http://books.toscrape.com/catalogue/its-only-the-himalayas_981/index.html'
28]

1nano pageScraper.js

 1[label ./book-scraper/pageScraper.js]
 2const scraperObject = {
 3    url: 'http://books.toscrape.com',
 4    async scraper(browser){
 5        let page = await browser.newPage();
 6    	console.log(`Navigating to ${this.url}...`);
 7    	// Navigate to the selected page
 8    	await page.goto(this.url);
 9    	// Wait for the required DOM to be rendered
10    	await page.waitForSelector('.page_inner');
11    	// Get the link to all the required books
12    	let urls = await page.$$eval('section ol > li', links => {
13    		// Make sure the book to be scraped is in stock
14    		links = links.filter(link => link.querySelector('.instock.availability > i').textContent !== "In stock")
15    		// Extract the links from the data
16    		links = links.map(el => el.querySelector('h3 > a').href)
17    		return links;
18    	});
19
20        // Loop through each of those links, open a new page instance and get the relevant data from them
21    	let pagePromise = (link) => new Promise(async(resolve, reject) => {
22    		let dataObj = {};
23    		let newPage = await browser.newPage();
24    		await newPage.goto(link);
25    		dataObj['bookTitle'] = await newPage.$eval('.product_main > h1', text => text.textContent);
26    		dataObj['bookPrice'] = await newPage.$eval('.price_color', text => text.textContent);
27    		dataObj['noAvailable'] = await newPage.$eval('.instock.availability', text => {
28    			// Strip new line and tab spaces
29    			text = text.textContent.replace(/(\r\n\t|\n|\r|\t)/gm, "");
30    			// Get the number of stock available
31    			let regexp = /^.*\((.*)\).*$/i;
32    			let stockAvailable = regexp.exec(text)[1].split(' ')[0];
33    			return stockAvailable;
34    		});
35    		dataObj['imageUrl'] = await newPage.$eval('#product_gallery img', img => img.src);
36    		dataObj['bookDescription'] = await newPage.$eval('#product_description', div => div.nextSibling.nextSibling.textContent);
37    		dataObj['upc'] = await newPage.$eval('.table.table-striped > tbody > tr > td', table => table.textContent);
38    		resolve(dataObj);
39    		await newPage.close();
40    	});
41
42    	for(link in urls){
43    		let currentPageData = await pagePromise(urls[link]);
44    		// scrapedData.push(currentPageData);
45    		console.log(currentPageData);
46    	}
47
48    }
49}
50
51module.exports = scraperObject;

1npm run start

 1[secondary_label Output]
 2Opening the browser......
 3Navigating to http://books.toscrape.com...
 4{
 5  bookTitle: 'A Light in the Attic',
 6  bookPrice: '£51.77',
 7  noAvailable: '22',
 8  imageUrl: 'http://books.toscrape.com/media/cache/fe/72/fe72f0532301ec28892ae79a629a293c.jpg',
 9  bookDescription: "It's hard to imagine a world without A Light in the Attic. [...]',
10  upc: 'a897fe39b1053632'
11}
12{
13  bookTitle: 'Tipping the Velvet',
14  bookPrice: '£53.74',
15  noAvailable: '20',
16  imageUrl: 'http://books.toscrape.com/media/cache/08/e9/08e94f3731d7d6b760dfbfbc02ca5c62.jpg',
17  bookDescription: `"Erotic and absorbing...Written with starling power."--"The New York Times Book Review " Nan King, an oyster girl, is captivated by the music hall phenomenon Kitty Butler [...]`,
18  upc: '90fa61229261140a'
19}
20{
21  bookTitle: 'Soumission',
22  bookPrice: '£50.10',
23  noAvailable: '20',
24  imageUrl: 'http://books.toscrape.com/media/cache/ee/cf/eecfe998905e455df12064dba399c075.jpg',
25  bookDescription: 'Dans une France assez proche de la nôtre, [...]',
26  upc: '6957f44c3847a760'
27}
28...

1nano pagescraper.js

 1[label ./book-scraper/pageScraper.js scraper()]
 2const scraperObject = {
 3    url: 'http://books.toscrape.com',
 4    async scraper(browser){
 5    	let page = await browser.newPage();
 6    	console.log(`Navigating to ${this.url}...`);
 7    	// Navigate to the selected page
 8    	await page.goto(this.url);
 9    	let scrapedData = [];
10    	// Wait for the required DOM to be rendered
11    	async function scrapeCurrentPage(){
12    		await page.waitForSelector('.page_inner');
13    		// Get the link to all the required books
14    		let urls = await page.$$eval('section ol > li', links => {
15    			// Make sure the book to be scraped is in stock
16    			links = links.filter(link => link.querySelector('.instock.availability > i').textContent !== "In stock")
17    			// Extract the links from the data
18    			links = links.map(el => el.querySelector('h3 > a').href)
19    			return links;
20    		});
21    		// Loop through each of those links, open a new page instance and get the relevant data from them
22    		let pagePromise = (link) => new Promise(async(resolve, reject) => {
23    			let dataObj = {};
24    			let newPage = await browser.newPage();
25    			await newPage.goto(link);
26    			dataObj['bookTitle'] = await newPage.$eval('.product_main > h1', text => text.textContent);
27    			dataObj['bookPrice'] = await newPage.$eval('.price_color', text => text.textContent);
28    			dataObj['noAvailable'] = await newPage.$eval('.instock.availability', text => {
29    				// Strip new line and tab spaces
30    				text = text.textContent.replace(/(\r\n\t|\n|\r|\t)/gm, "");
31    				// Get the number of stock available
32    				let regexp = /^.*\((.*)\).*$/i;
33    				let stockAvailable = regexp.exec(text)[1].split(' ')[0];
34    				return stockAvailable;
35    			});
36    			dataObj['imageUrl'] = await newPage.$eval('#product_gallery img', img => img.src);
37    			dataObj['bookDescription'] = await newPage.$eval('#product_description', div => div.nextSibling.nextSibling.textContent);
38    			dataObj['upc'] = await newPage.$eval('.table.table-striped > tbody > tr > td', table => table.textContent);
39    			resolve(dataObj);
40    			await newPage.close();
41    		});
42
43    		for(link in urls){
44    			let currentPageData = await pagePromise(urls[link]);
45    			scrapedData.push(currentPageData);
46    			// console.log(currentPageData);
47    		}
48    		// When all the data on this page is done, click the next button and start the scraping of the next page
49    		// You are going to check if this button exist first, so you know if there really is a next page.
50    		let nextButtonExist = false;
51    		try{
52    			const nextButton = await page.$eval('.next > a', a => a.textContent);
53    			nextButtonExist = true;
54    		}
55    		catch(err){
56    			nextButtonExist = false;
57    		}
58    		if(nextButtonExist){
59    			await page.click('.next > a');	
60    			return scrapeCurrentPage(); // Call this function recursively
61    		}
62    		await page.close();
63    		return scrapedData;
64    	}
65    	let data = await scrapeCurrentPage();
66    	console.log(data);
67    	return data;
68    }
69}
70
71module.exports = scraperObject;

1npm run start

1nano pageController.js

 1[label ./book-scraper/pageController.js]
 2const pageScraper = require('./pageScraper');
 3async function scrapeAll(browserInstance){
 4    let browser;
 5    try{
 6    	browser = await browserInstance;
 7    	let scrapedData = {};
 8    	// Call the scraper for different set of books to be scraped
 9    	scrapedData['Travel'] = await pageScraper.scraper(browser, 'Travel');
10    	await browser.close();
11    	console.log(scrapedData)
12    }
13    catch(err){
14    	console.log("Could not resolve the browser instance => ", err);
15    }
16}
17module.exports = (browserInstance) => scrapeAll(browserInstance)

1nano pageScraper.js

 1[label ./book-scraper/pageScraper.js]
 2const scraperObject = {
 3    url: 'http://books.toscrape.com',
 4    async scraper(browser, category){
 5        let page = await browser.newPage();
 6        console.log(`Navigating to ${this.url}...`);
 7        // Navigate to the selected page
 8        await page.goto(this.url);
 9        // Select the category of book to be displayed
10    	let selectedCategory = await page.$$eval('.side_categories > ul > li > ul > li > a', (links, _category) => {
11
12    		// Search for the element that has the matching text
13    		links = links.map(a => a.textContent.replace(/(\r\n\t|\n|\r|\t|^\s|\s$|\B\s|\s\B)/gm, "") === _category ? a : null);
14    		let link = links.filter(tx => tx !== null)[0];
15    		return link.href;
16    	}, category);
17    	// Navigate to the selected category
18    	await page.goto(selectedCategory);
19        let scrapedData = [];
20        // Wait for the required DOM to be rendered
21        async function scrapeCurrentPage(){
22            await page.waitForSelector('.page_inner');
23            // Get the link to all the required books
24            let urls = await page.$$eval('section ol > li', links => {
25                // Make sure the book to be scraped is in stock
26                links = links.filter(link => link.querySelector('.instock.availability > i').textContent !== "In stock")
27                // Extract the links from the data
28                links = links.map(el => el.querySelector('h3 > a').href)
29                return links;
30            });
31            // Loop through each of those links, open a new page instance and get the relevant data from them
32            let pagePromise = (link) => new Promise(async(resolve, reject) => {
33                let dataObj = {};
34                let newPage = await browser.newPage();
35                await newPage.goto(link);
36                dataObj['bookTitle'] = await newPage.$eval('.product_main > h1', text => text.textContent);
37                dataObj['bookPrice'] = await newPage.$eval('.price_color', text => text.textContent);
38                dataObj['noAvailable'] = await newPage.$eval('.instock.availability', text => {
39                    // Strip new line and tab spaces
40                    text = text.textContent.replace(/(\r\n\t|\n|\r|\t)/gm, "");
41                    // Get the number of stock available
42                    let regexp = /^.*\((.*)\).*$/i;
43                    let stockAvailable = regexp.exec(text)[1].split(' ')[0];
44                    return stockAvailable;
45                });
46                dataObj['imageUrl'] = await newPage.$eval('#product_gallery img', img => img.src);
47                dataObj['bookDescription'] = await newPage.$eval('#product_description', div => div.nextSibling.nextSibling.textContent);
48                dataObj['upc'] = await newPage.$eval('.table.table-striped > tbody > tr > td', table => table.textContent);
49                resolve(dataObj);
50                await newPage.close();
51            });
52
53            for(link in urls){
54                let currentPageData = await pagePromise(urls[link]);
55                scrapedData.push(currentPageData);
56                // console.log(currentPageData);
57            }
58            // When all the data on this page is done, click the next button and start the scraping of the next page
59            // You are going to check if this button exist first, so you know if there really is a next page.
60            let nextButtonExist = false;
61            try{
62                const nextButton = await page.$eval('.next > a', a => a.textContent);
63                nextButtonExist = true;
64            }
65            catch(err){
66                nextButtonExist = false;
67            }
68            if(nextButtonExist){
69                await page.click('.next > a');   
70                return scrapeCurrentPage(); // Call this function recursively
71            }
72            await page.close();
73            return scrapedData;
74        }
75        let data = await scrapeCurrentPage();
76        console.log(data);
77        return data;
78    }
79}
80
81module.exports = scraperObject;

1[label example page.$$eval() function]
2page.$$eval('selector', function(elem, args){
3    // .......
4}, args)

1npm run start

1nano pageController.js

 1[label ./book-scraper/pageController.js]
 2const pageScraper = require('./pageScraper');
 3async function scrapeAll(browserInstance){
 4    let browser;
 5    try{
 6    	browser = await browserInstance;
 7    	let scrapedData = {};
 8    	// Call the scraper for different set of books to be scraped
 9    	scrapedData['Travel'] = await pageScraper.scraper(browser, 'Travel');
10    	scrapedData['HistoricalFiction'] = await pageScraper.scraper(browser, 'Historical Fiction');
11    	scrapedData['Mystery'] = await pageScraper.scraper(browser, 'Mystery');
12    	await browser.close();
13    	console.log(scrapedData)
14    }
15    catch(err){
16    	console.log("Could not resolve the browser instance => ", err);
17    }
18}
19
20module.exports = (browserInstance) => scrapeAll(browserInstance)

1npm run start

1nano pageController.js

 1[label ./book-scraper/pageController.js]
 2const pageScraper = require('./pageScraper');
 3const fs = require('fs');
 4async function scrapeAll(browserInstance){
 5    let browser;
 6    try{
 7    	browser = await browserInstance;
 8    	let scrapedData = {};
 9    	// Call the scraper for different set of books to be scraped
10    	scrapedData['Travel'] = await pageScraper.scraper(browser, 'Travel');
11    	scrapedData['HistoricalFiction'] = await pageScraper.scraper(browser, 'Historical Fiction');
12    	scrapedData['Mystery'] = await pageScraper.scraper(browser, 'Mystery');
13    	await browser.close();
14    	fs.writeFile("data.json", JSON.stringify(scrapedData), 'utf8', function(err) {
15    	    if(err) {
16    	        return console.log(err);
17    	    }
18    	    console.log("The data has been scraped and saved successfully! View it at './data.json'");
19    	});
20    }
21    catch(err){
22    	console.log("Could not resolve the browser instance => ", err);
23    }
24}
25
26module.exports = (browserInstance) => scrapeAll(browserInstance)