Scraping across multiple pages (rvest) R

Question

I am trying to perform a webscrape using rvest to scrape data on a specific product category from a webshop. The product results are displayed across multiple webpages. When i use my code i only get the first 24 results which is equal to the products on the first page. How can I adjust my code to scrape across all results pages?

Thanks in advance!

url_bol <- 'https://www.bol.com/nl/l/lichtbronnen/N/14483'
webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
head(webpage_bol)

product_title_data_html <- html_nodes(webpage_bol, '.product-title')
product_title_data <- html_text(product_title_data_html)
head(product_title_data)
product_title_data<-gsub("\n","",product_title_data)
product_title_data<-gsub(" ","",product_title_data)
head(product_title_data)
length(product_title_data)

product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
product_brand_data <-html_text(product_brand_data_html)
head(product_brand_data)
product_brand_data<-gsub("\n","",product_brand_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_brand_data)
length(product_brand_data)

product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
product_price_data <- html_text(product_price_data_html)
head(product_price_data)
product_price_data<-gsub("\n","",product_price_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_price_data)
product_price_data
length(product_price_data)

bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)

View(bol.df)

Every other page has a different URL, "page2", "page3", etc.So you need to account for that in your code. — Robert
Hi Rob, thanks for your feedback. Do you have any ideas how I can do this? I used the code suggested by Barath below, but I still only get the first 24 results. — Karima Be
Id suggest something like a for loop to change the url of the website you are scraping — Robert

Bharath Bharath · Accepted Answer · 2018-09-11T22:25:25

url<-"https://www.bol.com/nl/l/lichtbronnen/N/14483/?page=3&view=list"
library(rvest)
page<-html_session(url)
total<-html_nodes(page,xpath='//*[@id="js_list_view"]/div[2]/p') %>% html_text()
total<-as.numeric(gsub("[^\\d]+", "", total, perl=TRUE))
all_df<-0
library(data.table)

for(i in 1:ceiling(total/24)){
  url_bol <- paste0('https://www.bol.com/nl/l/lichtbronnen/N/14483/?view=list&page=',i)
  webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
  head(webpage_bol)

  product_title_data_html <- html_nodes(webpage_bol, '.product-title')
  product_title_data <- html_text(product_title_data_html)
  head(product_title_data)
  product_title_data<-gsub("\n","",product_title_data)
  product_title_data<-gsub(" ","",product_title_data)
  head(product_title_data)
  length(product_title_data)

  product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
  product_brand_data <-html_text(product_brand_data_html)
  head(product_brand_data)
  product_brand_data<-gsub("\n","",product_brand_data)
  product_price_data<-gsub(" ","",product_price_data)
  head(product_brand_data)
  length(product_brand_data)

  product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
  product_price_data <- html_text(product_price_data_html)
  head(product_price_data)
  product_price_data<-gsub("\n","",product_price_data)
  product_price_data<-gsub(" ","",product_price_data)
  head(product_price_data)
  product_price_data
  length(product_price_data)
  bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)
  all_df[i]<-list(bol.df)
}

final<-rbindlist(all_df,fill = TRUE)




View(final)

Scraping across multiple pages (rvest) R

1 Answers