How do you pass the website where there's a panic and when you cross the page doesn't change the url in the browser line?
-
A couple of websites on the python: https://www.copart.com/search/runanddrive/ and when it comes to another page, the url is not changed, as I understand the AJAX requests, and they are met through XHR - queries when I move on to another page, the website sends XHR - search demand, which is the code in which it is stored, and I saw the specimen of page = 0, and when it comes to a second page, it changes to page 1. But I don't understand how to send a request to open my next page? 'With page code'
from bs4 import BeautifulSoup import requests from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait
URL = "https://www.copart.com/vehicle-search-featured/runanddrive" HEADERS = { "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36", "accept": "*/*" } def selen(url, params = None): driver = webdriver.Chrome(executable_path = "C:/Users/пк/OneDrive/Рабочий стол/Parser23/chromedriver.exe") driver.get(url) c = driver.page_source return c def get_content(html): cars = [] soup = BeautifulSoup(html,"html.parser") items = soup.find("div", class_ = "dataTables_wrapper form-inline dt-bootstrap no- footer").find("tbody").find_all("tr", {"role" : "row"}) for item in items: Retail_Value = item.find("span", {"data-uname" : "lotsearchLotestimatedretailvalue"}) link = item.find("a", class_ = "search-results") Sale_Date = item.find("span", {"data-uname" : "lotsearchLotauctiondate"}) if link: link = link.get("href") if Retail_Value: Retail_Value = Retail_Value.get_text(strip = True) else: Retail_Value = "Цены нет" if Sale_Date: Sale_Date = Sale_Date.get_text(strip = False) else: Sale_Date = "Цены нет!" cars.append({ "Lot# / link": f'{item.find("a", {"data-uname" : "lotsearchLotnumber"}).get_text(strip = True)} / {link}', "Year": item.find("span", {"data-uname" : "lotsearchLotcenturyyear"}).get_text(strip = True), "Make": item.find("span", {"data-uname" : "lotsearchLotmake"}).get_text(strip = True), "Model": item.find("span", {"data-uname": "lotsearchLotmodel"}).get_text(strip = True), "Item#": item.find("span", {"data-uname": "lotsearchItemnumber"}).get_text(strip = True), "Location/lane/ row": f'{item.find("span", {"data-uname" : "lotsearchLotyardname"}).get_text(strip = True)}/{item.find("span", {"pref- code" : "searchPreference.searchFields"}).get_text(strip = True)}', "Sale Date": f'{Sale_Date}', "Odometer": item.find("span", {"data-uname": "lotsearchLotodometerreading"}).get_text(strip = True), "Title code": item.find("span", {"data-uname": "lotsearchSaletitletype"}).get_text(strip = True), "Damage": item.find("span", {"data-uname" : "lotsearchLotdamagedescription"}).get_text(strip = True), "Est. Retail Value": Retail_Value, "Current Bid": item.find("ul", class_ = "list-unstyled").get_text(strip = True) }) print(cars) def parse(): html = selen(URL) get_content(html) parse()
-
So press the selenium-a on the "Next" button, the browser page is updated. We use the driver.page_source, and we have an updated code of the page so far.
P.S. We can go through the request, but the answer will be json.