How do you pass the website where there's a panic and when you cross the page doesn't change the url in the browser line?



  • A couple of websites on the python: https://www.copart.com/search/runanddrive/ and when it comes to another page, the url is not changed, as I understand the AJAX requests, and they are met through XHR - queries when I move on to another page, the website sends XHR - search demand, which is the code in which it is stored, and I saw the specimen of page = 0, and when it comes to a second page, it changes to page 1. But I don't understand how to send a request to open my next page? 'With page code'

        from bs4 import BeautifulSoup
        import requests
        from selenium import webdriver
        from selenium.webdriver.support.ui import WebDriverWait
    
    URL = "https://www.copart.com/vehicle-search-featured/runanddrive"
    HEADERS = {
    "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) 
    Chrome/92.0.4515.107 Safari/537.36",
    "accept": "*/*"
    }
    
    
    
    def selen(url, params = None):
    driver = webdriver.Chrome(executable_path = "C:/Users/пк/OneDrive/Рабочий 
    стол/Parser23/chromedriver.exe")
    driver.get(url)
    c = driver.page_source
    return c
    
    
    
    
    
    
    
    def get_content(html):
        cars = []
        soup = BeautifulSoup(html,"html.parser")
        items = soup.find("div", class_ = "dataTables_wrapper form-inline dt-bootstrap no- 
        footer").find("tbody").find_all("tr", {"role" : "row"})
        for item in items:
            Retail_Value = item.find("span", {"data-uname" : 
            "lotsearchLotestimatedretailvalue"})  
            link = item.find("a", class_ = "search-results")
            Sale_Date = item.find("span", {"data-uname" : "lotsearchLotauctiondate"})
            if link:    
                link = link.get("href")
            if Retail_Value:
                Retail_Value = Retail_Value.get_text(strip = True)
            else:
                Retail_Value = "Цены нет" 
            if Sale_Date:
                Sale_Date = Sale_Date.get_text(strip = False)
            else:
                Sale_Date = "Цены нет!"
            cars.append({
                "Lot# / link": f'{item.find("a", {"data-uname" : 
                "lotsearchLotnumber"}).get_text(strip = True)} / {link}',
                 "Year": item.find("span", {"data-uname" : 
                 "lotsearchLotcenturyyear"}).get_text(strip = True),
                 "Make": item.find("span", {"data-uname" : "lotsearchLotmake"}).get_text(strip 
                 = True),
                 "Model": item.find("span", {"data-uname": 
                 "lotsearchLotmodel"}).get_text(strip = True),
                 "Item#": item.find("span", {"data-uname": 
                 "lotsearchItemnumber"}).get_text(strip = True),
                 "Location/lane/ row": f'{item.find("span", {"data-uname" : 
                 "lotsearchLotyardname"}).get_text(strip = True)}/{item.find("span", {"pref- 
                 code" : "searchPreference.searchFields"}).get_text(strip = True)}',
                 "Sale Date": f'{Sale_Date}',
                 "Odometer": item.find("span", {"data-uname": 
                 "lotsearchLotodometerreading"}).get_text(strip = True),
                 "Title code": item.find("span", {"data-uname": 
                 "lotsearchSaletitletype"}).get_text(strip = True),
                 "Damage": item.find("span", {"data-uname" : 
                 "lotsearchLotdamagedescription"}).get_text(strip = True),
                 "Est. Retail Value": Retail_Value,
                 "Current Bid": item.find("ul", class_ = "list-unstyled").get_text(strip = 
                 True)
                 })
            print(cars)
    
    def parse():
        html = selen(URL)
        get_content(html)
    
    
    
    
    
    
    
    parse()
    



  • So press the selenium-a on the "Next" button, the browser page is updated. We use the driver.page_source, and we have an updated code of the page so far.

    P.S. We can go through the request, but the answer will be json.



Suggested Topics

  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2
  • 2