Selenium scrolls to the element but does not click

Question

Trying to click next button from navigation bar of website "https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe" using selenium in python.

from selenium.webdriver import Chrome
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import time 

URL = "https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe"
driver = Chrome(ChromeDriverManager().install())


class Scraper:
    def __init__(self, website):
        self.website = website
    
    def get_website(self):
        return driver.get(self.website)

    def ignore_cookie(self):
        try:
            ignore_cookies = driver.find_element(by=By.XPATH, value='//*[@id="onetrust-reject-all- handler"]')
            ignore_cookies.click()
        except AttributeError:
            pass



    def next_page(self):
        driver.find_element(by=By.NAME, value="pagination-button-next").click()

The ignore cookie function works fine. But next_page function scrolls to the next button but does not click it.

Barry the Platipus · Accepted Answer · 2022-11-21 21:55:39Z

0

Include the following imports:

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t

Edit your next_page function like so:

wait = WebDriverWait(driver, 25)

next_page_button = wait.until(EC.element_to_be_clickable((By.XPATH, '//a[@name="pagination-button-next"]')))
next_page_button.location_once_scrolled_into_view
t.sleep(2)
next_page_button.click()

See Selenium documentation at https://www.selenium.dev/documentation/

edited Nov 21, 2022 at 21:55

answered Nov 21, 2022 at 21:40

Barry the Platipus

10.5k2 gold badges9 silver badges33 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

MITHU · Accepted Answer · 2022-11-22 08:03:17Z

This should do it:

from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = "https://uk.trustpilot.com/categories/bars_cafes?subcategories=cafe"

class Scraper:
    def __init__(self, website):
        self.driver = Chrome(ChromeDriverManager().install())
        self.driver.get(website)
        self.wait = WebDriverWait(self.driver,20)


    def ignore_cookie(self):
        self.driver.find_element(By.CSS_SELECTOR, "button[class^='onetrust-close-btn-handler']").click()


    def fetch_content(self):
        while True:
            for item in self.driver.find_elements(By.CSS_SELECTOR, "section > [class*='card_card']"):
                shop_name = item.find_element(By.CSS_SELECTOR, "a[name='business-unit-card'] p[class*='displayName']").text
                yield shop_name

            try:
                self.next_page()
                self.wait.until(EC.staleness_of(item))
            except Exception as err:
                self.driver.quit()
                return


    def next_page(self):
        next_page = self.driver.find_element(By.CSS_SELECTOR, "a[name='pagination-button-next']")
        self.driver.execute_script("arguments[0].click();", next_page)


scrape = Scraper(url)
scrape.ignore_cookie()
for title in scrape.fetch_content():
    print(title)

Collectives™ on Stack Overflow

Selenium scrolls to the element but does not click

2 Answers 2

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related