ARTICLE AD BOX
Neither the requests library and nor the PyPi library is working for me. I have been trying constantly with selenium and I am unable to fetch the results, can somebody help me? is there any lib which I am not considering?
from pathlib import Path import platform import time from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.firefox.service import Service as FirefoxService from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.expected_conditions import presence_of_all_elements_located QUERY = input("Enter your Google search query: ") base_dir = Path(__file__).resolve().parent driver_path = base_dir.parent / "geckodriver.exe" def run_with_selenium() -> bool: options = FirefoxOptions() options.add_argument("--headless") service = FirefoxService(executable_path=str(driver_path)) driver = None try: driver = webdriver.Firefox(service=service, options=options) driver.get("https://www.google.com/search?q=" + QUERY) time.sleep(3) # Extract search results using different selectors results = [] # Try to find result containers result_containers = driver.find_elements(By.CSS_SELECTOR, "div[data-sokoban-container]") print(f"\nSearch Results for '{QUERY}':\n") for idx, container in enumerate(result_containers[:10], 1): # Get top 10 results try: # Try to find title title_elem = container.find_element(By.CSS_SELECTOR, "h3") title = title_elem.text # Try to find link link_elem = container.find_element(By.CSS_SELECTOR, "a") link = link_elem.get_attribute("href") # Try to find snippet/description snippet = "" try: snippet_elem = container.find_element(By.CSS_SELECTOR, "span[data-snippet]") snippet = snippet_elem.text except: # Fallback: try to find any descriptive text text_elems = container.find_elements(By.TAG_NAME, "span") for elem in text_elems: text = elem.text.strip() if text and len(text) > 20 and len(text) < 500: snippet = text break if title and link: print(f"{idx}. {title}") if link and "http" in link: print(f" URL: {link}") if snippet: print(f" Description: {snippet}") print() results.append({"title": title, "url": link, "snippet": snippet}) except Exception as e: continue if results: print(f"✓ Found {len(results)} results") return True else: print("No results extracted, but page loaded successfully") print(f"Page title: {driver.title}") return True except Exception as error: print(f"Selenium error: {type(error).__name__}: {str(error)[:200]}") return False finally: if driver is not None: try: driver.quit() except: pass run_with_selenium()