We Keep Coding

ValueError: unknown url type: ' ' (selenium)

I am going to download pictures from a clothing website for academic research, I use the code below
`
from ast import keyword
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
import time
import os
import wget
import random
import time
delay_choices = range(5,15)
delay = random.choice(delay_choices)
import requests
from fake_useragent import UserAgent
keyword = "jeans"
user_agent = UserAgent()
response = requests.get(url="https://www2.hm.com/en_asia3/ladies/shop-by-product/jeans.html", headers={ 'user-agent': user_agent.random })
driver = webdriver.Chrome("~~~~")
driver.get("https://www2.hm.com/en_asia3/ladies/shop-by-product/jeans.html")
time.sleep(4)
cookie = driver.find_element(By.ID, 'onetrust-accept-btn-handler')
cookie.click()
time.sleep(2)
for i in range(6):
driver.execute_script("window.scrollTo(0, 6900);")
time.sleep(delay)
loadmore = driver.find_element(By.XPATH,"/html/body/main/div/div/div/div[3]/div[2]/button")
loadmore.click()
imgs = driver.find_elements(By.CLASS_NAME, 'item-image')
path = os.path.join("H&M" + keyword)
os.mkdir(path)
count = 0
for img in imgs:
save_as = os.path.join(path, keyword + str(count) + '.jpg')
#print(img.get_attribute("src"))
wget.download(img.get_attribute("src"), save_as)
count += 1
time.sleep(6)
driver.quit()
`
and I got this issue:
in this line: wget.download(img.get_attribute('src'), save_as)
but I also use the "src" to download other website and didn't wrong.
I would wonder if anyone know what happen. 😢
Thanks a lot.
I have searched and couldn't solve this problem, and I hope someone can give me some advice.
enter image description here

The url you are trying to download using wget has lots of specific symbols and this can cause problems for the wget. This is an example of the URL you are attempting to download from: https://lp2.hm.com/hmgoepprod?set=source[/2b/bf/2bbf11a29fde773adcdOK],res[y],hmver[1]&call=url[file:/product/main]
Try to change the command a bit:
Instead of this:
wget.download(img.get_attribute("src"), save_as)
try this:
wget.download(f'"{img.get_attribute("src")}"', save_as)

Selenium Python Proxy

I want to use proxy with selenium, I watched the video from youtube how to do it, but it doesn't work. This is my code:
import names
import time as t
import random
import requests
import pyautogui
import smsactivateru
import tkinter as tk
import sys
import socket
from selenium import webdriver
from selenium.webdriver.common.proxy import *
from webdriver_manager.microsoft import EdgeChromiumDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
from smsactivateru import Sms, SmsTypes, SmsService, GetBalance, GetFreeSlots, GetNumber
from fake_useragent import UserAgent
options = webdriver.FirefoxOptions()
# set useragent
ua = UserAgent()
options.set_preference("general.useragent.override",ua.random)
# set proxy
firefox_capabilities = webdriver.DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
proxy = '91.214.31.234:8080'
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy
}
driver = webdriver.Firefox(options=options, proxy=proxy, capabilities=firefox_capabilities)
driver.get('https://2ip.ru')
t.sleep(5)
Why proxy doesn't work? Please help me. Kind people

The following code works for me in chrome.
PROXY = '91.214.31.234:8080'
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(options=chrome_options)
chrome.get("https://www.icanhazip.com")

Try this for FireFox.
from selenium import webdriver
webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
"httpProxy": PROXY,
"sslProxy": PROXY,
"proxyType": "MANUAL",
}
driver = webdriver.Firefox()
driver.get("https://www.icanhazip.com")

Python selenium how to keep browser open?

from bs4 import BeautifulSoup
import requests
from urllib.request import urlopen
import json
import time
from seleniumwire import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import ElementNotVisibleException
chrome_options = Options()
chrome_options.add_experimental_option("detach", True)
chrome_options.add_experimental_option("excludeSwitches", ["enable-logging"])
search_key = '성남 스터디카페'
url = "https://map.naver.com/v5/search/" + search_key
driverPath = "chromedriver.exe"
driver = webdriver.Chrome(driverPath, options=chrome_options)
driver.get(url)
I am making web crawler and I can see the page that I entered, but it keeps closing after 2~3 seconds.
I also used detach option.
Chrome version is 91.0.4472.124, so I downloaded 91 version webdriver, but the browser is still closing.
Is there any problem with my code?

It's closing because the main python process will stop after running the code.
If you want to keep the browser is opened, simply add in the end:
time.sleep(1000)

How to hide cmd.exe/console log of chromedriver in selenium in python?

How to hide cmd.exe/console log of chromedriver in selenium in python?
I tried:
driver.service.stop()
Full Code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
import time
print("Opening...")
driver = webdriver.Chrome()
driver.get('https://google.com')
driver.service.stop()
But it didn't close the console log/ cmd.exe

You cannot hide it completely in Chrome driver but you can suppress few and set minimum log level as below:
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument('log-level=2')
where log-level is
INFO = 0,
WARNING = 1,
LOG_ERROR = 2,
LOG_FATAL = 3.

#Combine this two options:
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
options = Options()
options.add_argument('--log-level=3')
dc = DesiredCapabilities.CHROME
dc['loggingPrefs'] = {'driver': 'OFF', 'server': 'OFF', 'browser': 'OFF'}
self.driver = webdriver.Chrome(chrome_options=options, desired_capabilities=dc, executable_path="C:\\path\\chromedriver.exe")

Proxy function works with requests, but not with a chrome browser?

I have no idea why the following code returns the proxy as invalid only for the chrome browser help is appreciated. Below are the imports.
import requests
import json
import time
import random
import threading
from threading import Thread
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from datetime import datetime
from proxymanager import ProxyManager
from random import randint
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.chrome.options import Options
def getProxy():
try:
proxy_manager = ProxyManager('proxies.txt')
proxydict = proxy_manager.random_proxy()
proxies = proxydict.get_dict()
except:
proxies = []
return proxies
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=https://%s' %getProxy)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://whatismyipaddress.com")

I'll go out of a limb and guess the problem is with the proxy expansion - you're trying to pass a dict to Chrome instead of the actual proxy address. You want to get the actual value from the Proxy() class in your getProxy() function, e.g.:
def get_proxy(string_only=True):
try:
proxy_manager = ProxyManager("proxies.txt")
proxy = proxy_manager.random_proxy()
if string_only:
return proxy.proxy_string
return proxy.get_dict()
except (OSError, IOError, IndexError) as e: # couldn't load the file / file is empty
return None
# With Chrome:
chrome_options = webdriver.ChromeOptions()
proxy = get_proxy()
if proxy:
chrome_options.add_argument("--proxy-server=" + proxy)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://whatismyipaddress.com")
# with requests:
response = requests.get("http://whatismyipaddress.com", proxies=get_proxy(False))
# etc.
I'd also recommend to load the proxy list only once if you intend to call this function often and if the proxies.txt is a static file.

This works
def get_proxy():
try:
proxy_manager = ProxyManager("proxies.txt")
return proxy_manager.random_proxy().proxy_string
except (OSError, IOError) as e: # couldn't load the file
return None
chrome_options = webdriver.ChromeOptions()
proxy = get_proxy()
if proxy:
chrome_options.add_argument("--proxy-server=" + proxy)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://whatismyipaddress.com")