We Keep Coding

Starting new HTTPS connection (1): 1.rome.api.flipkart.com:443

Hello I am running a script where we get data from certain websites and put them in a database using scrapy, the script runs well but in log text file it shows ''DEBUG: Starting new HTTPS connection (1): 1.rome.api.flipkart.com:443''
what can be the real problem here, any help is appreciated, here is the code
import os
import sys, getopt
import time
import datetime
import pytz
import mysql.connector
import configparser
import shutil
import time
import concurrent.futures
import pandas as pd
currentdir = os.path.dirname(os.path.realpath(__file__))
parentdir = os.path.dirname(currentdir)
sys.path.append(parentdir)
from datetime import datetime
from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess
from multiprocessing import Process
from db.db_action import DBAction
from utils import utils
from concurrent.futures import ThreadPoolExecutor, as_completed
tz = pytz.timezone("Asia/Kolkata")
crawl_inputs = dict()
crawl_inputs["env"] = "prod"
crawl_inputs["marketplace"] = "Amazon"
crawl_inputs["site"] = "amz_kw"
crawl_inputs["db_name"] = "asian_paints"
crawl_inputs["pf_id"] = "1"
crawl_inputs["location_search"] = "0"
crawl_inputs["limit"] = ""
crawl_inputs["page"] = 1
crawl_inputs["kw_snapshot"] = 0
crawl_inputs["pdp_snapshot"] = 0
crawl_inputs["quick_search"] = 1
crawl_inputs["client_id"] = 1241
crawl_inputs["client"] = "asian_paints"
crawl_inputs["start_time"] = datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
db_action = DBAction()
actual_count = 0
result_count = 0
archived_path = "C:/archives/"
connection = None
cursor = None
# Directory Create
try:
if not os.path.exists(archived_path):
os.makedirs(archived_path)
if not os.path.exists(archived_path + datetime.now(tz).strftime("%Y%m%d")+"/"+ crawl_inputs["site"]):
os.makedirs(archived_path + datetime.now(tz).strftime("%Y%m%d")+"/"+ crawl_inputs["site"])
if not os.path.exists("C:/var/logs/" + crawl_inputs["site"]):
os.makedirs("C:/var/logs/" + crawl_inputs["site"])
shutil.move("C:/var/logs/" + crawl_inputs["site"], archived_path + datetime.now(tz).strftime("%Y%m%d"), copy_function = shutil.copytree)
except Exception as e:
print(e)
print("File creation error: {0}:{1}".format(e.errno, e.strerror))
try:
if os.name == "nt":
log_path = "C:/var/logs"
base_dir = log_path+"/"+crawl_inputs["site"]
if not os.path.exists(base_dir):
os.makedirs(base_dir)
else:
log_path = "/var/logs/"
base_dir = log_path+"/"+crawl_inputs["site"]
if not os.path.exists(base_dir):
os.makedirs(base_dir)
directories = ["output", "run_log", "webpages"]
for directory in directories:
if not os.path.exists(base_dir+"/"+directory):
os.makedirs(base_dir+"/"+directory)
except OSError as oserr:
print("OS error occurred trying to open. Aborting.. Error{0}:{1}".format(oserr.errno, oserr.strerror))
sys.exit(1)
except IOError as ioerr:
print("I/O Error{0}: {1}".format(ioerr.errno, ioerr.strerror))
sys.exit(1)
except FileNotFoundError as fnfe:
print("File not found. Aborting.. Error: {0}:{1}".format(fnfe.errno, fnfe.strerror))
sys.exit(1)
except Exception as e:
print("File creation Error. Aborting.. Error: {0}:{1}".format(e.errno, e.strerror))
sys.exit(1)
crawl_inputs = db_action.get_kw_platform_inputs(crawl_inputs)
print(f"Total Executing Inputs : {len(crawl_inputs['inputs'])}")
print("Crawl ID: {0}".format(crawl_inputs["crawl_id"]))
def start(input):
pf_id = str(input["pf_id"])
keyword = str(input["keyword"])
brand_id = str(input["brand_id"])
brand_name = str(input["brand_name"])
keyword_id = str(input["keyword_id"])
location_id = str(input["location_id"])
location = str(input["location"])
pincode = str(input["pincode"])
location_search = str(crawl_inputs["location_search"])
env = str(crawl_inputs["env"])
db_name = str(crawl_inputs["db_name"])
crawl_id = str(crawl_inputs["crawl_id"])
site = str(crawl_inputs["site"])
page = str(crawl_inputs["page"])
command = 'python ' +currentdir+ '/main_kw.py --env="' +env+ '" --db_name="' +db_name+ '" --crawl_id="' +crawl_id+ '" --site="' +site+ '" --pf_id="'+pf_id+ '" --brand_id="' +brand_id+ '" --brand_name="' +brand_name+ '" --keyword_id="' +keyword_id+ '" --keyword="' +keyword+ '" --location_id="' +location_id+ '" --location="' +location+ '" --pincode="' +str(pincode)+ '" --page="' +page+ '" --location_search="' +location_search+ '"'
print("Executing Input :{0}".format(command))
os.system(command)
def runner():
threads = []
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for input in crawl_inputs.get("inputs"):
print(f"Input: {input}")
task = executor.submit(start, input)
threads.append(task)
for task in concurrent.futures.as_completed(threads):
print(task.result())
runner()
time.sleep(5)
crawl_inputs["finish_time"] = datetime.now(tz).strftime("%Y-%m-%d %H:%M:%S")
connection = db_action.db_connection(db_name=crawl_inputs["db_name"], env=crawl_inputs["env"])
cursor = connection.cursor()
try:
cursor.execute("update `amazon_crawl_kw` set `status` = 0 where `crawl_id` != " +str(crawl_inputs["crawl_id"])+ ";")
cursor.execute("select count(kw_crawl_data_id) as product_count from `amazon_crawl_kw` where status = 1 and pf_id = "+str(crawl_inputs["pf_id"])+ " and crawl_id=" +str(crawl_inputs["crawl_id"])+ ";")
row = cursor.fetchone()
print("row value: "+ str(row))
result_count = row["product_count"]
print("Crawled row count : " + str(result_count))
try:
sql2 = 'UPDATE rb_crawl SET status=1, end_time = "' +str(crawl_inputs["finish_time"])+ '" ,no_of_sku_parsed='+ str(result_count)+ ' WHERE crawl_id=' + str(crawl_inputs["crawl_id"])
cursor.execute(sql2)
except Exception as e:
print("The following exception occured while updating : "+ str(e))
sql3 = 'UPDATE rb_platform SET kw_crawl_data_date = "'+ str(crawl_inputs["start_time"]) + '", kw_crawl_data_id = ' +str(crawl_inputs["crawl_id"])+ ' WHERE pf_id = ' + str(crawl_inputs["pf_id"])
cursor.execute(sql3)
connection.commit()
connection.close()
print("Updated rb_platform successfully")
except Exception as e:
print("Updating crawl id failed with exception as :" + str(e))
# try:
# items_count = result_count
# subject = crawl_inputs["site"] +" crawling completed"
# body = "Hi Team,<br><br>" +crawl_inputs["site"]+ " crawling successfully completed for the plateform " +crawl_inputs["marketplace"]+ "...<br>Platform Id: " +str(crawl_inputs["pf_id"])+ "<br>Crawl Id: " +str(crawl_inputs["crawl_id"])+ "<br>Total crawled items: " +str(items_count)+ " <br>Total Actual Items: " + str(actual_count) +" <br>Please QC the data value..<br><br>Thanks<br>Trailytics Team"
# utils.send_mail("no-reply#trailytics.com", "vijay.kothawar#trailytics.com;ashish.rawat#trailytics.com;anirudh.varshney#trailytics.com;ashutosh.shukla#trailytics.com", subject, body)
# print("Crawling process has been completed")
# except Exception as e:
# print("Mail Sending error:" + str(e))
print("Finish")

How do I allow a client to connect to python server?

When I run the server, then the client, I Automatically get these messages:
**SERVER: **
Traceback (most recent call last):
File "C:\Users\Ryan\Desktop\testproj-server.py", line 142, in
handleConnectedSocket()
File "C:\Users\Ryan\Desktop\testproj-server.py", line 58, in handleConnectedSocket
rcvdStr = rcvdStr + fd.recv(1024)
TypeError: can only concatenate str (not "bytes") to str
**CLIENT: **
ERROR: Cannot connect to chat server [WinError 10056] A connect request was made on an already connected socket
Exiting... Goodbye! *
This is my code:
SERVER CODE:
import socket
import select
def runSelect():
selectUnsuccessful = True
while selectUnsuccessful:
try:
readyRecvList, readySendList, readyErrList = select.select(recvList, sendList, [])
selectUnsuccessful = False
except select.error:
for fd in recvList:
try:
tempRecvList, tempSendList, tempErrList = select.select([fd], [], [], 0)
except select.error:
if fd == serverSocket:
fd.close()
exit(1)
else:
if fd in recvList:
recvList.remove(fd)
fd.close()
return readyRecvList, readySendList
def handleListeningSocket():
try:
newConnectionSocket, addr = serverSocket.accept()
except socket.error as err:
print("\nERROR: Something went wrong in the accept() function call:", err)
exit(1)
try:
recvList.append(newConnectionSocket)
sendList.append(newConnectionSocket)
print("INFO: Connecting socket created between %s and %s" %
(newConnectionSocket.getsockname(), newConnectionSocket.getpeername()))
print("* Client %s is ready to chat *" % (str(newConnectionSocket.getpeername())))
except (socket.error, socket.gaierror) as err:
print("\nERROR: Something went wrong with the new connection socket:", err)
if newConnectionSocket in recvList:
recvList.remove(newConnectionSocket)
sendList.remove(newConnectionSocket)
newConnectionSocket.close()
def handleConnectedSocket():
try:
recvIsComplete = False
rcvdStr = ""
while not recvIsComplete:
rcvdStr = rcvdStr + fd.recv(1024)
if fd not in sendList:
sendList.append(fd)
# ~ is the delimiter used to indicate message start and finish
if rcvdStr.strip('~') != "":
if (rcvdStr[0] == "~") and (rcvdStr[-1] == "~"):
recvIsComplete = True
clientMessage = rcvdStr.strip('~')
else: # if empty string, connection has been terminated
if fd in recvList:
recvList.remove(fd)
if fd in sendList:
sendList.remove(fd)
del clientMessages[fd] # Delete connection information
fd.close()
if clientMessage == "quit()":
print ("\n* Client %s has left the chat room *\n" % (str(fd.getpeername())))
if fd in recvList:
recvList.remove(fd)
fd.close()
if fd in sendList:
sendList.remove(fd)
fd.close()
else:
print("\n%s: %s" % (fd.getpeername(), clientMessage))
# add message to dictionary, pending transmission
clientMessages[fd] = str(clientMessage)
except socket.error as err:
print("\nERROR: Connection to the client has abruptly ended:", err)
if fd in recvList:
recvList.remove(fd)
if fd in sendList:
sendList.remove(fd)
fd.close()
print("* I am ready to chat with a new client! *\n")
"""
main - Runs the Full Duplex Chat server
"""
# Global Variables
serverHost = 'localhost'
serverPort = 22222
recvList = []
sendList = []
clientMessages = {}
try:
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serverSocket.setblocking(0)
serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serverSocket.bind((serverHost, serverPort))
serverSocket.listen(3)
print ("INFO: I am listening at %s" % (str(serverSocket.getsockname())))
print ("* I am ready to chat with a new client! *\n")
except (socket.error, socket.gaierror) as err:
print ("\nERROR: Something went wrong in creating the listening socket:", err)
exit(1)
recvList = [serverSocket]
try:
while True:
serverSocket.setblocking(False)
readyForRecv, readyForSend = runSelect()
for fd in readyForRecv:
if fd == serverSocket:
handleListeningSocket()
else:
handleConnectedSocket()
for fd in readyForSend:
try:
if fd in clientMessages.keys(): # See if connection information exists
broadcast = str(clientMessages[fd]) # Add message to broadcast variable
if broadcast: # See if a message is actually there
for client in readyForSend: # Broadcast message to every connected client
if broadcast != "":
print ("* Broadcasting message \"%s\" to %s *" % (str(broadcast), client.getpeername()))
client.send(str(fd.getpeername()) + ": " + str(broadcast))
clientMessages[fd] = "" # Empty pending messages
#except:
# print "\nERROR: Something awful happened while broadcasting messages"
#break
except socket.error as err:
print ("\nERROR: Something awful happened with a connected socket:", err)
if fd in recvList:
recvList.remove(fd)
if fd in sendList:
sendList.remove(fd)
fd.close()
except KeyboardInterrupt:
for fd in recvList:
fd.close()
for fd in sendList:
fd.close()
print ("\nINFO: KeyboardInterrupt")
print ("* Closing all sockets and exiting... Goodbye! *")
exit(0)
CLIENT CODE:
import socket
import select
import sys
def main():
serverHost = 'localhost'
serverPort = 22222
try:
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error as err:
print("ERROR: Cannot create client side socket:", err)
exit(1)
while True:
try:
clientSocket.connect((serverHost, serverPort))
except socket.error as err:
print("ERROR: Cannot connect to chat server", err)
print("* Exiting... Goodbye! *")
exit(1)
except:
print("ERROR: Something awful happened!")
exit(1)
break
recvList = [clientSocket, sys.stdin]
print("* You are now connected to chat server %s as %s *" %
(clientSocket.getpeername(), clientSocket.getsockname()))
try:
while True:
readyRecvList, readySendList, readyErrList = select.select(recvList, [], [])
for fd in readyRecvList:
if fd == sys.stdin:
message = sys.stdin.readline().rstrip()
clientSocket.sendall("~" + str(message) + "~")
if (message == "quit()"):
print("* Exiting chat room! *")
clientSocket.close()
exit(0)
break
elif fd == clientSocket:
clientSocket.settimeout(3)
try:
message = clientSocket.recv(2048)
except socket.timeout as err:
print("ERROR: The recv() function timed out after 3 seconds! Try again.")
except:
print("ERROR: Something awful happened!")
else:
if message == "":
break
else:
print("%s\n" % (message))
clientSocket.settimeout(None)
break
except select.error as err:
for fd in recvList:
try:
tempRecvList, tempSendList, tempErrList = select.select([fd], [], [], 0)
except select.error:
if fd == clientSocket:
fd.close()
exit(1)
else:
if fd in recvList:
recvList.remove(fd)
fd.close()
except socket.error as err:
print("ERROR: Cannot connect to chat server", err)
print("* Exiting... Goodbye! *")
exit(1)
if fd in recvList:
fd.close()
except KeyboardInterrupt:
print("\nINFO: KeyboardInterrupt")
print("* Closing all sockets and exiting chat server... Goodbye! *")
clientSocket.close()
exit(0)
if __name__ == '__main__':
main()

You didn't decode the received bytes
rcvdStr = rcvdStr + fd.recv(1024).decode()
I haven't exactly checked, so tell me if this helps
EDIT:
Ok so for starters remove the while statement in line 17 of the client, once the client is connected to the server you can't connect again (make sure you remove the break in the new line 26)
The new line 34 is causinf errors (readyRecvList, readySendList, readyErrList = select.select(recvList, [], []))
Traceback (most recent call last):
File "C:\Users\User\Desktop\Client.py", line 93, in <module>
main()
File "C:\Users\User\Desktop\Client.py", line 34, in main
readyRecvList, readySendList, readyErrList = select.select(recvList, [], [])
io.UnsupportedOperation: fileno

Remove a file in Python

I'm trying to remove a file that physically exists on my system. I wrote the following function using Python 3 that does not work:
def remove_file(output_file):
print("Removing the output file: ", output_file)
try:
os.remove(output_file)
except RemoveFileError as e:
remove_stat = e.returncode
else:
remove_stat = 1
if remove_stat == 0:
print("File removed!")
else:
print("File not removed.")
When I try to remove the file using the above code this is the result I get:
Removing the output file: ../../../output_files/aws_instance_list/aws-master-list-03-21-2019.csv
File not removed.
What am I doing wrong?

The else clause of your try statement is being executed when no error occurred. See Handling Exceptions
Try this updated version of your code:
def remove_file(output_file):
print("Removing the output file: ", output_file)
try:
os.remove(output_file)
except RemoveFileError as e:
remove_stat = e.returncode
else:
remove_stat = 0
if remove_stat == 0:
print("File removed!")
else:
print("File not removed.")
print("File exists:", os.path.exists(output_file))

python ssh paramiko save to file

I would like to save to file all my output from ssh connection(s). The ssh connection works fine, the output is ok too to a stdout. I want to create a file for every connection individually in a file. I have changed the output lines to the lines below and also moved it above
output_filename = ip_address + ".txt"
file = open(output_filename, 'w')
file.write(output.decode)
file.close()
What is missing?
I receive this error:
line 100, in fractal
except 10060:
TypeError: catching classes that do not inherit from BaseException is not allowed
And it just don't save the output. the file is created, but in blank.
import socket
import paramiko
import time
import sys
def fractal(ip, username, passwd, enapass, command, command2, command3, command4, devtype):
ip_address = ip
user = username
password = passwd
enapw = enapass
commando = command
commando2 = command2
commando3 = command3
commando4 = command4
devtype = devtype
print("Connecting to: "+ip + " on Port 22")
try:
if ip:
global ssh_client
ssh_client = paramiko.client.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=ip_address, username=user, password=password, compress=True, look_for_keys=False, allow_agent=False, timeout=5)
print("########################## CONNECTED TO: "+ip_address +" ##########################")
remote_connection = ssh_client.invoke_shell()
if devtype == 'CISCO':
results = remote_connection.send("term len 0\n")
print(results)
time.sleep(2)
results = remote_connection.send("show run\n")
print(results)
time.sleep(6)
if devtype == 'F5':
remote_connection.send("term len 0\n")
time.sleep(2)
remote_connection.send("sh ver\n")
time.sleep(6)
if devtype == 'LINUX':
remote_connection.send("pwd\n")
time.sleep(2)
else:
#print("Please set IP Address first!!!")
pass
if enapass:
remote_connection.send("enable\n")
# remote_connection.send("conf t\n")
remote_connection.send(enapw)
remote_connection.send("\n")
else:
pass
if command:
#remote_connection.send("show run\n")
remote_connection.send(commando)
remote_connection.send("\n")
else:
print("Command not found!")
if command2:
remote_connection.send(commando2)
remote_connection.send("\n")
else:
pass
if command3:
remote_connection.send(commando3)
remote_connection.send("\n")
else:
pass
if command4:
remote_connection.send(commando4)
remote_connection.send("\n")
else:
pass
time.sleep(1)
output = remote_connection.recv(65535)
print(output.decode())
print("########################## END OF: " + ip_address + " ##########################")
reader = ssh_client.connect
ssh_client.close
output_filename = ip_address + ".txt"
file = open(output_filename, 'w')
file.write(output)
file.close()
except TypeError:
print('Please check your settings!')
except UnboundLocalError:
print('Please check IP Address!')
except paramiko.AuthenticationException:
print(ip+": Authentication failed, please verify your credentials.")
except paramiko.SSHException as sshException:
print(ip+": Unable to establish SSH connection: %s" % sshException)
except paramiko.BadHostKeyException as badHostKeyException:
print(ip+": Unable to verify server's host key: %s" % badHostKeyException)
except socket.error:
print(ip+": Couldn't connect to server. Check IP Address and Port")
# sys.exit()
except 10060:
print(ip+": The host was not reachable")
except socket.gaierror:
print(ip+': Check IP Address')
except 11004:
print(ip+": The host was not reachable")
except IOError as e:
print("I/O error({0}): {1}".format(e.errno, e.strerror))
except ValueError:
print("Could not convert data to an integer.")
except FileNotFoundError:
print("No File was selected!")
except:
print("Unexpected error:", sys.exc_info()[0])
raise
# countErr = paramiko.AuthenticationException
# countErr = 0
# for countErr in countErr:
# count = count + 1
# print ("Athentication failures: "+countErr)

This happens only in Python3, because except now expects a class that is a subclass of BaseException. And the integer 10060 is not.
>>> try:
... raise ValueError
... except 10080:
... print('dfsdf')
...
Traceback (most recent call last):
File "<stdin>", line 2, in <module>
ValueError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 3, in <module>
TypeError: catching classes that do not inherit from BaseException is not allowed
So Paramiko is raising some other error, but error handling dies when it tries to evaluate the except 10060 statement.

how to close the file after recording?

please help to fix the script.
import urllib
import re
import os
import pprint
import requests
import bs4
def make_catalog():
try:
os.mkdir('GRAB')
except FileExistsError:
print('FileExistsError')
except PermissionError :
print('PermissionError ')
except Exception:
print(Exception)
def change_catalog():
try:
os.chdir('GRAB')
except PermissionError :
print('PermissionError ')
except Exception:
print(Exception)
def download_image(path, name):
#urllib.URLopener().retrieve(prefix + path, name)
img = urllib.request.urlopen(prefix + path).read()
try:
f = open(name, "wb")
if f:
print('open!!!')
if f.write(img):
print('write!!!')
except OSError:
print('OSError')
except Exception:
print(Exception)
finally:
f.close()
beginIndex = 5794
endIndex = 5800
prefix = "http://www.inpic.ru"
rep_chars = ['\\', '/', ':', '*', '?', '"', '<', '>', '|', '-' , ' ']
make_catalog()
change_catalog()
for i in range(beginIndex, endIndex):
req = requests.get(prefix + '/image/' + str(i))
if req.status_code == requests.codes.ok:
#print(i, '\t', req.status_code, '\t', req, end='\n')
soup = bs4.BeautifulSoup(req.content)
#print(soup.prettify())
name = soup.find("td", {"class": "post_title"}).contents[1].contents
#author = soup.find("div", {"class": "date_author"}).contents[1].contents
print('NAME: ', name[0])
#print(author[0])
#name[0] = re.sub('[\\\\/:*?"<>|-]', '_', name[0])
for char in rep_chars:
name[0] = name[0].replace(char, '_')
print('newNAME: ', name[0])
mainImagePath = soup.find("img", {"class": "image last"})["src"]
mainImageExt = mainImagePath.split('.')[-1]
manyImages = soup.findAll("img", {"class": "image"})
print('MAINUMAGE: ', mainImagePath)
print('MAINIMAGE EXT: ',mainImageExt)
print('MANYIMAGE: \n')
pprint.pprint(manyImages)
if len(manyImages) > 1:
print('CATALOG MAKE')
try:
os.mkdir(name[0])
#except FileExistsError:
#print('FileExistsError')
except PermissionError :
print('PermissionError')
except Exception:
print(Exception)
os.chdir(name[0])
#download_image(mainImagePath, str(name[0]) + '_0.' + mainImageExt)
i = 0
for name in manyImages:
#print(name['src'], end='------------\n')
download_image(name['src'], str(name['src']))
i = i + 1
os.chdir('../')
else:
print('IMAGE MAKE')
download_image(mainImagePath, str(name[0]) + '.' + mainImageExt)
print('mainImagePath', mainImagePath)
print('name', str(name[0]) + '.' + mainImageExt)
print('==================================')
the problem that when recording images from the page group
http://www.inpic.ru/image/5797/
displays the following error message:
Traceback (most recent call last):
File "C:\VINT\OPENSERVER\OpenServer\domains\localhost\python\parse_html\1\q.py", line 98, in <module>
download_image(name['src'], str(name['src']))
File "C:\VINT\OPENSERVER\OpenServer\domains\localhost\python\parse_html\1\q.py", line 46, in download_image
f.close()
UnboundLocalError: local variable 'f' referenced before assignment

You are trying to close a file that failed to open. f was never assigned to because the open() call raised an exception.
Instead of closing the file object in the finally handler, use it as a context manager:
def download_image(path, name):
#urllib.URLopener().retrieve(prefix + path, name)
img = urllib.request.urlopen(prefix + path).read()
try:
with open(name, "wb") as f:
print('open!!!')
f.write(img)
print('write!!!')
except OSError:
print('OSError')
except Exception:
print(Exception)
Here the with statement will ensure that f is closed for you if it was opened successfully, whatever happens.