Not able to write in a file(python) - python

I'm trying to make an HTTP Server from scratch and wanted to write the log in a text file so I created this function.
def do_LOG(self, addr, request):
path = 'log/logging.txt'
host = addr[0]
port = addr[1]
method = request[:4]
headers = request.split('\n')
filename = headers[0].split()[1]
f = open(path, "a+")
f.writelines('Server used: ' + host + '\n'+'Port used: ' + port + '\n'+'Method Served: ' + method + '\n'+'Filename: ' + filename + '\n\n')
f.close()
return
This function only creates a file but is not able to write in the file. I'm overriding this function from the parent class.This is the definition in the parent class.
def do_LOG(self, addr, request):
return

Please Provide some more code
For better handling, add relative path of the directory by adding './' at the start.
Make Sure the 'log' directory exists in the project's parent directory
Still I'm providing a temporary fix-
class Logs:
def do_LOG(self, addr, request):
path = './log/logging.txt'
host = addr[0]
port = addr[1]
method = request[:4]
headers = request.split('\n')
filename = headers[0].split()[1]
f = open(path, "a+")
f.writelines('Server used: ' + host + '\n'+'Port used: ' + port + '\n'+'Method Served: ' + method + '\n'+'Filename: ' + filename + '\n\n')
f.close()
return
Logs().do_LOG("<addr>", "<request>")
Hope this helps!

First, be careful to use proper indentation (I suspect this is from copying your code).
Second, you chose the mode 'a+' which I don't know about. In order to write to a file, you should use the 'w' mode, I also recommend providing the encoding:
f = open(path, "w", encoding= "utf-8")
f.write('Server used: ' + host + '\nPort used: ' + port + '\n'+'Method Served: ' + method + '\n'+'Filename: ' + filename + '\n\n')
f.close()
If this does not work, maybe that's because there is a problem with the values of host, port or method, you should then try to write the file with multiple calls, to see where the problem occurs:
f.write('Server used')
f.write(host)
...

Related

100% cpu usage while iterating through dictionary

I have python script which synchronizes files from the remote server by comparing with the local files. It uses a file downloading tool written injava. I store filepath, file size and its name in dictionary which looks like:
eddsFilesDict = {'/part07_data07/Science/TGO/ACS/SDU_': ['860904
SDU__DACS_51FC_023D8101_2021-155T02-12-17__00001.EXM', '17660866
SDU__DACS_51FB_023D8101_2021-155T02-10-16__00001.EXM', '17660866
SDU__DACS_51FA_023D8101_2021-155T 02-02-18__00001.EXM', '17660866
SDU__DACS_51F9_023D8101_2021-155T02-00-16__00001.EXM']}
To list up files on the local machine I use next part of the code:
filenames = []
for top, dirs, files in os.walk('/data/local/'):
for fn in files:
filenames.append(os.stat(os.path.join(top, fn)).st_size.__str__() + ' ' + os.path.join(fn))
But in the following part of my script usage of CPU goes up to 100% and it lasts for 1-2 minutes. I cannot understand why. By experimental way I understood that subprocess is not the thing.
for pathname, fileName in eddsFilesDict.items():
for f in fileName:
if f not in filenames:
logger.info('REQUESTING FILE: ' + pathname + '/' + f.split('.')[0].split(' ')[1] + ' FILE SIZE: '
+ f.split(' ')[0])
if 'SDU' in f:
argsFile = shlex.split(
'/home/user1/client/bin/fs_client --arc tar') # here I call java tool to start file downloading
p1 = subprocess.Popen(argsFile, stderr=subprocess.PIPE)
out, err = p1.communicate()
logger.warning('REQUESTING FILE: ' + pathname + '/' + f.split('.')[0].split(' ')[1] + ' FILE SIZE: '
+ f.split(' ')[0] + '\n' + err.__str__())
elif 'SCI' in f:
argsFile = shlex.split(
'/home/user1/client/bin/fs_client --arc tar')
p2 = subprocess.Popen(argsFile, stderr=subprocess.PIPE)
out, err = p2.communicate()
logger.warning('REQUESTING FILE: ' + pathname + '/' + f.split('.')[0].split(' ')[1] + ' FILE SIZE: '
+ f.split(' ')[0] + '\n' + err.__str__())

Optimize the performance of retreiving file sizes with pysftp

I have a requirement to get the file details for certain locations (within the system and SFTP) and get the file size for some locations on SFTP which can be achieved using the shared code.
def getFileDetails(location: str):
filenames: list = []
if location.find(":") != -1:
for file in glob.glob(location):
filenames.append(getFileNameFromFilePath(file))
else:
with pysftp.Connection(host=myHostname, username=myUsername, password=myPassword) as sftp:
remote_files = [x.filename for x in sorted(sftp.listdir_attr(location), key=lambda f: f.st_mtime)]
if location == LOCATION_SFTP_A:
for filename in remote_files:
filenames.append(filename)
sftp_archive_d_size_mapping[filename] = sftp.stat(location + "/" + filename).st_size
elif location == LOCATION_SFTP_B:
for filename in remote_files:
filenames.append(filename)
sftp_archive_e_size_mapping[filename] = sftp.stat(location + "/" + filename).st_size
else:
for filename in remote_files:
filenames.append(filename)
sftp.close()
return filenames
There are more than 10000+ files in LOCATION_SFTP_A and LOCATION_SFTP_B. For each file, I need to get the file size. To get the size I am using
sftp_archive_d_size_mapping[filename] = sftp.stat(location + "/" + filename).st_size
sftp_archive_e_size_mapping[filename] = sftp.stat(location + "/" + filename).st_size
# Time Taken : 5 min+
sftp_archive_d_size_mapping[filename] = 1 #sftp.stat(location + "/" + filename).st_size
sftp_archive_e_size_mapping[filename] = 1 #sftp.stat(location + "/" + filename).st_size
# Time Taken : 20-30 s
If I comment sftp.stat(location + "/" + filename).st_size and assign static value It takes only 20-30 seconds to run the entire code. I am looking for a way How can optimize the time and get the file size details.
The Connection.listdir_attr already gives you the file size in SFTPAttributes.st_size.
There's no need to call Connection.stat for each file to get the size (again).
See also:
With pysftp or Paramiko, how can I get a directory listing complete with attributes?
How to fetch sizes of all SFTP files in a directory through Paramiko

Python - Path/Folder/File creation

I am running the following block of code to create the path to a new file:
# Opens/create the file that will be created
device_name = target_device["host"].split('.')
path = "/home/user/test_scripts/configs/" + device_name[-1] + "/"
print(path)
# Check if path exists
if not os.path.exists(path):
os.makedirs(path)
# file = open(time_now + "_" + target_device["host"] + "_config.txt", "w")
file = open(path + time_now + "_" + device_name[0] + "_config.txt", "w")
# Time Stamp File
file.write('\n Create on ' + now.strftime("%Y-%m-%d") +
' at ' + now.strftime("%H:%M:%S") + ' GMT\n')
# Writes output to file
file.write(output)
# Close file
file.close()
The code run as intended with the exception that it creates and saves the files on the directory: /home/user/test_scripts/configs/ instead on the indented one that should be: /home/user/test_scripts/configs/device_name[-1]/.
Please advise.
Regards,
./daq
Try using os.path.join(base_path, new_path) [Reference] instead of string concatenation. For example:
path = os.path.join("/home/user/test_scripts/configs/", device_name[-1])
os.makedirs(path, exist_ok=True)
new_name = time_now + "_" + device_name[0] + "_config.txt"
with open(os.path.join(path, new_name), "w+") as file:
file.write("something")
Although I don't get why you're creating a directory with device_name[-1] and as a file name using device_name[0].

Trouble with apostrophe in arcpy search cursor where clause

I've put together a tkinter form and python script for downloading files from an ftp site. The filenames are in the attribute table of a shapefile, as well as an overall Name that the filenames correspond too. In other words I look up a Name such as "CABOT" and download the filename 34092_18.tif. However, if a Name has an apostrophe, such as "O'KEAN", it's giving me trouble. I try to replace the apostrophe, like I've done in previous scripts, but it doesn't download anything....
whereExp = quadField + " = " + "'" + quadName.replace("'", '"') + "'"
quadFields = ["FILENAME"]
c = arcpy.da.SearchCursor(collarlessQuad, quadFields, whereExp)
for row in c:
tifFile = row[0]
tifName = quadName.replace("'", '') + '_' + tifFile
#fullUrl = ftpUrl + tifFile
local_filename = os.path.join(downloadDir, tifName)
lf = open(local_filename, "wb")
ftp.retrbinary('RETR ' + tifFile, lf.write)
lf.close()
Here is an example of a portion of a script that works fine by replacing the apostrophe....
where_clause = quadField + " = " + "'" + quad.replace("'", '"') + "'"
#out_quad = quad.replace("'", "") + ".shp"
arcpy.MakeFeatureLayer_management(quadTable, "quadLayer")
select_out_feature_class = arcpy.SelectLayerByAttribute_management("quadLayer", "NEW_SELECTION", where_clause)

FTP and python question

Can someone help me.
Why it is not working
import ftplib
import os
def readList(request):
machine=[]
login=[]
password=[]
for line in open("netrc"): #read netrc file
old=line.strip()
line=line.strip().split()
if old.startswith("machine"): machine.append(line[-1])
if old.startswith("login"): login.append(line[-1])
if old.startswith("password"): password.append(line[-1])
connectFtp(machine,login,password)
def connectFtp(machine,login,password):
for i in range(len(machine)):
try:
ftp = ftplib.FTP(machine[i])
print 'conected to ' + machine[i]
ftp.login(login[i],password[i])
print 'login - ' + login[i] + ' pasword -' + password[i]
except Exception,e:
print e
else:
ftp.cwd("PublicFolder")
print 'PublicFolder'
def upload(filename, file):
readList()
ext = os.path.splitext(file)[1]
if ext in (".txt", ".htm", ".html"):
ftp.storlines("STOR " + filename, open(file))
else:
ftp.storbinary("STOR " + filename, open(file, "rb"), 1024)
print 'success... yra'
upload('test4.txt', r'c:\example2\media\uploads\test4.txt')`
When it was together it was working. But when i separate it in to functions something happened, I cant understand what.
(Apart from the horrid indentation problems, which are presumably due to botched copy and paste otherwise you'd get syntax errors up the wazoo...!)...:
Scoping problem, first: connectFtp makes a local variable ftp so that variables goes away as soon as the function's done. Then upload tries using the variable, but of course it isn't there any more.
Add a return ftp at the end of connectFtp, a yield connectFtp instead of a plain call to the loop in readList, and use a for ftp in readList(): loop in upload.
Something like this?
import os
def readList(request):
machine = []
login = []
password = []
for line in open("netrc"): # read netrc file
old = line.strip()
line = line.strip().split()
if old.startswith("machine"): machine.append(line[-1])
if old.startswith("login"): login.append(line[-1])
if old.startswith("password"): password.append(line[-1])
yield connectFtp
def connectFtp(machine, login, password):
for i in range(len(machine)):
try:
ftp = ftplib.FTP(machine[i])
print 'conected to ' + machine[i]
ftp.login(login[i], password[i])
print 'login - ' + login[i] + ' pasword -' + password[i]
except Exception, e:
print e
else:
ftp.cwd("PublicFolder")
print 'PublicFolder'
return (ftp)
def upload(filename, file):
for ftp in readList():
ext = os.path.splitext(file)[1]
if ext in (".txt", ".htm", ".html"):
ftp.storlines("STOR " + filename, open(file))
else:
ftp.storbinary("STOR " + filename, open(file, "rb"), 1024)
print 'success... yra'
upload('test4.txt', r'c:\example2\media\uploads\test4.txt')
Error at line 19 something with try:
unindent does not math any outer indentation level

Categories