I'm writing a Python script to check if a file is added to a folder with watchdog, that file is going to be added to a queue.
My idea is to add the filename to a txt, then either run a new class that watches the txt and then executes a line in cmd and start for example FME.
Is it the best way to write a new .py for every new program I want to open. For example one for FME and one for notepad.
I still want the watchdog class to go into the background.
looking_for_files_and_adding_to_queue py
looking_in_queue_for_the_next_in_line_and_direct_to_3_party py
FME py
Notepad py
and so on...
Or on all.py
class looking_for_files_and_adding_to_queue
class looking_in_queue_for_the_next_in_line_and_direct_to_3_party
class FME
class Notepad
Today my script looks like this:
import time
import sys
import os
import datetime
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.tif"]
count_move = 0
def process(self, event):
if self.count_move == 1:
# the file will be processed there
folder = "P:\\03_auto\\Indata"
indata = event.src_path
#Makes a new folder in Utdata based on filename
newfolder = os.path.join(folder[:11], str("Utdata\\orto"), event.src_path[18:29])
if not os.path.exists(newfolder):
os.makedirs(newfolder)
#Logg and print start of FME
print(time.strftime('%a %H:%M:%S') + ": FME " + event.src_path[18:] + " startats i FME.")
log_file = open("P:\\03_auto\\log.txt", "a")
log_file.write(time.strftime('%a %H:%M:%S') + ": FME " + event.src_path[18:] + " startats i FME.\n")
log_file.close()
#Starting and excequting FME
var_fme = str('fme.exe "P:\\03_auto\\Script\\tiff_to_milti_jpg_tiff\\tif_to_multi-jpg-tiff.fmw" --SourceDataset_TIFF "') + indata + str('" --FEATURE_TYPES "" --DestDataset_JPEG "') + newfolder + str('" --DestDataset_JPEG_5 "') + newfolder + str('" --DestDataset_JPEG_4 "') + newfolder + str('" --DestDataset_GEOTIFF "') + newfolder + str('" --DestDataset_GEOTIFF_3 "') + newfolder + str('"')
os.system(var_fme)
#Logg and pring move file
print(time.strftime('%a %H:%M:%S') + ": Flytt " + event.src_path[18:] + " har flyttats till" + newfolder + "\nTransformering klar\n")
log_file = open("P:\\03_auto\\log.txt", "a")
log_file.write(time.strftime('%a %H:%M:%S') + ": Flytt " + event.src_path[18:] + " har flyttats till" + newfolder + "\nTransformering klar\n\n")
log_file.close()
#Move org file to Utdata\orto
file_move = newfolder + indata[17:]
os.rename(indata, file_move)
#Restets script
self.count_move = 0
else:
#Logg and pring loadning file while transfering
print(time.strftime('%a %H:%M:%S') + ": Laddar " + event.src_path[18:] + " startar inladdning.")
log_file = open("P:\\03_auto\\log.txt", "a")
log_file.write(time.strftime('%a %H:%M:%S') + ": Laddar " + event.src_path[18:] + " startar inladdning.\n")
log_file.close()
#Sets counter to 1 which enables the FME part
self.count_move += 1
def on_modified(self, event):
self.process(event)
if __name__ == '__main__':
path = "P:\\03_auto\\Indata"
observer = Observer()
observer.schedule(MyHandler(), path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
tl;dr keep everything in one file for now, split subsequently while refactoring when the file becomes huge.
Python does not force you split classes / functions into modules. We as programmers make that call for solely the purpose of readability and maintainability.
While refactoring I personally look at functions with more ~40 - 50 lines and files with ~ 1000 lines to split and try to keep closely related things together.
high cohesion and low coupling.
is a characteristic feature of good software.
Also, since you seem to be starting out with this project I would recommend you to first concentrate on making a version that works, thereafter refactor it to improve code quality.
premature optimization is the root of all evil.
I am assuming that you are looking for suggestions to improve code quality here, so here are a few things you might be also be interested in:
follow pep8 standards: https://pep8.org
make your functions / methods accept parameters instead of hardcoding them eg the path of the folder you are watching.
make your program capable of resuming operations even after erroneous / abrupt termination: eg store state with a file or database
instead of trying to implement a queue yourself use robust systems like rabbitmq or redis.
write functions / methods that perform only one operation and do it well.
This it how far I have done. Now I have to get the files from the que to FME
import time
import sys
import os
import datetime
import arrow
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from shutil import copy
class Queue:
def __init__(self):
self.items =[]
def isEmpty(self):
return self.items == []
def enqueue(self, item):
self.items.insert(0, item)
def dequeue(self):
self.items.pop()
def size(self):
return len(self.items)
def printqueue(self):
i = 0
for items in self.items:
i += 1
print(str(i) + ": " + items)
class MyHandler(PatternMatchingEventHandler):
patterns = ["*.tif","*.pdf"]
q = Queue()
def on_created(self, event):
file_name = os.path.basename(event.src_path)
file_type = file_name.split(".")[-1]
file_path = "path"
file_name_path = event.src_path
endwith = file_name.endswith("_mosaic_group1.tif")
new_folder = "C:\\FME_workdir\\"
new_path = new_folder + file_name
#create new temp folder for FME
if not os.path.exists(new_folder):
os.makedirs(new_folder)
#get tif file from project
if file_name.endswith("_mosaic_group1.tif") and not os.path.exists(new_path):
print("Queue:")
self.q.enqueue("[" + file_name + ", " + file_name_path + ", " + new_path + ", " + file_type + "]")
self.q.printqueue()
print("\n")
#fme = Fme()
#return fme.runfme(file_name, file_path, file_name_path)
#copy file to FME folder
if not os.path.exists(new_path):
copy(file_name_path, new_path)
#get the PDF report
elif file_name.endswith("_report.pdf") and "1_initial" in file_name_path:
pdf_path = os.path.dirname(file_name_path)
pdf_path_new_path = "\\".join(pdf_path.split("\\")[:3])
pdf_path_new_dir = "\\".join(pdf_path.split("\\")[5:6])
date_now = str(time.strftime("%y%m%d"))
pdf_new_path = pdf_path_new_path + "\\03_leverans\\" + pdf_path_new_dir + "_" + date_now
pdf_new_path_filename = pdf_new_path + "\\" + file_name
if not os.path.exists(pdf_new_path):
os.makedirs(pdf_new_path)
copy(file_name_path, pdf_new_path_filename)
#put inte que system
self.q.enqueue("[" + file_name + ", " + file_name_path + ", " + pdf_new_path + ", " + file_type + "]")
self.q.printqueue()
class Fme:
def runfme(self, file_name, file_path, file_name_path):
print("FME: " + self.file_name)
if __name__ == '__main__':
path = "P:\\"
observer = Observer()
observer.schedule(MyHandler(), path, recursive=True)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
print("stop")
observer.join()
input('Press ENTER to exit')
Related
Hello I am having a difficult time running a python script on startup that request user input in terminal to name a directory. The python script ask for a file name then creates a directory and csv file to store pictures and information of each picture such as GPS data in the csv code. My code works fine when I run it in Geany but I have tried every imaginable way to start the code on bootup. The code will work if I change direcname = str(input("name your file: ")) to direcname=str("file_name"). I have spent days trying to figure this out, I can not find a single way to open terminal on boot that works for my script.
#import packages
from gpiozero import Button, LED
from picamera import PiCamera
import os
import datetime
from gps import *
#define gpio pins and variables
pwd = os.getcwd()
camera = PiCamera()
led = LED(13)
previewbtn = Button(26, hold_time=2)
counter = 1
#GPS stuff
gpsd = gps(mode=WATCH_ENABLE|WATCH_NEWSTYLE)
#make new directory and create text file within
direcname = str(input("name your file: "))
newpath = pwd + '/' + direcname
os.makedirs(newpath)
txtfile = open(newpath + '/' + direcname + '.csv', 'w+')
txtfile.write('img, date/time, lat, lon, alt(m)')
txtfile.close()
#define functions
def capture():
global counter
camera.capture(newpath + '/' + direcname + str(counter) + '.jpg')
txtfile = open(newpath + '/' + direcname + '.csv', 'a')
txtfile.write("\n")
txtfile.write( direcname + str(counter) + ',' + str(datetime.datetime.now()) +
',' + lat1 + ',' + lon1 + ','+ alt1)
txtfile.close()
counter += 1
#run function
try:
while True:
#Setting lat,lon, and alt as variables
report = gpsd.next()
if report['class'] == 'TPV':
if getattr(report,'lat',0.0)!=0:
lat1 = str(getattr(report,'lat',0.0))
if getattr(report,'lon',0.0)!=0:
lon1 = str(getattr(report,'lon',0.0))
if getattr(report,'alt','nan')!= 'nan':
alt1 = str(getattr(report,'alt','nan'))
else:
lat1 = "ERROR"
lon1 = "ERROR"
alt1 = "ERROR"
#Everything else
led.source = previewbtn
previewbtn.when_pressed = camera.start_preview
previewbtn.when_held = capture
previewbtn.when_released = camera.stop_preview
except(KeyboardInterrupt, SystemExit):
print("Done.\nExiting")
If you are using window manager then you can add your python filename.py & in .xinitrc which starts executing anything which is written in that script.
If using any desktop environment then it comes with a default display manager which does the same things. Each display manager has their own way of auto-starting things.
And i recommend starting it while login rather than starting on boot.
SOLVED:
This issue has been solved by renaming of of my two loggers. My issue arose because i would do log.getLogger again in the main file. This caused 2 instances of the logger to be made. The solution is to remove the second call, OR rename one of the two.
I am attempting to setup a custom logger for my current project, and I am having difficulty with getting it to work properly outside of __init__.py file. The issue is that anything I log is logged twice.
My Code:
__Init__.py:
import datetime as date
import os
import platform as plt
import logging as log
prefsDirectory = 'prefs/'
prefsName = 'preferences.txt'
prefsLocation = prefsDirectory + prefsName
now = date.datetime.now()
# SETUP
if not(os.path.exists(prefsLocation)):
if not(plt.system() == "Darwin"):
os.mknod(prefsLocation)
with(open(prefsLocation, 'w+')) as f:
f.write('Log increment:\n' + str(1) + "\n")
f.close()
pass
else:
if not(os.path.exists(prefsDirectory)):
os.mkdir(prefsDirectory)
with(open(prefsLocation, 'w+')) as f:
f.close()
pass
with(open(prefsLocation, 'w+')) as f:
f.write('Log increment:\n' + str(0) + "\n")
f.write('\nCurrent Date:\n' + str(now.day) + "\n")
f.close()
pass
with(open(prefsLocation, "r")) as f:
data = f.readlines()
if not(str(now.day) == data[4]):
data[4] = str(now.day)
data[1] = str(0) + '\n'
# print('This ran')
else:
inc = str(int(data[1]) + 1)
data[1] = inc + "\n"
with(open(prefsLocation, "w")) as f:
lines = (str(item) for item in data)
for item in lines:
f.write(item)
dateC = "[" + str(now.year) + "-" + str(now.month) + "-" + data[4] + "]"
logDirectory = "logs/"
inc = int(data[1])
logName2 = str(dateC) + "-" + str(inc)
logName = logName2 + ".log"
logLocation = logDirectory + logName
if not(os.path.exists(logLocation)):
if not(plt.system() == "Darwin"):
os.mknod(logLocation)
else:
if not(os.path.isdir(logDirectory)):
os.mkdir(logDirectory)
with (open(logLocation, 'w+')) as f:
f.close()
pass
formatter = log.Formatter("[%(asctime)s][%(levelname)s][%(module)s] : %(message)s \n", "%H:%M-%S" + "s")
handler = log.StreamHandler()
handler.setFormatter(formatter)
handler.setLevel("DEBUG")
logger = log.getLogger("Main")
logger.addHandler(handler)
log.basicConfig(filename=logLocation, level=log.DEBUG, filemode="w",
format="[%(asctime)s][%(levelname)s][%(module)s] : %(message)s \n", datefmt="%H:%M-%S" + "s")
logger.info("[LOG NUMBER: " + str(inc) + "]")
logger.info("Found Settings file")
logger.info("Generated Log File")
__main__.py:
# IMPORTS
import logging as log
from main import variables as vrs
# VARIABLES
logg = vrs.logg
logg.addHandler(vrs.handlerMain)
log.basicConfig(filename=vrs.logLocation, level=log.DEBUG, filemode="w",
format="[%(asctime)s][%(levelname)s][%(module)s] : %(message)s \n", datefmt="%H:%M-%S" + "s")
with(open(vrs.prefsLocation, "r")) as f:
data = f.readlines()
# BODY
logg.info('Program Loading Completed.')
# Make a data holding file.
vrs.makefile('prefs/data.txt', 'prefs/', "Data File")
variables.py:
import datetime as date
import logging as log
import os
import platform as plt
prefsDirectory = 'prefs/'
prefsName = 'preferences.txt'
prefsLocation = prefsDirectory + prefsName
with(open(prefsLocation, "r")) as f:
data = f.readlines()
now = date.datetime.now()
dateC = "[" + str(now.year) + "-" + str(now.month) + "-" + data[4] + "]"
logDirectory = "logs/"
inc = int(data[1])
logName2 = str(dateC) + "-" + str(inc)
logName = logName2 + ".log"
logLocation = logDirectory + logName
formatter = log.Formatter("[%(asctime)s][%(levelname)s][%(module)s] : %(message)s \n", "%H:%M-%S" + "s")
handler = log.StreamHandler()
handler.setFormatter(formatter)
handler.setLevel("DEBUG")
handler.set_name('Main')
handlerMain = log.StreamHandler()
handlerMain.setFormatter(formatter)
handlerMain.setLevel("DEBUG")
logg = log.getLogger("Main")
def makefile(filelocation, filedirectory, filename):
if not (os.path.exists(filelocation)):
if not (plt.system() == "Darwin"):
os.mknod(filelocation)
with(open(filelocation, 'w+')) as file:
file.write('File Created:\n' + dateC + "\n")
file.close()
pass
else:
if not (os.path.exists(filedirectory)):
os.mkdir(filedirectory)
with(open(filelocation, 'w+')) as file:
file.write('File Created:\n' + dateC + "\n")
file.close()
pass
logg.info('Created file: ' + filename)
I am not sure what exactly causes the issue...I think it is something with defining a logger in the init file, and a second in the variables file.
If it helps, I will provide a copy of my file structure below:
<img src="https://i.gyazo.com/5cb1221a65a9ad50adf2a355f92f90e4.png" alt="Image from Gyazo" width="315"/>
<img src="https://i.gyazo.com/39f1b61ca09ed364080254a0f678db80.png" alt="Image from Gyazo" width="1280"/>
[ I Do not seem to be able to input gyazo images into the post, can one of you community moderator people put them in for me? ALSO, the folder to look at the one called AoC2018 ]
As jobevers aluded to in his comment, you are attaching two stream handlers to the 'Main' logger. First in the __init__.py and again in __main__.py. This also explains why your logging from within __init__.py is working correctly as __main__.py hasn't attached the second handler yet.
I suspect the reason you did not expect this behaviour is because you were expecting the loggers to be distinct. But the loggers you defined in __init__.py and in variables.py are actually the same. When you retrieve a logger using logging.getLogger(logger_name) with the same logger_name it returns the same logger. So when you call logging.getLogger('Main') in variables.py it still has the StreamHandler on it from when it was added in __init__.py.
Depending on what behaviour you want you should either give them distinct names or remove the second addHandler.
It should be noted that logger names follow a hierarchy. Log configuration for my_package will also configure logging for anything my_package.model, my_package.views, etc... The root logger is simply the logger with the empty string (logging.getLogger("")).
For more details I'd recommend just going through the official docs.
I've recently started using ffmpeg with the intention of converting my video library to h265 due to its compression benefits. I would like to run one command and have ffmpeg traverse the folder converting each file, one-by-one into h265. I've checked the documentation Here but I can't get my head around it. Does anybody have a template loop script for me to use?
I have ffmpeg installed on a Linux box and have successfully converted single files but I have around 400 files to convert, hence the looping question.
Thanks in advance.
EDIT:
The files I'm waiting to convert are videos with varying containers. I have bee using the python script below, which I have tweaked to suit my needs but isn't working. I will include the error I'm getting and a link to the original below my code.
import os
import sys
import re
import shutil
import subprocess
__author__ = 'Nikhil'
# Edit options here ##################################################
outmode = 'mp4' #Extension of file
remover = True # Delete original file after conversion complete
accept_ext = 'mp4 mkv avi divx m4v mpeg mpg wmv' #Extensions of video files to convert
ffmpeg_exe = "ffmpeg" #Path to ffmpeg executable
ffprobe_exe = "ffprobe" #Path to ffprobe executable
mkvextract_exe = "mkvextract" #Path to mkvextract executable
video_codec = 'libx265' #Video codec to use
video_type = 'h265' #Name of video codec to check for remux
audio_codec = 'aac' #Audio codec to use
audio_type = 'aac' #Name of audio codec to check for remux
crf = "28" #Video quality for libx264
vbr = '' #Audio quality
extract_subtitle = True #Extract subtitles?
subtitle_languages = "en eng english" #Codes for languages to extract
threads = 0 #Number of threads to use in ffmpeg, 0 defaults to all
additional_ffmpeg = '-preset slow -movflags +faststart' #Additional flags for ffmpeg, preset sets speed and compression, movflags to make file web optimized
## END OPTIONS - DO NOT EDIT BELOW THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING ##
outformat = 'mp4'
if outmode == 'mp4':
outformat = 'mp4'
elif outmode == 'mkv':
outformat = 'matroska'
def ffmpeg(*args, **kwargs):
largs = [ffmpeg_exe, ]
largs.extend(args)
try:
return subprocess.check_output(largs, **kwargs).decode('utf-8')
except:
return None
def getoutput(cmd):
if sys.version < '3':
try:
return subprocess.check_output(cmd.split(' '))
except:
return None
else:
return subprocess.getoutput(cmd)
formats = ""
if getoutput(ffmpeg_exe + ' -formats'):
formats = getoutput(ffmpeg_exe + ' -formats 2')
else:
exit(1)
if ('E mp4' in formats) and ('E matroska' in formats):
print("You have the suitable formats")
else:
print("You do not have both the mkv and mp4 formats...Exiting!")
exit(1)
codecs = getoutput(ffmpeg_exe + ' -codecs 2')
if video_codec in codecs:
print("Check " + video_codec + " Audio Encoder ... OK")
else:
print("Check " + video_codec + " Audio Encoder ... NOK")
exit(1)
if audio_codec in codecs:
print("Check " + audio_codec + " Audio Encoder ... OK")
else:
print("Check " + audio_codec + " Audio Encoder ... NOK")
exit(1)
print("Your FFMpeg is OK\nEntering File Processing\n")
subtitle_languages = subtitle_languages.lower()
def process_file(path, file):
extension = os.path.splitext(file)[1].replace(".", "")
filename = os.path.splitext(file)[0]
if extension in accept_ext:
print(file + " is an acceptable extension. Checking file...")
else:
print(file + " is not an acceptable extension. Skipping...")
return
if ffprobe_exe:
file_info = getoutput('"' + ffprobe_exe + '"' + " " + '"' + os.path.join(path, file) + '"')
else:
file_info = ffmpeg("-i", os.path.join(path, file))
if 'Invalid data found' in file_info:
print("File " + file + " is NOT A VIDEO FILE cannot be converted!")
return
encode_crf = []
if file_info.find("Video: " + video_type) != -1:
vcodec = 'copy'
print("Video is " + video_type + ", remuxing....")
else:
vcodec = video_codec
if crf:
encode_crf = ["-crf", "" + crf]
print("Video is not " + video_type + ", converting...")
encode_vbr = []
if "Audio: " + audio_type in file_info:
acodec = 'copy'
print("Audio is " + audio_type + ", remuxing....")
else:
acodec = audio_codec
if vbr:
encode_vbr = ["-vbr", "" + vbr]
print("Audio is not " + audio_type + ", converting...")
if extension == outmode and vcodec == 'copy' and acodec == 'copy':
print(file + " is already " + outmode + " and no conversion needed. Skipping...")
return
print(
"Using video codec: " + vcodec + " audio codec: " + acodec + " and Container format " + outformat + " for\nFile: " + file + "\nStarting Conversion...\n")
filename = filename.replace("XVID", video_type)
filename = filename.replace("xvid", video_type)
try:
args = ['-i', os.path.join(path, file), '-y', '-f', outformat, '-acodec', acodec]
if encode_vbr:
args.extend(encode_vbr)
args.extend(['-vcodec', vcodec])
if encode_crf:
args.extend(encode_crf)
if additional_ffmpeg:
args.extend(additional_ffmpeg.split(" "))
if threads:
args.extend(['-threads', str(threads)])
args.append(os.path.join(path, filename + '.temp'))
ffmpeg(*args)
print("")
except Exception as e:
print("Error: %s" % e)
print("Removing temp file and skipping file")
if os.path.isfile(os.path.join(path, filename + '.temp')):
os.remove(os.path.join(path, filename + '.temp'))
return
if extract_subtitle and (file_info.find("Subtitle:") != -1):
print("Extracting Subtitles")
matches = re.finditer("Stream #(\d+):(\d+)\((\w+)\): Subtitle: (.*)", file_info)
for m in matches:
if m.group(3).lower() not in subtitle_languages.split(" "):
continue
try:
if 'subrip' in m.group(4):
sub_format = 'copy'
sub_ext = '.srt'
elif mkvextract_exe and 'hdmv_pgs' in m.group(4):
subprocess.check_output([mkvextract_exe, 'tracks', os.path.join(path, file),
m.group(2) + ':' + os.path.join(path, filename + '.' + m.group(
3) + '.' + m.group(2) + '.sup')])
continue
else:
sub_format = 'srt'
sub_ext = '.srt'
ffmpeg("-i", os.path.join(path, file), '-y', '-map', m.group(1) + ':' + m.group(2), '-c:s:0',
sub_format,
os.path.join(path, filename + '.' + m.group(3) + '.' + m.group(2) + sub_ext))
print("")
except Exception as e:
print("Error: %s" % e)
print("Deleting subtitle.")
if os.path.isfile(os.path.join(path, filename + '.' + m.group(3) + '.' + m.group(2) + sub_ext)):
os.remove(os.path.join(path, filename + '.' + m.group(3) + '.' + m.group(2) + sub_ext))
if remover:
print("Deleting original file: " + file)
os.remove(os.path.join(path, file))
if outmode == extension:
shutil.move(os.path.join(path, filename + ".temp"), os.path.join(path, filename + ".enc." + outmode))
filename += ".enc"
else:
shutil.move(os.path.join(path, filename + ".temp"), os.path.join(path, filename + "." + outmode))
def process_directory(path):
if os.path.isfile(os.path.join(path, ".noconvert")):
return
for file in os.listdir(path):
filepath = os.path.join(path, file)
if os.path.isdir(filepath):
process_directory(filepath)
elif os.path.isfile(filepath):
process_file(path, file)
for arg in sys.argv[1:]:
if os.path.isdir(arg):
process_directory(arg)
elif os.path.isfile(arg):
process_file(os.path.dirname(arg), os.path.basename(arg))
The error I am getting is this:
Traceback (most recent call last):
File "/media/569f/ethan1878/bin/convert.py", line 209, in <module>
process_file(os.path.dirname(arg), os.path.basename(arg))
File "/media/569f/ethan1878/bin/convert.py", line 100, in process_file
if 'Invalid data found' in file_info:
TypeError: argument of type 'NoneType' is not iterable
and the original file is hosted Here (as a .txt file)
For a project of me, I need to be able to copy the output of a program (HTML) to my server with a script, so it can be done by just launching the script, no further handling.
The uploading should be done only for files where the content has been changed because the date is always changed after a new publish out of my program.
I've found a script online that was able to do what I've wanted with some minor changes. Now it works when I publish between 2 folders on my computer, but I should be able to replace my destination with an sFTP or SSH connection...
import os
import filecmp
import shutil
'''
Version: Python 3.3.2
Using python SyncFolder in CMD
Origin: https://gist.github.com/aortbals/5096365
Modified by Louisdj_15
'''
class Dispatch:
''' This class represents a synchronization object '''
def __init__(self, name=''):
self.name=name
self.node_list=[]
self.file_copied_count=0
self.folder_copied_count=0
self.file_removed_count=0
self.folder_removed_count=0
def add_node(self, node):
self.node_list.append(node)
def compare_nodes(self):
''' This method takes the nodes in the node_list and compares them '''
nodeListLength = len(self.node_list)
# For each node in the list
for node in self.node_list:
# If the list has another item after it, compare them
if self.node_list.index(node) + 1 < len(self.node_list):
node2 = self.node_list[self.node_list.index(node) + 1]
'''print(str('\nComparing node ') + str(self.node_list.index(node)) + str(' and Node ') + str(self.node_list.index(node) + 1) + ':')'''
print('Comparing localhost to server')
# Pass the two root directories of the nodes to the recursive _compare_directories
self._compare_directories(node.root_path, node2.root_path)
def _compare_directories(self, left, right):
''' This method compares directories. If there is a common directory, the
algorithm must compare what is inside of the directory by calling this
recursively.
'''
comparison = filecmp.dircmp(left, right)
if comparison.common_dirs:
for d in comparison.common_dirs:
self._compare_directories(os.path.join(left, d), os.path.join(right, d))
if comparison.left_only:
self._copy(comparison.left_only, left, right)
if comparison.right_only:
self._remove(comparison.right_only, right)
left_newer = []
right_newer = []
if comparison.diff_files:
for d in comparison.diff_files:
l_modified = os.stat(os.path.join(left, d)).st_mtime
r_modified = os.stat(os.path.join(right,d)).st_mtime
if l_modified > r_modified:
left_newer.append(d)
else:
right_newer.append(d)
self._copy(left_newer, left, right)
self._copy(right_newer, right, left)
def _copy(self, file_list, src, dest):
''' This method copies a list of files from a source node to a destination node '''
for f in file_list:
srcpath = os.path.join(src, os.path.basename(f))
if os.path.isdir(srcpath):
shutil.copytree(srcpath, os.path.join(dest, os.path.basename(f)))
self.folder_copied_count = self.folder_copied_count + 1
print('Copied directory \"' + os.path.basename(srcpath) + '\" from \"' + os.path.dirname(srcpath) + '\" to \"' + dest + '\"')
else:
shutil.copy2(srcpath, dest)
self.file_copied_count = self.file_copied_count + 1
print('Copied \"' + os.path.basename(srcpath) + '\" from \"' + os.path.dirname(srcpath) + '\" to \"' + dest + '\"')
def _remove(self, file_list, src):
''' This method removes a list of files from a destionation node if doesn't exist any longer on source '''
for f in file_list:
srcpath = os.path.join(src, os.path.basename(f))
if os.path.isdir(srcpath):
shutil.rmtree(srcpath)
self.folder_removed_count = self.folder_removed_count + 1
print('Removed directory \"' + os.path.basename(srcpath) + '\" from \"' + os.path.dirname(srcpath))
else:
os.remove(srcpath)
self.file_removed_count = self.file_removed_count + 1
print('Removed file \"' + os.path.basename(srcpath) + '\" from \"' + os.path.dirname(srcpath))
class Node:
''' This class represents a node in a dispathc synchronization '''
def __init__(self, path, name=''):
self.name=name
self.root_path = os.path.abspath(path)
self.file_list = os.listdir(self.root_path)
if __name__=="__main__":
node_source = Node('C:\\source_folder', 'source')
dispatch = Dispatch('Dispatcher')
node_dest = Node('HERE I NEED AN SFTP OR SSH LINK', 'dest')
dispatch.add_node(node_source)
dispatch.add_node(node_dest)
count=0
os.system('color 2')
os.system('cls')
print('------------------------------------')
print('| DISPATCHER: |')
print('------------------------------------')
while (count < 1):
dispatch.compare_nodes()
print('')
print('Total files copied ' + str(dispatch.file_copied_count))
print('Total folders copied ' + str(dispatch.folder_copied_count))
print('Total files removed ' + str(dispatch.file_removed_count))
print('Total folders removed ' + str(dispatch.folder_removed_count))
print('')
print('Dispatcher is done! Closing connections!')
count = count + 1
You can easily sftp files with fabrics put command.
from fabric.api import *
env.hosts = [<hosts>]
with cd('/tmp'):
put('/path/to/local/test.txt', 'files')
I am writing a program to perform file integrity checks of files in a directory. There are 3 nested loops in the code. When I run the code, the first two loops work great but the third loop does not run more than once.
import hashlib
import logging as log
import optparse
import os
import re
import sys
import glob
import shutil
def md5(fileName):
"""Compute md5 hash of the specified file"""
try:
fileHandle = open(fileName, "rb")
except IOError:
return
m5Hash = hashlib.md5()
while True:
data = fileHandle.read(8192)
if not data:
break
m5Hash.update(data)
fileHandle.close()
return m5Hash.hexdigest()
req = open("requested.txt")
for reqline in req:
reqName = reqline[reqline.rfind('/') + 1:len(reqline) - 1]
reqDir = reqline[0:reqline.rfind('/') + 1]
ezfimlog = open("ezfimlog.txt", 'a')
actFile = open("activefile.txt")
tempFile = open("activetemp.txt", 'w')
for name in glob.glob(reqDir + reqName):
fileHash = md5(name)
actInt = 0
if fileHash != None:
print fileHash
for actLine in actFile:
actNameDir = actLine[0:actLine.rfind(' : ')]
actHash = actLine[actLine.rfind(' : ') + 3:len(actLine) -1]
print (name + " " + actHash + " " + fileHash)
if actNameDir == name and actHash == fileHash:
tempFile.write(name + " : " + fileHash + "\n")
actInt = 1
if actNameDir == name and actHash != fileHash:
tempFile.write(name + " : " + actHash + "\n")
actInt = 1
ezfimlog.write("EzFIM Log: The file " + name + " was modified: " + actHash + "\n")
if actInt == 0:
ezfimlog.write("EzFIM Log: The file " + name + " was created: " + fileHash + "\n")
tempFile.write(name + " : " + fileHash + "\n")
shutil.copyfile("activetemp.txt", "activefile.txt")
You open actFile once and then try to read it many times. You'll need to open it each time you want to read it.
Move this line:
actFile = open("activefile.txt")
to just before this line:
for actLine in actFile: