How execute python3 script remotely via cURL? - python

I have this file here
# --------------------------------------------------------------
# Goal : Remove file base on input match
# Run : curl 45.55.88.57/code/fileModifier.py | python3
import os
import sys
rootdir = os.path.abspath(os.curdir)
print(rootdir)
#Give string to remove from folder names. Ensure that removing a string doens't make the folder name empty. It wont work
removeStringFromFolderName = input('Remove this from folder names :')
while removeStringFromFolderName == '':
print('Empty string not allowed')
removeStringFromFolderName = input('Remove this file if contain : ')
count = 0
subdir = [x for x in os.walk(rootdir)]
toRemove = []
for chunk in subdir:
folders = chunk[1]
if len(folders) > 0:
for aDir in folders:
if removeStringFromFolderName in aDir:
toRemove.append((chunk[0], aDir))
toRemove.reverse()
for folders in toRemove:
oldPath = (os.path.join(folders[0], folders[1]))
newPath = (os.path.join(folders[0], folders[1].replace(removeStringFromFolderName,'')))
os.rename(oldPath, newPath)
count +=1
subdir = [x for x in os.walk(rootdir)]
for chunk in subdir:
folders = chunk[1]
if len(folders) > 0:
for aDir in folders:
if removeStringFromFolderName in aDir:
print(os.path.join(chunk[0], aDir))
oldPath = (os.path.join(chunk[0], aDir))
newPath = (os.path.join(chunk[0], aDir.replace(removeStringFromFolderName,'')))
os.rename(oldPath, newPath)
count +=1
print('Renamed', count, 'files')
count = 0
#Give string to delete files which contain this string
removeThisFileNameIfContain = input('Enter string to delete files which contain this string: ')
while removeThisFileNameIfContain == '':
print('Empty string not allowed')
removeThisFileNameIfContain = input('Enter string to delete files which contain this string: ')
for subdir, dirs, files in os.walk(rootdir):
for aFile in files:
if '.py' in aFile:
continue
if removeThisFileNameIfContain in aFile:
os.remove(os.path.join(subdir, aFile))
count += 1
print('Deleted', count, 'files')
Work perfect when on local machine with python3, but when I uploaded into my VM, and executed remotely via cURL
I kept getting this
curl 45.55.88.57/code/fileModifier.py | python3
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 2266 100 2266 0 0 43381 0 --:--:-- --:--:-- --:--:-- 43576
/Users/bheng/Desktop/projects/bheng/fileModifier
Remove this from folder names :Traceback (most recent call last):
File "<stdin>", line 16, in <module>
EOFError: EOF when reading a line
What did I missed ?

Your usage is taking up stdout, which the input command needs.
Try this if your shell has the ability:
python3 <(curl 45.55.88.57/code/fileModifier.py)
Note: As Amadan said, your syntax (and mine) run a remote script locally, not vice versa.

First of all, you are not executing the script remotely. You are fetching it then executing it locally by piping into Python REPL.
When you are piping the script into Python, stdin is where the program comes from. You cannot use stdin to also get data from input().
To actually execute it remotely, you would need to either build in a web server in your code, or tell an existing web server to run your Python code (e.g. by registering a CGI handler).

Related

Python rclone check difference of two folders

I'm trying to code an automatic script for uploading to the gdrive with rclone.
I will not go through all the code only in this check statement, the rclone command checks files from the local folder and mounted folder something like this:
rclone check "local folder" "mounted folder" --ignore existing --onlyoneway
it returns in terminal some data that can't be stored in a text file or I don't now how.
def upload_check():
print(" check if all files are uploaded ")
global Error_upload
if :#I stuck here, rclone check and return true or false if all files are uploaded by name and size
Error_upload = True
return Error_upload
print("Not uploaded ")#---------------------------
else:# all good
Error_upload = False
return Error_upload
print("all files are online")#---------------------------
my question is how to properly check two directories if they are identical by all files inside and files size and returning Boolean True or False?
After a few days I come up with this complicated solution:
import shutil
import os
local = "Local/"
destination = "uploaded/"
checkfile = "logfile.txt"
def upload_check():
print(" check if all files are uploaded ")
global Error_upload
os.system("rclone check 'Local' 'gdrive' --one-way -vv -P --combined logfile.txt")
destination = "uploaded/"
checkfile = "logfile.txt"
search = "=" # move from the folder successfuly uplouded files
list_of_files = []
lines = []
folders = []
uniq_folder_list = []
shutil_l = []
shutil_f = []
for line in open(checkfile, "r"):
if search in line:
list_of_files = line.split("/")[1]
lines.append(list_of_files.rstrip())
list_of_folders = line.split(" ")[1].split("/")[0]
folders.append(list_of_folders.rstrip())
[uniq_folder_list.append(n) for n in folders if n not in uniq_folder_list]
for new_folder in uniq_folder_list:
if not os.path.exists(destination + new_folder):
os.makedirs(destination + new_folder)
for l, f in zip(lines, folders):
l1 = (local + f + "/" + l)
f1 = (destination + f)
shutil_l.append(l1.rstrip())
shutil_f.append(f1.rstrip())
for src, dest in zip(shutil_l, shutil_f):
shutil.move(src,dest)
os.system("rclone check 'Local' 'gdrive' --one-way -vv -P --combined logfile.txt")
with open(checkfile, 'r') as read_obj:
one_char = read_obj.read(1)
if not one_char:
Error_upload = False
return Error_upload
print("all files are online")
else:
Error_upload = True
return Error_upload
print("Not uploaded ")
First I created some files and a couple of them uploaded them to the drive, also one corrupted file. Than this scrip do the job.
The file logfile.txt contains a list generated with rclone
rclone check 'Local' 'gdrive' --one-way -vv -P --combined logfile.txt
this bash command will generate a logfile:
+ 20_10_10/IMG_1301-00006.jpg
+ 20_10_10/IMG_1640-00007.jpg
+ 20_10_10/IMG_1640-00008.jpg
+ 20_10_10/IMG_1640-00009.jpg
+ 20_10_10/IMG_1640-00010.jpg
+ 20_10_10/IMG_1640-00011.jpg #missing on remote
* 20_10_10/IMG_1301-00004.jpg #corrupted file
= 20_10_10/IMG_1301-00005.jpg
= 20_10_10/IMG_1301-00003.jpg
= 20_10_10/IMG_1301-00001.jpg
= 20_10_09/IMG_2145-00028.jpg
= 20_10_10/IMG_1301-00002.jpg
more info on rclone check help
on rclone. The files with "=" are identical on local and remote destination, so we want to move them from the source folder to an uploaded folder.
The script runs again and if the read function can't read anything, all files are online and the upload function does not need to run again. But since there are un uploaded files and a corrupted file (it can happened if the connection is lost while uploading) the script will run the upload function or what ever other function triggered by if function with variable "Error_upload"
just for reference:
if Error_upload == True:
print("All files are on the cloud")
else:
upload() #your upload function
upload_check()
I certainly know that this code could be simpler and improved.

Automatic Upload to Dropbox on Raspberry Pi

I setup the Dropbox uploader script for the Raspberry pi and was able to successfully upload a file to Dropbox, but now I need to setup an auto script to upload the files into a specific destination folder within Dropbox. I recently found a script that would do just that, but my problem with it is that I cant specify a Destination folder within Dropbox. Other users replied on the forum post asking for an input for a destination folder as well but the post has been inactive for months.
https://github.com/andreafabrizi/Dropbox-Uploader
https://www.raspberrypi.org/forums/viewtopic.php?t=164166
I researched other stackoverflow posts related to this problem, but they would not work in my circumstance. Both scripts work great but I would like to see if its possible to alter the script to specify a destination folder within dropbox.
syncdir is the local folder for upload.
I need an input for something like "/dropbox/TeamFolder" instead of just uploading the files straight into my Dropbox user directory.
import os
import subprocess
from subprocess import Popen, PIPE
#The directory to sync
syncdir="/home/pi/Dropbox-Files/"
#Path to the Dropbox-uploaded shell script
uploader = "/home/pi/Dropbox-Uploader/dropbox_uploader.sh"
#If 1 then files will be uploaded. Set to 0 for testing
upload = 1
#If 1 then don't check to see if the file already exists just upload it, if 0 don't upload if already exists
overwrite = 0
#If 1 then crawl sub directories for files to upload
recursive = 1
#Delete local file on successfull upload
deleteLocal = 0
#Prints indented output
def print_output(msg, level):
print((" " * level * 2) + msg)
#Gets a list of files in a dropbox directory
def list_files(path):
p = Popen([uploader, "list", path], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output = p.communicate()[0].decode("utf-8")
fileList = list()
lines = output.splitlines()
for line in lines:
if line.startswith(" [F]"):
line = line[5:]
line = line[line.index(' ')+1:]
fileList.append(line)
return fileList
#Uploads a single file
def upload_file(localPath, remotePath):
p = Popen([uploader, "upload", localPath, remotePath], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output = p.communicate()[0].decode("utf-8").strip()
if output.startswith("> Uploading") and output.endswith("DONE"):
return 1
else:
return 0
#Uploads files in a directory
def upload_files(path, level):
fullpath = os.path.join(syncdir,path)
print_output("Syncing " + fullpath,level)
if not os.path.exists(fullpath):
print_output("Path not found: " + path, level)
else:
#Get a list of file/dir in the path
filesAndDirs = os.listdir(fullpath)
#Group files and directories
files = list()
dirs = list()
for file in filesAndDirs:
filepath = os.path.join(fullpath,file)
if os.path.isfile(filepath):
files.append(file)
if os.path.isdir(filepath):
dirs.append(file)
print_output(str(len(files)) + " Files, " + str(len(dirs)) + " Directories",level)
#If the path contains files and we don't want to override get a list of files in dropbox
if len(files) > 0 and overwrite == 0:
dfiles = list_files(path)
#Loop through the files to check to upload
for f in files:
print_output("Found File: " + f,level)
if upload == 1 and (overwrite == 1 or not f in dfiles):
fullFilePath = os.path.join(fullpath,f)
relativeFilePath = os.path.join(path,f)
print_output("Uploading File: " + f,level+1)
if upload_file(fullFilePath, relativeFilePath) == 1:
print_output("Uploaded File: " + f,level + 1)
if deleteLocal == 1:
print_output("Deleting File: " + f,level + 1)
os.remove(fullFilePath)
else:
print_output("Error Uploading File: " + f,level + 1)
#If recursive loop through the directories
if recursive == 1:
for d in dirs:
print_output("Found Directory: " + d, level)
relativePath = os.path.join(path,d)
upload_files(relativePath, level + 1)
#Start
upload_files("",1)
When you use the dropbox_uploader.sh script you specify the folder to save the file to on the Dropbox account. However, that is limited to whatever settings you gave the "app" in the Dropbox settings to get your access token. You can set it to allow reading/writing anyplace in your Dropbox account or only in a specific folder.
Look for "Permission Type" and "App Folder Name" on the Dropbox apps setup page: https://www.dropbox.com/developers/apps

Python - Error related to path

I am using the code from here:
http://pythoncentral.io/finding-duplicate-files-with-python/
to find duplicated files in a folder.
Those are my first steps in Python (I come form VBA for Excel) and my problem is probably very simple, but I tried several things without success. After running the code I get the message:
-f is not a valid path, please verify
An exception has occurred, use %tb to see the full traceback.
%tb generates:
SystemExit Traceback (most recent call last)
<ipython-input-118-31268a802b4a> in <module>()
11 else:
12 print('%s is not a valid path, please verify' % i)
---> 13 sys.exit()
14 printResults(dups)
15 else:
SystemExit:
The code I am using is:
# dupFinder.py
import os, sys
import hashlib
def findDup(parentFolder):
# Dups in format {hash:[names]}
dups = {}
for dirName, subdirs, fileList in os.walk(parentFolder):
print('Scanning %s...' % dirName)
for filename in fileList:
# Get the path to the file
path = os.path.join(dirName, filename)
# Calculate hash
file_hash = hashfile(path)
# Add or append the file path
if file_hash in dups:
dups[file_hash].append(path)
else:
dups[file_hash] = [path]
return dups
# Joins two dictionaries
def joinDicts(dict1, dict2):
for key in dict2.keys():
if key in dict1:
dict1[key] = dict1[key] + dict2[key]
else:
dict1[key] = dict2[key]
def hashfile(path, blocksize = 65536):
afile = open(path, 'rb')
hasher = hashlib.md5()
buf = afile.read(blocksize)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(blocksize)
afile.close()
return hasher.hexdigest()
def printResults(dict1):
results = list(filter(lambda x: len(x) > 1, dict1.values()))
if len(results) > 0:
print('Duplicates Found:')
print('The following files are identical. The name could differ, but the content is identical')
print('___________________')
for result in results:
for subresult in result:
print('\t\t%s' % subresult)
print('___________________')
else:
print('No duplicate files found.')
if __name__ == '__main__':
path='C:/DupTestFolder/' #this is the path to analyze for duplicated files
if len(sys.argv) > 1:
dups = {}
folders = sys.argv[1:]
for i in folders:
# Iterate the folders given
if os.path.exists(i):
# Find the duplicated files and append them to the dups
joinDicts(dups, findDup(i))
else:
print('%s is not a valid path, please verify' % i)
sys.exit()
printResults(dups)
else:
print('Usage: python dupFinder.py folder or python dupFinder.py folder1 folder2 folder3')
I tried ending the path with and without "\" at the end, but the result is the same.
I am running Jupyter with Python 3.
Thanks in advance for your help!
The path variable is not used in your code.
All you do is an iteration over sys.argv[1:], which are the parameters of your script. You consider each parameter as a directory path.
On a Windows console, You can try:
python dupFinder.py C:\DupTestFolder
It should work.
Sys.argv works in command line window and takes arguments. It doesn't work with jupyter notebook naturally, or you need to figure out some commands in jupyter notebook.
Thanks!, I am able to run the code now. I have to do two things:
Save the dupFinder.py to the same folder that runs my python installation, in my case C:\Users\Pepe
Open the cmd window from Anaconda (that creates the cmd window in the folder where python is running), I presume I could do the same open the command window and navigate (cd\ command) to the folder location
Finally run python dupFinder.py C:\DupTestFolder.
Now I need to find out how to have the results saved to a .txt file for future use, I will search for it before posting. Thanks for your help!

Start script when new element on directory

I want to convert every new wav file coming into an input dir to another dir on mp3. I've been look how to convert those file but, I don't know how to add a listener on the input dir or if it's even possible?
Edit:
Sorry, I've forget to share you the code I already have. I use ffmpeg to convert audio file
import os, sys, glob
FFMPEG_PATH = "C:\\ffmpeg\\bin"
fileName = ""
fileExt = ""
wavdir = ""
mp3dir = ""
for file in glob.glob('wav/*.wav'):
# get the name without .ext
fileName = os.path.basename(file)
fileName = fileName.split(".")[0]
# verify if no mp3 file with thesame name exist
if not os.path.isfile('./mp3/'+fileName+".mp3"):
# set var with the 2 types files dir
wavdir = file
mp3dir = "mp3/"+fileName+".mp3"
# start the convertion with ffmpeg by commande line
os.system("ffmpeg -i "+wavdir+" "+mp3dir)
I'm not sure if this is the best idea, however you can assign a process or a thread to check if a file was added to the directory every X seconds:
import os
import time
wav_files_path = "/WAV_dir_path"
prev_files = os.listdir(wav_files_path)
x = 1 # time to sleep
while True:
files = os.listdir(wav_files_path)
if len(files) > len(prev_files): # if files are not deleted it's better to check if any files were added
# NEW FILE(S) ADDED
for f in files:
if f not in prev_files:
convert_file(f)
prev_files = files
time.sleep(x)
This is highly suboptimal, but should do the job
import os, time
SLEEPTIME = 0.5
TARGET_DIRECTORY = 'path_of_your_folder'
while True:
time.sleep(SLEEPTIME)
files = os.listdir(TARGET_DIRECTORY)
for file in files:
if file.endswith('.wav'):
CONVERT
Make a while True loop. Then in the loop you make another loop with for item in os.listdir(yourdir) in there you move every item and then you can make time.sleep(1) to reduce lag.

Cannot find the file specified when batch renaming files in a single directory

I've created a simple script to rename my media files that have lots of weird periods and stuff in them that I have obtained and want to organize further. My script kinda works, and I will be editing it to edit the filenames further but my os.rename line throws this error:
[Windows Error: Error 2: The system cannot find the file specified.]
import os
for filename in os.listdir(directory):
fcount = filename.count('.') - 1 #to keep the period for the file extension
newname = filename.replace('.', ' ', fcount)
os.rename(filename, newname)
Does anyone know why this might be? I have a feeling that it doesn't like me trying to rename the file without including the file path?
try
os.rename(filename, directory + '/' + newname);
Triton Man has already answered your question. If his answer doesn't work I would try using absolute paths instead of relative paths.
I've done something similar before, but in order to keep any name clashes from happening I temporarily moved all the files to a subfolder. The entire process happened so fast that in Windows Explorer I never saw the subfolder get created.
Anyhow if you're interested in looking at my script It's shown below. You run the script on the command line and you should pass in as a command-line argument the directory of the jpg files you want renamed.
Here's a script I used to rename .jpg files to multiples of 10. It might be useful to look at.
'''renames pictures to multiples of ten'''
import sys, os
debug=False
try:
path = sys.argv[1]
except IndexError:
path = os.getcwd()
def toint(string):
'''changes a string to a numerical representation
string must only characters with an ordianal value between 0 and 899'''
string = str(string)
ret=''
for i in string:
ret += str(ord(i)+100) #we add 101 to make all the numbers 3 digits making it easy to seperate the numbers back out when we need to undo this operation
assert len(ret) == 3 * len(string), 'recieved an invalid character. Characters must have a ordinal value between 0-899'
return int(ret)
def compare_key(file):
file = file.lower().replace('.jpg', '').replace('dscf', '')
try:
return int(file)
except ValueError:
return toint(file)
#files are temporarily placed in a folder
#to prevent clashing filenames
i = 0
files = os.listdir(path)
files = (f for f in files if f.lower().endswith('.jpg'))
files = sorted(files, key=compare_key)
for file in files:
i += 10
if debug: print('renaming %s to %s.jpg' % (file, i))
os.renames(file, 'renaming/%s.jpg' % i)
for root, __, files in os.walk(path + '/renaming'):
for file in files:
if debug: print('moving %s to %s' % (root+'/'+file, path+'/'+file))
os.renames(root+'/'+file, path+'/'+file)
Edit: I got rid of all the jpg fluff. You could use this code to rename your files. Just change the rename_file function to get rid of the extra dots. I haven't tested this code so there is a possibility that it might not work.
import sys, os
path = sys.argv[1]
def rename_file(file):
return file
#files are temporarily placed in a folder
#to prevent clashing filenames
files = os.listdir(path)
for file in files:
os.renames(file, 'renaming/' + rename_file(file))
for root, __, files in os.walk(path + '/renaming'):
for file in files:
os.renames(root+'/'+file, path+'/'+file)
Looks like I just needed to set the default directory and it worked just fine.
folder = r"blah\blah\blah"
os.chdir(folder)
for filename in os.listdir(folder):
fcount = filename.count('.') - 1
newname = filename.replace('.', ' ', fcount)
os.rename(filename, newname)

Categories