We Keep Coding

Trimming videos with 'ffmpeg and ffprobe'

I am working on an ETL process, and I'm now in the final stage of preprocessing my videos. I used the script below (reference: #FarisHijazi) to first auto detected black-screen frames using ffprobe and trim them out using ffmpeg.
The script worked for me but the problems are:
It cut off all other good frames together with the first bad frames. e.g. if gBgBgBgB represents a sequence of good and BAD frames for 5sec each, the script only returned the first g(5sec) and cut off the other BgBgBgB after it. I want to have only g g g g where all B B B B has been removed
I also want to detect other colors aside black-screen e.g. green-screen or red-screen or blurry part of video
Script doesn't work if video has no audio in it.
import argparse
import os
import shlex
import subprocess
parser = argparse.ArgumentParser(
__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument("input", type=str, help="input video file")
parser.add_argument(
"--invert",
action="store_true",
help="remove nonblack instead of removing black",
)
args = parser.parse_args()
##FIXME: sadly you must chdir so that the ffprobe command will work
os.chdir(os.path.split(args.input)[0])
args.input = os.path.split(args.input)[1]
spl = args.input.split(".")
outpath = (
".".join(spl[:-1])
+ "."
+ ("invert" if args.invert else "")
+ "out."
+ spl[-1]
)
def delete_back2back(l):
from itertools import groupby
return [x[0] for x in groupby(l)]
def construct_ffmpeg_trim_cmd(timepairs, inpath, outpath):
cmd = f'ffmpeg -i "{inpath}" -y -r 20 -filter_complex '
cmd += '"'
for i, (start, end) in enumerate(timepairs):
cmd += (
f"[0:v]trim=start={start}:end={end},setpts=PTS-STARTPTS,format=yuv420p[{i}v]; "
+ f"[0:a]atrim=start={start}:end={end},asetpts=PTS-STARTPTS[{i}a]; "
)
for i, (start, end) in enumerate(timepairs):
cmd += f"[{i}v][{i}a]"
cmd += f"concat=n={len(timepairs)}:v=1:a=1[outv][outa]"
cmd += '"'
cmd += f' -map [outv] -map [outa] "{outpath}"'
return cmd
def get_blackdetect(inpath, invert=False):
ffprobe_cmd = f'ffprobe -f lavfi -i "movie={inpath},blackdetect[out0]" -show_entries tags=lavfi.black_start,lavfi.black_end -of default=nw=1 -v quiet'
print("ffprobe_cmd:", ffprobe_cmd)
lines = (
subprocess.check_output(shlex.split(ffprobe_cmd))
.decode("utf-8")
.split("\n")
)
times = [
float(x.split("=")[1].strip()) for x in delete_back2back(lines) if x
]
assert len(times), "no black scene detected"
if not invert:
times = [0] + times[:-1]
timepairs = [
(times[i], times[i + 1]) for i in range(0, len(times) // 2, 2)
]
return timepairs
if __name__ == "__main__":
timepairs = get_blackdetect(args.input, invert=args.invert)
cmd = construct_ffmpeg_trim_cmd(timepairs, args.input, outpath)
print(cmd)
os.system(cmd)

How to integrate a bash command into a python code [duplicate]

This question already has answers here:
Variable interpolation in Python [duplicate]
(5 answers)
Closed 3 years ago.
everyone,
I'm looking to integrate a bash command into my python code to calculate indices. My problem is that I want to have an output image with a band for each of the calculated indices, but I can't integrate these indices by the bash command into my 'im_index' matrix created with my python code. I don't see how to link both of them... Do you have any idea?
import numpy as np
import sys
import os
import spectral as sp
from scipy import ndimage
import pylab as pl
from math import *
import spectral.io.envi as envi
#------------------------------------
def reject_outliers(data, m=1):
return data[abs(data - np.mean(data)) < m * np.std(data)]
#------------------------------------
def find_nearest(array, value):
#For a given value, find the nearest value in an array
array = np.asarray(array)
idx = (np.abs(array - value)).argmin()
return idx
#------------------------------------
#Open existing dataset
src_directory = "/d/afavro/Bureau/4_reflectance/"
dossier = os.listdir (src_directory)
print(dossier)
for fichier in dossier:
print (fichier)
ssrc_directory = "/d/afavro/Bureau/4_reflectance/" + fichier + "/"
rasters = os.listdir (ssrc_directory)
print(rasters)
OUTPUT_FOLDER = "/d/afavro/Bureau/5_indices2/" + 'indices_' + fichier + '/'
print(OUTPUT_FOLDER)
if not os.path.exists(OUTPUT_FOLDER):
os.makedirs(OUTPUT_FOLDER)
for image in rasters:
print(image)
name, ext = os.path.splitext(image)
if ext == '.hdr':
img = sp.open_image(ssrc_directory + image)
print(image)
im_HS = img[:,:,:]
cols = im_HS.shape[0] # Number of column
rows = im_HS.shape[1] # Number of lines
bands = im_HS.shape[2] # Number of bands
NbPix = cols * rows # Number of pixels
#Get wavelengths from hdr file
wv = np.asarray(img.bands.centers)
if len(wv) == 0 :
print("Wavelengths not defined in the hdr file")
sys.exit("Try again!")
if wv[0] > 100:
wv=wv*0.001 # Convert to micrometers if necessary
im_HS=im_HS.reshape(NbPix, bands)
#Compute HC index------------------------------------------------------
Nind=4 # Number of indice to be computed
im_index=np.zeros((cols*rows, Nind))
names = []
##NDVI computation
names.append('NDVI')
bande_ref=[0.67, 0.8]
bRef0 = find_nearest(wv,bande_ref[0])
bRef1 = find_nearest(wv,bande_ref[1])
#Check if the required specral bands are available
if (np.abs(wv[bRef0]-bande_ref[0])<=0.1 and np.abs(wv[bRef1]-bande_ref[1])<=0.1):
b0 = im_HS[:, bRef0]
b1 = im_HS[:, bRef1]
index = (b0 - b1) / (b0 + b1)
else:
index = np.zeros_like(im_HS[:,0])
print("Wavelengths selection problem, NDVI not computed")
im_index[:,0]= index
# bash command :
inRaster = ssrc_directory + image
print(inRaster)
outRaster = OUTPUT_FOLDER + 'indices_' + image
print (outRaster)
cmd = 'otbcli_RadiometricIndices -in inRaster -list Soil:BI Vegetation:MSAVI Vegetation:SAVI -out outRaster'
os.system(cmd)
#saving
im_index=im_index.reshape(cols, rows, Nind)
file_image = OUTPUT_FOLDER + "indices2_" + fichier
header = envi.read_envi_header(ssrc_directory + image)
header ['description'] = "fichier d'origine " + image
header ['band names'] = ['NDVI', 'Sober filter', 'NDWI', 'IB(1)', 'IB(2)']
del header['wavelength units']
del header['wavelength']
sp.envi.save_image(file_image + '.hdr', im_index, metadata=header, force = True, interleave = 'bsq')

Assuming this is the code you are actually asking about:
inRaster = ssrc_directory + image
print(inRaster)
outRaster = OUTPUT_FOLDER + 'indices_' + image
print (outRaster)
cmd = 'otbcli_RadiometricIndices -in inRaster -list Soil:BI Vegetation:MSAVI Vegetation:SAVI -out outRaster'
os.system(cmd)
Of course, inRaster inside of singe quotes is just a literal string; to interpolate the variable's value you can say
cmd = 'otbcli_RadiometricIndices -in ' + inRaster + \
' -list Soil:BI Vegetation:MSAVI Vegetation:SAVI -out ' + \
outRaster
or
cmd = 'otbcli_RadiometricIndices -in {0} -list Soil:BI Vegetation:MSAVI Vegetation:SAVI -out {1}'.format(
inRaster, outRaster)
or a number of other string interpolation techniques in Python (legacy % formatting, f-string, etc). But a better solution is to replace os.system with the more flexible and versatile subprocess, as suggested even in the os.system documentation.
subprocess.run([
'otbcli_RadiometricIndices',
'-in', inRaster,
'-list', 'Soil:BI', 'Vegetation:MSAVI', 'Vegetation:SAVI',
'-out', outRaster], check=True)
subprocess.run was introduced in Python 3.5; if you need compatibility with older versions, try subprocess.check_call or even the crude subprocess.call.

I think you might be looking for the subprocess package. An example:
>>> import subprocess as sp
>>> output = sp.check_output('echo hello world', shell=True)
>>> print(output)
b'hello world\n'
The check_output() method can be used to collect the stdout from a command. You'd need to parse the output to get integer indices afterwards.

Calling bash command inside Python returns error, but works in terminal

Here is the except of my code related to this:
def grd_commands(directory):
for filename in os.listdir(directory)[1:]:
print filename
new_filename = ''
first_letter = ''
second_letter = ''
bash_command = 'gmt grdinfo ' + filename + ' -I-'
print bash_command
coordinates = Popen(bash_command, stdout=PIPE, shell=True)
coordinates = coordinates.communicate()
latlong = re.findall(r'^\D*?([-+]?\d+)\D*?[-+]?\d+\D*?([-+]?\d+)', coordinates)
if '-' in latlong[1]:
first_letter = 'S'
else:
first_letter = 'N'
if '-' in latlong[0]:
second_letter = 'W'
else:
second_letter = 'E'
new_filename = first_letter + str(latlong[1]) + second_letter + str(latlong[0]) + '.grd'
Popen('gmt grdconvert ' + str(filename) + ' ' + new_filename, shell=True)
filenameis the name of the file that is is being passed to the function. When I run my code, I am receiving this error:
/bin/sh: gmt: command not found
Traceback (most recent call last):
File "/Users/student/Desktop/Code/grd_commands.py", line 38, in <module>
main()
File "/Users/student/Desktop/Code/grd_commands.py", line 10, in main
grd_commands(directory)
File "/Users/student/Desktop/Code/grd_commands.py", line 23, in grd_commands
latlong = re.findall(r'^\D*?([-+]?\d+)\D*?[-+]?\d+\D*?([-+]?\d+)', coordinates).split('\n')
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/re.py", line 177, in findall
return _compile(pattern, flags).findall(string)
TypeError: expected string or buffer
If I print out the string bash_command and try entering it into terminal it fully functions. Why doesn't it work when being called by my Python script?

The entire command line is being treated as a single command name. You need to either use shell=True to have the shell parse it as a command line:
coordinates = Popen(bash_command, stdout=PIPE, shell=True)
or preferably store the command name and its arguments as separate elements of a list:
bash_command = ['gmt', 'grdinfo', filename, '-I-']
coordinates = Popen(bash_command, stdout=PIPE)

Popen takes a list of arguments. There is a warning for using shell=True
Passing shell=True can be a security hazard if combined with untrusted input.
Try this:
from subprocess import Popen, PIPE
bash_command = 'gmt grdinfo ' + filename + ' -I-'
print(bash_command)
coordinates = Popen(bash_command.split(), stdout=PIPE)
print(coordinates.communicate()[0])
Ensure gmt is installed in a location specified by PATH in your /etc/environment file:
PATH=$PATH:/path/to/gmt
Alternatively, specify the path to gmt in bash_command:
bash_command = '/path/to/gmt grdinfo ' + filename + ' -I-'
You should be able to find the path with:
which gmt
As other people have suggested, an actual list would be the best approach instead of a string. Additionally, you must escape spaces with a '\' in order to actually access the file if there is a space in it.
for filename in os.listdir(directory)[1:]:
bash_command = ['gmt', 'grdinfo', filename.replace(" ", "\ "), '-I-']

'The filename, directory name, or volume label syntax is incorrect.\r\n', None

I am trying to have a Python script stage a file to a locally hosted Git:
gitExe = r"C:\Git\bin\git.exe"
gitdir = r' --git-dir="G:\QA\.git"'
worktree = r' --work-tree="G:\QA\"'
gitcmd = " add"
myCsv = "Reports.csv"
import subprocess
print gitdir
print gitcmd
print gitExe
cmd = 'C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv' ####WORKS####
#cmd = ['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv'] ####DOESNT WORK####
#cmd = ['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA" add Reports.csv'+""] ####DOESNT WORK####
#cmd = [gitExe + gitdir + worktree + gitcmd + myCsv] ####DOESNT WORK####
p = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
print cmd
print p.communicate()
When I pass the full command as a string (uncommendted line above) everything works fine but when I pass it as an array (the three commented out lines above) I get the following error message:
>>> ================================ RESTART ================================
>>>
--git-dir="G:\QA\.git"
add
C:\Git\bin\git.exe
['C:\\Git\\bin\\git.exe --git-dir="G:\\QA\\.git" --work-tree="G:\\QA\\" addReports.csv']
('The filename, directory name, or volume label syntax is incorrect.\r\n', None)
What gives?

When you pass the command as an array, each parameter must be a separate element.
cmd = ['C:\\Git\\bin\\git.exe', '--git-dir="G:\\QA\\.git"', '--work-tree="G:\\QA"',
'add', 'Reports.csv']
The first item in the array is the program name. In your case, python tried to make the entire command line into the program name.

i have this code. It is supposed to classify the segment files with the ratings in sys.argv[5].But it keeps having error

I have the following code tha uses FFmpeg . It has 5 argv and it takes in filename,video,segment size, start time, end time, ratings. thats supposed to let me classify segments many times with my ratings "PG G M18..." but there's this error,
"File "C:\c.py",line 92, in <module> os.rename<filename + str(x), filename + str(x) + classification)
WindowsError: [Error2] The system cannot find the file specified.
Ive tried to edit many times but this error still persists. Anyone have any idea what could this error mean and anyway to solve it?
import sys
import subprocess
import os
#change hh:mm:ss to seconds:
def getSeconds(sec):
l = sec.split(':')
return int(l[0])* 3600 + int(l[1])* 60 + float(l[2])
def get_total_time(filename):
proc = subprocess.Popen(["ffmpeg", "-i", filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = proc.communicate()[1]
target = [line for line in lines.split('\n') if 'Duration:' in line][0]
time = target.split('Duration: ')[-1].split(',', 1)[0]
return time
#check command line arguments
if len(sys.argv) < 5:
print "Error: not enough arguments"
sys.exit()
#save filename to file_name
file_name = sys.argv[1]
if not file_name.endswith('mpg'):
print 'Error! File extension not supported'
sys.exit()
# save a size of chunk in chunk
segsize = int(sys.argv[2])
chunk = (segsize * 1024)
# get time of starting censorship in seconds
start_censorship = getSeconds(sys.argv[3])
# get time of ending censorship in seconds
end_censorship = getSeconds(sys.argv[4])
classification = sys.argv[5]
if classification not in ['P','PG','G','NC16','M18','R21']:
print "Error: invalid classification"
sys.exit()
#initialize variable for extension
file_ext = ''
# if extension exists then save it into file_ext
if '.' in file_name:
# split file_name on two parts from right
file_ext = sys.argv[1].split('.')[-1]
# file_name without extension
filename = '.'.join(file_name.split('.')[:-1])
# total_time of file in seconds
total_time = getSeconds(get_total_time(file_name))
print total_time
#open file
in_file = open(file_name,'rb')
#read first chunks
s = in_file.read(chunk)
in_file.seek(0, 2)
file_size = in_file.tell()
chunks = (file_size / chunk) + 1
chunk_time = total_time/ file_size * chunk
#close input file
in_file.close()
#loop for each chunk
for x in range(0, chunks):
# starting time of current chunk
t1 = chunk_time*x
# ending time of current chunk
t2 = chunk_time*(x+1)
if t2 < start_censorship or t1 > end_censorship:
pass
else:
if os.path.exists(filename + str(x) + 'x'):
os.rename(filename + str(x) + 'x', filename + str(x))
os.rename(filename + str(x), filename + str(x) + classification)
#read next bytes

You're not checking whether filename + str(x) exists before you try to rename it. Either check first, or put it in a try block and catch OSError (which WindowsError subclasses). Either that or you're missing an indent on the second rename.

In that last for loop it looks like you are possibly renaming multiple files -- where do the original files come from?
Or, asked another way, for every chunk that is not censored you are renaming a file -- is that really what you want?