We Keep Coding

How can i optimize my Python loop for speed

I wrote some code that uses OCR to extract text from screenshots of follower lists and then transfer them into a data frame.
The reason I have to do the hustle with "name" / "display name" and removing blank lines is that the initial text extraction looks something like this:
Screenname 1
name 1
Screenname 2
name 2
(and so on)
So I know in which order each extraction will be.
My code works well for 1-30 images, but if I take more than that its gets a bit slow. My goal is to run around 5-10k screenshots through it at once. I'm pretty new to programming so any ideas/tips on how to optimize the speed would be very appreciated! Thank you all in advance :)
from PIL import Image
from pytesseract import pytesseract
import os
import pandas as pd
from itertools import chain
list_final = [""]
list_name = [""]
liste_anzeigename = [""]
list_raw = [""]
anzeigename = [""]
name = [""]
sort = [""]
f = r'/Users/PycharmProjects/pythonProject/images'
myconfig = r"--psm 4 --oem 3"
os.listdir(f)
for file in os.listdir(f):
f_img = f+"/"+file
img = Image.open(f_img)
img = img.crop((240, 400, 800, 2400))
img.save(f_img)
for file in os.listdir(f):
f_img = f + "/" + file
test = pytesseract.image_to_string(PIL.Image.open(f_img), config=myconfig)
lines = test.split("\n")
list_raw = [line for line in lines if line.strip() != ""]
sort.append(list_raw)
name = {list_raw[0], list_raw[2], list_raw[4],
list_raw[6], list_raw[8], list_raw[10],
list_raw[12], list_raw[14], list_raw[16]}
list_name.append(name)
anzeigename = {list_raw[1], list_raw[3], list_raw[5],
list_raw[7], list_raw[9], list_raw[11],
list_raw[13], list_raw[15], list_raw[17]}
liste_anzeigename.append(anzeigename)
reihenfolge_name = list(chain.from_iterable(list_name))
index_anzeigename = list(chain.from_iterable(liste_anzeigename))
sortieren = list(chain.from_iterable(sort))
print(list_raw)
sort_name = sorted(reihenfolge_name, key=sortieren.index)
sort_anzeigename = sorted(index_anzeigename, key=sortieren.index)
final = pd.DataFrame(zip(sort_name, sort_anzeigename), columns=['name', 'anzeigename'])
print(final)

Use a multiprocessing.Pool.
Combine the code under the for-loops, and put it into a function process_file.
This function should accept a single argument; the name of a file to process.
Next using listdir, create a list of files to process.
Then create a Pool and use its map method to process the list;
import multiprocessing as mp
def process_file(name):
# your code goes here.
return anzeigename # Or watever the result should be.
if __name__ is "__main__":
f = r'/Users/PycharmProjects/pythonProject/images'
p = mp.Pool()
liste_anzeigename = p.map(process_file, os.listdir(f))
This will run your code in parallel in as many cores as your CPU has.
For a N-core CPU this will take approximately 1/N times the time as doing it without multiprocessing.
Note that the return value of the worker function should be pickleable; it has to be returned from the worker process to the parent process.

How do i create a video from a static image and an audio file?

Im trying to write a program that creates an mp4 from a static image and an audio file.
Here is the code im using so far:
from moviepy.editor import AudioFileClip, ImageClip
def mp3PNGMerge(fileSaveName):
audio_clip = AudioFileClip("[PATH TO AUDIO]")
image_clip = ImageClip("PATH TO IMAGE")
video_clip = image_clip.set_audio(audio_clip)
video_clip.duration = audio_clip.duration
video_clip.fps = 1
video_clip.write_videofile(fileSaveName + '_CLIP.mp4')
mp3PNGMerge('[OUTPUT FILE NAME]')
However when this is run, the program creates a video the length of the audio clip but does not play the audio, it is completely silent.
Does anyone know what is going on here?

change the video_clip.fps to 30,
audio file can't be played on 1 fps
from moviepy.editor import AudioFileClip, ImageClip
def mp3PNGMerge(fileSaveName):
audio_clip = AudioFileClip("[PATH TO AUDIO]")
image_clip = ImageClip("PATH TO IMAGE")
video_clip = image_clip.set_audio(audio_clip)
video_clip.duration = audio_clip.duration
video_clip.fps = 30
video_clip.write_videofile(fileSaveName + '_CLIP.mp4')
mp3PNGMerge('[OUTPUT FILE NAME]')

The logic should be to set your sound on videoClip not imageClip.
I don't use MoviePY or Python but from looking at the docs, I would try as:
from moviepy.editor import AudioFileClip, ImageClip
def mp3PNGMerge(fileSaveName):
audio_clip = AudioFileClip("[PATH TO AUDIO]")
image_clip = ImageClip("PATH TO IMAGE")
video_clip = VideoClip(image_clip)
video_clip.set_audio(audio_clip)
video_clip.duration = audio_clip.duration
video_clip.fps = 30
video_clip.write_videofile(fileSaveName + '_CLIP.mp4')
mp3PNGMerge('[OUTPUT FILE NAME]')
I don't know if it will get you on the right track (eg: maybe video_clip.set_audio( ... ) is supposed to be set as video_clip.audio( ... ). The code logic should be correct.
Let me know how it goes (eg: I can delete this post if it's not a correct Answer).

Is there any way to add drawtext filter in video at the last 3 seconds by enable function in ffmpeg

I wanna ask that I want to add drawtext filter to the video in last 3 seconds only, may the video of any length like 2 minutes or 20 minutes, whatever. I want it in Python format.

With MoviePy you can get duration or end and use it to start text at end-3 or duration-3
from moviepy.editor import *
video = VideoFileClip("input.mp4")
print('duration:', video.duration)
text = TextClip('Hello', fontsize=70, color='white')
text = text.set_position('center')
text = text.set_start(video.end-3)
text = text.set_end(video.end)
#text = text.set_start(video.duration-3)
#text = text.set_end(video.duration)
result = CompositeVideoClip([video, text])
result.write_videofile("output.mp4")
EDIT:
The same method with ffmpeg-python.
First I get duration and later I use it with enable=f"between(t,{duration-3},{duration})"
I use x='(w-text_w)/2', y='(h-text_h)/2' to center text.
import ffmpeg
info = ffmpeg.probe('input.mp4')
duration = float(info['format']['duration'])
print('duration:', duration)
(
ffmpeg
.input('input.mp4')
.drawtext(text='Hello', x='(w-text_w)/2', y='(h-text_h)/2', fontsize=70, fontcolor='red', enable=f'between(t,{duration-3},{duration})')
.output('output.mp4')
.run(overwrite_output=True)
)
EDIT:
With ffmpeg-python you can even display arguments for program ffmpeg
video = (
ffmpeg
.input(filename)
.drawtext(text='Hello', x='(w-text_w)/2', y='(h-text_h)/2', fontsize=70, fontcolor='red', enable=f'between(t,{duration-3},{duration})')
.output('output.mp4')
)
video.run(overwrite_output=True)
video.view(detail=True) # image
print(video.get_args()) # text
Result:
['-i', 'input.mp4', '-filter_complex', '[0]drawtext=enable=between(t\\,4.036\\,7.036):fontcolor=red:fontsize=70:text=Hello:x=(w-text_w)/2:y=(h-text_h)/2[s0]', '-map', '[s0]', 'output.mp4']

Memory scanner for any program in Python

I am trying to create a memory scanner. similar to Cheat Engine. but only for extract information.
I know how to get the pid (in this case is "notepad.exe"). But I don't have any Idea about how to know wicht especific adress belong to the program that I am scanning.
Trying to looking for examples. I could see someone it was trying to scan every adress since one point to other. But it's to slow. Then I try to create a batch size (scan a part of memory and not one by one each adress). The problem is if the size is to short. still will take a long time. and if it is to long, is possible to lose many adress who are belong to the program. Because result from ReadMemoryScan is False in the first Adress, but It can be the next one is true. Here is my example.
import ctypes as c
from ctypes import wintypes as w
import psutil
from sys import stdout
write = stdout.write
import numpy as np
def get_client_pid(process_name):
pid = None
for proc in psutil.process_iter():
if proc.name() == process_name:
pid = int(proc.pid)
print(f"Found '{process_name}' PID = ", pid,f" hex_value = {hex(pid)}")
break
if pid == None:
print('Program Not found')
return pid
pid = get_client_pid("notepad.exe")
if pid == None:
sys.exit()
k32 = c.WinDLL('kernel32', use_last_error=True)
OpenProcess = k32.OpenProcess
OpenProcess.argtypes = [w.DWORD,w.BOOL,w.DWORD]
OpenProcess.restype = w.HANDLE
ReadProcessMemory = k32.ReadProcessMemory
ReadProcessMemory.argtypes = [w.HANDLE,w.LPCVOID,w.LPVOID,c.c_size_t,c.POINTER(c.c_size_t)]
ReadProcessMemory.restype = w.BOOL
GetLastError = k32.GetLastError
GetLastError.argtypes = None
GetLastError.restype = w.DWORD
CloseHandle = k32.CloseHandle
CloseHandle.argtypes = [w.HANDLE]
CloseHandle.restype = w.BOOL
processHandle = OpenProcess(0x10, False, int(pid))
# addr = 0x0FFFFFFFFFFF
data = c.c_ulonglong()
bytesRead = c.c_ulonglong()
start = 0x000000000000
end = 0x7fffffffffff
batch_size = 2**13
MemoryData = np.zeros(batch_size, 'l')
Size = MemoryData.itemsize*MemoryData.size
index = 0
Data_address = []
for c_adress in range(start,end,batch_size):
result = ReadProcessMemory(processHandle,c.c_void_p(c_adress), MemoryData.ctypes.data,
Size, c.byref(bytesRead))
if result: # Save adress
Data_address.extend(list(range(c_adress,c_adress+batch_size)))
e = GetLastError()
CloseHandle(processHandle)
I decided from 0x000000000000 to 0x7fffffffffff Because cheat engine scan this size. I am still a begginer with this kind of this about memory scan. maybe there are things that I can do to improve the efficiency.

I suggest you take advantage of existing python libraries that can analyse Windows 10 memory.
I'm no specialist but I've found Volatility. Seems to be pretty useful for your problem.
For running that tool you need Python 2 (Python 3 won't work).
For running python 2 and 3 in the same Windows 10 machine, follow this tutorial (The screenshots are in Spanish but it can easily be followed).
Then see this cheat sheet with main commands. You can dump the memory and then operate on the file.
Perhaps this leads you to the solution :) At least the most basic command pslist dumps all the running processes addresses.

psutil has proc.memory_maps()
pass the result as map to this function
TargetProcess eaxample 'Calculator.exe'
def get_memSize(self,TargetProcess,map):
for m in map:
if TargetProcess in m.path:
memSize= m.rss
break
return memSize
if you use this function, it returns the memory size of your Target Process
my_pid is the pid for 'Calculator.exe'
def getBaseAddressWmi(self,my_pid):
PROCESS_ALL_ACCESS = 0x1F0FFF
processHandle = win32api.OpenProcess(PROCESS_ALL_ACCESS, False, my_pid)
modules = win32process.EnumProcessModules(processHandle)
processHandle.close()
base_addr = modules[0] # for me it worked to select the first item in list...
return base_addr
to get the base address of your prog
so you search range is from base_addr to base_addr + memSize

How to create video thumbnails with Python and Gstreamer

I'd like to create thumbnails for MPEG-4 AVC videos using Gstreamer and Python. Essentially:
Open the video file
Seek to a certain point in time (e.g. 5 seconds)
Grab the frame at that time
Save the frame to disc as a .jpg file
I've been looking at this other similar question, but I cannot quite figure out how to do the seek and frame capture automatically without user input.
So in summary, how can I capture a video thumbnail with Gstreamer and Python as per the steps above?

To elaborate on ensonic's answer, here's an example:
import os
import sys
import gst
def get_frame(path, offset=5, caps=gst.Caps('image/png')):
pipeline = gst.parse_launch('playbin2')
pipeline.props.uri = 'file://' + os.path.abspath(path)
pipeline.props.audio_sink = gst.element_factory_make('fakesink')
pipeline.props.video_sink = gst.element_factory_make('fakesink')
pipeline.set_state(gst.STATE_PAUSED)
# Wait for state change to finish.
pipeline.get_state()
assert pipeline.seek_simple(
gst.FORMAT_TIME, gst.SEEK_FLAG_FLUSH, offset * gst.SECOND)
# Wait for seek to finish.
pipeline.get_state()
buffer = pipeline.emit('convert-frame', caps)
pipeline.set_state(gst.STATE_NULL)
return buffer
def main():
buf = get_frame(sys.argv[1])
with file('frame.png', 'w') as fh:
fh.write(str(buf))
if __name__ == '__main__':
main()
This generates a PNG image. You can get raw image data using gst.Caps("video/x-raw-rgb,bpp=24,depth=24") or something like that.
Note that in GStreamer 1.0 (as opposed to 0.10), playbin2 has been renamed to playbin and the convert-frame signal is named convert-sample.
The mechanics of seeking are explained in this chapter of the GStreamer Application Development Manual. The 0.10 playbin2 documentation no longer seems to be online, but the documentation for 1.0 is here.

An example in Vala, with GStreamer 1.0 :
var playbin = Gst.ElementFactory.make ("playbin", null);
playbin.set ("uri", "file:///path/to/file");
// some code here.
var caps = Gst.Caps.from_string("image/png");
Gst.Sample sample;
Signal.emit_by_name(playbin, "convert-sample", caps, out sample);
if(sample == null)
return;
var sample_caps = sample.get_caps ();
if(sample_caps == null)
return;
unowned Gst.Structure structure = sample_caps.get_structure(0);
int width = (int)structure.get_value ("width");
int height = (int)structure.get_value ("height");
var memory = sample.get_buffer().get_memory (0);
Gst.MapInfo info;
memory.map (out info, Gst.MapFlags.READ);
uint8[] data = info.data;

It's an old question but I still haven't found it documented anywhere.
I found that the following worked on a playing video with Gstreamer 1.0
import gi
import time
gi.require_version('Gst', '1.0')
from gi.repository import Gst
def get_frame():
caps = Gst.Caps('image/png')
pipeline = Gst.ElementFactory.make("playbin", "playbin")
pipeline.set_property('uri','file:///home/rolf/GWPE.mp4')
pipeline.set_state(Gst.State.PLAYING)
#Allow time for it to start
time.sleep(0.5)
# jump 30 seconds
seek_time = 30 * Gst.SECOND
pipeline.seek(1.0, Gst.Format.TIME,(Gst.SeekFlags.FLUSH | Gst.SeekFlags.ACCURATE),Gst.SeekType.SET, seek_time , Gst.SeekType.NONE, -1)
#Allow video to run to prove it's working, then take snapshot
time.sleep(1)
buffer = pipeline.emit('convert-sample', caps)
buff = buffer.get_buffer()
result, map = buff.map(Gst.MapFlags.READ)
if result:
data = map.data
pipeline.set_state(Gst.State.NULL)
return data
else:
return
if __name__ == '__main__':
Gst.init(None)
image = get_frame()
with open('frame.png', 'wb') as snapshot:
snapshot.write(image)
The code should run with both Python2 and Python3, I hope it helps someone.

Use playbin2. set the uri to the media file, use gst_element_seek_simple to seek to the desired time position and then use g_signal_emit to invoke the "convert-frame" action signal.