We Keep Coding

Simultaneous placement of camera images and buttons in GUI applications using Kivy

I am currently trying to create an app using Kivy that will turn the video into grayscale when the button is pressed.
At first, I was able to display the camera image on the GUI, but I'm having trouble figuring out how to place buttons and other parts from here.
import sys
import numpy as np
import cv2
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Label
from kivy.uix.button import Button
from kivy.uix.widget import Widget
from kivy.graphics import Rectangle
from kivy.graphics.texture import Texture
from kivy.clock import Clock
from kivy.core.window import Window
import datetime
import random
WINDOW_WIDTH = 1500
WINDOW_HEIGHT = 1008
### Setting of the window
Window.size = (WINDOW_WIDTH, WINDOW_HEIGHT)
class MyApp(App, Widget):
title = "opencv on kivy"
def __init__(self, **kwargs):
super(MyApp, self).__init__(**kwargs)
Clock.schedule_interval(self.update, 1.0 / 30)
self.widget = Widget()
# self.cap = cv2.VideoCapture(0)
self.cap = cv2.VideoCapture(1)
# The method by the intarval
def update(self, dt):
ret, img = self.cap.read()
### The color sequence in openCV is BGR, so fix it to RGB.
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
### The origin of Kivy's coordinates is the lower left, so flip it upside down.
img = cv2.flip(img, 0)
# if img is not None:
texture = Texture.create(size=(img.shape[1], img.shape[0]))
texture.blit_buffer(img.tostring())
with self.widget.canvas:
Rectangle(texture=texture ,pos=(0 + int(WINDOW_WIDTH/2) - int(img.shape[1]/2), WINDOW_HEIGHT - img.shape[0]), size=(img.shape[1], img.shape[0]))
return self.widget
def build(self):
return self.widget
# return MyApp()
if __name__ == '__main__':
MyApp().run()
Also, the application that displays the button works fine.
However, I have no idea how to combine these since build(self) can only return a button or a single widget for the camera image.
# import kivy module
import kivy
# this restrict the kivy version i.e
# below this kivy version you cannot
# use the app or software
kivy.require("1.9.1")
# base Class of your App inherits from the App class.
# app:always refers to the instance of your application
from kivy.app import App
# creates the button in kivy
# if not imported shows the error
from kivy.uix.button import Button
# class in which we are creating the button
class ButtonApp(App):
def build(self):
btn = Button(text ="Push Me !")
return btn
# creating the object root for ButtonApp() class
root = ButtonApp()
# run function runs the whole program
# i.e run() method which calls the
# target function passed to the constructor.
root.run()
thank you in advance.

You can use self.add_widget(..) to add other widgets inside Widget, Button, etc. but it doesn't have function to automatically arange layout.
But build() can use any widget - it doesn't have to Button or Widget.
If you use BoxLayout then you can use self.add_widget(...) to add widgets in rows or columns.
Other layout widgets can be useful to organize widgets in different way.
class MyApp(App, BoxLayout): # class MyApp(App, Widget)
def __init__(self, **kwargs):
# ... code ...
self.orientation = 'vertical' # BoxLayout
self.widget = Widget()
self.add_widget(self.widget)
self.button = Button(text='Gray', on_press=self.change_color)
self.add_widget(self.button)
Full working code.
I use BoxLayout with orientation = 'vertical' to organize in rows I put Widget in top row, and Button in bottom row.
Button runs function which switch value self.convert_to_grey - True/False - and update() uses this value to convert image to gray (and back to RGB but with gray color)
import sys
import numpy as np
import cv2
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.button import Label
from kivy.uix.button import Button
from kivy.uix.widget import Widget
from kivy.graphics import Rectangle
from kivy.graphics.texture import Texture
from kivy.clock import Clock
from kivy.core.window import Window
import datetime
import random
WINDOW_WIDTH = 1500
WINDOW_HEIGHT = 1008
### Setting of the window
Window.size = (WINDOW_WIDTH, WINDOW_HEIGHT)
class MyApp(App, BoxLayout):
title = "opencv on kivy"
def __init__(self, **kwargs):
super(MyApp, self).__init__(**kwargs)
self.orientation = 'vertical' # BoxLayout
self.convert_to_grey = False #
self.cap = cv2.VideoCapture(0)
#self.cap = cv2.VideoCapture(1)
self.widget = Widget()
self.add_widget(self.widget)
self.button = Button(text='Gray', on_press=self.change_color)
self.add_widget(self.button)
Clock.schedule_interval(self.update, 1.0 / 30)
def change_color(self, *args):
print('args:', args)
self.convert_to_grey = not self.convert_to_grey
def update(self, dt):
ret, img = self.cap.read()
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = cv2.flip(img, 0)
if self.convert_to_grey:
img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
texture = Texture.create(size=(img.shape[1], img.shape[0]))
texture.blit_buffer(img.tostring())
with self.widget.canvas:
Rectangle(texture=texture, pos=(0 + int(WINDOW_WIDTH/2) - int(img.shape[1]/2), WINDOW_HEIGHT - img.shape[0]), size=(img.shape[1], img.shape[0]))
return self.widget
def build(self):
return self
if __name__ == '__main__':
MyApp().run()
More layouts in Kivy doc:
Getting Started » Layouts
BTW: you can nested layouts - ie. inside BoxLayout you can use one row with GridLayout and other row with different layout or widget.

Click screenshot of part of the screen using python kivy

At the moment the below code shows a camera layout using laptop webcam just fine - I want to show a rectangle frame within the camera window - user will hold a book aligned to the frame and I need to capture an image of the book i.e capture image of the part within the frame. I am struggling to
Show a transparent rectangle as a frame (this is inside a boxlayout inside a camera floatlayout)
Grab an image of only the part within the frame
There is a button below the camera layout at click of which the image will be saved to a folder on the machine
Please can somebody guide me how to proceed and whether this can be achieved in any other way using any other module
import kivy
from PIL import ImageGrab
from kivy.uix.boxlayout import BoxLayout
from numpy import shape
kivy.require('1.7.2')
from kivy.app import App
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.camera import Camera
from kivy.uix.button import Button
from kivy.core.window import Window
from kivy.graphics import Color, Rectangle
class CamApp(App):
# Function to take a screenshot
def screengrab(self, *largs):
im2 = ImageGrab.grab(bbox=None)
im2.show()
#outname = self.fileprefix + '_%(counter)04d.png'
#Window.screenshot(name=outname)
def build(self):
# create a floating layout as base
camlayout = FloatLayout(size=(600, 600))
cam = Camera() # Get the camera
cam = Camera(resolution=(1024, 1024), size=(300, 300))
cam.play = True # Start the camera
camlayout.add_widget(cam)
boxlayout = BoxLayout(id='imageBox', size_hint=[0.5, 0.7], pos_hint={'center_x': .5, 'center_y': .5})
boxlayout.bind(size=self.update_rect, pos=self.update_rect)
with boxlayout.canvas.before:
#Color(0, 1, 0, 1) # green; colors range from 0-1 instead of 0-255
self.rect = Rectangle(size=boxlayout.size,
pos=boxlayout.pos, outline='black')
camlayout.add_widget(boxlayout)
button = Button(text='Take Picture', size_hint=(0.12, 0.12))
button.bind(on_press=self.screengrab)
camlayout.add_widget(button) # Add button to Camera Layout
self.fileprefix = 'snap'
return camlayout
def update_rect(self,instance, value):
self.rect.pos = instance.pos
self.rect.size = instance.size
if __name__ == '__main__':
CamApp().run()
Click to see output

In kivy, the Color() function takes arguments representing RGBA. That means the last argument is alpha, and passing it a float will make it transparent.
So while Color(0, 1, 0, 1) is a solid color, Color(0, 1, 0, .5) will be semitransparent.
As for grabbing an image of only part of the frame, you might consider grabbing the whole frame, then using PIL to crop the image with
from PIL import Image
im = Image.open(r"C:\path\to\picture\my_screenshot.png")
im.crop((left, top, right, bottom))
Just replace left, top, right, and bottom with the dimensions you want to cut out.

Kivy segmentation fault on blitting opencv picture

I am trying to cast video from my external cam, that is captured through some aliexpress EasyCap, to my kivy app. One issue I've faced is that crashes with segmentation fault on trying to
texture = Texture.create(size=(frame.shape[0], frame.shape[1]))
I've found that the problem is on kivy's side. It sometimes can't create NPOT textures. So, I've changed it to POT shape and copied what is possible to another numpy array.
flipped = cv2.flip(frame, 0)
buf = np.zeros((512, 512, 3), dtype=np.uint8)
for i in range(min(frame.shape[0], 512)):
for j in range(min(frame.shape[1], 512)):
buf[i, j] = flipped[i, j]
buf = buf.tostring()
texture = Texture.create(size=(512, 512))
texture.blit_buffer(buf, colorfmt="bgr", bufferfmt="ubyte")
self.texture = texture
But it still crashes with the same old segmentation fault on the following line:
texture.blit_buffer(buf, colorfmt="bgr", bufferfmt="ubyte")
If it is relevant, cv2.imshow("image", buf) before buf.tostring() show the image correctly.
Here's the original code:
from kivy.app import App
from kivy.uix.image import Image
from kivy.clock import Clock
from kivy.graphics.texture import Texture
import cv2
import threading
from time import sleep
import numpy as np
class KivyCamera(Image):
def __init__(self, **kwargs):
super(KivyCamera, self).__init__(**kwargs)
self.fps = 30
self.capture = cv2.VideoCature(0)
threading.Thread(target=self.update).start()
def update(self):
while True:
ret, frame = self.capture.read()
if ret:
buf = cv2.flip(frame, 0).tostring()
texture = Texture.create(size=(frame.shape[0], frame.shape[1])
texture.blit_buffer(buf, colorfmt="bgr", bufferfmt="ubyte")
self.texture = texture
sleep(1.0 / self.fps)
class CamApp(App):
def build(self):
return KivyCamera()
if __name__ == "__main__":
CamApp().run()

Few issues with your code:
The GUI updates should always be done in the mainthread as indicated by John Anderson. Hence instead of separate thread, the update() function should be called through Clock schedule.
The size tuple in the Texture.create statement should be reversed and the colorfmt="bgr" should be added. It should be: texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt="bgr").
There is no layout defined. Layouts like BoxLayout should be added as a placeholder of Image widget.
Below is the modified working version of your code:
from kivy.app import App
from kivy.uix.image import Image
from kivy.uix.boxlayout import BoxLayout
from kivy.clock import Clock
from kivy.graphics.texture import Texture
import cv2
class KivyCamera(BoxLayout):
def __init__(self, **kwargs):
super(KivyCamera, self).__init__(**kwargs)
self.img1=Image()
self.add_widget(self.img1)
self.capture = cv2.VideoCapture(0)
Clock.schedule_interval(self.update, 1.0/33.0)
def update(self, *args):
ret, frame = self.capture.read()
if ret:
buf = cv2.flip(frame, 0).tostring()
texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt="bgr")
texture.blit_buffer(buf, colorfmt="bgr", bufferfmt="ubyte")
self.img1.texture = texture
class CamApp(App):
def build(self):
return KivyCamera()
if __name__ == "__main__":
CamApp().run()
The while loop is removed and Clock.schedule_interval is used with 1 / 33. steps.
Hope this helps.

Not well documented, but I believe the Texture manipulation (specifically the blit_buffer()) must be done on the main thread. Your update() method is being run in another thread, so use Clock.schedule_once() to call a method that does the Texture manipulation (and the self.texture = texture) back in the main thread.

Is there a way to integrate the imshow() function of OpenCV into kivy or kv file in python

I'm new here and hope for a little help and would be very happy about that.
I write a small program in python, kivy and opencv.
The problem is that I would like to integrate my webcam with opencv and not via the existing camera function from kivy.
I have already found a similar problem here Integrate OpenCV webcam into a Kivy user interface but this does not solve my problem.
In my OpenCV code, also runs a code for facial recognition (https://github.com/ageitgey/face_recognition/blob/master/examples/facerec_from_webcam_faster.py). So it is therefore important that the command imshow() is issued. How can I integrate the webcam version of imshow() from Opencv into kivy or into a kv file?
Unfortunately, I don't know if something like that might work. Can one of you help me or has a idea. Thank you very much for your help.
Python file:
import cv2
from kivy.app import App
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen
class MainScreen(Screen):
pass
class Manager(ScreenManager):
pass
kv = Builder.load_file("file.kv")
class Main(App):
def build(self):
return kv
if __name__ == '__main__':
Main().run()
OpenCV - Code:
import cv2
cam = cv2.VideoCapture(0)
while(True):
ret, frame = cam.read()
# ...
# more code
# ...
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
My Kivy file (minimal):
MainScreen:
MainScreen:
<MainScreen>:
name: "Test"
FloatLayout:
Label:
text: "Webcam from OpenCV?"
pos_hint: {"x":0.0, "y":0.8}
size_hint: 1.0, 0.2
Button:
text: 'Click me!!'
pos_hint: {"x":0.0, "y":0.0}
size_hint: 1.0, 0.2
font_size: 50

Here is a hack that sort of does what I think you want:
import threading
from functools import partial
import cv2
from kivy.app import App
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen
class MainScreen(Screen):
pass
class Manager(ScreenManager):
pass
Builder.load_string('''
<MainScreen>:
name: "Test"
FloatLayout:
Label:
text: "Webcam from OpenCV?"
pos_hint: {"x":0.0, "y":0.8}
size_hint: 1.0, 0.2
Image:
# this is where the video will show
# the id allows easy access
id: vid
size_hint: 1, 0.6
allow_stretch: True # allow the video image to be scaled
keep_ratio: True # keep the aspect ratio so people don't look squashed
pos_hint: {'center_x':0.5, 'top':0.8}
Button:
text: 'Stop Video'
pos_hint: {"x":0.0, "y":0.0}
size_hint: 1.0, 0.2
font_size: 50
on_release: app.stop_vid()
''')
class Main(App):
def build(self):
# start the camera access code on a separate thread
# if this was done on the main thread, GUI would stop
# daemon=True means kill this thread when app stops
threading.Thread(target=self.doit, daemon=True).start()
sm = ScreenManager()
self.main_screen = MainScreen()
sm.add_widget(self.main_screen)
return sm
def doit(self):
# this code is run in a separate thread
self.do_vid = True # flag to stop loop
# make a window for use by cv2
# flags allow resizing without regard to aspect ratio
cv2.namedWindow('Hidden', cv2.WINDOW_NORMAL | cv2.WINDOW_FREERATIO)
# resize the window to (0,0) to make it invisible
cv2.resizeWindow('Hidden', 0, 0)
cam = cv2.VideoCapture(0)
# start processing loop
while (self.do_vid):
ret, frame = cam.read()
# ...
# more code
# ...
# send this frame to the kivy Image Widget
# Must use Clock.schedule_once to get this bit of code
# to run back on the main thread (required for GUI operations)
# the partial function just says to call the specified method with the provided argument (Clock adds a time argument)
Clock.schedule_once(partial(self.display_frame, frame))
cv2.imshow('Hidden', frame)
cv2.waitKey(1)
cam.release()
cv2.destroyAllWindows()
def stop_vid(self):
# stop the video capture loop
self.do_vid = False
def display_frame(self, frame, dt):
# display the current video frame in the kivy Image widget
# create a Texture the correct size and format for the frame
texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')
# copy the frame data into the texture
texture.blit_buffer(frame.tobytes(order=None), colorfmt='bgr', bufferfmt='ubyte')
# flip the texture (otherwise the video is upside down
texture.flip_vertical()
# actually put the texture in the kivy Image widget
self.main_screen.ids.vid.texture = texture
if __name__ == '__main__':
Main().run()
This hides the imshow() window (by making its size 0x0), then displays the frame in an Image Widget. Not sure if a window size of 0x0 messes with your other code.

I used the code from above and modified it a bit. Problems arise when I use/return a kivy file instead of the ScreenManger. In my example there is no video output.
What is the problem/fail in the code?
main.py
import threading
from functools import partial
import cv2
from kivy.app import App
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.lang import Builder
from kivy.uix.screenmanager import ScreenManager, Screen
class Scan(Screen):
pass
class WindowManager(ScreenManager):
pass
kv = Builder.load_file("kivy.kv")
class Main(App):
def build(self):
threading.Thread(target=self.doit, daemon=True).start()
self.new_screen = Scan() # ?
return kv
def doit(self):
self.do_vid = True
cv2.namedWindow('Hidden', cv2.WINDOW_NORMAL | cv2.WINDOW_FREERATIO)
cv2.resizeWindow('Hidden', 0, 0)
cam = cv2.VideoCapture(0)
while (self.do_vid):
ret, frame = cam.read()
# ...
Clock.schedule_once(partial(self.display_frame, frame))
cv2.imshow('Hidden', frame)
cv2.waitKey(1)
cam.release()
cv2.destroyAllWindows()
def display_frame(self, frame, dt):
texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')
texture.blit_buffer(frame.tobytes(order=None), colorfmt='bgr', bufferfmt='ubyte')
texture.flip_vertical()
# No output of the Video Stream
# Scan().ids.vid.texture = texture also doesn't work
self.new_screen.ids.vid.texture = texture
if __name__ == '__main__':
Main().run()
kivy.kv
WindowManager:
Scan:
<Scan>:
name: 'scan'
FloatLayout:
Image:
id: vid
allow_stretch: True
keep_ratio: True
pos_hint: {'x':0.0, 'y':0.2}
size_hint: 1.0, 0.8
Button:
id: button_start
text: 'Button'
pos_hint: {'x':0.0, 'y':0.0}
size_hint: 0.7, 0.2
background_color: 0.7, 0.9, 0.0, 1
font_size: 50
Image:
id: folder
source: 'pic/folder.png'
pos_hint: {'x':0.7, 'y':0.0}
size_hint: 0.3, 0.2

Kivy - Adding elements in a Screen

import os,sys,random
from kivy.app import App
from kivy.uix.widget import Widget
from kivy.uix.button import Button
from kivy.graphics import Color, Rectangle
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.image import Image
from kivy.uix.screenmanager import ScreenManager, Screen , SlideTransition
from kivy.animation import Animation
from kivy.properties import StringProperty
class Page(Screen):
source = StringProperty()
class Imglayout(FloatLayout):
def __init__(self,**args):
super(Imglayout,self).__init__(**args)
with self.canvas.before:
Color(0,0,0,0)
self.rect=Rectangle(size=self.size,pos=self.pos)
self.rootpath = os.path.dirname(os.path.realpath(sys.argv[0]))
self.images=[]
for img in os.listdir(self.rootpath+'/images'):
self.images.append(self.rootpath+'/images/'+img)
self.index=random.randint(0,len(self.images)-1)
self.im=Image(source=self.images[self.index])
self.im.keep_ratio= False
self.im.allow_stretch = True
#self.add_widget(self.im)
self.sm = ScreenManager(transition=SlideTransition())
self.page1=Page(name='page1', source = self.images[self.index])
self.page2=Page(name='page2', source = self.images[self.index+1])
self.sm.add_widget(self.page1)
self.sm.add_widget(self.page2)
self.bind(size=self.updates,pos=self.updates)
def updates(self,instance,value):
self.rect.size=instance.size
self.rect.pos=instance.pos
def on_touch_down(self,touch):
if self.collide_point(*touch.pos):
if(self.sm.current == 'page1'):
next='page2'
page=self.page2
else:
next='page1'
page=self.page1
self.index=(self.index+1)%len(self.images)
page.source = self.images[self.index]
page.background.scale = 1.0
self.sm.transition=SlideTransition()
self.sm.current = next
anim = Animation(
scale=page.background.scale*1.3,
duration=15.0
)
anim.start(page.background)
return True
return False
class MainTApp(App):
def build(self):
root = BoxLayout(orientation='vertical',spacing=10)
c = Imglayout()
root.add_widget(c)
cat=Button(text="Categories",size_hint=(1,.05))
cat.bind(on_press=self.callback)
root.add_widget(cat);
return root
def callback(self,value):
print "CALLBACK CAT"
if __name__ == '__main__':
MainTApp().run()
Taking some hints from here i made the above program. It says that Page does not have a background attribute in both my and the referenced code. Its kind of obvious since there is no background attribute. I thought it inherited that from Screen. I am trying to make a slideshow kind of thing. But i cant find any information on how to set the background of a screen.
And if i comment out everything with .background and run the app. click on the black space, then i start getting this error continuously on the terminal
[ERROR ] [OSC ] Address 127.0.0.1:3333 already in use, retry
in 2 second
And i still dont get any background on the app.(its all black.)
and if i print self.sm.current on the touich function. Then i find that its always page1, it never changes.

The Kivy Guide explains how to add a background to a Widget. Briefly, you can do it in Python using the following code which binds to the position and size of the widget to make sure the background moves with the widget.
with widget_instance.canvas.before:
Color(0, 1, 0, 1) # green; colors range from 0-1 instead of 0-255
self.rect = Rectangle(size=layout_instance.size,
pos=layout_instance.pos)
widget_instance.bind(size=self._update_rect, pos=self._update_rect)
...
def _update_rect(self, instance, value):
self.rect.pos = instance.pos
self.rect.size = instance.size
Or you can do it more naturally with the kv language e.g.
MyWidget:
canvas.before:
Color:
rgba: 0, 1, 0, 1
Rectangle:
# self here refers to the widget i.e screen
pos: self.pos
size: self.size