I’m using Windows 7 with PANDA3D 1.9.2, OPENCV 3.1 and Python 2.7.5
What I am trying to do is create a video feed from a digital camera and overlay a 3D image created with PANDA3D (similar to augmented reality).
I have started with the following code, which gives me a camera feed and displays it to the screen.
import numpy as np
import cv2
cap = cv2.VideoCapture(0)
while(True):
# Capture frame-by-frame
ret, frame = cap.read()
# Our operations on the frame come here
# gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Display the resulting frame
#cv2.imshow('frame',gray)
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()However, I have found no way of making the CV frame a PANDA3D texture which I can attach to a cardmaker and place at the back of my scene. I have tried converting the CV2 frame to a buffer, using code I found on this site from an older post:
from pandac.PandaModules import WebcamVideo
from panda3d.core import MovieTexture, Texture, CardMaker, NodePath
from direct.showbase.ShowBase import ShowBase
import cv2
import numpy as np
class MediaPlayer(ShowBase):
def __init__(self,cam_no):
print "initialised"
ShowBase.__init__(self)
self.cap = cv2.VideoCapture(cam_no)
ret, frame = self.cap.read()
h,w,d = frame.shape
self.tex = Texture()
self.tex.setup2dTexture(h, w, Texture.T_unsigned_byte, Texture.F_luminance)
buf = frame[:,:,0].T.tostring() # slice RGB to gray scale, transpose 90 degree, convert to text buffer
self.tex.setRamImage(buf) # overwriting the memory with new buffer
cm = CardMaker("My Fullscreen Card")
cm.setFrameFullscreenQuad()
# Tell the CardMaker to create texture coordinates that take into
# account the padding region of the texture.
cm.setUvRange(self.tex)
# Now place the card in the scene graph and apply the texture to it.
self.card = NodePath(cm.generate())
self.card.setTexture(self.tex)
self.card.reparentTo(base.render2d)
taskMgr.add(self.updateTex, 'video frame update')
def updateTex(self, task):
ret, frame = self.cap.read()
h,w,d = frame.shape
#frame = np.random.randint(0,255,(h,w,3)).astype(np.uint8)
buf = frame[:,:,0].T.tostring() # slice RGB to gray scale, transpose 90 degree, convert to text buffer
self.tex.setRamImage(buf) # overwriting the memory with new buffer
return task.cont
player = MediaPlayer(0)
#print "got here 1"
player.run()This simply gives me a white box on the screen, even when using the randomly created frame. I can’t say as I completely understand the transformation from a CV frame to a panda3d buffer as I have taken this directly from an example.
I then tried using the Panda3D API calls to read directly from the camera, without using CV2:
from pandac.PandaModules import WebcamVideo
from panda3d.core import MovieTexture, Texture, CardMaker, NodePath
from direct.showbase.ShowBase import ShowBase
class MediaPlayer(ShowBase):
def __init__(self,cam_no):
print "initialised"
ShowBase.__init__(self)
option = WebcamVideo.getOption(cam_no)
self.cursor = option.open()
self.tex = MovieTexture("myTexture")
print "Init"
print self.tex
if self.cursor.ready:
print "I'm here"
status = self.cursor.setTime(0,0)
print "buffer full = %s" % status
self.cursor.setupTexture(self.tex)
#buf = self.cursor.fetchBuffer()
#print "Buffer = %s" % buf
x = self.cursor.sizeX()
y = self.cursor.sizeY()
print "x= %d y=%d" % (x ,y)
print "is streaming %s" % self.cursor.streaming()
print "Aborted = %s" % self.cursor.aborted()
# Set up a fullscreen card to set the video texture on.
cm = CardMaker("My Fullscreen Card")
cm.setFrameFullscreenQuad()
# Tell the CardMaker to create texture coordinates that take into
# account the padding region of the texture.
#cm.setUvRange(self.tex)
# Now place the card in the scene graph and apply the texture to it.
self.card = NodePath(cm.generate())
self.card.setTexture(self.tex)
self.card.reparentTo(base.render2d)
taskMgr.add(self.updateTex, 'video frame update')
def updateTex(self, task):
#print "update called"
if self.cursor.ready: # video input
#self.cursor.setupTexture(self.tex)
self.card.setTexture(self.tex)
#print "new frame"
return task.cont
player = MediaPlayer(0)
#print "got here 1"
player.run()This gives me a completely black screen. The cursor reports that there is a frame ready, and the frame x and y sizes are correct. However, if I try to read from the buffer (with or without the setTime call) the cursorFetchBuffer call returns a value of NONE, indicating that the camera has no data to return.
Has anybody ever managed to get a digital camera working using either of these methods. I’ve spent days on this and am about to give up, so any help would be very gratefully received – example code would be most useful as I am learning both Python and the Panda3D API.
Thanks in advance, Paul. 