spatialcomputing.info

Hi everybody,

i’m developing an AR projection, but i run out into a problem. I can’t make my object stay exactly on the marker, it seems to travel all around the image…

I’ve used ARuco and OpenCV for detection of rvec and tvec and OpenGL for rendering. I know that a change of prospective matrix is needeed due to opengl different coordinate system but i can’t find the right one and how to apply it.

Can you help me?

Here’s my code

​

############################### # GLOBAL VARIABLES width = 1280 height = 720 texture_background = None chess_piece = None webcam = cv2.VideoCapture(0) camera_matrix, dist_coefs, rvecs, tvecs = calibrate() znear = 10 zfar = 10000.0 ############################### ############################### INIT OPENGL def init_gl(): global texture_background, chess_piece glClearColor(0.0, 0.0, 0.0, 0.0) glClearDepth(1.0) # glDepthFunc(GL_LESS) glDepthFunc(GL_LEQUAL) glEnable(GL_DEPTH_TEST) glShadeModel(GL_SMOOTH) glMatrixMode(GL_PROJECTION) glLoadIdentity() gluPerspective(40.34, 1.78, znear, zfar) #Cambiare all'occorrenza glViewport(0,0,width,height) ppm = getOpenglProjectionMatrix(camera_matrix, znear, zfar, width, height) # # glMatrixMode(GL_PROJECTION) # # glPushMatrix() glLoadMatrixf(ppm) glMatrixMode(GL_MODELVIEW) # assign shapes chess_piece = OBJ(obj_path, True) # assign texture glEnable(GL_TEXTURE_2D) texture_background = glGenTextures(1) ############################### WEBCAM HANDLER def getNextFrame(): global webcam ret, frame = webcam.read() if not ret or frame is None: webcam.release() print("Released webcam") return None return frame ############################### WINDOWS FUNCTION def draw_background(image): global texture_background bg_image = cv2.flip(image, 0) bg_image = Image.fromarray(bg_image) ix = bg_image.size[0] iy = bg_image.size[1] bg_image = bg_image.tobytes("raw", "BGRX", 0, -1) # create background texture glBindTexture(GL_TEXTURE_2D, texture_background) glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST) glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST) glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image) # draw background glBindTexture(GL_TEXTURE_2D, texture_background) glPushMatrix() glTranslatef(0.0,0.0,-10.0) glBegin(GL_QUADS) glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 3.0) glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 3.0) glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 3.0) glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 3.0) glEnd( ) glPopMatrix() def draw_obj(rvec, tvec, center, p_k): ### OBTAINING PROJECTION MATRIX view_matrix = getCorrectPPM(rvec,tvec) # view_matrix = tryThis(p_k) ############################### lightfv = ctypes.c_float * 4 glDisable(GL_TEXTURE_2D) glEnable(GL_DEPTH_TEST) glLightfv(GL_LIGHT0, GL_POSITION, lightfv(-1.0, 1.0, 1.0, 0.0)) glEnable(GL_LIGHT0) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadMatrixd(view_matrix) glTranslatef(0.0, 0.0, -100.0) # glTranslatef(-center[0], center[1], 0.0) # glRotatef(-90, 1.0, 0.0, 0.0) glEnable(GL_LIGHTING) glCallList(chess_piece.gl_list) glColor3f(1.0, 1.0, 1.0) glPopMatrix() glEnable(GL_TEXTURE_2D) glDisable(GL_DEPTH_TEST) glDisable(GL_LIGHTING) def display(): global chess_piece, camera_matrix, dist_coefs glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) glLoadIdentity() image = getNextFrame() rvec, tvec, frame, center, p_k = detect(image, camera_matrix, dist_coefs, chess_piece) draw_background(frame) if len(rvec) != 0 and len(tvec) != 0: draw_obj(rvec, tvec, center, p_k) glutSwapBuffers() def reshape(): if height == 0: height = 1 glViewport(0, 0, width, height) glMatrixMode(GL_PROJECTION) glLoadIdentity() if width <= height: glOrtho(-nRange, nRange, -nRange*h/w, nRange*h/w, -nRange, nRange) else: glOrtho(-nRange*w/h, nRange*w/h, -nRange, nRange, -nRange, nRange) glMatrixMode(GL_MODELVIEW) glLoadIdentity() ############################### APPLICATION def init_application(): print("Initializing app") glutInit() glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGBA | GLUT_DEPTH) glutInitWindowSize(width, height) glutInitWindowPosition(0, 0) glutCreateWindow("AR Chess") glutDisplayFunc(display) glutIdleFunc(display) # glutReshapeFunc(reshape) init_gl() print("Inizialization completed") glutMainLoop() init_application() 

The function to get the „corrected ppm“ is:

def getOpenglProjectionMatrix(camera_matrix, near, far, width, height): K00 = camera_matrix[0][0] K11 = camera_matrix[1][1] K01 = camera_matrix[0][1] K02 = camera_matrix[0][2] K12 = camera_matrix[1][2] height = 720 width = 1280 fx = K00 fy = K11 fovy = 2*np.arctan(0.5*height/fy)*180/np.pi aspect = (width*fy)/(height*fx) near = 0.1 far = 100.0 print("FOVY:",fovy) print("ASPECT:",aspect) result00 = 2*K00 / width # result[0,1] = -2*K01 / width result01 = 0 result02 = (width - 2*K02 + 2*0) / width result03 = 0 result10 = 0 result11 = -2*K11 / height result12 = (height - 2*K12 + 2*0) / height result13 = 0 result20 = 0 result21 = 0 result22 = (-far - near) / (far - near) result23 = -2*far*near/(far-near) result30 = 0 result31 = 0 result32 = -1 result33 = 0 result = np.array([[result00,result01,result02,result03], [result10,result11,result12,result13], [result20,result21,result22,result23], [result30,result31,result32,result33]]) print(result) print(type(result)) m = map(float,result.T.flat) m = (GLfloat * 16)(*m) return m 

I’m working on a project that involves image tracking on laser cut/engraved materials (wood, acrylic, etc.) and I’ve gotten to a point where it’s functional but fickle. There’s a lot of jittering/blinking in and out due to things like burn marks or spots where the lines are too thin and the engraved area ‚fell out‘. Does anyone have experience with using laser cut materials for image tracking that could give me some pointers to get more consistent image tracking?