LadybugVirtualCamera.cpp

Go to the documentation of this file.

/*
 * LadybugVirtualCamera.cpp
 *
 *  Created on: Aug 5, 2010
 *      Author: petrito1@cmp.felk.cvut.cz
 */

#include "LadybugVirtualCamera.h"

#include <fstream>
#include <GL/glu.h>
#include <GL/glut.h>
#include <opencv2/highgui/highgui.hpp>

#include "CameraException.h"

namespace nifti {
namespace ladybug {

LadybugVirtualCamera::LadybugVirtualCamera() :
  viewportWidth(640), viewportHeight(480), pan(0.0), tilt(0.0), horizontalFov(60.0), verticalFov(45.0), meshCols(0),
      meshRows(0), alphamaskAvailable(false), initialized(false), offScreenRendering(false), frameBuffer(0),
      colorRenderBuffer(0), depthRenderBuffer(0) {

  // TODO: Remove...?
  for (int i = 0; i < NUM_CAMERAS; i++) {
    mesh[i] = NULL;

    alphaMaskWidths[i] = 0;
    alphaMaskHeights[i] = 0;
    alphamasks[i] = NULL;

    imageWidths[i] = 0;
    imageHeights[i] = 0;

    glTextureWidths[i] = 0;
    glTextureHeights[i] = 0;
    glTextures[i] = (GLuint) 0;
  }
}

LadybugVirtualCamera::~LadybugVirtualCamera() {
  clearMesh();
  clearAlphamask();
}

void LadybugVirtualCamera::initializeTextures() {
  glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);
  glEnable(GL_TEXTURE_2D);
  glShadeModel(GL_FLAT);
  // Generate and setup textures and blending.
  glGenTextures(6, glTextures);
}

void LadybugVirtualCamera::initializeFramebuffers() {
  glGenFramebuffers(1, &frameBuffer);
  glBindFramebuffer(GL_FRAMEBUFFER, frameBuffer);

  // Create and attach a color buffer.
  glGenRenderbuffers(1, &colorRenderBuffer);
  // We must bind colorRenderBuffer before we call glRenderbufferStorage.
  glBindRenderbuffer(GL_RENDERBUFFER, colorRenderBuffer);
  // TODO: The storage has to be set according to viewport size.
  glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, viewportWidth, viewportHeight);
  // Attach color buffer to FBO.
  glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, colorRenderBuffer);

  glGenRenderbuffers(1, &depthRenderBuffer);
  glBindRenderbuffer(GL_RENDERBUFFER, depthRenderBuffer);
  // TODO: The storage has to be set according to viewport size.
  glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, viewportWidth, viewportHeight);
  // Attach depth buffer to FBO.
  glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, depthRenderBuffer);

  // Does the GPU support current FBO configuration?
  GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
  switch (status) {
  case GL_FRAMEBUFFER_COMPLETE:
    break;
  default:
    deleteBuffers();
    throw std::runtime_error("Could not setup frame buffer.\n");
  }
}

void LadybugVirtualCamera::initialize(bool offScreenRendering) {
  this->offScreenRendering = offScreenRendering;

  if (this->offScreenRendering) {
    int glutWin = glutCreateWindow("temp");
    GLenum glew = glewInit();
    if (GLEW_OK != glew) {
      glutDestroyWindow(glutWin);
      throw std::runtime_error((char*) glewGetErrorString(glew));
    }

    initializeFramebuffers();
  }

  initializeTextures();
  this->initialized = true;
}

void LadybugVirtualCamera::updateBuffers() {
  glBindRenderbuffer(GL_RENDERBUFFER, colorRenderBuffer);
  glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, viewportWidth, viewportHeight);

  glBindRenderbuffer(GL_RENDERBUFFER, depthRenderBuffer);
  glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, viewportWidth, viewportHeight);
}

void LadybugVirtualCamera::updateViewport() {

  glViewport(0, 0, viewportWidth, viewportHeight);
}

void LadybugVirtualCamera::updateProjection() {

  glMatrixMode(GL_PROJECTION);
  glLoadIdentity();
  gluPerspective((GLdouble) verticalFov, (GLdouble) horizontalFov / (GLdouble) verticalFov, 0.1, 100.0);
}

void LadybugVirtualCamera::updateModelView() {

  glMatrixMode(GL_MODELVIEW);
  glLoadIdentity();
  glRotated(-tilt, 1.0, 0.0, 0.0); // Tilt the view.
  glRotated(-pan, 0.0, 1.0, 0.0); // Pan the view.
  // TODO: Do not rotate in case of portrait images?
  glRotated(-90.0, 1.0, 0.0, 0.0); // Convert Ladybug 3D coordinate system to OpenGL's one.
}

void LadybugVirtualCamera::clearBuffers() {

  // Clear the buffers to transparent black at "infinity".
  glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
  glClearDepth(1.0);
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
}

void LadybugVirtualCamera::render() {
  glMatrixMode(GL_TEXTURE);
  glLoadIdentity();

//  printf("Mesh rows: %d, cols: %d.\n", meshRows, meshCols); // TODO: Remove line.

  // TODO: Avoid rendering textures outside the frustum?
  for (int iCamera = 0; iCamera < NUM_CAMERAS; iCamera++) // for each camera
  {
//    printf("Rendering cam %d.\n", iCamera); // TODO: Remove line.
    double validTextureWidth = (double) imageWidths[iCamera] / glTextureWidths[iCamera];
    double validTextureHeight = (double) imageHeights[iCamera] / glTextureHeights[iCamera];

    glBindTexture(GL_TEXTURE_2D, glTextures[iCamera]);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

    for (int iRow = 0; iRow < meshRows - 1; iRow++) // for each row
    {
      glBegin(GL_TRIANGLE_STRIP);
      for (int iCol = 0; iCol < meshCols; iCol++) // for each column
      {
        // Texture coordinates.
        double p1 = (double) iCol / (meshCols - 1) * validTextureWidth;
        double q1 = (double) iRow / (meshRows - 1) * validTextureHeight;
        double q2 = (double) (iRow + 1.0) / (meshRows - 1) * validTextureHeight;

        // 3-D world coordinates.
        int ptr1 = iRow * meshCols + iCol;
        double x1 = mesh[iCamera][ptr1 * 3 + 0];
        double y1 = mesh[iCamera][ptr1 * 3 + 1];
        double z1 = mesh[iCamera][ptr1 * 3 + 2];

        int ptr2 = (iRow + 1) * meshCols + iCol;
        double x2 = mesh[iCamera][ptr2 * 3 + 0];
        double y2 = mesh[iCamera][ptr2 * 3 + 1];
        double z2 = mesh[iCamera][ptr2 * 3 + 2];

        glTexCoord2d(p1, q1);
        glVertex3d(x1, y1, z1);

        glTexCoord2d(p1, q2);
        glVertex3d(x2, y2, z2);
      }
      glEnd();
    }
  }
}

void LadybugVirtualCamera::renderView(cv::Mat& image) {

  clearBuffers();
  updateViewport();
  updateProjection();
  updateModelView();
  render();

  // Copy pixels to main memory.
  cv::Mat flippedImage(viewportHeight, viewportWidth, CV_8UC4);
  glReadPixels(0, 0, viewportWidth, viewportHeight, GL_BGRA, GL_UNSIGNED_BYTE, flippedImage.data);

  // OpenGL y coordinate rises towards top of the image... flip the image.
  cv::flip(flippedImage, image, 0);
}

void LadybugVirtualCamera::deleteBuffers() {
  if (!this->offScreenRendering) {
    // Nothing to delete.
    return;
  }

  if (colorRenderBuffer > 0) {
    glDeleteRenderbuffers(1, &colorRenderBuffer);
  }
  if (depthRenderBuffer > 0) {
    glDeleteRenderbuffers(1, &depthRenderBuffer);
  }
  // Bind 0 framebuffer, which means render to back buffer.
  // As a result, our framebuffer is unbound.
  glBindFramebuffer(GL_FRAMEBUFFER, 0);
  if (frameBuffer > 0) {
    glDeleteFramebuffers(1, &frameBuffer);
  }
}

void LadybugVirtualCamera::getViewport(int* viewportWidth, int* viewportHeight) {
  *viewportWidth = this->viewportWidth;
  *viewportHeight = this->viewportHeight;
}
void LadybugVirtualCamera::setViewport(int viewportWidth, int viewportHeight) {
  // Regenerate the graphics buffers only when necessary.
  if (this->viewportWidth != viewportWidth || this->viewportHeight != viewportHeight) {

    this->viewportWidth = viewportWidth;
    this->viewportHeight = viewportHeight;

    if (this->offScreenRendering) {
      updateBuffers();
    }
  }
}

double LadybugVirtualCamera::getPan() {
  return this->pan;
}
void LadybugVirtualCamera::setPan(double pan) {
  this->pan = pan;
}
double LadybugVirtualCamera::getTilt() {
  return this->tilt;
}
void LadybugVirtualCamera::setTilt(double tilt) {
  this->tilt = tilt;
}
double LadybugVirtualCamera::getHorizontalFov() {
  return this->horizontalFov;
}
void LadybugVirtualCamera::setHorizontalFov(double horizontalFov) {
  this->horizontalFov = horizontalFov;
}
double LadybugVirtualCamera::getVerticalFov() {
  return this->verticalFov;
}
void LadybugVirtualCamera::setVerticalFov(double verticalFov) {
  this->verticalFov = verticalFov;
}

void LadybugVirtualCamera::loadMesh(const std::string& meshFile) {
  printf("Loading mesh from %s...\n", meshFile.data());
  // TODO: Use temp buffer until we know that no exception will occur.
  clearMesh();

  FILE *fp = fopen(meshFile.data(), "r");
  if (fp == NULL) {
    throw std::runtime_error("Opening mesh file failed: " + meshFile);
  }

  if (fscanf(fp, "cols %d rows %d\n", &meshCols, &meshRows) != 2) {
    fclose(fp);
    throw std::runtime_error("Reading mesh file header failed.");
  }
  printf("Header: rows: %d, columns: %d\n", meshRows, meshCols);

  for (int iImage = 0; iImage < NUM_CAMERAS; iImage++) {
    mesh[iImage] = new double[meshCols * meshRows * 3];

    for (int iRow = 0; iRow < meshRows; iRow++) {
      for (int iCol = 0; iCol < meshCols; iCol++) {
        double x, y, z;
        int scanned;
        if ((scanned = fscanf(fp, "%lf, %lf, %lf", &x, &y, &z)) != 3) {
          fclose(fp);
          printf("Image %d, row %d, column %d. Values scanned: %d\n", iImage, iRow, iCol, scanned);
          printf("%lf\n", x);
          throw std::runtime_error("Reading mesh file failed.");
        }
        mesh[iImage][(iRow * meshCols + iCol) * 3 + 0] = x;
        mesh[iImage][(iRow * meshCols + iCol) * 3 + 1] = y;
        mesh[iImage][(iRow * meshCols + iCol) * 3 + 2] = z;
      }
    }
  }

  fclose(fp);
}

void LadybugVirtualCamera::clearMesh() {
  printf("Clearing mesh...\n");
  for (int i = 0; i < NUM_CAMERAS; i++) {
    delete[] mesh[i];
    mesh[i] = NULL;
  }
}

void LadybugVirtualCamera::loadAlphamask(const std::string& alphamaskFilePrefix) {
  clearAlphamask();
  printf("Loading alphamask using prefix %s...\n", alphamaskFilePrefix.data());
  alphamaskAvailable = true;
  for (int iCamera = 0; iCamera < NUM_CAMERAS; iCamera++) {
    char pgmPath[1024];
    sprintf(pgmPath, "%s%d.pgm", alphamaskFilePrefix.data(), iCamera);

    // Read alpha mask and set it to the 4th (U) byte of the BGRU buffer while scaling.
    cv::Mat alphamaskMat = cv::imread(pgmPath, 0);
    // TODO: Check elem size?
    //printf("Alphamask element size: %lu.", static_cast<long unsigned int> (alphamaskMat.elemSize()));
    if (alphamaskMat.elemSize() > 1) {
      alphamaskAvailable = false;
      throw std::runtime_error("Invalid alphamask type.");
    }
    alphaMaskWidths[iCamera] = alphamaskMat.cols;
    alphaMaskHeights[iCamera] = alphamaskMat.rows;
    int alphamaskSize = alphamaskMat.cols * alphamaskMat.rows * alphamaskMat.elemSize();
    alphamasks[iCamera] = new unsigned char[alphamaskSize];
    memcpy(alphamasks[iCamera], alphamaskMat.data, alphamaskSize);

    // Enable blending.
//    glEnable(GL_BLEND);
//    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
  }
}

void LadybugVirtualCamera::clearAlphamask() {
  printf("Clearing alphamask...\n");
  for (int iCamera = 0; iCamera < NUM_CAMERAS; iCamera++) {
    //    free(alphamasks[iCamera]);
    delete[] alphamasks[iCamera];
    alphamasks[iCamera] = NULL;
  }
  alphamaskAvailable = false;
  glDisable(GL_BLEND);
}

void LadybugVirtualCamera::setImage(const int& imageIndex, const cv::Mat& image) {

  imageWidths[imageIndex] = image.cols;
  imageHeights[imageIndex] = image.rows;

  // 3-channel BGR image is assumed.
  int channels = image.channels();
  if (channels != 3) {
    throw std::runtime_error("Image must have 3 channels.");
  }

  unsigned char* bgraBuffer = new unsigned char[imageWidths[imageIndex] * imageHeights[imageIndex] * 4];
  // Copy color bands from the BGR buffer to the BGRA buffer.
  for (int iData = 0; iData < imageWidths[imageIndex] * imageHeights[imageIndex] * channels; iData++) {
    // TODO: Is data a pointer to ROI or whole data?
    bgraBuffer[iData + (iData / channels)] = image.data[iData];
  }

  if (alphamaskAvailable) {
    for (int y = 0; y < imageHeights[imageIndex]; y++) {
      for (int x = 0; x < imageWidths[imageIndex]; x++) {
        int alphamaskX = x * alphaMaskWidths[imageIndex] / imageWidths[imageIndex];
        int alphamaskY = y * alphaMaskHeights[imageIndex] / imageHeights[imageIndex];
        bgraBuffer[(y * imageWidths[imageIndex] + x) * 4 + 3] = alphamasks[imageIndex][alphamaskY
            * alphaMaskWidths[imageIndex] + alphamaskX];
      }
    }
  }

  glBindTexture(GL_TEXTURE_2D, glTextures[imageIndex]);

  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);

  // Make a texture from the PPM image.
  // Texture size is set to the minimum of power of two which can contain the PPM
  // so that this program works on lower OpenGL versions.
  glTextureWidths[imageIndex] = getMinimumPowerOfTwo(imageWidths[imageIndex]);
  glTextureHeights[imageIndex] = getMinimumPowerOfTwo(imageHeights[imageIndex]);

  // Copy PPM image pixels to the valid region of the texture.
  int textureElems = glTextureWidths[imageIndex] * glTextureHeights[imageIndex] * 4;
  unsigned char* textureBuffer = new unsigned char[textureElems];
//  memset(textureBuffer, 0, textureElems);

  for (int y = 0; y < imageHeights[imageIndex]; y++) {
    for (int x = 0; x < imageWidths[imageIndex]; x++) {
      textureBuffer[(y * glTextureWidths[imageIndex] + x) * 4 + 0] = bgraBuffer[(y * imageWidths[imageIndex] + x) * 4
          + 0]; // B
      textureBuffer[(y * glTextureWidths[imageIndex] + x) * 4 + 1] = bgraBuffer[(y * imageWidths[imageIndex] + x) * 4
          + 1]; // G
      textureBuffer[(y * glTextureWidths[imageIndex] + x) * 4 + 2] = bgraBuffer[(y * imageWidths[imageIndex] + x) * 4
          + 2]; // R
      textureBuffer[(y * glTextureWidths[imageIndex] + x) * 4 + 3] = bgraBuffer[(y * imageWidths[imageIndex] + x) * 4
          + 3]; // A
    }
  }

  // Transfer the RGB buffer to graphics card.
//  printf("Calling glTexImage2D.\n");
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, glTextureWidths[imageIndex], glTextureHeights[imageIndex], 0, GL_BGRA_EXT,
      GL_UNSIGNED_BYTE, textureBuffer);
//  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, glTextureWidths[imageIndex], glTextureHeights[imageIndex], 0, GL_RGBA,
//        GL_UNSIGNED_BYTE, textureBuffer);
//  printf("Calling gluBuild2DMipmaps.\n");
//  gluBuild2DMipmaps(GL_TEXTURE_2D, GL_RGBA,
//        glTextureWidths[imageIndex], glTextureHeights[imageIndex], GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV, textureBuffer);
//  printf("GL error: %s.\n", glGetString(glGetError()));

  // TODO: Reset after every texture update?
  glTexEnvf(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE);
  glEnable(GL_TEXTURE_2D);
  glShadeModel(GL_FLAT);
  glEnable(GL_BLEND);
  glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

  // Now the RGB texture is transferred to graphics, we won't need these.
  delete[] bgraBuffer;
  delete[] textureBuffer;
}

void LadybugVirtualCamera::setImages(const cv::Mat(&images)[NUM_CAMERAS]) {
  for (int iImage = 0; iImage < NUM_CAMERAS; iImage++) {
    setImage(iImage, images[iImage]);
  }
}

void LadybugVirtualCamera::setImages(const cv::Mat& compositeImage) {
  int imageHeight = compositeImage.rows / NUM_CAMERAS;
  cv::Mat images[NUM_CAMERAS];
  for (int iImage = 0; iImage < NUM_CAMERAS; iImage++) {
    images[iImage] = compositeImage.rowRange(iImage * imageHeight, (iImage + 1) * imageHeight);
  }
  setImages(images);
}

int LadybugVirtualCamera::getMinimumPowerOfTwo(int n) {
  int i = 1;
  while (i < n) {
    i *= 2;
  }
  return i;
}

}
}