During the semester I have been developing a project which creates a relationship between material and image. The project consists in a screen made of a material capable to be distorted, and at the same time to produce a distortion on the image that the user see on it.

According to this I made a screen using lycra, where I’m going to project in real time the image of the people who is in front of it. To generate the distortion I started to work based on a code of an ITP alumni which using the position of the mouse generates a distortion on the image captured by the web cam of the laptop.

The first prototype that I made was using foam and pressure sensors, then I changed the values of X and Y of the mouse by the values of the sensors to produce the change on the image. The next step was starting to use the Kinect as a sensor to map the depth and the position where the material were distorted, the idea is to use a webcam to capture the face of the user and project from behind over the screen at the same time that the kinect is mapping the distortion of the screen reading as a distance from the plane of the screen when is not distorted.

Processing Code:

/*
Kinect code for Marcela’s IGOE DEV STUDIO project

Based on Malleable Matrix by Robert Carlsen
Maps a capture object onto a grid of quads which are
translated in z-space based on the position of the mouse.

FIRST THING:
Press ‘c’ to start calibrating…click mouse once at top left corner of square, then at bottom right corner

press ‘v’ to turn on/off video image in background

press ‘u’/'d’ to set the THRESHOLD closer or further

Press ‘b’ to hide/show the ellipses which shows the depth

*/
//video library
import processing.video.*;

//fullscreen library
import fullscreen.*;
import japplemenubar.*;

// kinect library
import librarytests.*;
import org.openkinect.*;
import org.openkinect.processing.*;

Kinect kinect;
Capture video;
FullScreen fs;

boolean flip = true;
float deep_x, deep_y;
float threshold = 0.5;
boolean displayImage = true;
boolean displayEllipses = true;
int x1, x2, y1, y2;
boolean calibrating = false;
boolean gettingFirstPoint = true;
boolean calibrated = false;
float[] depthLookUp;

void setup() {
size(640,480,P3D);
frameRate(60);
noStroke();
fs = new FullScreen(this); // Create the fullscreen object
fs.enter(); // enter fullscreen mode

depthLookUp = new float[2048];

kinect = new Kinect(this); //Create a new Kinect object
kinect.start();
kinect.enableDepth(true);
kinect.processDepthImage(true);

video = new Capture(this,640,480,30);

for (int i = 0; i < depthLookUp.length; i++){
depthLookUp[i] = rawDepthToMeters(i);
}

}

void draw() {
background(0);
lights();

int dia = 10;
float a;
float b;

if (displayImage)
image(kinect.getDepthImage(), 0, 0);

if (calibrating){
fill(255);
ellipse(mouseX, mouseY, 10, 10);
if (gettingFirstPoint){
//Fill in if neccesary
}
else{
rect(x1, y1, mouseX-x1, mouseY-y1);
}
}
else{
processKinectDepth(); // function to transform the position of depth into a point X,Y
}

//start distorting the video
for(int j=0 ; j < height ; j+=dia){

for(int i=0 ; i < width ; i+=dia){

float i2 = width - i ; //flip de image
pushMatrix();
translate(i,j);

a = deep_x;
b = deep_y;

//println ("x: " + a + " y: " + b);

//calculate the z-pos of the corners
float myDepth1 = dist(a,b,i,j);
float myDepth2 = dist(a,b,i+dia,j);
float myDepth3 = dist(a,b,i+dia,j+dia);
float myDepth4 = dist(a,b,i,j+dia);

if(flip){
myDepth1=-myDepth1 + 100;
myDepth2=-myDepth2 + 100;
myDepth3=-myDepth3 + 100;
myDepth4=-myDepth4 + 100;
}

beginShape();
texture(video);

vertex(0,0,myDepth1,i,j);
vertex(dia,0,myDepth2,i+dia,j);
vertex(dia,dia,myDepth3,i+dia,j+dia);
vertex(0,dia,myDepth4,i,j+dia);

endShape();
popMatrix();
}
}
//end distorting the video

}

// function to transform the position of depth into a point X,Y
void processKinectDepth(){

if (calibrated){
int[] depth = kinect.getRawDepth(); //get the raw depth pixel array
int lowest_depth = 2047;

ArrayList deep_pixels = new ArrayList();

for (int r = y1; r < y2; r++){
for(int c = x1; c< x2; c++){
int offset = (c) + (r * width);
if (depth[offset] < lowest_depth){
lowest_depth = depth[offset];
deep_pixels.clear();
deep_pixels.add(new PVector(c, r));//add c also

}
else if (depth[offset] == lowest_depth){
deep_pixels.add(new PVector(c, r));
}

}
}
if (displayEllipses)
show();

//Average all deep pixels together
deep_x = 0;
deep_y = 0;
for (PVector pnt : deep_pixels){
deep_x += pnt.x;
deep_y += pnt.y;
}

deep_x = deep_x / deep_pixels.size();
deep_y = deep_y / deep_pixels.size();
}
}

//show video
void captureEvent(Capture video) {
video.read();
}

void mousePressed(){
if (calibrating){
if (mousePressed){
if (gettingFirstPoint){
x1 = mouseX;
y1 = mouseY;
gettingFirstPoint = false;
}
else{
x2 = mouseX;
y2 = mouseY;
gettingFirstPoint = true;
calibrating = false;
calibrated = true;
}
}
}
}

float rawDepthToMeters(int depthValue){
if (depthValue < 2047){
return (float)(1.0 / ((double)(depthValue) * -0.0030711016 + 3.3309495161));
}
return 0.0f;

}

void calibrate(){
calibrating = true;
}

// function to help to callibrate the distance between the kinect and the screen
void show(){
int[] depth = kinect.getRawDepth(); //get the raw depth pixel array
for (int y = y1; y < y2; y+=5){
for (int x = x1; x < x2; x+=5){
int offset = (y * width) + (width - x );
if (depthLookUp[depth[offset]] < threshold){
fill(251);
float val = 10 * (threshold - depthLookUp[depth[offset]]); //15 - map(depthLookUp[depth[offset]], 0, threshold, 5, 10);
ellipse(x, y,val, val);
}
}
}
}

void keyPressed(){
if (key == 'u'){
threshold += 0.01;
println("Threshold: " + threshold);
}
else if (key == 'd')
{
threshold -= 0.01;
println("Threshold: " + threshold);
}
else if (key == 'c'){
calibrate();
}
else if (key == 'v'){
displayImage = !displayImage;
}
else if (key == 'b'){
displayEllipses = !displayEllipses;
}
}

Filed under P.Development.