This was the final applet in motion. Using the minim library for processing, each waveform is generated in realtime as the two sounds play over eachother creating a pretty chaotic sound, but there are some instances of overlapping patterns where the mashup works pretty well. In the third version of the code, the boolean of the two waveforms is generated, producing a new way to visualize the waveforms. View the youtube video here, but I really need to figure out a way to add sound to the video, silence doesn’t do it justice. Charlie Parker, Iggy Pop and Richard Wagner comparison + code:

import processing.dxf.*;
import ddf.minim.analysis.*;
import ddf.minim.*;
FFT fftLog1;
FFT fftLog2;

Waveform myRects;

Minim minim;
AudioPlayer groove1;
AudioPlayer groove2;

boolean record;

PFont font;
PFont fontoutline;

void setup(){
size(1200,600,P3D);
noStroke();
minim = new Minim(this);
groove1 = minim.loadFile(“groove_iggy.mp3″);
groove2 = minim.loadFile(“groove_wagner.mp3″);

groove1.loop();//repeat each song
groove2.loop();

font = loadFont(“HelveticaNeueLT-Bold-18.vlw”);
fontoutline = loadFont(“HelveticaNeueLT-Bold-18.vlw”);

fftLog1 = new FFT(groove1.bufferSize(),groove1.sampleRate()); //create the FFT logarithmic scale
fftLog2 = new FFT(groove2.bufferSize(),groove2.sampleRate());
fftLog1.logAverages(22,4); //adjust numbers to adjust spacing
fftLog2.logAverages(22,4);

float w1 = float ((fftLog1.avgSize()+fftLog2.avgSize())/2);
float x = w1;
float y = 0;
float z = 50;
myRects = new Waveform(x,y,z);

}

void draw(){
background(15);
directionalLight(126,126,126,sin(radians(frameCount)),cos(radians(frameCount)),1);
ambientLight(152,152,152);

for(int i = 0; i < fftLog1.avgSize(); i++){
int w = int(width/fftLog1.avgSize());
float zoom = 1;
float jitter = (max(fftLog1.getAvg(i)*200,fftLog2.getAvg(i)*200 )); //jitter in camera influenced by waveform
PVector foc = new PVector((myRects.x*.5+jitter*.5), myRects.y+jitter, 0);
PVector cam = new PVector(zoom, zoom, -zoom);
if (frameCount < 260){
camera(foc.x+cam.x,foc.y+(cam.y-1500*(cos(radians(-frameCount+60)))),foc.z+cam.z-400,
foc.x,foc.y,foc.z-100,
0,0,1);
//println(-1500*(cos(radians(-frameCount+60))));
}
else {
camera(foc.x+cam.x,foc.y+(cam.y+1418.278),foc.z+cam.z-400,foc.x,foc.y,foc.z-100,0,0,1);
}
}
fftLog1.forward(groove1.mix); //play each song
fftLog2.forward(groove2.mix);

myRects.update1(); //update each waveform+boolean
myRects.update2();
myRects.update3();

myRects.textdraw1(); //draw z height for song waveforms
myRects.textdraw2();

if(record){
beginRaw(DXF, "output.dxf");
}
// DXF will export the stuff drawn between here.

myRects.plotBoolean(); //create surfaces
myRects.plotTrace1();
myRects.plotTrace2();

if(record){
endRaw();
record = false;
println("Done DXF~!");
}
}

void stop() {
groove1.close(); // always close Minim audio classes when you finish with them
groove2.close();

minim.stop(); // always stop Minim before exiting
super.stop();
}

class Waveform{
float x,y,z;

PVector[] pts1 = new PVector[fftLog1.avgSize()];
PVector[] pts2 = new PVector[fftLog2.avgSize()];
PVector[] pts3 = new PVector[fftLog1.avgSize()]; //needed for boolean waveform

PVector[] trace1 = new PVector[0];
PVector[] trace2 = new PVector[0];
PVector[] trace3 = new PVector[0]; //needed for boolean waveform

Waveform(float incomingX, float incomingY, float incomingZ){
x = incomingX;
y = incomingY;
z = incomingZ;
}

void update1(){ //plot boolean waveform
plotB();
}

void plotB(){
for(int i = 0; i < fftLog1.avgSize(); i++){
int w = int(width/fftLog1.avgSize());

x = i*w-1050; //adjust the x position of the waveform here
y = frameCount*5;
z = height/4-fftLog1.getAvg(i)*10;

stroke(0);
point(x, y, z);
pts1[i] = new PVector(x, y, z);
//increase size of array trace by length+1
trace1 = (PVector[]) expand(trace1, trace1.length+1);
//always get the next to last
trace1[trace1.length-1] = new PVector(pts1[i].x, pts1[i].y, pts1[i].z);
}
}

void plotBoolean(){
stroke(255,80);
int inc = (fftLog1.avgSize()+fftLog2.avgSize())/2;

for(int i=1; i<(trace1.length+trace2.length)/2-inc; i++){
if(i%inc != 0){
beginShape(TRIANGLE_STRIP);
float value = (trace1[i].z*100);
float m = map(value, -500, 20000, 0, 255);
fill(m*2, 125, -m*2, 140);
int threshold = 15;
if (trace1[i].z

vertex(trace1[i].x, trace1[i].y, (max(trace1[i].z,trace2[i].z))+(min(trace1[i].z,trace2[i].z)));
vertex(trace1[i-1].x, trace1[i-1].y, (max(trace1[i-1].z,trace2[i-1].z))+(min(trace1[i-1].z,trace2[i-1].z)));
vertex(trace1[i+inc].x, trace1[i+inc].y, (max(trace1[i+inc].z,trace2[i+inc].z))+(min(trace1[i+inc].z,trace2[i+inc].z)));
vertex(trace1[i-1+inc].x, trace1[i-1+inc].y,(max(trace1[i-1+inc].z,trace2[i-1+inc].z))+(min(trace1[i-1+inc].z,trace2[i-1+inc].z)));
endShape(CLOSE);
}
}
}
}

void update3(){ // plot first song's waveform
plot1();
}

void plot1(){
for(int i = 0; i < fftLog1.avgSize(); i++){
int w = int(width/fftLog1.avgSize());

x = -2500+i*w; //adjust the x position of the waveform here
y = frameCount*5;
z = height/4-fftLog1.getAvg(i)*10;

stroke(0);
point(x, y, z);
pts3[i] = new PVector(x, y, z);
//increase size of array trace by length+1
trace3 = (PVector[]) expand(trace3, trace3.length+1);
//always get the next to last
trace3[trace3.length-1] = new PVector(pts3[i].x, pts3[i].y, pts3[i].z);
}
}

void textdraw1(){ //plot first song's z height waveform

for(int i =0; i

textMode(SCREEN);
textFont(font, 10);
float theX = screenX(pts3[i].x+5, pts3[i].y+5, pts3[i].z+5);
float theY = screenY(pts3[i].x+5, pts3[i].y+5, pts3[i].z+5);
fill(255);
text(round(fftLog1.getAvg(i)*100), theX, theY, 0);
if(frameCount>220){
textFont(fontoutline, 24);
fill(155);
text(“wagner”,200,500,0);
text(“iggy”,900,500,0);
text(“max(iggy-wagner)”,500,500,0);
}
}
}
void plotTrace1(){
stroke(255,80);
int inc = fftLog1.avgSize();

for(int i=1; i
if(i%inc != 0){
beginShape(TRIANGLE_STRIP);
float value = (trace3[i].z*100);
float m = map(value, -500, 20000, 0, 255);
fill(m*2, 125, -m*2, 140);
vertex(trace3[i].x, trace3[i].y, trace3[i].z);
vertex(trace3[i-1].x, trace3[i-1].y, trace3[i-1].z);
vertex(trace3[i+inc].x, trace3[i+inc].y, trace3[i+inc].z);
vertex(trace3[i-1+inc].x, trace3[i-1+inc].y, trace3[i-1+inc].z);
endShape(CLOSE);
}
}
}

void update2(){ //plot second song's waveform
plot2();
}

void plot2(){
for(int i = 0; i < fftLog2.avgSize(); i++){
int w = int(width/fftLog2.avgSize());

x = i*w; //adjust the x position of the waveform here
y = frameCount*5;
z = height/4-fftLog2.getAvg(i)*10;

stroke(0);
point(x, y, z);
pts2[i] = new PVector(x, y, z);
//increase size of array trace by length+1
trace2 = (PVector[]) expand(trace2, trace2.length+1);
//always get the next to last
trace2[trace2.length-1] = new PVector(pts2[i].x, pts2[i].y, pts2[i].z);
}
}

void textdraw2(){
for(int i =0; i

textFont(font);
float theX = screenX(pts2[i].x+5, pts2[i].y+5, pts2[i].z+5);
float theY = screenY(pts2[i].x+5, pts2[i].y+5, pts2[i].z+5);
fill(255);
text(round(fftLog2.getAvg(i)*100), theX, theY, 0);
}
}

void plotTrace2(){
stroke(255,80);
int inc = fftLog2.avgSize();

for(int i=1; i if(i%inc != 0){
beginShape(TRIANGLE_STRIP);
float value = (trace2[i].z*100);
float m = map(value, -500, 20000, 0, 255);
fill(m*2, 125, -m*2, 140);
vertex(trace2[i].x, trace2[i].y, trace2[i].z);
vertex(trace2[i-1].x, trace2[i-1].y, trace2[i-1].z);
vertex(trace2[i+inc].x, trace2[i+inc].y, trace2[i+inc].z);
vertex(trace2[i-1+inc].x, trace2[i-1+inc].y, trace2[i-1+inc].z);
endShape(CLOSE);
}
}
}
}

void keyPressed() {
// use a key press so that it doesn't make a million files
if (key == 'r') record = true;
}

4 Responses to “visualizing sound in processing”

1. Andrew Mallis Says:
   October 24th, 2010 at 1:59 am

   I am interested in studying your code. However a bunch of your for loops are incomplete in what is posted.

   for(int i=1; i

   also, one of the vertex statements trails off:

   void plotBoolean(){
   [...]
   vertex(trace1[i].x

   Would you kindly repost your code?
   Thanks much!

2. Rebecca Lee Says:
   May 8th, 2011 at 9:30 am

   Hi, I just emailed you about this source code which I could not find previously. Though I am having errors as per Andrew, though I am too much of a beginner to fix them. Is there any chance that you might share the code?

   Thank you
   Rebecca

3. John Locke Says:
   May 8th, 2011 at 2:26 pm

   This post has a link to the the code live in action: http://gracefulspoon.com/blog/2009/03/06/iggy-wave/

4. Rebecca Lee Says:
   May 9th, 2011 at 11:45 am

   Thank you so much! It is beautiful.

reply