I completed an edit of this with the design suggestions. Basically avoiding sending vectors to the language and back to the server. The result and performance are about the same. It would still be great to add the features desired above, but feel free to check out this if it’s useful.
There is one question that came out of this. I was sometimes getting this error: “Error: Already processing”. I think it had to do with creating a FluidDataSet, maybe one whose name was already used on the server? Is there a way to create a FluidDataSet and avoid this?
// if you highlight all and cmd-return, it should run...
(
s.options.device_("Scarlett 6i6 USB");
s.waitForBoot({
Task({
var updatePeriod = 5 /*in seconds*/; // how often to recompute the k means
var totalHistory = 10 /*in seconds*/; // how long into the past should we keep track of for clustering into k means
var bus = Bus.audio(s); // for sending audio to the analysis synth
var synth; // plays a buffer
var analysis_synth; // does analysis
var trigRate = 30; // how many times to update every second
var audioBuf = Buffer.read(s,"/Users/ted/Documents/_CREATING/_PROJECT FILES/Machine Learning/Training Data/Audio/source audio/ml examples/elec/stereos/no_input_mixer 01.wav");
var nb_of_dims = 12; // just the 12 mfccs for now...
var dataset;// = FluidDataSet(s,\raw_data,nb_of_dims);
var normed_dataset;// = FluidDataSet(s,\normed_dataset,nb_of_dims);
var scaler = FluidNormalize(s);
var kmeans = FluidKMeans(s);
var k_ = 4; // what the k will be for k means
var get_cluster; // a function that will take in a buffer (vector) update the view with it's color
var nHistories = trigRate * totalHistory; // total number of histories to keep track of
var updateBuf = Buffer.alloc(s,nb_of_dims); // a buffer to use for sending new points to the FluidDataSet
var normedBuf = Buffer.alloc(s,nb_of_dims); // a buffer to put a normalized datapoint in
var clustered = false; // is true once at least one clustering has happened (at one point one can predict on new points)
var buffer_filled = false;
var uvs; // userviews that will flash showing current cluster prediction
var win; // window where the colors flash
var serverBuf = Buffer.alloc(s,nb_of_dims);
var dummyBuf = Buffer.alloc(s,nb_of_dims);
var update_counter = 0;
// the colors that flash (4 for now because 'k_' = 4, would need more with a larger k)
var colors = [Color.cyan,Color.yellow,Color.green,Color.red];
s.sync;
dataset = FluidDataSet(s,\raw_data,nb_of_dims);
normed_dataset = FluidDataSet(s,\normed_dataset,nb_of_dims);
s.sync;
/*
because the datasets are "named" on the server, does it make sense that i should probably clear them each time,
just to be sure that there isn't any residual data lingering in these "names"
*/
dataset.clear;
normed_dataset.clear;
dummyBuf.setn(0,0.dup(nb_of_dims));
s.sync;
// initialize the size of the dataset
/* nHistories.do({
arg i;
i.postln;
dataset.addPoint(i.asString,dummyBuf);
s.sync;
/*
i decided against this dataset size initialization strategy, since it took a little bit of time to
complete and as i was iterating on this code, i decided it was too long...
*/
});*/
s.sync;
// play sound
synth = {
arg rate = 1;
var sig = PlayBuf.ar(audioBuf.numChannels,audioBuf,rate,loop:1);
//var sig = SoundIn.ar(0);
Out.ar(bus,Mix(sig));
sig;// * -20.dbamp;
}.play;
s.sync;
// define this function that will take in a buffer (vector) update the view with it's color
get_cluster = {
arg unnormed_buf; // take in the raw vector that is on the server
//normalize it into the normedBuf
scaler.normalizePoint(unnormed_buf,normedBuf,{
// use that to predict
kmeans.predictPoint(normedBuf,{
arg cluster;
// update views.
uvs.do({
arg uv, i;
if(cluster == i,{
defer{
// using alpha here because it might be nice to set the alpha to the
// amplitude or something so it flickers with the sound...
uv.background_(uv.background.alpha_(1));
};
},{
defer{
// alpha = 0 means don't show me this.
uv.background_(uv.background.alpha_(0));
};
})
});
});
});
};
// synth that does the analysis
analysis_synth = {
arg in_bus;
var input = In.ar(in_bus);
var vector = FluidMFCC.kr(input); // 13 by default
vector = vector[1..12];
// a "clock" to send to the language... explained below at OSCFunc...
SendReply.kr(Impulse.kr(trigRate),"/clock");
//vector.poll;
// i wish...
//RecordBuf.kr(vector,serverBuf);
vector.do({
arg val, i;
BufWr.kr(val,serverBuf,i);
/*
has to work this way because FluidDataSet.updatePoint needs to have a 1 channel buf with
nFrames = num dimensions
the RecordBuf.kr above would be simpler, however then the buffer would need to be a 1 frame
buf with nChannels = num dimensions
maybe ".updatePoint" could be modified to take a buffer that is either *1 channel x nDimensions frames* OR
*nDimensions channels x 1 frame* ?
*/
});
Silent.ar;
}.play(synth,addAction:\addAfter,args:[\in_bus,bus]);
OSCFunc({
var label = (update_counter % nHistories).asString;
if(update_counter % trigRate == 0,{
"currently updating data point: %".format(label).postln;
});
if(buffer_filled,{
dataset.updatePoint(label,serverBuf);
if(clustered,{
get_cluster.(serverBuf);
});
},{
dataset.addPoint(label,serverBuf);
if(update_counter == (nHistories - 1),{buffer_filled = true;});
});
update_counter = update_counter + 1;
/*
i originally had this update functionality (i.e., the "updatePoint" and the "get_cluster") happening just in a loop in a routine with
"trigRate.reciprocal.wait" in the loop, but it turn sout that "updatePoint" takes some time, so
each iteration of the loop was much longer than trigRate.reciprocal.
so now i'm using an Impulse.kr on the server to "clock" my "addPoint" and "get_cluster" through an OSCFunc
this is better though because then i'm not sending the vector back to the language, loading it back to the server in
*another* buffer, to finally add to the dataset
(for what it's worth, this way of clocking something in the language with an OSCFunc is something i find myself
doing often... it works quite well, however does seem a little weird. does anyone know of a better way to be
thinking about this?)
*/
},"/clock");
Routine({
// wait for the whole buffer to fill up!
totalHistory.wait;
/*
infinitely loop this: do the things to get the kmeans...
*/
inf.do({
arg i;
"clustering for the % th time".format(i).postln;
scaler.fit(dataset.asString,{
"fitting to the scaler complete".postln;
scaler.normalize(dataset.asString,normed_dataset.asString,{
"normalizing the dataset complete".postln;
kmeans.fit(normed_dataset,k_,action:{
"fitting kmeans complete".postln;
clustered = true; // this way after this has happened at least once, we'll know we can do "predict" above
});
});
});
updatePeriod.wait; // only update this ever so often...
});
}).play;
// window for showing the current cluster prediction
Window.closeAll;
win = Window("",Rect(Window.screenBounds.width + 100,100,600,800)).front;
win.background_(Color.black);
uvs = k_.collect({
arg i;
var width = win.bounds.width / k_;
var uv = UserView(win,Rect(width * i,0,width,win.bounds.height));
uv.background_(colors[i].alpha_(0));
//uv.visible_(false);
});
},AppClock).play;
});
)