hey, after attending the flucoma workshop at stanford early July, I have been creating a more advanced implementation of the MLPRegressor example for interpolating between presets in SC.
It can be used with an arbitrary Synthdef and without doing the scaling to 0 and 1 for the MLPRegressor manually inside the SynthDef. The scaling is done automatically via .map and .unmap of ControlSpecs.
The SynthDef is primed to a NodeProxy and you can specify which parameters should be controlled by the MLPRegressor. I got alot of help by @elgiano on this.
The most convenient use without creating a custom GUI with Sliders for all the parameters of your SynthDef is to use NodeProxyGui2.
You can also map and unmap MIDIcontrol of the Slider2D which is unfortunately atm just normal MIDI and not 14Bit Midi with higher resolution.
1.) first declare a SynthDef you want to use with the MLPRegressor and make sure to specify the specs for all of your arguments / NamedControls.
(
var multiChannelTrigger = { |numChannels, trig|
numChannels.collect{ |chan|
PulseDivider.ar(trig, numChannels, chan);
};
};
var multiChannelPhase = { |triggers, rates|
triggers.collect{ |localTrig, i|
var hasTriggered = PulseCount.ar(localTrig) > 0;
Sweep.ar(localTrig, rates[i] * hasTriggered);
};
};
var multiChannelDemand = { |triggers, demandUgen|
var demand = demandUgen;
triggers.collect{ |localTrig|
Demand.ar(localTrig, 0, demand)
};
};
var channelMask = { |triggers, numChannels, channelMask, centerMask|
var panChannels = Array.series(numChannels, -1 / numChannels, 2 / numChannels).wrap(-1.0, 1.0);
var panPositions = panChannels.collect { |pos| Dser([pos], channelMask) };
panPositions = panPositions ++ Dser([0], centerMask);
Demand.ar(triggers, 0, Dseq(panPositions, inf)).lag(0.001);
};
var hanningWindow = { |phase|
(1 - (phase * 2pi).cos) / 2 * (phase < 1);
};
SynthDef(\test, {
var numChannels = 5;
var tFreq, trig, triggers, chanMask;
var grainFreqs, overlaps, maxOverlaps, windowRates;
var windowPhases, grainWindows, pulsaretPhases, pulsarets, sig;
tFreq = \tFreq.kr(5, spec: ControlSpec(1, 500, \exp));
trig = Impulse.ar(tFreq.lag(0.02));
triggers = multiChannelTrigger.(numChannels, trig);
chanMask = channelMask.(triggers, numChannels - 1,
\channelMask.kr(1, spec: ControlSpec(0, 1, \lin, 1)),
\centerMask.kr(1, spec: ControlSpec(0, 1, \lin, 1))
);
grainFreqs = multiChannelDemand.(triggers, Dwhite(0, 1)).linexp(0, 1,
\freqLo.kr(100, spec: ControlSpec(0.1, 1000, \lin)),
\freqHi.kr(100, spec: ControlSpec(0.1, 1000, \lin))
);
overlaps = multiChannelDemand.(triggers, Dwhite(0, 1)).linexp(0, 1,
\overlapLo.kr(1, spec: ControlSpec(1, 50, \lin)),
\overlapHi.kr(1, spec: ControlSpec(1, 50, \lin))
);
maxOverlaps = min(overlaps, numChannels * (grainFreqs / tFreq));
windowRates = grainFreqs / maxOverlaps;
windowPhases = multiChannelPhase.(triggers, windowRates);
grainWindows = hanningWindow.(windowPhases);
pulsaretPhases = multiChannelPhase.(triggers, grainFreqs);
pulsarets = sin(pulsaretPhases * 2pi);
pulsarets = pulsarets * grainWindows;
pulsarets = PanAz.ar(2, pulsarets, chanMask * \panMax.kr(0.8));
sig = pulsarets.sum;
sig = sig * \amp.kr(-25, spec: ControlSpec(-35, -5, \lin)).dbamp;
sig = LeakDC.ar(sig);
OffsetOut.ar(\out.kr(0), sig);
}).add;
)
2.) then evaluate the ~makeMLP function which creates an Enviroment and the necessary methods. It takes a SynthDef name as the first argument which is then primed to a NodeProxy. With the second argument params you define which arguments or NamedControls from your SynthDef should be controlled by the MLPRegressor. With initArgs you can define the initial parameters of the Synth.
(
~makeMLP = { |synthDefName, params, initArgs=#[], numChannels = 2|
Environment.make { |self|
~synthDef = SynthDescLib.global[synthDefName].def ?? {
Error("SynthDef '%' not found".format(synthDefName)).throw
};
~params = params;
if (params.isNil or: params.isEmpty) {
Error("MLP: pass a non-empty array of parameter names to be controlled, % was given.".format(params)).throw
};
~dsXY = FluidDataSet(s);
~dsParams = FluidDataSet(s);
~bufXY = Buffer.alloc(s, 2);
~bufParams = Buffer.alloc(s, params.size);
~mlp = FluidMLPRegressor(
server: s,
hiddenLayers: [7],
activation: FluidMLPRegressor.sigmoid,
outputActivation: FluidMLPRegressor.sigmoid,
maxIter: 1000,
learnRate: 0.1,
batchSize: 1,
validation: 0,
);
~nodeProxy = NodeProxy.audio(s, numChannels);
~nodeProxy.prime(~synthDef).set(*initArgs);
// methods
~startControl = { |self|
self.stopControl;
self.mlpSynth = {
var val, xy, trig;
xy = FluidBufToKr.kr(self.bufXY);
trig = Changed.kr(xy).sum;
self.mlp.kr(trig, self.bufXY, self.bufParams);
val = FluidBufToKr.kr(self.bufParams);
SendReply.kr(Changed.kr(val).sum, "/paramsChanged", val);
Silent.ar;
}.play;
self.responder = OSCFunc({ |msg|
var vals = msg.drop(3);
self.params.do { |key, n|
self.nodeProxy.set(key, self.synthDef.specs[key].map(vals[n]))
}
},"/paramsChanged", argTemplate:[self.mlpSynth.nodeID]);
};
~stopControl = { |self|
self.mlpSynth !? { self.mlpSynth.free; self.mlpSynth = nil };
self.responder !? { self.responder.free; self.responder = nil }
};
~getParamValues = { |self|
self.params.collect(self.nodeProxy.get(_));
};
~getParamValuesUni = { |self|
self.params.collect { |k|
self.synthDef.specs[k].unmap(self.nodeProxy.get(k));
}
};
~addPoint = { |self, id|
self.dsXY.addPoint(id, self.bufXY);
self.bufParams.loadCollection(self.getParamValuesUni, 0) {
self.dsParams.addPoint(id, self.bufParams);
};
};
~train = { |self, train|
self.trainTask = self.trainTask ?? {
Task {
loop {
var done = false;
var c = CondVar();
self.mlp.fit(self.dsXY, self.dsParams) { |loss|
done = true;
c.signalOne;
"> Training done. Loss: %".format(loss).postln;
};
c.wait { done };
0.1.wait;
}
}
};
if (train) {
self.trainTask.play;
} {
self.trainTask.pause;
};
};
// GUI
~makeGui = { |self|
var win, graphView, pointView, buttons, refreshPlotter;
var counter = 0;
refreshPlotter = {
self.dsXY.dump {|v|
if (v["cols"] == 0) {
v = Dictionary["cols" -> 2, "data" -> Dictionary[]]
};
pointView.dict = v
};
};
pointView = FluidPlotter(standalone: false);
pointView.pointSizeScale = 20/6;
refreshPlotter.value();
win = Window(self.synthDef.name, Rect(10, 500, 440, 320)).front;
self.xySlider = Slider2D()
.background_(Color.white.alpha_(0))
.action_{ |view|
self.setLatent(0, view.x, view.y)
};
SkipJack({ self.xySlider.setXY(*self.coords.asArray)
}.defer, 0.01, { self.xySlider.isClosed });
graphView = StackLayout(self.xySlider, View().layout_(
VLayout(pointView).margins_(10)
)).mode_(\stackAll);
buttons = VLayout(
Button(bounds: 100@20).states_([["Add Points"]])
.action_{
var id = "point-%".format(counter);
self.addPoint(id);
counter = counter + 1;
refreshPlotter.value();
},
Button(bounds: 100@20).states_([["Save Data"]])
.action_{
FileDialog({|folder|
self.dsXY.write(folder +/+ "xydata.json");
self.dsParams.write(folder +/+ "paramsdata.json");
}, {}, 2, 0, true);
},
Button(bounds: 100@20).states_([["Load Data"]])
.action_{
FileDialog({|folder|
self.dsXY.read(folder +/+ "xydata.json");
self.dsParams.read(folder +/+ "paramsdata.json");
}, fileMode: 2, acceptMode: 0, stripResult: true);
refreshPlotter.value();
},
Button(bounds: 100@20).states_([
["Train"], ["Train", Color.white, Color.yellow(0.5)]])
.action_{ |v|
self.train(v.value > 0)
},
Button(bounds: 100@20).states_([["Save MLP"]])
.action_{
Dialog.savePanel({|path|
if(PathName(path).extension != "json"){
path = "%.json".format(path);
};
self.mlp.write(path);
});
},
Button(bounds: 100@20).states_([["Load MLP"]])
.action_{
Dialog.openPanel({|path|
self.mlp.read(path, action: {
defer { refreshPlotter.value() }
});
});
},
Button(bounds: 100@20).states_([["Not Predicting"],
["Predicting", Color.white, Color.red(0.8)]])
.action_{ |view|
if (view.value > 0) { self.startControl } { self.stopControl };
}.value_(self.mlpSynth.notNil),
Button(bounds: 100@20).states_([["Clear"]])
.action_{|state|
self.mlp.clear;
self.dsXY.clear;
self.dsParams.clear;
refreshPlotter.value();
}
);
win.layout = HLayout([graphView, stretch: 2], buttons);
};
~setLatent = { |self, n ...coords|
self.coords = self.coords ?? { Order[] };
coords.do { |c, k| self.coords[n + k] = c };
self.bufXY.setn(n, coords);
};
~mapMidi = { |self, ccs|
self.midiResponders = self.midiResponders ?? { Order[] };
self.unmapMidi;
ccs.do { |cc, n|
self.midiResponders[cc] = MIDIFunc.cc({ |v, c|
self.setLatent(n, v / 127)
}, cc).fix
};
};
~unmapMidi = { |self|
self.midiResponders.do(_.free);
};
}.know_(true);
};
)
3.) pass the SynthDef name to the ~makeMLP function and declare all the parameters you want to have controlled by the MLPRegressor and set if you like initial parameters for the Synth and the number of output channels of the NodeProxy. Calling .makeGui will create the MLPRegressor GUI and accessing the NodeProxy by calling .nodeProxy you can create a GUI for all the parameters most easily here with either .gui for NodeProxyGui or .gui2 for NodeProxyGui2
(
x = ~makeMLP.(\test,
params: [
\tFreq,
\overlapLo,
\overlapHi,
\freqLo,
\freqHi
],
initArgs: [
\amp, -5.dbamp,
]
);
x.makeGui;
x.nodeProxy.gui2;
)
4.) map and unmap MIDI to control the Slider2D and declare the CCs as an argument.
x.mapMidi([0, 1]);
x.unmapMidi;
but make Sure you have initialised all the necesary Midi configurations for your Midi device:
~initMIDI = {
MIDIClient.init;
MIDIIn.connectAll;
m = MIDIOut.newByName("Ableton to SC", "Ableton to SC").latency_(0);
};
~initMIDI.();
The current implementation however cannot be used with array NamedControls like in the following example, if you specify \freqRange or \ampRange for the parameters which you want to control and are both array NamedControls you get ERROR: Primitive '_ArrayAdd' failed.
I think the problem is in ~addPoints. But using params.size to create the frames for ~bufParams is a already problem, because its not taking into account the amount of channels of your list of parameters.
I think if the current paramValueUni which is added to self.dsParams here:
self.bufParams.loadCollection(self.getParamValuesUni, 0) {
self.dsParams.addPoint(id, self.bufParams);
};
is an array, then the items in the array should added one after another as Points or?
(
SynthDef(\test, {
var freqRange = \freqRange.kr([2500, 3500], spec: ControlSpec(100, 8000));
var ringRange = \ringRange.kr([0.1, 0.15], spec: ControlSpec(0.1, 2.0));
var ampRange = \ampRange.kr([0.1, 0.2], spec: ControlSpec(0.1, 1.0));
var sig = Splay.ar(
Array.fill(3, {
Ringz.ar(
Dust.ar(\dens.kr(5, spec: ControlSpec(1, 10))),
exprand(freqRange[0], freqRange[1]),
exprand(ringRange[0], ringRange[1]),
exprand(ampRange[0], ampRange[1])
)
})
).distort;
sig = sig * \amp.kr(0.5, spec: ControlSpec(0, 1));
Out.ar(\out.kr(0), sig);
}).add;
)
(
~makeMLP = { |synthDefName, params|
Environment.make { |self|
~synthDef = SynthDescLib.global[synthDefName].def ?? {
Error("SynthDef '%' not found".format(synthDefName)).throw
};
~params = params;
if (params.isNil or: params.isEmpty) {
Error("MLP: pass a non-empty array of parameter names to be controlled, % was given.".format(params)).throw
};
~dsXY = FluidDataSet(s);
~dsParams = FluidDataSet(s);
~bufXY = Buffer.alloc(s, 2);
~bufParams = Buffer.alloc(s, params.size);
~nodeProxy = NodeProxy.audio(s, 2);
~nodeProxy.prime(~synthDef);
~getParamValues = { |self|
self.params.collect(self.nodeProxy.get(_));
};
~getParamValuesUni = { |self|
self.params.collect { |k|
self.synthDef.specs[k].unmap(self.nodeProxy.get(k));
};
};
~addPoint = { |self, id|
self.dsXY.addPoint(id, self.bufXY);
self.bufParams.loadCollection(self.getParamValuesUni, 0) {
self.dsParams.addPoint(id, self.bufParams);
};
};
}.know_(true);
};
)
(
x = ~makeMLP.(\test,
params: [
\freqRange,
\ringRange,
\ampRange,
\dens,
],
);
x.addPoint;
)