/**
* Activation functions
*
* Activation functions determine what activation value neurons should get. Depending on your network's environment, choosing a suitable activation function can have a positive impact on the learning ability of the network.
*
* @see [Activation Function on Wikipedia](https://en.wikipedia.org/wiki/Activation_function)
* @see [Beginners Guide to Activation Functions](https://towardsdatascience.com/secret-sauce-behind-the-beauty-of-deep-learning-beginners-guide-to-activation-functions-a8e23a57d046)
* @see [Understanding activation functions in neural networks](https://medium.com/the-theory-of-everything/understanding-activation-functions-in-neural-networks-9491262884e0)
* @see [List of activation functions in neural networks with pros/cons](https://stats.stackexchange.com/questions/115258/comprehensive-list-of-activation-functions-in-neural-networks-with-pros-cons)
*
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.<ACTIVATION_FUNCTION>;
*
* // eg.
* A.squash = methods.activation.LOGISTIC;
*
* @namespace
*/
const activation = {
/**
* [Logistic function.](https://en.wikipedia.org/wiki/Logistic_function)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.LOGISTIC;
*/
LOGISTIC: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => Math.exp(-x) / (Math.pow((1 + Math.exp(-x)), 2)) : (x) => 1 / (1 + Math.exp(-x));
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [TanH function.](https://en.wikipedia.org/wiki/Hyperbolic_function#Hyperbolic_tangent)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.TANH;
*/
TANH: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => 1 - (Math.tanh(x) * Math.tanh(x)) : (x) => Math.tanh(x);
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Identity function.](https://en.wikipedia.org/wiki/Identity_function)
*
* Returns input as output, used for [memory neurons](Layer#.Memory).
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.IDENTITY;
*/
IDENTITY: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => 1 : (x) => x;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Step function.](https://en.wikipedia.org/wiki/Heaviside_step_function)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.STEP;
*/
STEP: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => 0 : (x) => x > 0 ? 1 : 0;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [ReLU function.]{@link https://en.wikipedia.org/wiki/Rectifier_(neural_networks)}
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.RELU;
*/
RELU: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => x > 0 ? 1 : 0 : (x) => x > 0 ? x : 0;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Softsign function.](https://en.wikipedia.org/wiki/Activation_function#Comparison_of_activation_functions)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.SOFTSIGN;
*/
SOFTSIGN: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => x / ((1 + Math.abs(x)) * (1 + Math.abs(x))) : (x) => x / (1 + Math.abs(x));
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
// SOFTCLIP: function(x, derivate) {
// if(x == undefined) throw new ReferenceError("Parameter 'x' is required, but it was not defined");
//
// const f = derivate ? (x) => (Math.exp(1) - 1) * Math.exp(x) / (Math.exp(x) + Math.exp(1)) * (1 + Math.exp(1)) : (x) => Math.log((1 + Math.exp(x)) / (1 + Math.exp(x-1)))
//
// return Array.isArray(x) ? x.map(f) : f(x)
// },
/**
* [Sinusoid function.](https://en.wikipedia.org/wiki/Sine_wave)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.SINUSOID;
*/
SINUSOID: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => Math.cos(x) : (x) => Math.sin(x);
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Guassian function.](https://en.wikipedia.org/wiki/Gaussian_function)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.GAUSSIAN;
*/
GAUSSIAN: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => -2 * x * Math.exp(-(x * x)) : (x) => Math.exp(-(x * x));
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Bent identity function.](https://en.wikipedia.org/wiki/Activation_function#Comparison_of_activation_functions)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.BENT_IDENTITY;
*/
BENT_IDENTITY: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => x / (2 * Math.sqrt((x * x) + 1)) + 1 : (x) => (Math.sqrt((x * x) + 1) - 1) / 2 + x;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Bipolar function](https://wagenaartje.github.io/neataptic/docs/methods/activation/), if x > 0 then returns 1, otherwise returns -1
*
* @member
* @function
* @param {number | number[]} x Input value to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.BIPOLAR;
*/
BIPOLAR: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => 0 : (x) => x > 0 ? 1 : -1;
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Bipolar sigmoid function.](https://wagenaartje.github.io/neataptic/docs/methods/activation/)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.BIPOLAR_SIGMOID;
*/
BIPOLAR_SIGMOID: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => (2 * Math.exp(-x)) / (Math.pow((1 + Math.exp(-x)), 2)) : (x) => 2 / (1 + Math.exp(-x)) - 1;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Hard tanh function.](https://wagenaartje.github.io/neataptic/docs/methods/activation/)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.HARD_TANH;
*/
HARD_TANH: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => x > -1 && x < 1 ? 1 : 0 : (x) => Math.max(-1, Math.min(1, x));
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Absolute function.](https://wagenaartje.github.io/neataptic/docs/methods/activation/)
*
* Avoid using this activation function on a node with a selfconnection
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.ABSOLUTE;
*/
ABSOLUTE: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => x < 0 ? -1 : 1 : (x) => Math.abs(x);
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Inverse function.](https://wagenaartje.github.io/neataptic/docs/methods/activation/)
*
* @member
* @function
* @param {number | number[]} x Input value(s) to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.INVERSE;
*/
INVERSE: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const f = derivate ? (x) => -1 : (x) => 1 - x;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
/**
* [Scaled exponential linear unit.](https://towardsdatascience.com/selu-make-fnns-great-again-snn-8d61526802a9)
*
* Exponential linear units try to make the mean activations closer to zero which speeds up learning. It has been shown that ELUs can obtain higher classification accuracy than ReLUs. α is a hyper-parameter here and to be tuned and the constraint is α ≥ 0(zero).
*
* @see {@link https://arxiv.org/pdf/1706.02515.pdf|Self-Normalizing Neural Networks}
*
* @member
* @function
* @param {number | number[]} x Input value to activation function
* @param {boolean} [derivate] Flag to select derivative function
*
* @example
* let { methods, Node } = require("@liquid-carrot/carrot");
*
* // Changing a neuron's activation function
* let A = new Node();
* A.squash = methods.activation.SELU;
*/
SELU: function(x, derivate) {
// Dirty but neccessary to support Network.standalone as currently written
const clamp = function(x) {
const max = Number.MAX_VALUE;
return x === Infinity
? max
: x === -Infinity
? -max
: x;
};
if (x == undefined) throw new ReferenceError('Parameter \'x\' is required, but it was not defined');
const alpha = 1.6732632423543772848170429916717;
const scale = 1.0507009873554804934193349852946;
const f = derivate ? (x) => x > 0 ? scale : ((x > 0 ? x : alpha * Math.exp(x) - alpha) + alpha) * scale : (x) => (x > 0 ? x : alpha * Math.exp(x) - alpha) * scale;
// return Array.isArray(x) ? x.map(f) : f(x); unsafe mode
return Array.isArray(x) ? x.map(clamp(f)) : clamp(f(x));
},
};
module.exports = activation;
