Applies a scaled TanH function element-wise to the input.
| Equation |
![\[ f(x) = tanh(x \times 0.66666667) \times 1.7159\]](form_2.png)
|
LeCun et al. suggest scale_in=2/3 and scale_out=1.7159. By carefully matching scale_in and scale_out, the nonlinearity can also be tuned to preserve the mean and variance of its input:
- scale_in=0.5, scale_out=2.4: If the input is a random normal variable, the output will have zero mean and unit variance.
- scale_in=1, scale_out=1.6: Same property, but with a smaller linear regime in input space.
- scale_in=0.5, scale_out=2.27: If the input is a uniform normal variable, the output will have zero mean and unit variance.
- scale_in=1, scale_out=1.48: Same property, but with a smaller linear regime in input space.
|
| | GBlockScaledTanh (size_t size=0) |
| |
| | GBlockScaledTanh (GDomNode *pNode) |
| |
| virtual double | derivative (double x, double f_x) const override |
| | Evaluates the derivative of the activation function. x is the net input, and f_x is the output activation–the value obtained by calling eval(x). More...
|
| |
| virtual double | eval (double x) const override |
| | Evaluates the activation function. More...
|
| |
| virtual std::string | name () const override |
| | Returns the name of this block in the form of a string. More...
|
| |
| virtual BlockType | type () const override |
| | Returns the type of this block. More...
|
| |
| | GBlockActivation (size_t size=0) |
| |
| | GBlockActivation (GDomNode *pNode) |
| |
| virtual void | backProp (GContext &ctx, const GVec &input, const GVec &output, const GVec &outBlame, GVec &inBlame) const override |
| | Evaluates outBlame, and adds to inBlame. (Note that it "adds to" the inBlame because multiple blocks may fork from a common source.) More...
|
| |
| virtual bool | elementWise () const override |
| | Returns true iff this block operates only on individual elements. More...
|
| |
| virtual void | forwardProp (GContext &ctx, const GVec &input, GVec &output) const override |
| | Evaluate the input, set the output. More...
|
| |
| virtual size_t | inputs () const override |
| | Returns the number of inputs this block consumes. More...
|
| |
| virtual size_t | outputs () const override |
| | Returns the number of outputs this block produces. More...
|
| |
| virtual void | resize (size_t inputs, size_t outputs) override |
| | Resizes this block. More...
|
| |
| virtual GDomNode * | serialize (GDom *pDoc) const override |
| | Marshall this block into a DOM. More...
|
| |
| | GBlockWeightless () |
| |
| | GBlockWeightless (GDomNode *pNode) |
| |
| virtual | ~GBlockWeightless () |
| |
| virtual void | copyWeights (const GBlock *pSource) override |
| | Copy the weights from pSource to this block. (Assumes pSource is the same type of block.) More...
|
| |
| virtual void | diminishWeights (double amount, bool regularizeBiases) override |
| | Moves all weights in the direction of zero by the specified amount. More...
|
| |
| virtual void | maxNorm (double min, double max) override |
| | Scales weights if necessary such that the manitude of the weights (not including the bias) feeding into each unit are >= min and <= max. More...
|
| |
| virtual void | perturbWeights (GRand &rand, double deviation) override |
| | Perturbs the weights that feed into the specifed units with Gaussian noise. The default values apply the perturbation to all units. More...
|
| |
| virtual void | resetWeights (GRand &rand) override |
| | Initialize the weights, usually with small random values. More...
|
| |
| virtual void | scaleWeights (double factor, bool scaleBiases) override |
| | Multiplies all the weights by the specified factor. More...
|
| |
| virtual void | step (double learningRate, const GVec &gradient) override |
| | Add the weight and bias gradient to the weights. More...
|
| |
| virtual void | updateGradient (GContext &ctx, const GVec &input, const GVec &outBlame, GVec &gradient) const override |
| | Evaluate the input and outBlame, update the gradient for updating the weights by gradient descent. More...
|
| |
| virtual size_t | vectorToWeights (const double *pVector) override |
| | Deserialize from a vector to the weights in this block. Return the number of elements consumed. More...
|
| |
| virtual size_t | weightCount () const override |
| | Returns the number of double-precision elements necessary to serialize the weights of this block into a vector. More...
|
| |
| virtual size_t | weightsToVector (double *pOutVector) const override |
| | Serialize the weights in this block into a vector. Return the number of elements written. More...
|
| |
| | GBlock () |
| |
| | GBlock (GDomNode *pNode) |
| |
| virtual | ~GBlock () |
| |
| size_t | inPos () const |
| | Returns the offset in the previous layer's output where values are fed as input to this block. More...
|
| |
| virtual bool | isRecurrent () const |
| | Returns true iff this block is recurrent. More...
|
| |
| void | setInPos (size_t n) |
| | Sets the starting offset in the previous layer's output where values will be fed as input to this block. More...
|
| |
| virtual std::string | to_str () const |
| | Returns a string representation of this block. More...
|
| |
| virtual bool | usesGPU () |
| | Returns true iff this block does its computations in parallel on a GPU. More...
|
| |
Public Member Functions inherited from GClasses::GBlockActivation