% A two-layer Neural Network with self-teaching via backpropagation
% Learning rate decreases over time to avoid overfitting.
% This network's inputs are an n-by-m matrix representing pixels of
% a letter drawing, up/down-sampled to match a resolution of m-by-n.

classdef NeuralNet < handle
    properties
       x_size
       y_size
       learning_rate_init
       learning_rate
       hidden_node_ct
       hidden_layer_weights
       output_node_ct
       output_layer_weights
    end
    methods
        % Creates a new Neural Net with given parameters
        function obj = NeuralNet(x, y, learning_init, hidden_node_ct, classification_ct)
            % Size of input drawings
            obj.x_size = x;
            obj.y_size = y;
            % Initial learning rate value - a number between 0 and 1
            obj.learning_rate_init = learning_init;
            % This should reduce with each new training example
            % Possible ways to scale:
            % - init*(example#)/(example_ct)
            % - init/sqrt(example#)
            obj.learning_rate = learning_init;
            % Number of nodes in the "hidden" layer
            obj.hidden_node_ct = hidden_node_ct;
            % Weights from input layer to hidden layer
            % One weight per input pixel, plus one for bias
            % Hidden node indexes column
            % Input node weight / bias indexes row
            obj.hidden_layer_weights = rand(x*y+1, obj.hidden_node_ct)-0.5;
            % Number of nodes in the "output" layer
            % Corresponds to number of possible classifications for the input,
            % i.e., 'a', 'b', 'c', ...
            obj.output_node_ct = classification_ct;
            % Weights from hidden layer to output layer
            % One weight per hidden node, plus one for bias
            % Output node indexes column
            % Hidden node weight / bias indexes row
            obj.output_layer_weights = rand(obj.hidden_node_ct+1, classification_ct)-0.5;
        end
        % Calculates output of this neural network n_net for input_drawing
        % provided.
        function [winner, classification] = calculate_output(n_net, input)
            % Change input drawing size to match neural network size if
            % necessary
            s = size(input);
            if (s(1) ~= n_net.y_size || s(2) ~= n_net.x_size)
                input_drawing = change_size(input,n_net.x_size,n_net.y_size);
            else
                input_drawing = input;
            end
            
            % Input matrix to hidden layer (shaped specifically for matrix
            % multiplication w/ weight matrix)
            hidden_layer_input = [reshape(input_drawing, 1, n_net.y_size*n_net.x_size) 1];

            % Calculate sigmoid squashed output for the hidden layer
            % Sigmoid with range 0 to 1: 1/(1+exp(-1*x))
            hidden_layer_output_raw = hidden_layer_input*n_net.hidden_layer_weights;
            hidden_layer_output = 1./(1+exp(-1.*hidden_layer_output_raw));

            % Input matrix to output layer (shaped specifically for matrix
            % multiplication w/ weight matrix)
            output_layer_input = [hidden_layer_output 1];

            % Calculate sigmoid squashed output for the output layer
            % Sigmoid with range 0 to 1: 1/(1+exp(-1*x))
            output_layer_output_raw = output_layer_input*n_net.output_layer_weights;
            output_layer_output = 1./(1+exp(-1.*output_layer_output_raw));
            
            [val, winner] = max(output_layer_output);
            % Threshold output for classification
            classification = output_layer_output;
        end
        % Updates weights for this Neural Net based on the given
        % input_drawing and its given correct_classification, using the
        % currently set learning_rate. Note: running this function does not
        % update the learning rate -- this must be done by the user with
        % the set_learning_rate function.
        function teach_network(n_net, input, correct_classification)
            % Change input drawing size to match neural network size if
            % necessary
            s = size(input);
            if (s(1) ~= n_net.y_size || s(2) ~= n_net.x_size)
                input_drawing = change_size(input,n_net.x_size,n_net.y_size);
            else
                input_drawing = input;
            end
            
            % Input matrix to hidden layer (shaped specifically for matrix
            % multiplication w/ weight matrix)
            hidden_layer_input = [reshape(input_drawing, 1, n_net.y_size*n_net.x_size) 1];

            % Calculate sigmoid squashed output for the hidden layer
            % Sigmoid with range 0 to 1: 1/(1+exp(-1*x))
            hidden_layer_output_raw = hidden_layer_input*n_net.hidden_layer_weights;
            hidden_layer_output = 1./(1+exp(-1.*hidden_layer_output_raw));

            % Input matrix to output layer (shaped specifically for matrix
            % multiplication w/ weight matrix)
            output_layer_input = [hidden_layer_output 1];

            % Calculate sigmoid squashed output for the output layer
            % Sigmoid with range 0 to 1: 1/(1+exp(-1*x))
            output_layer_output_raw = output_layer_input*n_net.output_layer_weights;
            output_layer_output = 1./(1+exp(-1.*output_layer_output_raw));

            % Determine errors, propagating backwards from output.
            output_layer_error = correct_classification - output_layer_output;
            hidden_layer_error = output_layer_error*n_net.output_layer_weights(1:end-1,:)';

            % Update output layer weights with backpropagation algorithm.
            sigmoid_deriv = (1./(1+exp(-1*output_layer_output_raw))).*(1 - 1./(1+exp(-1*output_layer_output_raw)));
            deltas = n_net.learning_rate*output_layer_input'*(output_layer_error.*sigmoid_deriv);
            n_net.output_layer_weights = n_net.output_layer_weights + deltas;

            % Update hidden layer weights with backpropagation algorithm.
            sigmoid_deriv = (1./(1+exp(-1*hidden_layer_output_raw))).*(1 - 1./(1+exp(-1*hidden_layer_output_raw)));
            deltas = n_net.learning_rate*hidden_layer_input'*(hidden_layer_error.*sigmoid_deriv);
            n_net.hidden_layer_weights = n_net.hidden_layer_weights + deltas;
        end
        function set_learning_rate(n_net, new_rate)
            n_net.learning_rate = new_rate;
        end
        function weights_c_code(n_net)
            fprintf('#define OUTPUT_NODE_CT %d\n', n_net.output_node_ct);
            fprintf('#define HIDDEN_NODE_CT %d\n', n_net.hidden_node_ct);
            fprintf('#define X_SCALED %d\n', n_net.x_size);
            fprintf('#define Y_SCALED %d\n', n_net.y_size);
            fprintf('\n');
            fprintf('double hidden_layer_weights[HIDDEN_NODE_CT][X_SCALED*Y_SCALED] = {\n');
            for i=1:n_net.hidden_node_ct
                fprintf('/*Hidden node %3d*/{', i);
                for j=1:n_net.x_size*n_net.y_size
                    fprintf('%9.5f', n_net.hidden_layer_weights(j,i));
                    if(j<n_net.x_size*n_net.y_size)
                        fprintf(', ');
                    else
                        fprintf('}');
                    end
                end
                if(i<n_net.hidden_node_ct)
                    fprintf(',\n');
                else
                    fprintf('\n');
                end
            end
            fprintf('};\n');
            fprintf('double output_layer_weights[OUTPUT_NODE_CT][HIDDEN_NODE_CT] = {\n');
            for i=1:n_net.output_node_ct
                fprintf('/*Output node %3d*/{', i);
                for j=1:n_net.hidden_node_ct
                    fprintf('%9.5f', n_net.output_layer_weights(j,i));
                    if(j<n_net.hidden_node_ct)
                        fprintf(', ');
                    else
                        fprintf('}');
                    end
                end
                if(i<n_net.output_node_ct)
                    fprintf(',\n');
                else
                    fprintf('\n');
                end
            end
            fprintf('};\n');
        end
    end
end