% Gives a rough estimation of overfitting on the neural net

disp('Alphabet data loading from files...');
loadalphs
disp('Cleaning and normalizing alphabets...');
thrnormalall

x_net_sz = 15;
y_net_sz = 12;

alphs_scaled = change_size_all(alphs, x_net_sz, y_net_sz);
s = size(alphs);

% Parameters that may be varied
hidden_node_ct = ceil(8*sqrt(x_net_sz*y_net_sz));
sweeps_max = 2500;
sweep_inc = 10;

c = floor(sweeps_max/sweep_inc);
alph_trains = zeros(1,c);
fail_cts = zeros(1,c);
err_scores = zeros(1,c);

% Number of classifications
classf_ct = 26;

% Initial learning rate (0 to 1)
learning_rate_init = 0.5;

for alp_num = 1:s(2)
    
    % Create Neural Network.
    net = NeuralNet(x_net_sz,y_net_sz,learning_rate_init, hidden_node_ct, classf_ct);

    for i=1:c

        alph_trains(i) = sweep_inc*i*(s(2)-1);
        fprintf('Training Neural Net for %d sweeps total\n', sweep_inc*i);

        for sets=1:sweep_inc
            % train on all alphabets supplied
            for file=1:s(2)
                % Get this from loaded alphabets, new one each time.
                % Loads alphabet letters into cell matrix alp.
                if(file ~= alp_num)
                    alp = alphs_scaled{file};
                    for let = 1:26
                        input = alp{let};
                        % Set this to be whatever the correct output should be for the above
                        % input. -1 = false, 1 = true.
                        classification_correct = zeros(1,26);
                        classification_correct(let) = 1;

                        % Teach the network with this input example.
                        teach_network(net, input, classification_correct);
                    end
                end
            end
            % adjust learning rate
            set_learning_rate(net, learning_rate_init/sqrt(sweep_inc*(i-1)+sets));
        end

        [fails err] = showNetPerformance(net, alphs_scaled, alp_num);
        close;
        err_scores(i) = err_scores(i) + err;
        fail_cts(i) = fail_cts(i) + length(fails);
    end
    figure;
    plot(alph_trains, err_scores, 'g');
    title(['Classification Error v. Number of Training Iterations', sprintf('Alphabet %d',alp_num)]);
    xlabel('Number of Training Examples');
    ylabel('Classification Error');
end

figure;
plot(alph_trains, err_scores, 'r');
title(['Overall Classification Error v. Number of Training Iterations', '[Cross-Validated]']);
xlabel('Number of Training Examples');
ylabel('Overall Classification Error');
figure;
plot(alph_trains, fail_cts, 'b');
title(['Number of Incorrectly Classified Letters v.', 'Number of Training Iterations']);
xlabel('Number of Training Examples');
ylabel('Letters Incorrectly Classified');