Network/test_gradient_check.cpp at master · oulrich1/Network · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
#include <iostream>
#include <cmath>
#include "allheader.h"
#include "network.h"

using namespace ml;
using namespace std;

void printMatrix(const Mat<double>& mat, const string& name) {
    cout << name << " (" << mat.size().cy << "x" << mat.size().cx << "):" << endl;
    for (int i = 0; i < mat.size().cy; i++) {
        cout << "  [ ";
        for (int j = 0; j < mat.size().cx; j++) {
            cout << mat.getAt(i, j) << " ";
        }
        cout << "]" << endl;
    }
}

int main() {
    cout << "=== GRADIENT CHECK: Verify backprop is computing gradients ===" << endl;

    // Create simple 2-2-1 network
    Layer<double>* input = new Layer<double>(2, "input");
    Layer<double>* hidden = new Layer<double>(2, "hidden");
    Layer<double>* output = new Layer<double>(1, "output");

    Network<double>* net = new Network<double>();
    net->setInputLayer(input);
    net->setOutputLayer(output);
    net->connect(input, hidden);
    net->connect(hidden, output);
    net->init();

    cout << "\n--- Set specific weights for reproducibility ---" << endl;

    // Input->Hidden weights: (2, 3) - 2 hidden nodes, 3 inputs (2 + bias)
    Mat<double> w_ih(2, 3, 0);
    w_ih.setAt(0, 0, 0.5);
    w_ih.setAt(0, 1, 0.5);
    w_ih.setAt(0, 2, 0.1);
    w_ih.setAt(1, 0, -0.5);
    w_ih.setAt(1, 1, -0.5);
    w_ih.setAt(1, 2, 0.2);
    input->setWeights(hidden, w_ih);
    printMatrix(w_ih, "Initial input->hidden weights");

    // Hidden->Output weights: (1, 3) - 1 output node, 3 inputs (2 + bias)
    Mat<double> w_ho(1, 3, 0);
    w_ho.setAt(0, 0, 1.0);
    w_ho.setAt(0, 1, 1.0);
    w_ho.setAt(0, 2, 0.5);
    hidden->setWeights(output, w_ho);
    printMatrix(w_ho, "Initial hidden->output weights");

    // Test input: [1.0, 0.0]
    Mat<double> inputData(1, 2, 0);
    inputData.setAt(0, 0, 1.0);
    inputData.setAt(0, 1, 0.0);
    printMatrix(inputData, "\nInput");

    // Forward pass
    Mat<double> result = net->feed(inputData);
    printMatrix(result, "Output");

    // Target: [0.0]
    Mat<double> target(1, 1, 0.0);
    Mat<double> error = Diff<double>(target, result);
    printMatrix(error, "Error (target - output)");

    // Backward pass
    output->setErrors(error);
    net->backprop();

    cout << "\n--- After backprop, check if layers have errors set ---" << endl;
    Mat<double> output_errors = output->getErrors();
    Mat<double> hidden_errors = hidden->getErrors();
    Mat<double> input_errors = input->getErrors();

    printMatrix(output_errors, "Output layer errors");
    printMatrix(hidden_errors, "Hidden layer errors");
    printMatrix(input_errors, "Input layer errors");

    cout << "\n--- Update weights with small learning rate ---" << endl;
    Mat<double> w_ih_before = input->getWeights(hidden);
    Mat<double> w_ho_before = hidden->getWeights(output);

    net->updateWeights(0.1);

    Mat<double> w_ih_after = input->getWeights(hidden);
    Mat<double> w_ho_after = hidden->getWeights(output);

    cout << "\nInput->Hidden weight changes:" << endl;
    for (int i = 0; i < w_ih_before.size().cy; i++) {
        cout << "  Node " << i << ": ";
        for (int j = 0; j < w_ih_before.size().cx; j++) {
            double delta = w_ih_after.getAt(i, j) - w_ih_before.getAt(i, j);
            cout << delta << " ";
        }
        cout << endl;
    }

    cout << "\nHidden->Output weight changes:" << endl;
    for (int i = 0; i < w_ho_before.size().cy; i++) {
        cout << "  Node " << i << ": ";
        for (int j = 0; j < w_ho_before.size().cx; j++) {
            double delta = w_ho_after.getAt(i, j) - w_ho_before.getAt(i, j);
            cout << delta << " ";
        }
        cout << endl;
    }

    // Check if ANY weights changed
    bool weights_changed = false;
    for (int i = 0; i < w_ih_before.size().cy; i++) {
        for (int j = 0; j < w_ih_before.size().cx; j++) {
            if (abs(w_ih_after.getAt(i, j) - w_ih_before.getAt(i, j)) > 1e-10) {
                weights_changed = true;
            }
        }
    }
    for (int i = 0; i < w_ho_before.size().cy; i++) {
        for (int j = 0; j < w_ho_before.size().cx; j++) {
            if (abs(w_ho_after.getAt(i, j) - w_ho_before.getAt(i, j)) > 1e-10) {
                weights_changed = true;
            }
        }
    }

    cout << "\n========================================" << endl;
    if (weights_changed) {
        cout << "RESULT: Weights ARE changing - backprop working" << endl;
    } else {
        cout << "RESULT: Weights NOT changing - backprop BROKEN!" << endl;
    }
    cout << "========================================" << endl;

    delete net;
    delete output;
    delete hidden;
    delete input;

    return weights_changed ? 0 : 1;
}