Okay, I wrote the same thing from scratch to see where you could get stuck, and I have no problems at all; I get really good results right from the beginning. The problem now is that I see no differences between my code and yours. Here's my program code if you want to compare (it's Java because... reasons).
package neural;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Random;
public class NeuralNetwork {
private HashSet<Node> nodes = new HashSet<Node>();
public NeuralNetwork() {
}
public void addNode(Node node) {
nodes.add(node);
}
public void train(HashMap<Node, Double> inputs, Node outputNode, double target, double learnSpeed) {
for (Node node : nodes) {
node.reset();
if (inputs.containsKey(node)) {
node.isInput = true;
node.input = inputs.get(node);
}
}
double result = outputNode.output();
outputNode.setAsOutput();
HashMap<Node, Double> errors = new HashMap<Node, Double>();
for (Node node : nodes) {
errors.put(node, node.calculateError(target, result));
}
for (Node node : nodes) {
node.applyError(learnSpeed, errors.get(node));
errors.put(node, 0.0);
}
}
public double ask(HashMap<Node, Double> inputs, Node outputNode) {
for (Node node : nodes) {
node.reset();
if (inputs.containsKey(node)) {
node.isInput = true;
node.input = inputs.get(node);
}
}
return outputNode.output();
}
public static class Node {
private final HashMap<Node, Double> inputs = new HashMap<Node, Double>();
private final HashMap<Node, Double> outputs = new HashMap<Node, Double>();
private final ActivationFunction phi;
public double input = 0;
public boolean isInput = false;
public double affinity = 0;
private boolean inputTotalDone = false;
private double inputTotal;
private boolean do_dresDone = false;
private double do_dres;
private String name;
public Node(String name, ActivationFunction af) {
this.name = name;
this.phi = af;
}
public void reset() {
inputTotalDone = do_dresDone = isInput = false;
}
public double inputTotal() {
if (inputTotalDone) {
return inputTotal;
} else {
inputTotalDone = true;
inputTotal = affinity;
for (Node in : inputs.keySet()) {
inputTotal += inputs.get(in)*in.output();
}
return inputTotal;
}
}
public double output() {
if (isInput) return input;
return phi.f(inputTotal());
}
public double d_output() {
return phi.df(inputTotal());
}
public void setAsOutput() {
do_dresDone = true;
do_dres = 1;
}
// Total output change per node output change
public double do_dres() {
if (do_dresDone) {
return do_dres;
} else {
do_dresDone = true;
do_dres = 0;
for (Node on : outputs.keySet()) {
do_dres += on.do_dres()*outputs.get(on)*on.d_output();
}
return do_dres;
}
}
public double calculateError(double target, double result) {
/*
* E = result error;
* o = total result;
* res = this output;
* net = this net sum;
*/
double dE_do = result - target;
double dres_dnet = d_output();
return dE_do * do_dres() * dres_dnet;
}
public void applyError(double learnSpeed, double error) {
for (Node in : inputs.keySet()) {
double w = inputs.get(in);
w -= in.output()*learnSpeed*error;
setInput(in, w);
}
affinity -= learnSpeed*error;
}
private void setInput(Node inputtingNode, double weight) {
inputs.put(inputtingNode, weight);
inputtingNode.outputs.put(this, weight);
}
public void printInfo() {
System.out.println("Node " + name + " (input " + input + ", affinity " + affinity + "):");
for (Node inputNode : inputs.keySet()) {
System.out.println("\tWeight from " + inputNode.name + ": " + inputs.get(inputNode));
}
System.out.println("\tOutput: " + output());
for (Node outputNode : outputs.keySet()) {
System.out.println("\tWeight to " + outputNode.name + ": " + outputs.get(outputNode));
}
}
public static interface ActivationFunction {
double f(double x);
double df(double x);
}
public static class LinAF implements ActivationFunction {
@Override
public double f(double x) {
return x;
}
@Override
public double df(double x) {
return 1;
}
}
public static class LogisticAF implements ActivationFunction {
@Override
public double f(double x) {
return 1/(1+Math.exp(-x));
}
@Override
public double df(double x) {
return f(x)*(1-f(x));
}
}
}
public static void main(String[] args) {
Random rand = new Random();
Node a = new Node("a", new Node.LinAF());
Node z = new Node("z", new Node.LinAF());
NeuralNetwork n = new NeuralNetwork();
n.addNode(a);
n.addNode(z);
for (int i = 1; i <= 8; ++i) {
Node q = new Node("q"+i, new Node.LogisticAF());
n.addNode(q);
q.setInput(a, rand.nextDouble());
q.affinity = rand.nextDouble();
z.setInput(q, rand.nextDouble());
}
z.affinity = rand.nextDouble();
HashMap<Node, Double> inputs = new HashMap<Node, Double>();
for (int i = 0; i < 1000000; ++i) {
double j = 2*rand.nextDouble()-1;
inputs.put(a, j);
n.train(inputs, z, targetFunc(j), 0.3);
if (i % 200000 == 0) {
for (Node node : n.nodes) {
node.printInfo();
}
System.out.println("Input: " + j + ", Target: " + targetFunc(j) + ", Result: " + z.output());
System.out.println();
System.out.flush();
}
}
for (double j = -1; j < 1; j += 0.05) {
inputs.put(a, j);
System.out.println(targetFunc(j) + ", " + n.ask(inputs, z) + " ;");
}
}
public static double targetFunc(double x) {
return Math.sin(8*x);
}
}
Author
Topic: if self.isCoder(): post() #Programming Thread (Read 886269 times)