1 package net.bmahe.genetics4j.samples.neat;
2
3 import java.util.List;
4 import java.util.Map;
5 import java.util.Set;
6
7 import org.apache.logging.log4j.LogManager;
8 import org.apache.logging.log4j.Logger;
9
10 import net.bmahe.genetics4j.core.EASystem;
11 import net.bmahe.genetics4j.core.EASystemFactory;
12 import net.bmahe.genetics4j.core.Fitness;
13 import net.bmahe.genetics4j.core.evolutionlisteners.EvolutionListeners;
14 import net.bmahe.genetics4j.core.spec.EAConfiguration;
15 import net.bmahe.genetics4j.core.spec.EAConfiguration.Builder;
16 import net.bmahe.genetics4j.core.spec.Optimization;
17 import net.bmahe.genetics4j.core.spec.mutation.CreepMutation;
18 import net.bmahe.genetics4j.core.spec.mutation.MultiMutations;
19 import net.bmahe.genetics4j.core.spec.mutation.MutationPolicy;
20 import net.bmahe.genetics4j.core.spec.mutation.RandomMutation;
21 import net.bmahe.genetics4j.core.spec.selection.SelectionPolicy;
22 import net.bmahe.genetics4j.core.spec.selection.Tournament;
23 import net.bmahe.genetics4j.core.spec.statistics.distributions.NormalDistribution;
24 import net.bmahe.genetics4j.core.termination.Terminations;
25 import net.bmahe.genetics4j.neat.Activations;
26 import net.bmahe.genetics4j.neat.Connection;
27 import net.bmahe.genetics4j.neat.NeatEAExecutionContexts;
28 import net.bmahe.genetics4j.neat.NeatUtils;
29 import net.bmahe.genetics4j.neat.RecurrentNetwork;
30 import net.bmahe.genetics4j.neat.chromosomes.NeatChromosome;
31 import net.bmahe.genetics4j.neat.spec.NeatChromosomeSpec;
32 import net.bmahe.genetics4j.neat.spec.combination.NeatCombination;
33 import net.bmahe.genetics4j.neat.spec.mutation.AddConnection;
34 import net.bmahe.genetics4j.neat.spec.mutation.AddNode;
35 import net.bmahe.genetics4j.neat.spec.mutation.DeleteConnection;
36 import net.bmahe.genetics4j.neat.spec.mutation.DeleteNode;
37 import net.bmahe.genetics4j.neat.spec.mutation.NeatConnectionWeight;
38 import net.bmahe.genetics4j.neat.spec.mutation.SwitchStateMutation;
39 import net.bmahe.genetics4j.neat.spec.selection.NeatSelection;
40
41 public final class RecurrentControlExample {
42
43 private static final Logger logger = LogManager.getLogger(RecurrentControlExample.class);
44
45 private RecurrentControlExample() {
46 }
47
48
49 private static final List<Float> START_POSITIONS = List.of(-4.0f, -2.5f, 2.5f, 4.0f);
50 private static final float TARGET_POSITION = 0.0F;
51 private static final float DT = 0.25F;
52 private static final int HORIZON_STEPS = 28;
53 private static final float MAX_THRUST = 1.0F;
54 private static final float POSITION_NORMALIZER = 5.0F;
55
56
57 public static void main(final String[] args) {
58 final EAConfiguration<Float> configuration = buildControlConfiguration();
59 final var executionContext = NeatEAExecutionContexts.<Float>standard()
60 .populationSize(200)
61 .addEvolutionListeners(EvolutionListeners.ofLogTopN(logger, 2))
62 .build();
63
64 final EASystem<Float> eaSystem = EASystemFactory.from(configuration, executionContext);
65 final var result = eaSystem.evolve();
66
67 logger.info("Best controller fitness: {}", result.bestFitness());
68 describeBestController(result.bestIndividual().genotype().getChromosome(0, NeatChromosome.class));
69 }
70
71
72 private static EAConfiguration<Float> buildControlConfiguration() {
73 final Builder<Float> builder = new EAConfiguration.Builder<>();
74 builder.chromosomeSpecs(NeatChromosomeSpec.of(2, 1, -4, 4))
75 .parentSelectionPolicy(buildSelectionPolicy())
76 .combinationPolicy(NeatCombination.build())
77 .mutationPolicies(buildMutations())
78 .fitness(createControllerFitness(false))
79 .optimization(Optimization.MAXIMIZE)
80 .termination(Terminations.<Float>or(Terminations.ofStableFitness(400), Terminations.ofMaxGeneration(2500)));
81 return builder.build();
82 }
83
84
85 private static SelectionPolicy buildSelectionPolicy() {
86 return NeatSelection.builder()
87 .minSpeciesSize(1)
88 .perSpeciesKeepRatio(0.25f)
89 .speciesSelection(Tournament.of(3))
90 .speciesPredicate(
91 (i1, i2) -> NeatUtils.compatibilityDistance(i1.genotype(), i2.genotype(), 0, 2, 2, 1f) < 4.0)
92 .build();
93 }
94
95 private static List<MutationPolicy> buildMutations() {
96 return List.of(
97 MultiMutations.of(
98 CreepMutation.of(0.75f, NormalDistribution.of(0.0, 0.333)),
99 RandomMutation.of(0.65f),
100 NeatConnectionWeight.builder().populationMutationProbability(0.8f).build()),
101 MultiMutations.of(
102 SwitchStateMutation.of(0.02f),
103 AddNode.of(0.2f),
104 DeleteNode.of(0.15f),
105 AddConnection.builder().populationMutationProbability(0.6f).allowOutputNodeAsSource(true).build(),
106 DeleteConnection.of(0.4f)));
107 }
108
109
110 private static Fitness<Float> createControllerFitness(final boolean verbose) {
111 return genotype -> {
112 final NeatChromosome chromosome = genotype.getChromosome(0, NeatChromosome.class);
113 if (chromosome.getConnections().isEmpty()) {
114 return 0f;
115 }
116
117 final RecurrentNetwork network = buildNetwork(chromosome);
118 final Integer outputNodeIndex = chromosome.getOutputNodeIds().getFirst();
119
120 float cumulativeError = 0.0F;
121 for (final float startPosition : START_POSITIONS) {
122 cumulativeError += simulateEpisode(network, outputNodeIndex, startPosition, verbose);
123 }
124
125 final float maxScore = START_POSITIONS.size() * HORIZON_STEPS * 5.0f;
126 return Math.max(0.0f, maxScore - cumulativeError);
127 };
128 }
129
130
131 private static float simulateEpisode(final RecurrentNetwork network, final Integer outputNodeIndex,
132 final float startPosition, final boolean verbose) {
133 float position = startPosition;
134 float velocity = 0.0F;
135 float episodeError = 0.0F;
136
137 network.resetState();
138
139 for (int step = 0; step < HORIZON_STEPS; step++) {
140 final Map<Integer, Float> outputs = network.step(Map.of(0, normalizedPosition(position), 1, 1.0f));
141 float thrust = outputs.getOrDefault(outputNodeIndex, 0.0f);
142 thrust = clamp(thrust, MAX_THRUST);
143
144 velocity += thrust * DT;
145 position += velocity * DT;
146
147 final float error = Math.abs(position - TARGET_POSITION);
148 episodeError += error;
149
150 if (verbose) {
151 logger.info("start={} step={} pos={} vel={} thrust={}", startPosition, step, position, velocity, thrust);
152 }
153 }
154
155 return episodeError;
156 }
157
158 private static RecurrentNetwork buildNetwork(final NeatChromosome chromosome) {
159 final Set<Integer> inputIndices = Set.copyOf(chromosome.getInputNodeIds());
160 final Set<Integer> outputIndices = Set.copyOf(chromosome.getOutputNodeIds());
161 final List<Connection> connections = chromosome.getConnections();
162
163 return new RecurrentNetwork(inputIndices, outputIndices, connections, Activations.tanhFloat, 16, 1e-4f, 0.0f);
164 }
165
166 private static float normalizedPosition(final float position) {
167 return clamp(position / POSITION_NORMALIZER, 1.0f);
168 }
169
170 private static float clamp(final float value, final float limit) {
171 return Math.max(-limit, Math.min(limit, value));
172 }
173
174 private static void describeBestController(final NeatChromosome chromosome) {
175 final RecurrentNetwork network = buildNetwork(chromosome);
176 final Integer outputNodeIndex = chromosome.getOutputNodeIds().getFirst();
177
178 for (final float startPosition : START_POSITIONS) {
179 simulateEpisode(network, outputNodeIndex, startPosition, true);
180 }
181 }
182 }