1 package net.bmahe.genetics4j.samples.mixturemodel;
2
3 import java.io.IOException;
4 import java.nio.charset.StandardCharsets;
5 import java.nio.file.Path;
6 import java.util.HashSet;
7 import java.util.Map;
8 import java.util.Map.Entry;
9 import java.util.Objects;
10 import java.util.Set;
11 import java.util.TreeMap;
12 import java.util.stream.IntStream;
13
14 import org.apache.commons.csv.CSVFormat;
15 import org.apache.commons.csv.CSVPrinter;
16 import org.apache.commons.lang3.Validate;
17 import org.apache.commons.math3.distribution.MultivariateNormalDistribution;
18 import org.apache.commons.math3.exception.MathUnsupportedOperationException;
19 import org.apache.commons.math3.linear.NonPositiveDefiniteMatrixException;
20 import org.apache.commons.math3.linear.SingularMatrixException;
21 import org.apache.logging.log4j.LogManager;
22 import org.apache.logging.log4j.Logger;
23
24 import net.bmahe.genetics4j.core.Genotype;
25 import net.bmahe.genetics4j.core.Individual;
26 import net.bmahe.genetics4j.core.chromosomes.DoubleChromosome;
27 import net.bmahe.genetics4j.core.chromosomes.FloatChromosome;
28 import net.bmahe.genetics4j.core.spec.EvolutionResult;
29 import net.bmahe.genetics4j.moo.FitnessVector;
30
31 public class ClusteringUtils {
32 public static final Logger logger = LogManager.getLogger(ClusteringUtils.class);
33
34 public static int[] assignClustersDoubleChromosome(final int distributionNumParameters, final double[][] samples,
35 final Genotype genotype) {
36
37 final var fChromosome = genotype.getChromosome(0, DoubleChromosome.class);
38 final int[] clusters = new int[samples.length];
39 final double[] bestProb = new double[samples.length];
40
41 for (int c = 0; c < clusters.length; c++) {
42 clusters[c] = 0;
43 bestProb[c] = Double.MIN_VALUE;
44 }
45
46 double sumAlpha = 0.0F;
47 int k = 0;
48 while (k < fChromosome.getSize()) {
49 sumAlpha += fChromosome.getAllele(k);
50 k += distributionNumParameters;
51 }
52
53 int i = 0;
54 int clusterIndex = 0;
55 while (i < fChromosome.getSize()) {
56
57 final double alpha = fChromosome.getAllele(i) / sumAlpha;
58 final double[] mean = new double[] { fChromosome.getAllele(i + 1), fChromosome.getAllele(i + 2) };
59 final double[][] covariance = new double[][] {
60 { fChromosome.getAllele(i + 3) - 15, fChromosome.getAllele(i + 4) - 15 },
61 { fChromosome.getAllele(i + 4) - 15, fChromosome.getAllele(i + 5) - 15 } };
62
63 try {
64 final var multivariateNormalDistribution = new MultivariateNormalDistribution(mean, covariance);
65
66 for (int j = 0; j < samples.length; j++) {
67 float likelyhood = (float) (alpha * multivariateNormalDistribution.density(samples[j]));
68
69 if (clusters[j] < 0 || bestProb[j] < likelyhood) {
70 bestProb[j] = likelyhood;
71 clusters[j] = clusterIndex;
72 }
73 }
74 } catch (NonPositiveDefiniteMatrixException | SingularMatrixException | MathUnsupportedOperationException e) {
75 }
76
77 i += distributionNumParameters;
78 clusterIndex++;
79 }
80
81 return clusters;
82 }
83
84 public static int[] assignClustersFloatChromosome(final int distributionNumParameters, final double[][] samples,
85 final Genotype genotype) {
86
87 final var fChromosome = genotype.getChromosome(0, FloatChromosome.class);
88 final int[] clusters = new int[samples.length];
89 final double[] bestProb = new double[samples.length];
90
91 for (int c = 0; c < clusters.length; c++) {
92 clusters[c] = 0;
93 bestProb[c] = Double.MIN_VALUE;
94 }
95
96 double sumAlpha = 0.0F;
97 int k = 0;
98 while (k < fChromosome.getSize()) {
99 sumAlpha += fChromosome.getAllele(k);
100 k += distributionNumParameters;
101 }
102
103 int i = 0;
104 int clusterIndex = 0;
105 while (i < fChromosome.getSize()) {
106
107 final double alpha = fChromosome.getAllele(i) / sumAlpha;
108 final double[] mean = new double[] { fChromosome.getAllele(i + 1), fChromosome.getAllele(i + 2) };
109 final double[][] covariance = new double[][] {
110 { fChromosome.getAllele(i + 3) - 15, fChromosome.getAllele(i + 4) - 15 },
111 { fChromosome.getAllele(i + 4) - 15, fChromosome.getAllele(i + 5) - 15 } };
112
113 try {
114 final var multivariateNormalDistribution = new MultivariateNormalDistribution(mean, covariance);
115
116 for (int j = 0; j < samples.length; j++) {
117 float likelyhood = (float) (alpha * multivariateNormalDistribution.density(samples[j]));
118
119 if (clusters[j] < 0 || bestProb[j] < likelyhood) {
120 bestProb[j] = likelyhood;
121 clusters[j] = clusterIndex;
122 }
123 }
124 } catch (NonPositiveDefiniteMatrixException | SingularMatrixException | MathUnsupportedOperationException e) {
125 }
126
127 i += distributionNumParameters;
128 clusterIndex++;
129 }
130
131 return clusters;
132 }
133
134 public static void persistClusters(final float[] x, final float[] y, final int[] cluster, final String filename)
135 throws IOException {
136 Validate.isTrue(x.length == y.length);
137 Validate.isTrue(x.length == cluster.length);
138 logger.info("Saving clusters to CSV: {}", filename);
139
140 final CSVPrinter csvPrinter;
141 try {
142 csvPrinter = CSVFormat.DEFAULT.withAutoFlush(true)
143 .withHeader(new String[] { "cluster", "x", "y" })
144 .print(Path.of(filename), StandardCharsets.UTF_8);
145 } catch (IOException e) {
146 logger.error("Could not open {}", filename, e);
147 throw new RuntimeException("Could not open file " + filename, e);
148 }
149
150 for (int i = 0; i < cluster.length; i++) {
151 try {
152 csvPrinter.printRecord(cluster[i], x[i], y[i]);
153 } catch (IOException e) {
154 throw new RuntimeException("Could not write data", e);
155 }
156 }
157 csvPrinter.close(true);
158 }
159
160
161 public static void persistClusters(final double[] x, final double[] y, final int[] cluster, final String filename)
162 throws IOException {
163 Validate.isTrue(x.length == y.length);
164 Validate.isTrue(x.length == cluster.length);
165 logger.info("Saving clusters to CSV: {}", filename);
166
167 final CSVPrinter csvPrinter;
168 try {
169 csvPrinter = CSVFormat.DEFAULT.withAutoFlush(true)
170 .withHeader(new String[] { "cluster", "x", "y" })
171 .print(Path.of(filename), StandardCharsets.UTF_8);
172 } catch (IOException e) {
173 logger.error("Could not open {}", filename, e);
174 throw new RuntimeException("Could not open file " + filename, e);
175 }
176
177 for (int i = 0; i < cluster.length; i++) {
178 try {
179 csvPrinter.printRecord(cluster[i], x[i], y[i]);
180 } catch (IOException e) {
181 throw new RuntimeException("Could not write data", e);
182 }
183 }
184 csvPrinter.close(true);
185 }
186
187 public static Map<Integer, Individual<FitnessVector<Float>>> groupByNumClusters(final double[][] samplesDouble,
188 final EvolutionResult<FitnessVector<Float>> evolutionResult) {
189 Validate.notEmpty(samplesDouble);
190 Objects.requireNonNull(evolutionResult);
191
192 final Map<Integer, Individual<FitnessVector<Float>>> groups = new TreeMap<>();
193
194 final var listFitnessResult = evolutionResult.fitness();
195 final var populationResult = evolutionResult.population();
196
197 for (int i = 0; i < populationResult.size(); i++) {
198
199 final var genotype = populationResult.get(i);
200 final var fitness = listFitnessResult.get(i);
201
202 groups.compute(
203 Math.round(fitness.get(1)),
204 (k, currentBestIndividual) -> currentBestIndividual == null
205 || currentBestIndividual.fitness().get(0) < fitness.get(0) ? Individual.of(genotype, fitness)
206 : currentBestIndividual);
207 }
208
209 return groups;
210 }
211
212 public static void categorizeByNumClusters(final int distributionNumParameters, final int maxPossibleDistributions,
213 final float[] x, final float[] y, final double[][] samplesDouble,
214 final EvolutionResult<FitnessVector<Float>> evolutionResult, final String baseDir, final String type)
215 throws IOException {
216 Validate.notEmpty(samplesDouble);
217 Objects.requireNonNull(evolutionResult);
218 Validate.notBlank(baseDir);
219 Validate.notBlank(type);
220
221 final var groupedByNumClusters = groupByNumClusters(samplesDouble, evolutionResult);
222 logger.info("Groups:");
223 for (Entry<Integer, Individual<FitnessVector<Float>>> entry : groupedByNumClusters.entrySet()) {
224 final int numUnusedClusters = entry.getKey();
225 final var individual = entry.getValue();
226
227 final int numClusters = maxPossibleDistributions - numUnusedClusters;
228
229 logger.info(
230 "\tNum Clusters: {} - Unused Clusters: {} - Fitness: {}",
231 numClusters,
232 numUnusedClusters,
233 individual.fitness());
234
235 final int[] assignedClusters = ClusteringUtils
236 .assignClustersFloatChromosome(distributionNumParameters, samplesDouble, individual.genotype());
237 final Set<Integer> uniqueAssigned = new HashSet<>();
238 uniqueAssigned.addAll(IntStream.of(assignedClusters).boxed().toList());
239
240 ClusteringUtils.persistClusters(
241 x,
242 y,
243 assignedClusters,
244 baseDir + "assigned-" + type + "-" + uniqueAssigned.size() + ".csv");
245 }
246 }
247
248 public static void writeCSVReferenceValue(final String filename, final int generations, final Number value)
249 throws IOException {
250 Validate.notBlank(filename);
251 Validate.isTrue(generations > 0);
252 Objects.requireNonNull(value);
253
254 final var csvPrinter = CSVFormat.DEFAULT.withAutoFlush(true)
255 .withHeader("generation", "fitness")
256 .print(Path.of(filename), StandardCharsets.UTF_8);
257
258 for (int i = 0; i < generations; i++) {
259 csvPrinter.printRecord(i, value);
260 }
261 csvPrinter.close();
262 }
263 }