dna_multiclust and dna_getVariables

* Added dna_multiclust back in.
* Added print method.
* Added dna_getVariables back in.
* plot functions for multiclust still missing.

dna/src/main/java/dna/Dna.java

@@ -27,7 +27,7 @@ public class Dna {
 	public static Dna dna;
 	public static Logger logger;
 	public static Sql sql;
-	public static final String date = "2023-08-31";
+	public static final String date = "2023-09-18";
 	public static final String version = "3.0.10";
 	public static final String operatingSystem = System.getProperty("os.name");
 	public static File workingDirectory = null;

dna/src/main/java/dna/HeadlessDna.java

@@ -12,6 +12,7 @@
 
 import export.*;
 import me.tongfei.progressbar.ProgressBar;
+import model.Value;
 import org.jasypt.exceptions.EncryptionOperationNotPossibleException;
 import org.rosuda.JRI.RConsoleOutputStream;
 import org.rosuda.JRI.Rengine;
@@ -746,6 +747,31 @@ private void saveJsonXml(String fileFormat, String outfile) {
 		}
 	}
 
+	/* =================================================================================================================
+	 * Functions for managing variables
+	 * =================================================================================================================
+	 */
+
+	/**
+	 * Retrieve variables and data type definitions for a given statement type (via label).
+	 *
+	 * @param statementTypeLabel  Label of the statement type for which variables should be retrieved.
+	 * @return                    Array list of {@link Value} objects representing variables.
+	 */
+	public ArrayList<Value> getVariables(String statementTypeLabel) {
+		return Dna.sql.getStatementType(statementTypeLabel).getVariables();
+	}
+
+	/**
+	 * Retrieve variables and data type definitions for a given statement type (via ID).
+	 *
+	 * @param statementTypeId  ID of the statement type for which variables should be retrieved.
+	 * @return                 Array list of {@link Value} objects representing variables.
+	 */
+	public ArrayList<Value> getVariables(int statementTypeId) {
+		return Dna.sql.getStatementType(statementTypeId).getVariables();
+	}
+
 	/* =================================================================================================================
 	 * Functions for managing attributes
 	 * =================================================================================================================

dna/src/main/java/export/Exporter.java

@@ -2601,111 +2601,6 @@ public BarplotResult generateBarplotData() {
 		return new BarplotResult(this.variable1, values, counts, attributes, intScale, attributeVariables);
 	}
 
-	/**
-	 * Partition the discourse network into a backbone and redundant set of second-mode entities using penalised
-	 * spectral distances and simulated annealing. This method prepares the data before the algorithm starts.
-	 *
-	 * @param p Penalty parameter.
-	 * @param T Number of iterations.
-	 */
-	/*
-	public void initializePenaltyBackbone(double p, int T) {
-		this.p = p;
-		this.T = T;
-		this.isolates = false; // no isolates initially for full matrix; will be set to true after full matrix has been computed
-
-		// initial values before iterations start
-		this.originalStatements = this.filteredStatements; // to ensure not all isolates are included later
-
-		// full set of concepts C
-		fullConcepts = this.extractLabels(this.filteredStatements, this.variable2, this.variable2Document);
-
-		// full network matrix Y against which we compare in every iteration
-		fullMatrix = this.computeOneModeMatrix(this.filteredStatements, this.qualifierAggregation, this.startDateTime, this.stopDateTime);
-		this.isolates = true; // include isolates in the iterations; will be adjusted to full matrix without isolates manually each time
-
-		// compute normalised eigenvalues for the full matrix; no need to recompute every time as they do not change
-		eigenvaluesFull = computeNormalizedEigenvalues(fullMatrix.getMatrix());
-
-		// pick a random concept c_j from C and save its index
-		int randomConceptIndex = ThreadLocalRandom.current().nextInt(0, fullConcepts.length);
-
-		// final backbone list B, which contains only one random concept initially but will contain the final backbone set in the end
-		finalBackboneList = new ArrayList<String>();
-
-		// add the one uniformly sampled concept c_j to the backbone as the initial solution at t = 0: B <- {c_j}
-		finalBackboneList.add(fullConcepts[randomConceptIndex]);
-
-		// final redundant set R, which is initially C without c_j
-		finalRedundantList = Arrays
-				.stream(fullConcepts)
-				.filter(c -> !c.equals(fullConcepts[randomConceptIndex]))
-				.collect(Collectors.toCollection(ArrayList::new));
-
-		// final statement list: filter the statement list by only retaining those statements that are in the final backbone set B
-		finalStatementList = this.filteredStatements
-				.stream()
-				.filter(s -> finalBackboneList.contains(((Entity) s.get(this.variable2)).getValue()))
-				.collect(Collectors.toCollection(ArrayList::new));
-
-		// final matrix based on the initial final backbone set, Y^B, which is initially identical to the previous matrix
-		finalMatrix = this.computeOneModeMatrix(finalStatementList, this.qualifierAggregation, this.startDateTime, this.stopDateTime);
-
-		// create an initial current backbone set B_0, also with the one c_j concept like in B: B_0 <- {c_j}
-		currentBackboneList = new ArrayList<String>();
-		currentBackboneList.add(fullConcepts[randomConceptIndex]);
-
-		// create an initial current redundant set R_t, which is C without c_j
-		currentRedundantList = Arrays
-				.stream(fullConcepts)
-				.filter(c -> !c.equals(fullConcepts[randomConceptIndex]))
-				.collect(Collectors.toCollection(ArrayList::new));
-
-		// filter the statement list by only retaining those statements that are in the initial current backbone set B_0
-		currentStatementList = this.filteredStatements
-				.stream()
-				.filter(s -> currentBackboneList.contains(((Entity) s.get(this.variable2)).getValue()))
-				.collect(Collectors.toCollection(ArrayList::new));
-
-		// create initial current matrix at t = 0
-		currentMatrix = this.computeOneModeMatrix(currentStatementList, this.qualifierAggregation, this.startDateTime, this.stopDateTime);
-
-		// initialise (empty) action set S
-		this.actionList = new ArrayList<String>();
-
-		// initialise selected action s
-		selectedAction = "";
-
-		// initialise the candidate backbone set at t, B^*_t
-		candidateBackboneList = new ArrayList<String>();
-
-		// initialise the candidate redundant set at t, R^*_t
-		candidateRedundantList = new ArrayList<String>();
-
-		// declare loss comparison result variables
-		oldLoss = 0.0;
-		newLoss = 0.0;
-		finalLoss = 0.0;
-		accept = false;
-
-		// reporting
-		temperatureLog = new ArrayList<Double>();
-		acceptanceProbabilityLog = new ArrayList<Double>();
-		acceptedLog = new ArrayList<Integer>();
-		penalizedBackboneLossLog = new ArrayList<Double>();
-		proposedBackboneSizeLog = new ArrayList<Integer>();
-		acceptedBackboneSizeLog = new ArrayList<Integer>();
-		finalBackboneSizeLog = new ArrayList<Integer>();
-		acceptanceRatioLastHundredIterationsLog = new ArrayList<Double>();
-
-		// matrix algebra declarations
-		eigenvaluesCurrent = new double[0];
-
-		// set to first iteration and start simulated annealing
-		t = 1;
-	}
-	*/
-
 	/**
 	 * Get the current iteration {@code t} of the simulated annealing algorithm.
 	 *
@@ -2724,110 +2619,6 @@ public void setCurrentT(int t) {
 		this.t = t;
 	}
 
-	/**
-	 * Execute the next iteration of the simulated annealing backbone algorithm.
-	 */
-	/*
-	public void iteratePenaltyBackbone() {
-		// first step: make a random move by adding, removing, or swapping a concept and computing a new candidate
-		actionList.clear(); // clear the set of possible actions and repopulate, depending on solution size
-		if (currentBackboneList.size() < 2) { // if there is only one concept, don't remove it because empty backbones do not work
-			actionList.add("add");
-			actionList.add("swap");
-		} else if (currentBackboneList.size() > fullConcepts.length - 2) { // do not create a backbone with all concepts because it would be useless
-			actionList.add("remove");
-			actionList.add("swap");
-		} else { // everything in between one and |C| - 1 concepts: add all three possible moves to the action set
-			actionList.add("add");
-			actionList.add("remove");
-			actionList.add("swap");
-		}
-		Collections.shuffle(actionList); // randomly re-order the action set...
-		selectedAction = actionList.get(0); // and draw the first action (i.e., pick a random action)
-		candidateBackboneList.clear(); // create a candidate copy of the current backbone list, to be modified
-		candidateBackboneList.addAll(currentBackboneList);
-		candidateRedundantList.clear(); // create a candidate copy of the current redundant list, to be modified
-		candidateRedundantList.addAll(currentRedundantList);
-		if (selectedAction.equals("add")) { // if we add a concept...
-			Collections.shuffle(candidateRedundantList); // randomly re-order the current redundant list...
-			candidateBackboneList.add(candidateRedundantList.get(0)); // add the first concept from the redundant list to the backbone...
-			candidateRedundantList.remove(0); // and delete it in turn from the redundant list
-		} else if (selectedAction.equals("remove")) { // if we remove a concept...
-			Collections.shuffle(candidateBackboneList); // randomly re-order the backbone list to pick a random concept for removal as the first element...
-			candidateRedundantList.add(candidateBackboneList.get(0)); // add the selected concept to the redundant list...
-			candidateBackboneList.remove(0); // and remove it from the backbone list
-		} else if (selectedAction.equals("swap")) { //if we swap out a concept...
-			Collections.shuffle(candidateBackboneList); // re-order the backbone list...
-			Collections.shuffle(candidateRedundantList); // re-order the redundant list...
-			candidateBackboneList.add(candidateRedundantList.get(0)); // add the first (random) redundant concept to the backbone list...
-			candidateRedundantList.remove(0); // then remove it from the redundant list...
-			candidateRedundantList.add(candidateBackboneList.get(0)); // add the first (random) backbone concept to the redundant list...
-			candidateBackboneList.remove(0); // then remove it from the backbone list
-		}
-		proposedBackboneSizeLog.add(candidateBackboneList.size()); // log number of concepts in candidate backbone in the current iteration
-		// after executing the action, filter the statement list based on the candidate backbone set B^*_t in order to create the candidate matrix
-		candidateStatementList = this.filteredStatements
-				.stream()
-				.filter(s -> candidateBackboneList.contains(((Entity) s.get(this.variable2)).getValue()))
-				.collect(Collectors.toCollection(ArrayList::new));
-		// create candidate matrix after filtering the statements based on the action that was executed
-		candidateMatrix = this.computeOneModeMatrix(candidateStatementList, this.qualifierAggregation, this.startDateTime, this.stopDateTime);
-		candidateMatrix = this.reduceCandidateMatrix(candidateMatrix, fullMatrix.getRowNames()); // ensure it has the right dimensions by purging isolates relative to the full matrix
-
-		// second step: compare loss between full and previous matrix to loss between full and candidate matrix and accept or reject candidate
-		temperature = 1 - (1 / (1 + Math.exp(-(-5 + (12.0 / T) * t)))); // temperature
-		temperatureLog.add(temperature);
-		eigenvaluesCurrent = computeNormalizedEigenvalues(currentMatrix.getMatrix()); // normalised eigenvalues for the current matrix
-		eigenvaluesCandidate = computeNormalizedEigenvalues(candidateMatrix.getMatrix()); // normalised eigenvalues for the candidate matrix
-		oldLoss = penalizedLoss(eigenvaluesFull, eigenvaluesCurrent, p, currentBackboneList.size(), fullConcepts.length); // spectral distance between full and previous matrix
-		newLoss = penalizedLoss(eigenvaluesFull, eigenvaluesCandidate, p, candidateBackboneList.size(), fullConcepts.length); // spectral distance between full and candidate matrix
-		penalizedBackboneLossLog.add(newLoss); // log the penalised spectral distance between full and candidate solution
-		accept = false;
-		if (newLoss < oldLoss) { // if candidate is better than previous matrix, adopt it as current solution
-			accept = true; // flag this solution for acceptance
-			acceptanceProbabilityLog.add(-1.0); // log the acceptance probability as -1.0; technically it should be 1.0 because the solution was better and hence accepted, but it would be useless for plotting the acceptance probabilities as a diagnostic tool
-			eigenvaluesFinal = computeNormalizedEigenvalues(currentMatrix.getMatrix()); // normalised eigenvalues for the current matrix
-			finalLoss = penalizedLoss(eigenvaluesFull, eigenvaluesFinal, p, finalBackboneList.size(), fullConcepts.length); // test if also better than global optimum so far
-			if (newLoss <= finalLoss) { // if better than the best solution, adopt candidate as new final backbone solution
-				finalBackboneList.clear(); // clear the best solution list
-				finalBackboneList.addAll(candidateBackboneList); // and populate it with the concepts from the candidate solution instead
-				finalRedundantList.clear(); // same with the redundant list
-				finalRedundantList.addAll(candidateRedundantList);
-				finalStatementList.clear(); // same with the final list of statements
-				finalStatementList.addAll(candidateStatementList);
-				finalMatrix = new Matrix(candidateMatrix); // save the candidate matrix as best solution matrix
-			}
-		} else { // if the solution is worse than the previous one, apply Hastings ratio and temperature and compare with random number
-			r = Math.random(); // random double between 0 and 1
-			acceptance = Math.exp(-(newLoss - oldLoss)) * temperature; // acceptance probability
-			acceptanceProbabilityLog.add(acceptance); // log the acceptance probability
-			if (r < acceptance) { // apply probability rule
-				accept = true;
-			}
-		}
-		if (accept) { // if candidate is better than previous matrix...
-			currentBackboneList.clear(); // create candidate copy and save as new current matrix
-			currentBackboneList.addAll(candidateBackboneList);
-			currentRedundantList.clear(); // also save the redundant candidate as new current redundant list
-			currentRedundantList.addAll(candidateRedundantList);
-			currentStatementList.clear(); // save candidate statement list as new current statement list
-			currentStatementList.addAll(candidateStatementList);
-			currentMatrix = new Matrix(candidateMatrix); // save candidate matrix as new current matrix
-			acceptedLog.add(1); // log the acceptance of the proposed candidate
-		} else {
-			acceptedLog.add(0); // log the non-acceptance of the proposed candidate
-		}
-		acceptedBackboneSizeLog.add(currentBackboneList.size()); // log how many concepts are in the current iteration after the decision
-		finalBackboneSizeLog.add(finalBackboneList.size()); // log how many concepts are in the final backbone solution in the current iteration
-		log = 0.0; // compute ratio of acceptances in last up to 100 iterations
-		for (int i = t - 1; i >= t - Math.min(100, t); i--) {
-			log = log + acceptedLog.get(i);
-		}
-		acceptanceRatioLastHundredIterationsLog.add(log / Math.min(100, t)); // log ratio of accepted candidates in the last 100 iterations
-		t = t + 1; // go to next iteration
-	}
-	*/
-
 	/**
 	 * Reduce the dimensions of a candidate matrix with all isolate nodes to the dimensions of the full matrix, which
 	 * does not contain isolate nodes.
@@ -3007,8 +2798,6 @@ public void writeBackboneToFile(String filename) {
 		}
 	}
 
-	/* NEW BACKBONE ATTEMPT FROM HERE */
-
 	/**
 	 * Compute penalized Euclidean spectral distance.
 	 *

dna/src/main/java/gui/BackboneExporter.java

@@ -5,7 +5,6 @@
 import com.github.lgooddatepicker.components.TimePickerSettings;
 import dna.Dna;
 import export.Exporter;
-import export.SimulatedAnnealingBackboneResult;
 import logger.LogEvent;
 import logger.Logger;
 import me.tongfei.progressbar.ProgressBar;
@@ -15,7 +14,6 @@
 import javax.swing.*;
 import javax.swing.event.ListSelectionEvent;
 import javax.swing.event.ListSelectionListener;
-import javax.swing.filechooser.FileNameExtensionFilter;
 import java.awt.*;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
@@ -27,7 +25,6 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * GUI for network export. Allows the user to set up options for exporting

rDNA/rDNA/NAMESPACE

@@ -9,14 +9,17 @@ S3method(autoplot,dna_network_twomode)
 S3method(plot,dna_backbone)
 S3method(print,dna_backbone)
 S3method(print,dna_barplot)
+S3method(print,dna_multiclust)
 S3method(print,dna_network_onemode)
 S3method(print,dna_network_twomode)
 export(dna_backbone)
 export(dna_barplot)
 export(dna_closeDatabase)
 export(dna_getAttributes)
+export(dna_getVariables)
 export(dna_init)
 export(dna_jar)
+export(dna_multiclust)
 export(dna_network)
 export(dna_openConnectionProfile)
 export(dna_openDatabase)