Skip to content

Commit

Permalink
Autotrack fixes and improvements (#23)
Browse files Browse the repository at this point in the history 
- Fixes incorrectly labeled planes in TrackingView
- Add new setting for vertical setup of the Aurora Field generator
- Add new setting for changing the video resolution
- Add new setting for extensive video device search
- New settings view 
- Fix issues with Filechoosers when last file location is no longer available
- Fix issues when Autotrack and Visualization use one device simultaneously. Now the video stream is shared among all views.
- Temporarily removed sensor curve visualization
- Multiple other fixes
  • Loading branch information
@TheRisenPhoenix
TheRisenPhoenix authored Nov 15, 2023
1 parent 16a8593 commit 812671f
Show file tree
Hide file tree
Showing 16 changed files with 596 additions and 457 deletions.
2 changes: 1 addition & 1 deletion src/main/java/algorithm/ImageDataProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public boolean openConnection(VideoSource source){
public boolean openConnection(VideoSource source, int deviceId){
switch (source) {
case LIVESTREAM:
imgSrc = new LivestreamSource(deviceId);
imgSrc = LivestreamSource.forDevice(deviceId);
break;
case OPENIGTLINK:
imgSrc = new OIGTImageSource();
Expand Down
88 changes: 65 additions & 23 deletions src/main/java/controller/AutoTrackController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import java.nio.file.Path;
import java.util.*;
import java.util.List;
import java.util.logging.Logger;
import java.util.prefs.Preferences;

public class AutoTrackController implements Controller {
Expand All @@ -49,6 +50,8 @@ public class AutoTrackController implements Controller {
private final Map<String, Integer> deviceIdMapping = new LinkedHashMap<>();
private final Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().create();
private static final Preferences userPreferences = Preferences.userRoot().node("AutoTrack");
private static final Preferences userPreferencesGlobal = Preferences.userRoot().node("IGT_Settings");
private final Logger logger = Logger.getLogger(this.getClass().getName());

@FXML
public ChoiceBox<String> sourceChoiceBox;
Expand Down Expand Up @@ -98,6 +101,7 @@ public class AutoTrackController implements Controller {

private final ObservableList<Point3> clicked_image_points = FXCollections.observableArrayList();
private final ObservableList<Point3> clicked_tracker_points = FXCollections.observableArrayList();
private Mat cachedTransformMatrix = null;

@Override
public void initialize(URL location, ResourceBundle resources) {
Expand All @@ -108,6 +112,7 @@ public void initialize(URL location, ResourceBundle resources) {
connectionProgressSpinner.setVisible(false);
captureProgressSpinner.setVisible(false);
sourceChoiceBox.getSelectionModel().selectedItemProperty().addListener(x -> changeVideoView());
sourceChoiceBox.setTooltip(new Tooltip("If you have multiple cameras connected, enable \"Search for more videos\" in the settings view to see all available devices"));
captureRateComboBox.getItems().addAll("1000", "2000", "5000", "10000", "30000");
captureRateComboBox.getSelectionModel().select(0);

Expand Down Expand Up @@ -163,7 +168,7 @@ public void setStatusLabel(Label statusLabel) {
private void loadAvailableVideoDevicesAsync() {
connectionProgressSpinner.setVisible(true);
new Thread(() -> {
createDeviceIdMapping(true);
createDeviceIdMapping(userPreferencesGlobal.getBoolean("searchForMoreVideos", false));
Platform.runLater(() -> {
sourceChoiceBox.getItems().addAll(deviceIdMapping.keySet());
if (!deviceIdMapping.isEmpty()) {
Expand All @@ -179,10 +184,10 @@ private void loadAvailableVideoDevicesAsync() {
/**
* Tests out available video device ids. All devices that don't throw an error are added to the list.
* This is bad style, but openCV does not offer to list available devices.
* @param fast Whether all available devices shall be enumerated. If set to true, there's a minimal performance gain.
* @param exhaustiveSearch Whether all available devices shall be enumerated. If set to false, there's a minimal performance gain.
*/
private void createDeviceIdMapping(boolean fast) {
if(fast){
private void createDeviceIdMapping(boolean exhaustiveSearch) {
if(!exhaustiveSearch){
deviceIdMapping.put("Default Camera",0);
return;
}
Expand Down Expand Up @@ -270,7 +275,8 @@ private void updateTrackingData(){
series.getData().add(new XYChart.Data<>(0,0)); // Workaround to display legend
dataSeries.add(series);
series.getData().remove(0);
videoImagePlot.initSensorCurve(series);
// TODO: The sensor curve needs reworking (apply on transformed data and dont shrink)
// videoImagePlot.initSensorCurve(series);
}

var series = dataSeries.get(i);
Expand All @@ -280,9 +286,12 @@ private void updateTrackingData(){
var max_num_points = 4; // 1

var shifted_points = use3dTransformCheckBox.isSelected() ? applyTrackingTransformation3d(point.getX(), point.getY(), point.getZ()) : applyTrackingTransformation2d(point.getX(), point.getY(), point.getZ());
lastTrackingData.add(new ExportMeasurement(tool.getName(), point.getX(), point.getY(), point.getZ(), shifted_points[0], shifted_points[1], shifted_points[2]));
var x_normalized = shifted_points[0] / currentShowingImage.getWidth();
var y_normalized = shifted_points[1] / currentShowingImage.getHeight();
lastTrackingData.add(new ExportMeasurement(tool.getName(), point.getX(), point.getY(), point.getZ(), shifted_points[0], shifted_points[1], shifted_points[2], x_normalized, y_normalized));

data.add(new XYChart.Data<>(shifted_points[0], shifted_points[1]));

data.add(new XYChart.Data<>(shifted_points[0],shifted_points[1]));
if(data.size() > max_num_points){
data.remove(0);
}
Expand Down Expand Up @@ -314,6 +323,7 @@ private void saveCapturedData(BufferedImage image, List<ExportMeasurement> track
gson.toJson(trackingData, fw);
}
} catch (IOException e) {
logger.log(java.util.logging.Level.SEVERE, "Error saving captured data", e);
e.printStackTrace();
}
}
Expand All @@ -325,8 +335,8 @@ private void saveCapturedData(BufferedImage image, List<ExportMeasurement> track
public void on_browseOutputDirectory() {
DirectoryChooser directoryChooser = new DirectoryChooser();
directoryChooser.setTitle("Select Output Directory");
var lastLocation = userPreferences.get("outputDirectory",System.getProperty("user.home"));
directoryChooser.setInitialDirectory(new File(lastLocation));
var lastLocationFile = getLastKnownFileLocation("outputDirectory",System.getProperty("user.home"));
directoryChooser.setInitialDirectory(lastLocationFile);
var directory = directoryChooser.showDialog(null);
if (directory != null) {
outputPathField.setText(directory.getAbsolutePath());
Expand All @@ -345,6 +355,7 @@ public void on_openOutputDirectory() {
try {
Desktop.getDesktop().open(new File(directory));
} catch (IOException e) {
logger.log(java.util.logging.Level.SEVERE, "Error opening output directory", e);
e.printStackTrace();
}
}
Expand Down Expand Up @@ -390,8 +401,8 @@ public void on_doAutoCapture() {
public void on_importMatrix() {
FileChooser fileChooser = new FileChooser();
fileChooser.setTitle("Select Matrix JSON");
var lastLocation = userPreferences.get("matrixDirectory",System.getProperty("user.home"));
fileChooser.setInitialDirectory(new File(lastLocation));
var lastLocationFile = getLastKnownFileLocation("matrixDirectory",System.getProperty("user.home"));
fileChooser.setInitialDirectory(lastLocationFile);
var inputFile = fileChooser.showOpenDialog(null);
if(inputFile == null){
return;
Expand All @@ -402,8 +413,10 @@ public void on_importMatrix() {
transformationMatrix = TransformationMatrix.loadFromJSON(path);
roiDirty = true;
regMatrixStatusBox.setSelected(true);
cachedTransformMatrix = null;
userPreferences.put("matrixDirectory", inputFile.getAbsoluteFile().getParent());
}catch (FileNotFoundException e) {
logger.log(java.util.logging.Level.SEVERE, "Error loading matrix", e);
e.printStackTrace();
}
}
Expand All @@ -418,7 +431,9 @@ public void on_reloadMatrix(){
transformationMatrix = TransformationMatrix.loadFromJSON(lastMatrixPath);
roiDirty = true;
regMatrixStatusBox.setSelected(true);
cachedTransformMatrix = null;
}catch (FileNotFoundException e) {
logger.log(java.util.logging.Level.SEVERE, "Error loading matrix", e);
e.printStackTrace();
}
}
Expand All @@ -436,29 +451,43 @@ public void on_generateMatrix(){

FileChooser fileChooser = new FileChooser();
fileChooser.setTitle("Set save location for matrix json");
var lastLocation = userPreferences.get("matrixDirectory",System.getProperty("user.home"));
fileChooser.setInitialDirectory(new File(lastLocation));
var lastLocationFile = getLastKnownFileLocation("matrixDirectory",System.getProperty("user.home"));
fileChooser.setInitialDirectory(lastLocationFile);
fileChooser.setInitialFileName("transformationMatrix.json");
var saveFile = fileChooser.showSaveDialog(null);
if(saveFile != null){
try {
transformationMatrix.saveToJSON(saveFile);
this.transformationMatrix = transformationMatrix;
regMatrixStatusBox.setSelected(true);
cachedTransformMatrix = null;
userPreferences.put("matrixDirectory", saveFile.getAbsoluteFile().getParent());
} catch (IOException e) {
logger.log(java.util.logging.Level.SEVERE, "Error saving matrix", e);
e.printStackTrace();
}
}
}

private File getLastKnownFileLocation(String key, String defaultLocation){
var lastLocation = userPreferences.get(key,defaultLocation);
var lastLocationFile = new File(lastLocation);
if(!lastLocationFile.exists()){
logger.log(java.util.logging.Level.WARNING, "Last directory does not exist, default directory instead");
lastLocationFile = new File(defaultLocation);
}
return lastLocationFile;
}

/**
* Applies the transformation matrix to the image
* @param mat The image to be transformed
* @return The transformed image
*/
private Mat applyImageTransformations(Mat mat){
Imgproc.warpAffine(mat, mat, transformationMatrix.getTranslationMat(), mat.size());
Imgproc.warpAffine(mat, mat, transformationMatrix.getRotationMat(), mat.size());
Imgproc.warpAffine(mat, mat, transformationMatrix.getScaleMat(), mat.size());
// Imgproc.warpAffine(mat, mat, transformationMatrix.getTranslationMat(), mat.size());
// Imgproc.warpAffine(mat, mat, transformationMatrix.getRotationMat(), mat.size());
// Imgproc.warpAffine(mat, mat, transformationMatrix.getScaleMat(), mat.size());

/*
var imagePoints = transformationMatrix.getImagePoints();
Expand Down Expand Up @@ -496,19 +525,27 @@ private Mat applyImageTransformations(Mat mat){
* @param z Z-Coordinate of the point - Ignored in the 2d version
* @return The transformed point as array of length 3 (xyz)
*/
private double[] applyTrackingTransformation2d(double x, double y, double z){
var matrix = transformationMatrix.getTransformMatOpenCvEstimated2d();
private double[] applyTrackingTransformation2d(double x, double y, double z) {
// TODO: Cache matrix
if (cachedTransformMatrix == null){
cachedTransformMatrix = transformationMatrix.getTransformMatOpenCvEstimated2d();
}
var vector = new Mat(3,1, CvType.CV_64F);
vector.put(0,0,x);

if(userPreferencesGlobal.getBoolean("verticalFieldGenerator", false)){
vector.put(0,0,z);
}else{
vector.put(0,0,x);
}
vector.put(1,0,y);
vector.put(2,0,1);

var pos_star = new Mat(2,1,CvType.CV_64F);
Core.gemm(matrix, vector,1, new Mat(),1,pos_star);
Core.gemm(cachedTransformMatrix, vector,1, new Mat(),1,pos_star);
double[] out = new double[3];
out[0] = pos_star.get(0,0)[0];
out[1] = pos_star.get(1,0)[0];
out[2] = z;
out[2] = 0;
return out;
}

Expand All @@ -527,7 +564,7 @@ private double[] applyTrackingTransformation3d(double x, double y, double z){
vector.put(2,0,z);
vector.put(3,0,1);

var pos_star = new Mat(3,1,CvType.CV_64F);
var pos_star = new Mat();
Core.gemm(matrix, vector,1, new Mat(),1,pos_star);
//Core.perspectiveTransform(); // Maybe try this?
double[] out = new double[3];
Expand All @@ -553,7 +590,12 @@ private void onImageClicked(double x, double y){
}

clicked_image_points.add(new Point3(x, y, 0.0));
clicked_tracker_points.add(new Point3(trackingData.get(0).x_raw, trackingData.get(0).y_raw, trackingData.get(0).z_raw));
if(userPreferencesGlobal.getBoolean("verticalFieldGenerator", false)) {
clicked_tracker_points.add(new Point3(trackingData.get(0).z_raw, trackingData.get(0).y_raw, trackingData.get(0).x_raw));
}else {
clicked_tracker_points.add(new Point3(trackingData.get(0).x_raw, trackingData.get(0).y_raw, trackingData.get(0).z_raw));
}

}
}
}
65 changes: 53 additions & 12 deletions src/main/java/controller/SettingsController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,82 @@

import java.net.URL;
import java.util.ResourceBundle;
import java.util.prefs.Preferences;

import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.fxml.FXML;
import javafx.scene.control.CheckBox;
import javafx.scene.control.ListView;
import javafx.scene.control.TextField;
import javafx.scene.control.Tooltip;
import util.CustomLogger;

public class SettingsController implements Controller {
private static final Preferences userPreferences = Preferences.userRoot().node("IGT_Settings");

private static final ObservableList<String> listItems = FXCollections.
observableArrayList();
@FXML public ListView<String> listView;
@FXML public CheckBox consoleOutput;
@FXML public CheckBox searchForMoreVideos;
@FXML public CheckBox verticalFG;

@FXML public TextField videoWidth;
@FXML public TextField videoHeight;

@FXML
private void changeConsoleOutput() {
CustomLogger.changeConsoleOutput();
userPreferences.putBoolean("logToConsole", consoleOutput.isSelected());
}

@FXML
private void onSearchForMoreVideosClicked() {
userPreferences.putBoolean("searchForMoreVideos", searchForMoreVideos.isSelected());
}

@FXML
private void onVerticalFGClicked(){
userPreferences.putBoolean("verticalFieldGenerator", verticalFG.isSelected());
}

@Override
public void initialize(URL location, ResourceBundle resources) {
registerController();
listItems.addAll("Logging");
listView.setItems(listItems);
listView.getSelectionModel().select(0);
if (CustomLogger.isConsoleOn()) {
consoleOutput.setSelected(true);

var consolePreference = userPreferences.getBoolean("logToConsole", true);
consoleOutput.setSelected(consolePreference);
if (!CustomLogger.isConsoleOn() && consolePreference) {
CustomLogger.changeConsoleOutput();
}

var searchForMoreVideosPreference = userPreferences.getBoolean("searchForMoreVideos", false);
searchForMoreVideos.setSelected(searchForMoreVideosPreference);
var tooltip = new Tooltip("When enabled, the autotrack view will try to find and enumerate all video devices that are connected to the computer. This is helpful if you have more than one camera. However, this will increase the time needed before the view is ready");
tooltip.setWrapText(true);
searchForMoreVideos.setTooltip(tooltip);

var exchangeYZPreference = userPreferences.getBoolean("verticalFieldGenerator", false);
verticalFG.setSelected(exchangeYZPreference);

var videoWidthPreference = userPreferences.getInt("videoWidth", 640);
videoWidth.setText(String.valueOf(videoWidthPreference));
var videoHeightPreference = userPreferences.getInt("videoHeight", 480);
videoHeight.setText(String.valueOf(videoHeightPreference));

videoWidth.textProperty().addListener((observable, oldValue, newValue) -> {
if (!newValue.matches("\\d*")) {
videoWidth.setText(newValue.replaceAll("[^\\d]", ""));
}
userPreferences.putInt("videoWidth", Integer.parseInt(videoWidth.getText()));
});

videoHeight.textProperty().addListener((observable, oldValue, newValue) -> {
if (!newValue.matches("\\d*")) {
videoHeight.setText(newValue.replaceAll("[^\\d]", ""));
}
userPreferences.putInt("videoHeight", Integer.parseInt(videoHeight.getText()));
});
}

@FXML
@Override
public void close() {
listItems.clear();
unregisterController();
}
}
8 changes: 4 additions & 4 deletions src/main/java/controller/VideoController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,10 +167,10 @@ private void setInitialImageSize() {
ivHeight.setText(Double.toString(height));
ivWidth.setText(Double.toString(width));

topSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) height));
bottomSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) height));
rightSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) width));
leftSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) width));
topSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) height-1));
bottomSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) height-1));
rightSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) width-1));
leftSpinner.setValueFactory(new SpinnerValueFactory.IntegerSpinnerValueFactory(0,(int) width-1));

// We don't need to remove the old listeners since the valueProperty will be a new one than before (because of the new ValueFactory)
setCropListener();
Expand Down
9 changes: 8 additions & 1 deletion src/main/java/inputOutput/ExportMeasurement.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@ public class ExportMeasurement {
public double y_raw;
@Expose
public double z_raw;
@Expose
public double x_normalized;
@Expose
public double y_normalized;

public ExportMeasurement(String toolName, double x_raw, double y_raw, double z_raw, double x_shifted, double y_shifted, double z_shifted) {
public ExportMeasurement(String toolName, double x_raw, double y_raw, double z_raw, double x_shifted, double y_shifted, double z_shifted, double x_normalized, double y_normalized) {
this.toolName = toolName;
this.x_raw = x_raw;
this.y_raw = y_raw;
Expand All @@ -27,5 +31,8 @@ public ExportMeasurement(String toolName, double x_raw, double y_raw, double z_r
this.x_shifted = x_shifted;
this.y_shifted = y_shifted;
this.z_shifted = z_shifted;

this.x_normalized = x_normalized;
this.y_normalized = y_normalized;
}
}
Loading

0 comments on commit 812671f

Please sign in to comment.