Skip to content

Commit

Permalink
Adds migration and column for doc_type_label and fixes virus scanni…
Browse files Browse the repository at this point in the history
…ng logic (#424)

* Adds migration and column for doc type label for user_files table. 
* Fixes an issue with virus scanning if using the noop scanner (would always say the files was scanned even if feature was disabled.)
* Creates annotation for @DynamicField and some tests for that work.
* Adds dynamic field handling for validation
* Updates readme 
Co-Authored-By: Lauren Kemperman; lkemperman@codeforamerica.org
  • Loading branch information
lkemperman-cfa authored Nov 2, 2023
1 parent a8626c9 commit 5ca8004
Show file tree
Hide file tree
Showing 31 changed files with 536 additions and 151 deletions.
219 changes: 175 additions & 44 deletions README.md

Large diffs are not rendered by default.

34 changes: 22 additions & 12 deletions src/main/java/formflow/library/FileController.java
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ public class FileController extends FormFlowController {
private final String SESSION_USERFILES_KEY = "userFiles";
private final Integer maxFiles;

@Value("${form-flow.uploads.default-doc-type-label:#{null}}")
private String defaultDocType;

@Value("${form-flow.uploads.virus-scanning.enabled:false}")
private boolean isVirusScanningEnabled;

private final ObjectMapper objectMapper = new ObjectMapper();

public FileController(
Expand Down Expand Up @@ -123,19 +129,22 @@ public ResponseEntity<?> upload(
return new ResponseEntity<>(message, HttpStatus.UNSUPPORTED_MEDIA_TYPE);
}

boolean wasScannedForVirus = true;
try {
if (fileVirusScanner.virusDetected(file)) {
String message = messageSource.getMessage("upload-documents.error-virus-found", null, locale);
return new ResponseEntity<>(message, HttpStatus.UNPROCESSABLE_ENTITY);
}
} catch (WebClientResponseException | TimeoutException e) {
if (blockIfClammitUnreachable) {
log.error("The virus scan service could not be reached. Blocking upload.");
String message = messageSource.getMessage("upload-documents.error-virus-scanner-unavailable", null, locale);
return new ResponseEntity<>(message, HttpStatus.SERVICE_UNAVAILABLE);
boolean wasScannedForVirus = false;
if (isVirusScanningEnabled) {
try {
if (fileVirusScanner.virusDetected(file)) {
String message = messageSource.getMessage("upload-documents.error-virus-found", null, locale);
return new ResponseEntity<>(message, HttpStatus.UNPROCESSABLE_ENTITY);
}
wasScannedForVirus = true;
} catch (WebClientResponseException | TimeoutException e) {
wasScannedForVirus = false;
if (blockIfClammitUnreachable) {
log.error("The virus scan service could not be reached. Blocking upload.");
String message = messageSource.getMessage("upload-documents.error-virus-scanner-unavailable", null, locale);
return new ResponseEntity<>(message, HttpStatus.SERVICE_UNAVAILABLE);
}
}
wasScannedForVirus = false;
}

if (fileValidationService.isTooLarge(file)) {
Expand Down Expand Up @@ -174,6 +183,7 @@ public ResponseEntity<?> upload(
.filesize((float) file.getSize())
.mimeType(file.getContentType())
.virusScanned(wasScannedForVirus)
.docTypeLabel(defaultDocType)
.build();

uploadedFile = userFileRepositoryService.save(uploadedFile);
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/formflow/library/ScreenController.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import formflow.library.data.SubmissionRepositoryService;
import formflow.library.data.UserFileRepositoryService;
import formflow.library.file.FileValidationService;
import formflow.library.inputs.UnvalidatedField;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpSession;
import java.io.IOException;
Expand Down Expand Up @@ -43,6 +42,8 @@
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.view.RedirectView;

import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS;

/**
* A controller to render any screen in flows, including subflows.
*/
Expand Down Expand Up @@ -686,7 +687,7 @@ private void handleAddressValidation(Submission submission, FormSubmission formS
formSubmission.setValidatedAddress(validatedAddresses);
// clear lingering address(es) from the submission stored in the database.
formSubmission.getAddressValidationFields().forEach(item -> {
String inputName = item.replace(UnvalidatedField.VALIDATE_ADDRESS, "");
String inputName = item.replace(UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS, "");
submission.clearAddressFields(inputName);
});
}
Expand Down
44 changes: 39 additions & 5 deletions src/main/java/formflow/library/ValidationService.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -17,7 +18,9 @@
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.apache.commons.lang3.StringUtils;

import static formflow.library.inputs.FieldNameMarkers.DYNAMIC_FIELD_MARKER;

/**
* A service that validates flow inputs based on input definition.
Expand Down Expand Up @@ -45,8 +48,8 @@ public class ValidationService {
/**
* Autoconfigured constructor.
*
* @param validator Validator from Jakarta package.
* @param actionManager the <code>ActionManager</code> that manages the logic to be run at specific points
* @param validator Validator from Jakarta package.
* @param actionManager the <code>ActionManager</code> that manages the logic to be run at specific points
* @param inputConfigPath the package path where inputs classes are located
*/
public ValidationService(Validator validator, ActionManager actionManager,
Expand Down Expand Up @@ -95,18 +98,48 @@ private Map<String, List<String>> performFieldLevelValidation(String flowName, F
}

formSubmission.getFormData().forEach((key, value) -> {
boolean dynamicField = false;
var messages = new ArrayList<String>();
List<String> annotationNames = null;

if (key.contains("[]")) {
key = key.replace("[]", "");
}

String originalKey = key;

if (key.contains(DYNAMIC_FIELD_MARKER)) {
dynamicField = true;
key = StringUtils.substringBefore(key, DYNAMIC_FIELD_MARKER);
}

try {
annotationNames = Arrays.stream(flowClass.getDeclaredField(key).getDeclaredAnnotations())
.map(annotation -> annotation.annotationType().getName()).toList();
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
if (dynamicField) {
throw new RuntimeException(
String.format(
"Input field '%s' has dynamic field marker '%s' in its name, but we are unable to " +
"find the field in the input file. Is it a dynamic field?",
originalKey, DYNAMIC_FIELD_MARKER
)
);
} else {
throw new RuntimeException(e);
}
}

// if it's acting like a dynamic field, then ensure that it is marked as one
if (dynamicField) {
if (!annotationNames.contains("formflow.library.data.annotations.DynamicField")) {
throw new RuntimeException(
String.format(
"Field name '%s' (field: '%s') acts like it's a dynamic field, but the field does not contain the @DynamicField annotation",
key, originalKey
)
);
}
}

if (Collections.disjoint(annotationNames, requiredAnnotationsList) && value.equals("")) {
Expand All @@ -118,7 +151,8 @@ private Map<String, List<String>> performFieldLevelValidation(String flowName, F
.forEach(violation -> messages.add(violation.getMessage()));

if (!messages.isEmpty()) {
validationMessages.put(key, messages);
// uses original key to accommodate dynamic input names
validationMessages.put(originalKey, messages);
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
import com.smartystreets.api.us_street.Batch;
import com.smartystreets.api.us_street.Lookup;
import formflow.library.data.FormSubmission;
import formflow.library.inputs.UnvalidatedField;
import formflow.library.inputs.AddressParts;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS;

@Slf4j
@Component
public class ValidationRequestFactory {
Expand All @@ -21,8 +22,11 @@ public ValidationRequestFactory() {
public Batch create(FormSubmission formSubmission) {
Batch smartyBatch = new Batch();
List<String> addressInputNames = formSubmission.getFormData().keySet().stream()
.filter(key -> key.startsWith(UnvalidatedField.VALIDATE_ADDRESS) && formSubmission.getFormData().get(key).equals("true"))
.map(key -> key.substring(UnvalidatedField.VALIDATE_ADDRESS.length())).toList();
.filter(key ->
key.startsWith(UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS.toString()) &&
formSubmission.getFormData().get(key).equals("true")
)
.map(key -> key.substring(UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS.toString().length())).toList();

addressInputNames.forEach(inputName -> {
Lookup lookup = new Lookup();
Expand Down
22 changes: 14 additions & 8 deletions src/main/java/formflow/library/data/FormSubmission.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package formflow.library.data;

import formflow.library.address_validation.ValidatedAddress;
import formflow.library.inputs.UnvalidatedField;
import formflow.library.inputs.AddressParts;
import java.util.ArrayList;
import java.util.List;
Expand All @@ -11,11 +10,18 @@
import lombok.Data;
import org.springframework.util.MultiValueMap;

import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_CSRF;
import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS;
import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_VALIDATED;

@Data
public class FormSubmission {

public Map<String, Object> formData;
private List<String> unvalidatedFields = List.of(UnvalidatedField.CSRF, UnvalidatedField.VALIDATE_ADDRESS);
private List<String> unvalidatedFields = List.of(
UNVALIDATED_FIELD_MARKER_CSRF,
UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS
);

public FormSubmission(MultiValueMap<String, String> formData) {
this.formData = removeEmptyValuesAndFlatten(formData);
Expand Down Expand Up @@ -54,19 +60,19 @@ public Map<String, Object> getValidatableFields() {
*/
public List<String> getAddressValidationFields() {
return formData.entrySet().stream()
.filter(entry -> entry.getKey().startsWith(UnvalidatedField.VALIDATE_ADDRESS))
.filter(entry -> entry.getKey().startsWith(UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS))
.filter(entry -> entry.getValue().toString().equalsIgnoreCase("true"))
.map(Entry::getKey).toList();
}

public void setValidatedAddress(Map<String, ValidatedAddress> validatedAddresses) {
validatedAddresses.forEach((key, value) -> {
if (value != null) {
formData.put(key + AddressParts.STREET_ADDRESS_1 + UnvalidatedField.VALIDATED, value.getStreetAddress());
formData.put(key + AddressParts.STREET_ADDRESS_2 + UnvalidatedField.VALIDATED, value.getApartmentNumber());
formData.put(key + AddressParts.CITY + UnvalidatedField.VALIDATED, value.getCity());
formData.put(key + AddressParts.STATE + UnvalidatedField.VALIDATED, value.getState());
formData.put(key + AddressParts.ZIPCODE + UnvalidatedField.VALIDATED, value.getZipCode());
formData.put(key + AddressParts.STREET_ADDRESS_1 + UNVALIDATED_FIELD_MARKER_VALIDATED, value.getStreetAddress());
formData.put(key + AddressParts.STREET_ADDRESS_2 + UNVALIDATED_FIELD_MARKER_VALIDATED, value.getApartmentNumber());
formData.put(key + AddressParts.CITY + UNVALIDATED_FIELD_MARKER_VALIDATED, value.getCity());
formData.put(key + AddressParts.STATE + UNVALIDATED_FIELD_MARKER_VALIDATED, value.getState());
formData.put(key + AddressParts.ZIPCODE + UNVALIDATED_FIELD_MARKER_VALIDATED, value.getZipCode());
}
});
}
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/formflow/library/data/Submission.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package formflow.library.data;

import static formflow.library.inputs.FieldNameMarkers.UNVALIDATED_FIELD_MARKER_VALIDATED;
import static jakarta.persistence.TemporalType.TIMESTAMP;

import formflow.library.inputs.AddressParts;
import formflow.library.inputs.UnvalidatedField;
import io.hypersistence.utils.hibernate.type.json.JsonType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
Expand Down Expand Up @@ -203,11 +203,11 @@ public Boolean getIterationIsCompleteStatus(String subflow, String iterationUuid
*/
public void clearAddressFields(String inputName) {
// we want to clear out
inputData.remove(inputName + AddressParts.STREET_ADDRESS_1 + UnvalidatedField.VALIDATED);
inputData.remove(inputName + AddressParts.STREET_ADDRESS_2 + UnvalidatedField.VALIDATED);
inputData.remove(inputName + AddressParts.CITY + UnvalidatedField.VALIDATED);
inputData.remove(inputName + AddressParts.STATE + UnvalidatedField.VALIDATED);
inputData.remove(inputName + AddressParts.ZIPCODE + UnvalidatedField.VALIDATED);
inputData.remove(inputName + AddressParts.STREET_ADDRESS_1 + UNVALIDATED_FIELD_MARKER_VALIDATED);
inputData.remove(inputName + AddressParts.STREET_ADDRESS_2 + UNVALIDATED_FIELD_MARKER_VALIDATED);
inputData.remove(inputName + AddressParts.CITY + UNVALIDATED_FIELD_MARKER_VALIDATED);
inputData.remove(inputName + AddressParts.STATE + UNVALIDATED_FIELD_MARKER_VALIDATED);
inputData.remove(inputName + AddressParts.ZIPCODE + UNVALIDATED_FIELD_MARKER_VALIDATED);
}

/**
Expand Down
10 changes: 6 additions & 4 deletions src/main/java/formflow/library/data/UserFile.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,6 @@ public class UserFile {

@Id
@GeneratedValue
//@Type(type = "org.hibernate.type.UUIDCharType")
//@Type(type = "pg-uuid")
private UUID fileId;

@ManyToOne
Expand All @@ -65,10 +63,13 @@ public class UserFile {

@Column
private Float filesize;

@Column(name = "virus_scanned")
private boolean virusScanned;

@Column(name = "doc_type_label")
private String docTypeLabel;

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down Expand Up @@ -102,6 +103,7 @@ public static HashMap<String, String> createFileInfo(UserFile userFile, String t
fileInfo.put("filesize", userFile.getFilesize().toString());
fileInfo.put("thumbnailUrl", thumbBase64String);
fileInfo.put("type", userFile.getMimeType());
fileInfo.put("docTypeLabel", userFile.getDocTypeLabel());
return fileInfo;
}
}
}
24 changes: 24 additions & 0 deletions src/main/java/formflow/library/data/annotations/DynamicField.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package formflow.library.data.annotations;

import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
import static java.lang.annotation.ElementType.TYPE;
import static java.lang.annotation.RetentionPolicy.RUNTIME;

import jakarta.validation.Payload;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;

@Target({ElementType.FIELD, TYPE, ANNOTATION_TYPE})
@Retention(RUNTIME)
@Documented
public @interface DynamicField {

String message() default "";

Class<?>[] groups() default {};

Class<? extends Payload>[] payload() default {};

}
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package formflow.library.data.validators;
package formflow.library.data.annotations;

import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.RetentionPolicy.RUNTIME;

import formflow.library.data.validators.MoneyValidator;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package formflow.library.data.validators;
package formflow.library.data.annotations;

import static java.lang.annotation.ElementType.FIELD;
import static java.lang.annotation.RetentionPolicy.RUNTIME;

import formflow.library.data.validators.PhoneValidator;
import jakarta.validation.Constraint;
import jakarta.validation.Payload;
import java.lang.annotation.Documented;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package formflow.library.data.validators;

import formflow.library.data.annotations.Money;
import java.util.regex.Pattern;
import jakarta.validation.ConstraintValidator;
import jakarta.validation.ConstraintValidatorContext;
Expand All @@ -8,6 +9,7 @@
* This class validates that Money has been passed in the correct format - one or more digits, a dot and 2 digits after the dot.
*/
public class MoneyValidator implements ConstraintValidator<Money, String> {

@Override
public boolean isValid(String value, ConstraintValidatorContext context) {
return Pattern.matches("(^(0|([1-9]\\d*))?(\\.\\d{1,2})?$)?", value);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package formflow.library.data.validators;

import formflow.library.data.annotations.Phone;
import jakarta.validation.ConstraintValidator;
import jakarta.validation.ConstraintValidatorContext;
import java.util.regex.Pattern;
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/formflow/library/inputs/FieldNameMarkers.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package formflow.library.inputs;

public class FieldNameMarkers {

public static final String UNVALIDATED_FIELD_MARKER_CSRF = "_csrf";
public static final String UNVALIDATED_FIELD_MARKER_VALIDATE_ADDRESS = "validate_";
public static final String UNVALIDATED_FIELD_MARKER_VALIDATED = "_validated";
public static final String DYNAMIC_FIELD_MARKER = "_wildcard_";
}
Loading

0 comments on commit 5ca8004

Please sign in to comment.