diff --git a/ddm/static/ddm/css/research-interface-base.css b/ddm/static/ddm/css/research-interface-base.css index 024304e..c865427 100644 --- a/ddm/static/ddm/css/research-interface-base.css +++ b/ddm/static/ddm/css/research-interface-base.css @@ -21,6 +21,10 @@ align-items: center; } +.fs-08 { + font-size: 0.8rem !important; +} + .ddm-nav-icon { color: #1a1a1a !important; font-size: 1.5rem; @@ -168,7 +172,6 @@ .ddm-admin-section-body { padding: 20px; border-radius: 0 0 5px 5px; - background: #fbfbfb; } .ddm-admin-section-body tbody { border-top: none !important; @@ -391,4 +394,52 @@ } .version-info { color: lightgray; +} + +.ddm-info-accordion { + border-left: 2px solid black; + border-right: 2px solid black; + margin: 15px 10PX 25px 10px; +} + +.ddm-info-accordion-header { + background: #f2f2f2; +} + +.ddm-info-accordion-button { + padding: 5px 15px; + background: none; + border: none; + width: 100%; + text-align: left; +} + +.ddm-info-accordion-button:after { + content: "\F229"; + font-family: "bootstrap-icons"; + display: inline-block; + float: right; + transition-duration: .35s; +} + +.ddm-info-accordion-button:not(.collapsed):after { + content: "\F229"; + font-family: "bootstrap-icons"; + display: inline-block; + transform: rotate(-180deg); + transition-duration: .35s; + float: right; +} + +.ddm-info-accordion-body { + padding: 15px 21px; + border-bottom: 1px solid lightgrey; +} + +.ddm-info-accordion-body ol { + line-height: 1.5rem; +} + +.ddm-info-accordion-body li { + padding-left: 5px; } \ No newline at end of file diff --git a/ddm/static/ddm/js/processing-rule-modal.js b/ddm/static/ddm/js/processing-rule-modal.js index a384956..2ac7521 100644 --- a/ddm/static/ddm/js/processing-rule-modal.js +++ b/ddm/static/ddm/js/processing-rule-modal.js @@ -1,10 +1,10 @@ -hideOrShowReplacementAndComparisonValue = function( id ) { +hideOrShowReplacementAndComparisonValue = function (id) { let replacementInput = $("#id_processingrule_set-" + id + "-replacement_value"); let comparisonValue = $("#id_processingrule_set-" + id + "-comparison_value"); let val = $("[id$=" + id + "-comparison_operator]").val(); - if (val.indexOf("regex-replace-match") >= 0 ) { + if (val.indexOf("regex-replace-match") >= 0) { replacementInput.parent().show(); comparisonValue.parent().show(); } else if (val === "") { @@ -14,21 +14,27 @@ hideOrShowReplacementAndComparisonValue = function( id ) { replacementInput.parent().hide(); comparisonValue.parent().show(); } + + if (val.indexOf("regex") >= 0) { + $('label[for="id_processingrule_set-' + id + '-comparison_value"]').text("Regular expression (regex):"); + } else { + $('label[for="id_processingrule_set-' + id + '-comparison_value"]').text("Comparison value:"); + } } -$( "body" ).on("change", "select[id$='-comparison_operator']", function() { +$("body").on("change", "select[id$='-comparison_operator']", function () { const current_id = $(this).attr("id").match(/\d+/)[0]; hideOrShowReplacementAndComparisonValue(current_id); }); -$(document).ready(function() { +$(document).ready(function () { let IDs = new Set(); - $("[id$=-comparison_operator]").each(function() { - if( /\d+/.test($( this ).attr("id")) ) { - IDs.add($( this ).attr("id").match(/\d+/)[0]); + $("[id$=-comparison_operator]").each(function () { + if (/\d+/.test($(this).attr("id"))) { + IDs.add($(this).attr("id").match(/\d+/)[0]); } }); - for ( const id of IDs ) { + for (const id of IDs) { hideOrShowReplacementAndComparisonValue(id); } }); diff --git a/ddm/templates/ddm/admin/components/info_collapsible.html b/ddm/templates/ddm/admin/components/info_collapsible.html new file mode 100644 index 0000000..bdac3b3 --- /dev/null +++ b/ddm/templates/ddm/admin/components/info_collapsible.html @@ -0,0 +1,19 @@ +
+ +
+ +
+ +
+
+ {% if include_path != "" %} + {% include include_path %} + {% else %} + {{ body|safe }} + {% endif %} +
+
+ +
diff --git a/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_data_extraction.html b/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_data_extraction.html index 9b1cc09..99a728a 100644 --- a/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_data_extraction.html +++ b/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_data_extraction.html @@ -1,15 +1,90 @@ -
Data Extraction
- \ No newline at end of file + + +

When a file is uploaded, the configured rules are applied row-wise/entry-wise. This means, for every data row/entry in the uploaded data, the rules are applied in the configured order.

+ +

Available extraction operators

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Extraction OperatorDescriptionNote
Keep FieldKeep this field in the uploaded data.
Equal (==)Delete row/entry if the value contained in the given field equals the comparison value.Works for strings, integers, and dates1.
Not Equal !=Delete row/entry if the value contained in the given field does not equal the comparison value.Works for strings, integers, and dates1.
Greater than (>)Delete row/entry if the value contained in the given field is greater than the comparison value.Works for integers and dates1. String values are skipped and the row will be kept in the data.
Smaller than (<)Delete row/entry if the value contained in the given field is smaller than the comparison value.Works for integers and dates1. String values are skipped and the row will be kept in the data.
Greater than or equal (>=)Delete row/entry if the value contained in the given field is greater than or equal to the comparison value.Works for integers and dates1. String values are skipped and the row will be kept in the data.
Smaller than or equal (<=)Delete row/entry if the value contained in the given field is smaller than or equal to the comparison value.Works for integers and dates1. String values are skipped and the row will be kept in the data.
Delete match (regex)Delete parts of the value contained in the given field that match the given regular expression (regex) + (e.g., if the regular expression (regex) = "^Watched " and a field contains the value "Watched video XY" the following + value will be kept in the uploaded data: "video XY").All field values are converted to strings before this operation is applied.
Replace match (regex)Replace parts of the value contained in the given field that match the given regular expression (regex) + (e.g., if the regular expression (regex) = "[\w-\.]+@([\w-]+\.)+[\w-]{2,4}" and + the replacement value = "_anonymized_" and a field contains the value "some text email@address.com" the following + value will be kept in the uploaded data: "some text _anonymized_").All field values are converted to strings before this operation is applied.
Delete row when match (regex)Delete row/entry if the value contained in the given field matches the given regular expression (regex) (e.g., if + regular expression (regex) = "^Watched " and a field contains the value "Watched video XY" + the row/entry will be deleted from the uploaded data).All field values are converted to strings before this operation is applied.
+ +

+ 1Dates are inferred from string values if they are formatted according to ISO, RFC2822, or HTTP standards, + and only if both the field value and the comparison value follow the same format. Otherwise, the entry will be treated as a regular string. +

diff --git a/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_file_validation.html b/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_file_validation.html index e78287e..ea302f8 100644 --- a/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_file_validation.html +++ b/ddm/templates/ddm/admin/data_donation/donation_blueprint/block_file_validation.html @@ -1,17 +1,11 @@ -
-

Data Extraction Settings

- -

Data Extraction is a two-step process consisting of first the file validation and second the data extraction.

- -
File Validation
- -
\ No newline at end of file +

+ When a file is uploaded, it is first validated following these steps: +

+
    +
  1. [Only applies to ZIP uploads] It is checked whether the uploaded ZIP container contains the expected file as specified in the file path setting.
  2. +
  3. It is checked whether the uploaded file is in the expected file format.
  4. +
  5. It is checked whether the identified file contains all expected fields.
  6. +
+

+ If any of these validation steps fail, no data is extracted and an exception message explaining what went wrong will be shown to the participant. +

diff --git a/ddm/templates/ddm/admin/data_donation/donation_blueprint/edit.html b/ddm/templates/ddm/admin/data_donation/donation_blueprint/edit.html index 10ad38c..7d535a9 100644 --- a/ddm/templates/ddm/admin/data_donation/donation_blueprint/edit.html +++ b/ddm/templates/ddm/admin/data_donation/donation_blueprint/edit.html @@ -13,6 +13,8 @@ {{ form.non_field_errors }}
+
General Settings
+ {% for field in form %} {% if field.name not in "expected_fields,expected_fields_regex_matching,regex_path,exp_file_format,csv_delimiter,json_extraction_root" %}

@@ -25,9 +27,13 @@ {% endfor %}

- {% include "ddm/admin/data_donation/donation_blueprint/block_file_validation.html" %} +
+
File Validation Settings
+ + {% include "ddm/admin/components/info_collapsible.html" with element_id="filevalidation" title="How the file validation works" include_path="ddm/admin/data_donation/donation_blueprint/block_file_validation.html" body="" %} + {% for field in form %} {% if field.name in "expected_fields,expected_fields_regex_matching,regex_path,exp_file_format,csv_delimiter,json_extraction_root" %}

@@ -40,12 +46,16 @@ {% endfor %}

- {% include "ddm/admin/data_donation/donation_blueprint/block_data_extraction.html" %} +
- {{ formset.management_form }}
-
Extraction Rules
- +
Data Extraction Settings
+ {% include "ddm/admin/components/info_collapsible.html" with element_id="dataextraction" title="How the data extraction works" include_path="ddm/admin/data_donation/donation_blueprint/block_data_extraction.html" body="" %} + + {{ formset.management_form }} + + Extraction Rules +
@@ -138,8 +148,8 @@
Extraction Rules
{% for field in formset.empty_form %} {% if field.name in 'name,field,regex_field,execution_order,input_type,comparison_operator,comparison_value,replacement_value' %}

- {{ field.label }} -
{{ field.help_text }}
+ {{ field.label_tag }} + {{ field.help_text }} {{ field }} {{ field.errors }} @@ -149,7 +159,9 @@

Extraction Rules
- +
+ +

🠐 Back

Order Rule name