🐛 Validate delete rows the way import runs them

app/assets/javascripts/bulkrax/importers_stepper.js

@@ -1556,6 +1556,7 @@
       works: data.works,
       fileSets: data.fileSets || data.file_sets,
       totalItems: data.totalItems != null ? data.totalItems : data.total_items,
+      deletionCount: data.deletionCount || 0,
       headers: data.headers,
       missingRequired: data.missingRequired || data.missing_required,
       unrecognized: data.unrecognized,
@@ -2211,6 +2212,10 @@
       ? '<span class="tree-existing-badge" title="' + t('existing_record_title') + '">' +
         t('existing_record_badge') + '</span>'
       : ''
+    var deletingBadge = item.deleting
+      ? '<span class="tree-deleting-badge" title="' + t('deleting_record_title') + '">' +
+        t('deleting_record_badge') + '</span>'
+      : ''
 
     var html =
       '<div class="tree-item' + (isExisting ? ' tree-item-existing' : '') + '" data-item-id="' +
@@ -2234,6 +2239,7 @@
         : '') +
       count +
       existingBadge +
+      deletingBadge +
       '</div>'
 
     if (hasChildren) {
@@ -2471,6 +2477,9 @@
           file_sets: data.fileSets.length
         }) +
         '</p>'
+      if (data.deletionCount > 0) {
+        recordsHtml += '<p>' + t('review_deletions', { count: data.deletionCount }) + '</p>'
+      }
     } else {
       recordsHtml = '<p class="text-muted">' + t('review_skipped') + '</p>'
     }

app/assets/stylesheets/bulkrax/stepper/_summary.scss

@@ -122,6 +122,12 @@ $summary-variants: (
   vertical-align: middle;
 }
 
+.tree-deleting-badge {
+  @extend .tree-existing-badge;
+  color: $color-error-dark;
+  background: $bg-error;
+}
+
 .tree-shared-badge {
   display: inline-flex;
   align-items: center;

app/controllers/bulkrax/entries_controller.rb

@@ -28,6 +28,7 @@ def update
              end
       # do not run counters as it loads the whole parser
       item = @entry.importerexporter
+      item.clear_deleted_identifiers_unless_importing if item.is_a?(Bulkrax::Importer)
       current_run = item.current_run(skip_counts: true)
       @entry.set_status_info('Pending', current_run)
       ScheduleRelationshipsJob.set(wait: 5.minutes).perform_later(importer_id: @entry.importer.id)

app/models/bulkrax/importer.rb

@@ -239,6 +239,17 @@ def import_relationships
 
     DEFAULT_OBJECT_TYPES = %w[collection work file_set relationship].freeze
 
+    # A single entry re-run after the import finished may relate records a
+    # later import recreated, so the finished run's deleted identifiers
+    # (CsvParser#record_deleted_identifiers) no longer apply. While entries
+    # are still Pending, the run's own jobs still need them.
+    def clear_deleted_identifiers_unless_importing
+      return if parser_fields['deleted_identifiers'].blank? || entries.pending.exists?
+
+      parser_fields['deleted_identifiers'] = []
+      save
+    end
+
     def import_objects(types_array = nil)
       self.only_updates ||= false
       self.save if self.new_record? # Object needs to be saved for statuses

app/models/concerns/bulkrax/import_behavior.rb

@@ -41,20 +41,27 @@ def add_user_to_permission_templates!
 
     def parent_jobs
       self.parsed_metadata[related_parents_parsed_mapping].each do |parent_identifier|
-        next if parent_identifier.blank?
+        next if parent_identifier.blank? || deleted_identifier?(parent_identifier)
 
         PendingRelationship.create!(child_id: self.identifier, parent_id: parent_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 
     def child_jobs
       self.parsed_metadata[related_children_parsed_mapping].each do |child_identifier|
-        next if child_identifier.blank?
+        next if child_identifier.blank? || deleted_identifier?(child_identifier)
 
         PendingRelationship.create!(parent_id: self.identifier, child_id: child_identifier, importer_run_id: importerexporter.last_run.id, order: self.id)
       end
     end
 
+    # A row this import deletes (CsvParser#record_deleted_identifiers) can
+    # never be related, so no pending relationship to it is written.
+    def deleted_identifier?(identifier)
+      @deleted_identifiers ||= Array(importerexporter.parser_fields['deleted_identifiers']).to_set
+      @deleted_identifiers.include?(identifier.to_s)
+    end
+
     def find_collection_ids
       self.collection_ids
     end

app/parsers/bulkrax/application_parser.rb

@@ -258,12 +258,18 @@ def record_raw_metadata(record)
 
     def record_deleted?(record)
       return false unless record.key?(:delete)
-      ActiveModel::Type::Boolean.new.cast(record[:delete])
+      self.class.flag_set?(record[:delete])
     end
 
     def record_remove_and_rerun?(record)
       return false unless record.key?(:remove_and_rerun)
-      ActiveModel::Type::Boolean.new.cast(record[:remove_and_rerun])
+      self.class.flag_set?(record[:remove_and_rerun])
+    end
+
+    # Whether a row's flag cell, such as `delete`, is on. Guided validation
+    # reads delete rows with the same rule.
+    def self.flag_set?(value)
+      ActiveModel::Type::Boolean.new.cast(value) || false
     end
 
     def create_entry_and_job(current_record, type, identifier = nil)

app/parsers/bulkrax/csv_parser.rb

@@ -23,6 +23,47 @@ def self.file_split_pattern
       Bulkrax::SplitPatternCoercion.coerce(split_value) || Bulkrax.multi_value_element_split_on
     end
 
+    def create_objects(types_array = nil)
+      record_deleted_identifiers(types_array)
+      super
+    rescue StandardError => e
+      set_status_info(e)
+    end
+
+    # Each row's import job skips a relationship to a row this import deletes
+    # (ImportBehavior#parent_jobs), even one whose own job has not run yet, so
+    # the identifiers are saved before any job is queued.
+    def record_deleted_identifiers(types_array = nil)
+      types = (types_array || %w[collection work file_set]) - ['relationship']
+      return if types.empty?
+
+      deleted = queued_records(types).select { |record| record_deleted?(record) }
+      mapping = deleted.empty? ? {} : importerexporter.mapping || {}
+      identifiers = deleted.flat_map { |record| [record[source_identifier], Bulkrax::FieldResolver.row_id(record.to_h.transform_keys(&:to_s), mapping)&.last] }
+                           .map { |value| value.to_s.strip }.reject(&:empty?)
+      fields = importerexporter.parser_fields
+      # Each record type may be submitted separately; keep the run's earlier ones.
+      earlier = fields['deleted_identifiers_run_id'] == current_run.id ? Array(fields['deleted_identifiers']) : []
+      fields['deleted_identifiers'] = (earlier + identifiers).uniq
+      fields['deleted_identifiers_run_id'] = current_run.id
+      importerexporter.save
+    end
+
+    # An Existing Entries run submits entries as imports and deletes nothing.
+    def rebuild_entries(types_array = nil)
+      importerexporter.parser_fields['deleted_identifiers'] = []
+      importerexporter.save
+      super
+    end
+
+    # The rows #create_objects queues: those with a source identifier, or one
+    # Bulkrax fills in, up to the import limit.
+    def queued_records(types)
+      fills_in = Bulkrax.fill_in_blank_source_identifiers.present?
+      queued = types.flat_map { |type| send(type.pluralize) }.select { |record| record[source_identifier].present? || fills_in }
+      limit.nil? || limit.zero? ? queued : queued.first(limit)
+    end
+
     def records(_opts = {})
       return @records if @records.present?
 
@@ -112,9 +153,13 @@ def missing_elements(record)
     end
 
     def valid_import?
-      compressed_record = records.flat_map(&:to_a).partition { |_, v| !v }.flatten(1).to_h
-      error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(compressed_record).join(', ')}"
-      raise StandardError, error_alert unless required_elements?(compressed_record)
+      # A row that only deletes needs no metadata: its record is never built.
+      kept = records.reject { |record| record_deleted?(record) }
+      if kept.any?
+        compressed_record = kept.flat_map(&:to_a).partition { |_, v| !v }.flatten(1).to_h
+        error_alert = "Missing at least one required element, missing element(s) are: #{missing_elements(compressed_record).join(', ')}"
+        raise StandardError, error_alert unless required_elements?(compressed_record)
+      end
 
       file_paths.is_a?(Array)
     rescue StandardError => e
@@ -357,7 +402,7 @@ def file_paths
 
       @file_paths ||= records.map do |r|
         raw_values = r.select { |key, _| Bulkrax::FieldResolver.header_for_any?(key, file_headers) }.values.reject(&:blank?)
-        next if raw_values.empty?
+        next if raw_values.empty? || record_deleted?(r)
         raise StandardError, "Record references local files but no files directory could be resolved from the import path" if files_dir.nil?
 
         raw_values.flat_map { |v| v.split(split_pattern) }.map(&:strip).reject(&:empty?).map { |f| existing_file_path(files_dir, f) }

app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb

@@ -27,7 +27,8 @@ def validate_csv(csv_file:, zip_file: nil, admin_set_id: nil, csv_in_zip: false)
             parse_csv_inputs(csv_file, admin_set_id)
 
           all_ids          = csv_data.map { |r| r[:source_identifier] }.compact.to_set
-          header_issues    = check_headers(headers, raw_csv, mapping_manager, mappings, field_metadata, field_analyzer)
+          header_issues    = check_headers(headers, raw_csv, mapping_manager, mappings, field_metadata, field_analyzer,
+                                           required_models: required_header_models(csv_data, headers, mappings))
           missing_required = header_issues[:missing_required]
           notices, row_errors, collections, works, file_sets =
             run_validations(csv_data, all_ids, headers, source_id_key, mappings, field_metadata, missing_required, zip_file,
@@ -102,7 +103,7 @@ def parse_csv_inputs(csv_file, admin_set_id)
         end
 
         # Runs all header-level checks and returns a hash of results.
-        def check_headers(headers, raw_csv, mapping_manager, mappings, field_metadata, field_analyzer) # rubocop:disable Metrics/ParameterLists
+        def check_headers(headers, raw_csv, mapping_manager, mappings, field_metadata, field_analyzer, required_models: field_metadata.keys) # rubocop:disable Metrics/ParameterLists
           all_models    = field_metadata.keys
           valid_headers = build_valid_validation_headers(mapping_manager, field_analyzer,
                                                          all_models, mappings, field_metadata)
@@ -118,7 +119,7 @@ def check_headers(headers, raw_csv, mapping_manager, mappings, field_metadata, f
           valid_headers = (valid_headers + suffixed).uniq
 
           {
-            missing_required: find_missing_required_headers(headers, field_metadata, mapping_manager),
+            missing_required: find_missing_required_headers(headers, field_metadata.slice(*required_models), mapping_manager),
             unrecognized: find_unrecognized_validation_headers(headers, valid_headers,
                                                                mapping_manager: mapping_manager,
                                                                field_metadata: field_metadata),
@@ -144,6 +145,7 @@ def extract_hierarchy_items(csv_data, all_ids, find_record, mappings, find_exist
           extract_validation_items(
             csv_data, all_ids, find_record,
             find_existing_record: find_existing_record,
+            deleted_ids: deleted_identifiers(csv_data, mappings, find_existing_record),
             parent_split_pattern: resolve_parent_split_pattern(mappings),
             child_split_pattern: resolve_children_split_pattern(mappings),
             parent_columns: parent_columns,
@@ -158,30 +160,24 @@ def relationship_columns(mappings)
 
         # Runs all registered row validators and returns the collected errors.
         def run_row_validators(csv_data, all_ids, source_id_key, mappings, field_metadata, find_record, notices = [], mapping_manager: nil, zip_plan: nil, find_existing_record: nil) # rubocop:disable Metrics/ParameterLists
-          parent_columns, children_columns = relationship_columns(mappings)
           context = {
             errors: [],
             warnings: [],
             seen_ids: {},
             all_ids: all_ids,
             source_identifier: source_id_key.to_s,
-            parent_split_pattern: resolve_parent_split_pattern(mappings),
-            child_split_pattern: resolve_children_split_pattern(mappings),
-            parent_column: parent_columns.first,
-            children_column: children_columns.first,
-            parent_columns: parent_columns,
-            children_columns: children_columns,
             mappings: mappings,
             mapping_manager: mapping_manager,
             field_metadata: field_metadata,
             find_record_by_source_identifier: find_record,
             find_existing_record: find_existing_record,
-            relationship_graph: build_relationship_graph(csv_data, mappings),
             notices: notices,
-            zip_plan: zip_plan
+            zip_plan: zip_plan,
+            **relationship_context(csv_data, mappings, find_existing_record)
           }
           csv_data.each_with_index do |record, index|
-            Bulkrax.csv_row_validators.each { |v| v.call(record, index + 2, context) } # 1-indexed, plus header row
+            row_number = index + 2 # 1-indexed, plus header row
+            Bulkrax.csv_row_validators.each { |v| v.call(record, row_number, context) }
           end
           context[:errors]
         end