CDPT-1984 Remove students from upload

Gemfile

@@ -34,3 +34,7 @@ end
 group :test, :development do
   gem "brakeman"
 end
+
+group :development do
+  gem "rack-console"
+end

Gemfile.lock

@@ -161,6 +161,9 @@ GEM
       nio4r (~> 2.0)
     racc (1.8.1)
     rack (2.2.9)
+    rack-console (1.3.2)
+      rack (>= 1.1)
+      rack-test
     rack-protection (2.2.3)
       rack
     rack-test (2.1.0)
@@ -173,19 +176,19 @@ GEM
     reline (0.5.9)
       io-console (~> 0.5)
     rexml (3.3.7)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.1)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
     rubocop (1.64.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
@@ -290,6 +293,7 @@ DEPENDENCIES
   pg
   poltergeist (>= 1.4.0)
   puma
+  rack-console
   rack-test
   rake
   rspec

README.md

@@ -77,6 +77,12 @@ $ bundle exec rackup
 
 The site will be accessible at http://localhost:9292/api/v1/mediators.
 
+#### Local console:
+
+```
+$ bundle exec rack-console
+```
+
 ### Run tests:
 
 ```
@@ -146,7 +152,7 @@ All dates are in ISO 8601 standard. All booleans are a string 'Yes' or 'No', to
 
 #### Simple detail
 * **id** *int*: internal ID of the mediator
-* **urn** *string*: registration number of the mediator, composed of a numeric root followed by a letter [ATP] indicating the status of the mediator. 'A' means certified, 'P' provisionally certified, and 'T' in training.
+* **urn** *string*: registration number of the mediator, composed of a numeric root followed by a letter [ATPS] indicating the status of the mediator. 'A' means certified, 'P' provisionally certified 'T' in training and 'S' student. Students are removed on import.
 * **dcc** *boolean*: Whether the mediator is habilitated to work with children directly.
 * **title** *string*: 'Mr', 'Mrs', etc.
 * **first_name** *string*: Given name - 'Sue', 'Peter', etc.

config/kubernetes/staging/ingress.yml

@@ -13,8 +13,9 @@ metadata:
       location ~* \.(php|cgi|xml)$ { deny all; access_log off; }
     nginx.ingress.kubernetes.io/enable-modsecurity: "true"
     nginx.ingress.kubernetes.io/modsecurity-snippet: |
-      SecRuleEngine On
-      SecDefaultAction "phase:2,pass,log,tag:github_team=central-digital-product-team,tag:namespace=family-mediators-api-staging"
+      SecAuditEngine On
+      SecRuleEngine DetectionOnly
+      SecDefaultAction "phase:2,pass,log,tag:github_team=central-digital-product-team,tag:namespace=family-mediators-api-production"
 spec:
   ingressClassName: modsec
   tls:

lib/admin/admin.rb

@@ -10,6 +10,7 @@
 require_relative "processing/headings"
 require_relative "processing/marshaler"
 require_relative "processing/confidential_field_remover"
+require_relative "processing/record_remover"
 require_relative "services/process_file"
 require_relative "services/process_data"
 require_relative "helpers"

lib/admin/parsers/workbook.rb

@@ -21,26 +21,15 @@ def call
     private
 
       def parse_mediators
-        as_hashes = transform_mediators
-        remove_blank_rows(as_hashes)
+        Processing::RecordRemover.new(transform_mediators).call
       end
 
       def parse_blacklist
         blacklist_worksheet[1..].map { |row| row.cells.first.value }
       end
 
       def processed_blacklist
-        @headings_processor.process(parse_blacklist)
-      end
-
-      def remove_blank_rows(hashes)
-        hashes.each_with_object([]) do |row, compacted|
-          compacted << row unless all_values_blank?(row)
-        end
-      end
-
-      def all_values_blank?(row)
-        row.values.all? { |val| (val && val.empty?) || val.nil? }
+        @headings_processor.call(parse_blacklist)
       end
 
       def transform_mediators
@@ -56,7 +45,7 @@ def transform_mediators
       end
 
       def processed_headings
-        @processed_headings ||= @headings_processor.process(
+        @processed_headings ||= @headings_processor.call(
           mediators_worksheet[0].cells.map(&:value),
         )
       end

lib/admin/processing/headings.rb

@@ -1,7 +1,7 @@
 module Admin
   module Processing
     class Headings
-      def self.process(headings)
+      def self.call(headings)
         headings.map! { |heading|
           heading ? heading.downcase.gsub(/[^a-z0-9]/, "_").squeeze("_").to_sym : nil
         }.compact

lib/admin/processing/record_remover.rb

@@ -0,0 +1,37 @@
+module Admin
+  module Processing
+    class RecordRemover
+      def initialize(records)
+        @records = records
+      end
+
+      def call
+        @records = remove_blank_records
+        @records = remove_students
+        @records
+      end
+
+    private
+
+      def remove_blank_records
+        @records.each_with_object([]) do |record, compacted|
+          compacted << record unless all_values_blank?(record)
+        end
+      end
+
+      def all_values_blank?(record)
+        record.values.all? { |val| (val && val.empty?) || val.nil? }
+      end
+
+      def remove_students
+        @records.each_with_object([]) do |record, without_students|
+          without_students << record unless is_student?(record)
+        end
+      end
+
+      def is_student?(record)
+        record[:urn]&.last == "S"
+      end
+    end
+  end
+end

lib/admin/validators/mediator_validator.rb

@@ -3,7 +3,7 @@
 module Admin
   module Validators
     class MediatorValidator < Dry::Validation::Contract
-      URN_FORMAT = /^\d{4}[TAP]$/
+      URN_FORMAT = /^\d{4}[TAPS]$/
       YES_NO_VALUES = %w[Yes No].freeze
 
       schema do

spec/admin/processing/headings_spec.rb

@@ -2,24 +2,24 @@ module Admin
   module Processing
     describe Headings do
       it "Replaces spaces with underscores" do
-        expect(described_class.process(["foo bar"])).to eq [:foo_bar]
+        expect(described_class.call(["foo bar"])).to eq [:foo_bar]
       end
 
       it "Downcases" do
-        expect(described_class.process(%w[FooBar])).to eq [:foobar]
+        expect(described_class.call(%w[FooBar])).to eq [:foobar]
       end
 
       it "Replaces slashes with underscores" do
-        expect(described_class.process(["foo/bar"])).to eq [:foo_bar]
+        expect(described_class.call(["foo/bar"])).to eq [:foo_bar]
       end
 
       it "Replaces multiple non alpha chars with ONE underscore" do
-        expect(described_class.process(["foo@@@£££££bar"])).to eq [:foo_bar]
-        expect(described_class.process(["foo     bar"])).to eq [:foo_bar]
+        expect(described_class.call(["foo@@@£££££bar"])).to eq [:foo_bar]
+        expect(described_class.call(["foo     bar"])).to eq [:foo_bar]
       end
 
       it "Handles an array correctly" do
-        expect(described_class.process(%w[foo bar])).to eq %i[foo bar]
+        expect(described_class.call(%w[foo bar])).to eq %i[foo bar]
       end
     end
   end

spec/admin/processing/record_remover_spec.rb

@@ -0,0 +1,27 @@
+module Admin
+  module Processing
+    describe RecordRemover do
+      subject(:remover) { described_class.new(records) }
+
+      describe "when there are blank records" do
+        let(:records) { [record_blanks, record_normal] }
+        let(:record_blanks) { { first_name: "", last_name: "", urn: nil } }
+        let(:record_normal) { { first_name: "John", last_name: "Smith", urn: "1234A" } }
+
+        it "Removes blank fields" do
+          expect(remover.call).to eq([record_normal])
+        end
+      end
+
+      describe "when there are student records" do
+        let(:records) { [record_certified, record_student] }
+        let(:record_certified) { { first_name: "Jane", last_name: "Jones", urn: "1234A" } }
+        let(:record_student) { { first_name: "John", last_name: "Smith", urn: "1234S" } }
+
+        it "Removes students" do
+          expect(remover.call).to eq([record_certified])
+        end
+      end
+    end
+  end
+end