From c19e14992fe81cef1a17ab1dceda4e81888a8979 Mon Sep 17 00:00:00 2001
From: Andrew Cain <macite@gmail.com>
Date: Fri, 25 Oct 2024 18:05:28 +1100
Subject: [PATCH] fix: enhance substitutions for ipynb

---
 app/helpers/file_helper.rb | 73 ++++++++++++++++++++------------------
 1 file changed, 38 insertions(+), 35 deletions(-)

diff --git a/app/helpers/file_helper.rb b/app/helpers/file_helper.rb
index 362d9597c..3772291ff 100644
--- a/app/helpers/file_helper.rb
+++ b/app/helpers/file_helper.rb
@@ -547,40 +547,43 @@ def move_compressed_task_to_new(task)
     task.extract_file_from_done student_work_dir(:new), '*', ->(_task, to_path, name) { "#{to_path}#{name}" }
   end
 
-  REPLACEMENTS_SED_COMMAND = [
-    ['[\\]u0000','␀'],
-    ['[\\]u0001','␁'],
-    ['[\\]u0002','␂'],
-    ['[\\]u0003','␃'],
-    ['[\\]u0004','␄'],
-    ['[\\]u0005','␅'],
-    ['[\\]u0006','␆'],
-    ['[\\]u0007','␇'],
-    ['[\\]u0008','␈'],
-    ['[\\]u0009','␉'],
-    ['[\\]u000A','␊'],
-    ['[\\]u000B','␋'],
-    ['[\\]u000C','␌'],
-    ['[\\]u000D','␍'],
-    ['[\\]u000E','␎'],
-    ['[\\]u000F','␏'],
-    ['[\\]u0010','␐'],
-    ['[\\]u0011','␑'],
-    ['[\\]u0012','␒'],
-    ['[\\]u0013','␓'],
-    ['[\\]u0014','␔'],
-    ['[\\]u0015','␕'],
-    ['[\\]u0016','␖'],
-    ['[\\]u0017','␗'],
-    ['[\\]u0018','␘'],
-    ['[\\]u0019','␙'],
-    ['[\\]u001A','␚'],
-    ['[\\]u001B','␛'],
-    ['[\\]u001C','␜'],
-    ['[\\]u001D','␝'],
-    ['[\\]u001E','␞'],
-    ['[\\]u001F','␟']
-  ].map { |r| "s/#{r[0]}/#{r[1]}/gI" }.join('; ').freeze
+  REPLACEMENTS_PERL_COMMAND = [
+    ['[\\\\]u0000','[NUL]'],
+    ['[\\\\]u0001','[SOH]'],
+    ['[\\\\]u0002','[STX]'],
+    ['[\\\\]u0003','[ETX]'],
+    ['[\\\\]u0004','[EOT]'],
+    ['[\\\\]u0005','[ENQ]'],
+    ['[\\\\]u0006','[ACK]'],
+    ['[\\\\]u0007','[BEL]'],
+    ['[\\\\]u0008','[BS]'],
+    ['(?<![\\\\])[\\\\]b','[BS]'],
+    ['[\\\\]u0009','[HT]'],
+    ['[\\\\]u000A','[LF]'],
+    ['[\\\\]u000B','[VT]'],
+    ['[\\\\]u000C','[FF]'],
+    ['(?<![\\\\])[\\\\]f','[FF]'],
+    ['[\\\\]u000D','[CR]'],
+    ['(?<![\\\\])[\\\\]r','[CR]'],
+    ['[\\\\]u000E','[SO]'],
+    ['[\\\\]u000F','[SI]'],
+    ['[\\\\]u0010','[DLE]'],
+    ['[\\\\]u0011','[DC1]'],
+    ['[\\\\]u0012','[DC2]'],
+    ['[\\\\]u0013','[DC3]'],
+    ['[\\\\]u0014','[DC4]'],
+    ['[\\\\]u0015','[NAK]'],
+    ['[\\\\]u0016','[SYN]'],
+    ['[\\\\]u0017','[ETB]'],
+    ['[\\\\]u0018','[CSN]'],
+    ['[\\\\]u0019','[EM]'],
+    ['[\\\\]u001A','[SUB]'],
+    ['[\\\\]u001B','[ESC]'],
+    ['[\\\\]u001C','[FS]'],
+    ['[\\\\]u001D','[GS]'],
+    ['[\\\\]u001E','[RS]'],
+    ['[\\\\]u001F','[US]']
+  ].map { |r| "s/#{r[0]}/#{r[1]}/gi" }.join(';').freeze
 
   #
   # Ensure that the contents of a file appear to be valid UTF8, on retry convert to ASCII to ensure
@@ -597,7 +600,7 @@ def ensure_utf8_code(output_filename, force_ascii)
     end
 
     # Remove utf8 control character sequences
-    `sed -i '#{FileHelper::REPLACEMENTS_SED_COMMAND}' "#{tmp_filename}"`
+    `perl -i -pe '#{FileHelper::REPLACEMENTS_PERL_COMMAND}' "#{tmp_filename}"`
 
     # Move into place
     FileUtils.mv(tmp_filename, output_filename)