chg: Move prodigy functions to prodigy module

Author: ivyleavedtoadflax

deep_reference_parser/prodigy/labels_to_prodigy.py

@@ -0,0 +1,58 @@
+
+def labels_to_prodigy(tokens, labels):
+    """
+    Converts a list of tokens and labels like those used by Rodrigues et al,
+    and converts to prodigy format dicts.
+
+    Args:
+        tokens (list): A list of tokens.
+        labels (list): A list of labels relating to `tokens`.
+
+    Returns:
+        A list of prodigy format dicts containing annotated data.
+    """
+
+    prodigy_data = []
+
+    all_token_index = 0
+
+    for line_index, line in enumerate(tokens):
+        prodigy_example = {}
+
+        tokens = []
+        spans = []
+        token_start_offset = 0
+
+        for token_index, token in enumerate(line):
+
+            token_end_offset = token_start_offset + len(token)
+
+            tokens.append(
+                {
+                    "text": token,
+                    "id": token_index,
+                    "start": token_start_offset,
+                    "end": token_end_offset,
+                }
+            )
+
+            spans.append(
+                {
+                    "label": labels[line_index][token_index : token_index + 1][0],
+                    "start": token_start_offset,
+                    "end": token_end_offset,
+                    "token_start": token_index,
+                    "token_end": token_index,
+                }
+            )
+
+            prodigy_example["text"] = " ".join(line)
+            prodigy_example["tokens"] = tokens
+            prodigy_example["spans"] = spans
+            prodigy_example["meta"] = {"line": line_index}
+
+            token_start_offset = token_end_offset + 1
+
+        prodigy_data.append(prodigy_example)
+
+    return prodigy_data

deep_reference_parser/prodigy/misc.py

@@ -0,0 +1,36 @@
+import spacy
+
+def _join_prodigy_tokens(text):
+    """Return all prodigy tokens in a single string
+    """
+
+    return "\n".join([str(i) for i in text])
+
+def prodigy_to_conll(docs):
+    """
+    Expect list of jsons loaded from a jsonl
+    """
+
+    nlp = spacy.load("en_core_web_sm")
+    texts = [doc["text"] for doc in docs]
+    docs = list(nlp.tokenizer.pipe(texts))
+
+    out = [_join_prodigy_tokens(i) for i in docs]
+
+    out_str = "DOCSTART\n\n" + "\n\n".join(out)
+
+    return out_str
+
+
+def prodigy_to_lists(docs):
+    """
+    Expect list of jsons loaded from a jsonl
+    """
+
+    nlp = spacy.load("en_core_web_sm")
+    texts = [doc["text"] for doc in docs]
+    docs = list(nlp.tokenizer.pipe(texts))
+
+    out = [[str(token) for token in doc] for doc in docs]
+
+    return out

tests/test_labels_to_prodigy.py renamed to tests/prodigy/test_labels_to_prodigy.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # coding: utf-8
 
-from deep_reference_parser.reference_utils import labels_to_prodigy
+from deep_reference_parser.prodigy import labels_to_prodigy
 
 
 def test_labels_to_prodigy():

tests/prodigy/test_misc.py

@@ -0,0 +1,18 @@
+from deep_reference_parser.prodigy import prodigy_to_conll
+
+def test_prodigy_to_conll():
+
+    before = [
+        {"text": "References",},
+        {"text": "37. No single case of malaria reported in"},
+        {
+            "text": "an essential requirement for the correct labelling of potency for therapeutic"
+        },
+        {"text": "EQAS, quality control for STI"},
+    ]
+
+    after = "DOCSTART\n\nReferences\n\n37\n.\nNo\nsingle\ncase\nof\nmalaria\nreported\nin\n\nan\nessential\nrequirement\nfor\nthe\ncorrect\nlabelling\nof\npotency\nfor\ntherapeutic\n\nEQAS\n,\nquality\ncontrol\nfor\nSTI"
+
+    out = prodigy_to_conll(before)
+
+    assert after == out