feat: Added HPLC Reader (as tar archive) (#102)

build: added dependency: [BinaryParser](https://github.com/ComPlat/BinaryParser)

split requirements for dev and prod environments:

* production: use pre-built parser :  
 target  sys_platform == "linux" (linux_x86_64) and python_version == "3.12"

* dev : build using wheel pybind11

---------

Co-authored-by: Martin <m.starman@live.comdd>

.github/workflows/pylint.yml

@@ -16,7 +16,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        pip install wheel setuptools pip pybind11 --upgrade
+        pip install pip --upgrade
         pip install -r ./requirements/dev.txt
         pip install pylint
     - name: Analysing the code with pylint

.github/workflows/pytest.yml

@@ -16,7 +16,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          pip install wheel setuptools pip pybind11 --upgrade
+          pip install pip --upgrade
           pip install -r ./requirements/dev.txt
           pip install pytest
       - name: Build tests

converter_app/models.py

@@ -6,7 +6,6 @@
 import tempfile
 import uuid
 from collections import defaultdict
-
 from pathlib import Path
 
 import magic

converter_app/readers/afm.py

@@ -1,8 +1,8 @@
 import logging
 import re
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix
 
 logger = logging.getLogger(__name__)

converter_app/readers/asc_zip.py

@@ -2,9 +2,10 @@
 import os
 import tempfile
 from zipfile import ZipFile
+
+from converter_app.readers.helper.asc_helper import AscHelper
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
-from converter_app.readers.helper.asc_helper import AscHelper
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/ascii.py

@@ -1,5 +1,6 @@
 import logging
 import re
+
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
 

converter_app/readers/csv_reader.py

@@ -2,8 +2,8 @@
 import io
 import logging
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/ebl.py

@@ -1,6 +1,7 @@
 import logging
 import re
 from enum import Enum
+
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
 

converter_app/readers/excel.py

@@ -3,8 +3,8 @@
 
 import openpyxl
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader, Table
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/hplc_reader.py

@@ -0,0 +1,62 @@
+import logging
+
+import hplc as ph
+
+from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
+
+logger = logging.getLogger(__name__)
+
+
+class HplcReader(Reader):
+    """
+    Reads tarballed hplc files with extension .tar.gz
+    """
+    identifier = 'hplc_reader'
+    priority = 5
+
+    def __init__(self, file, *tar_files):
+        super().__init__(file, *tar_files)
+        self.df = None
+        self.temp_dir = None
+
+    def check(self):
+        """
+        :return: True if it fits
+        """
+
+        if self.is_tar_ball:
+            try:
+                if len(self.file_content) > 1:
+                    self.df = ph.read_chromatograms(self.file_content[0].file_path)
+                else:
+                    return False
+                return True
+            except ValueError:
+                pass
+        return False
+
+    def prepare_tables(self):
+        tables = []
+
+        keys = list(self.df.keys())
+        waves = [x for x in keys if x.startswith('Wave')]
+        waves.sort()
+        time = self.df['time']
+        for wave_key in waves:
+            wave = self.df[wave_key]
+            table = self.append_table(tables)
+            kv = wave_key.split('_')
+            table['metadata'][kv[0]] = str(kv[1])
+            table['metadata']['AllWaves'] = str(waves)
+            for i, t in enumerate(time):
+                table['rows'].append([t, float(wave[i])])
+
+            table['columns'] = [{
+                'key': str(idx),
+                'name': f'{value}'
+            } for idx, value in enumerate(['Time', 'Wavelength'])]
+        return tables
+
+
+Readers.instance().register(HplcReader)

converter_app/readers/html.py

@@ -1,9 +1,8 @@
 import logging
+from html.parser import HTMLParser
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
-from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix
-from html.parser import HTMLParser
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/json_reader.py

@@ -1,6 +1,7 @@
 import copy
 import json
 import logging
+
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
 

converter_app/readers/old_excel.py

@@ -1,8 +1,9 @@
 import logging
+
 import xlrd
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/pdf.py

@@ -1,6 +1,8 @@
 import logging
 import tempfile
+
 import fitz
+
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
 

converter_app/readers/sec.py

@@ -1,7 +1,8 @@
 import logging
 import re
-from converter_app.readers.helper.reader import Readers
+
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/tif.py

@@ -1,5 +1,6 @@
 import logging
 import re
+
 from converter_app.readers.helper.base import Reader
 from converter_app.readers.helper.reader import Readers
 

converter_app/readers/uvvis.py

@@ -1,7 +1,7 @@
 import logging
 
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.csv_reader import CSVReader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/uxd_reader.py

@@ -1,8 +1,8 @@
 import logging
 
 from converter_app.models import File
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/vsi.py

@@ -1,5 +1,4 @@
 import logging
-
 import xml.etree.ElementTree as ET
 
 from converter_app.models import File

converter_app/readers/xml_reader.py

@@ -1,10 +1,9 @@
 import logging
-
 import xml.etree.ElementTree as ET
 
 from converter_app.models import File
-from converter_app.readers.helper.reader import Readers
 from converter_app.readers.helper.base import Reader
+from converter_app.readers.helper.reader import Readers
 
 logger = logging.getLogger(__name__)
 

converter_app/readers/xrdml.py

@@ -1,8 +1,8 @@
 import xml.etree.ElementTree as ET
 
 from converter_app.models import File
-from converter_app.readers.xml_reader import XMLReader
 from converter_app.readers.helper.reader import Readers
+from converter_app.readers.xml_reader import XMLReader
 
 
 class XRDMLReader(XMLReader):

converter_app/router.py

@@ -8,11 +8,11 @@
 """
 import json
 import os
-
 from pathlib import Path
+
 from flask import Flask, Response, abort, jsonify, make_response, request
-from flask_httpauth import HTTPBasicAuth
 from flask_cors import CORS
+from flask_httpauth import HTTPBasicAuth
 
 from converter_app.converters import Converter
 from converter_app.datasets import Dataset

requirements/common.txt

@@ -1,15 +1,4 @@
-defusedxml~=0.7.1
-Flask~=2.0.2
-Flask-Cors~=3.0.10
-Flask-HTTPAuth~=4.5.0
-gunicorn~=20.1.0
-openpyxl~=3.0.9
-python-dotenv~=0.19.2
-python-magic~=0.4.24
-gemmi~=0.6.6
-xlrd~=2.0.1
-Werkzeug~=2.2.2
-jcamp~=1.2.2
-PyMuPDF==1.23.7
-pylint==3.0.3
-str2bool~=1.1
+-r default.txt
+
+binaryparser @ https://github.com/ComPlat/BinaryParser/releases/download/v1.0.0/BinaryParser-0.0.1-cp312-cp312-linux_x86_64.whl ; sys_platform == "linux" and python_version == "3.12"
+

requirements/default.txt

@@ -0,0 +1,15 @@
+defusedxml~=0.7.1
+Flask~=2.0.2
+Flask-Cors~=3.0.10
+Flask-HTTPAuth~=4.5.0
+gunicorn~=20.1.0
+openpyxl~=3.0.9
+python-dotenv~=0.19.2
+python-magic~=0.4.24
+gemmi~=0.6.6
+xlrd~=2.0.1
+Werkzeug~=2.2.2
+jcamp~=1.2.2
+PyMuPDF==1.23.7
+pylint==3.0.3
+str2bool~=1.1

requirements/dev.txt

@@ -1,8 +1,12 @@
--r common.txt
+-r default.txt
 
 coverage
 pytest
 pytest-dotenv
 pytest-cov
 pytest-flask
 GitPython
+wheel
+setuptools
+pybind11
+binaryparser @ git+https://github.com/ComPlat/BinaryParser@main

test_manager/__init__.py

@@ -10,7 +10,6 @@
 
 from werkzeug.datastructures import FileStorage
 
-
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test_manager.profile_test_generator import generate_profile_tests, generate_expected_profiles_results

test_manager/profile_test_generator.py

@@ -2,19 +2,18 @@
 import json
 import os
 import re
-import shutil
 import traceback
 import zipfile
 from json import JSONDecodeError
 
-from converter_app.writers.jcampzip import JcampZipWriter
-from test_manager.test_file_manager import CURRENT_DIR, RES_PROFILE_PATH
-from test_manager.utils import basic_walk
 from werkzeug.datastructures import FileStorage
 
+from converter_app.converters import Converter
 from converter_app.models import File
 from converter_app.readers import READERS as registry
-from converter_app.converters import Converter
+from converter_app.writers.jcampzip import JcampZipWriter
+from test_manager.test_file_manager import CURRENT_DIR
+from test_manager.utils import basic_walk
 from test_manager.utils_test import set_flask_test_config
 
 TEST_IDX = 0

test_manager/test_file_manager.py

@@ -1,6 +1,7 @@
 import os
 import shutil
 import tempfile
+
 import git
 
 # Get the current script's directory