diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index fdd7a492..1e274232 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -16,7 +16,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install wheel setuptools pip pybind11 --upgrade + pip install pip --upgrade pip install -r ./requirements/dev.txt pip install pylint - name: Analysing the code with pylint diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6af9461e..6c1c1803 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,7 +16,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install wheel setuptools pip pybind11 --upgrade + pip install pip --upgrade pip install -r ./requirements/dev.txt pip install pytest - name: Build tests diff --git a/converter_app/models.py b/converter_app/models.py index 7fdb4d76..998a6e44 100644 --- a/converter_app/models.py +++ b/converter_app/models.py @@ -6,7 +6,6 @@ import tempfile import uuid from collections import defaultdict - from pathlib import Path import magic diff --git a/converter_app/readers/afm.py b/converter_app/readers/afm.py index 2905da05..ed54cf8c 100644 --- a/converter_app/readers/afm.py +++ b/converter_app/readers/afm.py @@ -1,8 +1,8 @@ import logging import re -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix logger = logging.getLogger(__name__) diff --git a/converter_app/readers/asc_zip.py b/converter_app/readers/asc_zip.py index 1b7db7bd..d1fdd2d6 100644 --- a/converter_app/readers/asc_zip.py +++ b/converter_app/readers/asc_zip.py @@ -2,9 +2,10 @@ import os import tempfile from zipfile import ZipFile + +from converter_app.readers.helper.asc_helper import AscHelper from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers -from converter_app.readers.helper.asc_helper import AscHelper logger = logging.getLogger(__name__) diff --git a/converter_app/readers/ascii.py b/converter_app/readers/ascii.py index a4faaeb7..15358c1a 100644 --- a/converter_app/readers/ascii.py +++ b/converter_app/readers/ascii.py @@ -1,5 +1,6 @@ import logging import re + from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers diff --git a/converter_app/readers/csv_reader.py b/converter_app/readers/csv_reader.py index e88557d8..2d17f11c 100644 --- a/converter_app/readers/csv_reader.py +++ b/converter_app/readers/csv_reader.py @@ -2,8 +2,8 @@ import io import logging -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/ebl.py b/converter_app/readers/ebl.py index 1ea0a330..a2d44e93 100644 --- a/converter_app/readers/ebl.py +++ b/converter_app/readers/ebl.py @@ -1,6 +1,7 @@ import logging import re from enum import Enum + from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers diff --git a/converter_app/readers/excel.py b/converter_app/readers/excel.py index 7a3a8ab6..68f1fa54 100644 --- a/converter_app/readers/excel.py +++ b/converter_app/readers/excel.py @@ -3,8 +3,8 @@ import openpyxl -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader, Table +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/hplc_reader.py b/converter_app/readers/hplc_reader.py new file mode 100644 index 00000000..a9437cf7 --- /dev/null +++ b/converter_app/readers/hplc_reader.py @@ -0,0 +1,62 @@ +import logging + +import hplc as ph + +from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers + +logger = logging.getLogger(__name__) + + +class HplcReader(Reader): + """ + Reads tarballed hplc files with extension .tar.gz + """ + identifier = 'hplc_reader' + priority = 5 + + def __init__(self, file, *tar_files): + super().__init__(file, *tar_files) + self.df = None + self.temp_dir = None + + def check(self): + """ + :return: True if it fits + """ + + if self.is_tar_ball: + try: + if len(self.file_content) > 1: + self.df = ph.read_chromatograms(self.file_content[0].file_path) + else: + return False + return True + except ValueError: + pass + return False + + def prepare_tables(self): + tables = [] + + keys = list(self.df.keys()) + waves = [x for x in keys if x.startswith('Wave')] + waves.sort() + time = self.df['time'] + for wave_key in waves: + wave = self.df[wave_key] + table = self.append_table(tables) + kv = wave_key.split('_') + table['metadata'][kv[0]] = str(kv[1]) + table['metadata']['AllWaves'] = str(waves) + for i, t in enumerate(time): + table['rows'].append([t, float(wave[i])]) + + table['columns'] = [{ + 'key': str(idx), + 'name': f'{value}' + } for idx, value in enumerate(['Time', 'Wavelength'])] + return tables + + +Readers.instance().register(HplcReader) diff --git a/converter_app/readers/html.py b/converter_app/readers/html.py index f5ebaff5..9f28bf11 100644 --- a/converter_app/readers/html.py +++ b/converter_app/readers/html.py @@ -1,9 +1,8 @@ import logging +from html.parser import HTMLParser -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader -from converter_app.readers.helper.unit_converter import convert_units, search_terms_matrix -from html.parser import HTMLParser +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/json_reader.py b/converter_app/readers/json_reader.py index 2604c71b..9e7e4d21 100644 --- a/converter_app/readers/json_reader.py +++ b/converter_app/readers/json_reader.py @@ -1,6 +1,7 @@ import copy import json import logging + from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers diff --git a/converter_app/readers/old_excel.py b/converter_app/readers/old_excel.py index e8efdf17..8553611b 100644 --- a/converter_app/readers/old_excel.py +++ b/converter_app/readers/old_excel.py @@ -1,8 +1,9 @@ import logging + import xlrd -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/pdf.py b/converter_app/readers/pdf.py index e2b707e3..3b9bc9aa 100644 --- a/converter_app/readers/pdf.py +++ b/converter_app/readers/pdf.py @@ -1,6 +1,8 @@ import logging import tempfile + import fitz + from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers diff --git a/converter_app/readers/sec.py b/converter_app/readers/sec.py index fa6995e5..e09cf45c 100644 --- a/converter_app/readers/sec.py +++ b/converter_app/readers/sec.py @@ -1,7 +1,8 @@ import logging import re -from converter_app.readers.helper.reader import Readers + from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/tif.py b/converter_app/readers/tif.py index 47de418e..cb6dd33a 100644 --- a/converter_app/readers/tif.py +++ b/converter_app/readers/tif.py @@ -1,5 +1,6 @@ import logging import re + from converter_app.readers.helper.base import Reader from converter_app.readers.helper.reader import Readers diff --git a/converter_app/readers/uvvis.py b/converter_app/readers/uvvis.py index d635f8a3..2851d77d 100644 --- a/converter_app/readers/uvvis.py +++ b/converter_app/readers/uvvis.py @@ -1,7 +1,7 @@ import logging -from converter_app.readers.helper.reader import Readers from converter_app.readers.csv_reader import CSVReader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/uxd_reader.py b/converter_app/readers/uxd_reader.py index 3a1e5dc3..0b55fa9d 100644 --- a/converter_app/readers/uxd_reader.py +++ b/converter_app/readers/uxd_reader.py @@ -1,8 +1,8 @@ import logging from converter_app.models import File -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/vsi.py b/converter_app/readers/vsi.py index 3c5b552f..65d7589f 100644 --- a/converter_app/readers/vsi.py +++ b/converter_app/readers/vsi.py @@ -1,5 +1,4 @@ import logging - import xml.etree.ElementTree as ET from converter_app.models import File diff --git a/converter_app/readers/xml_reader.py b/converter_app/readers/xml_reader.py index 52292c37..2d4e3756 100644 --- a/converter_app/readers/xml_reader.py +++ b/converter_app/readers/xml_reader.py @@ -1,10 +1,9 @@ import logging - import xml.etree.ElementTree as ET from converter_app.models import File -from converter_app.readers.helper.reader import Readers from converter_app.readers.helper.base import Reader +from converter_app.readers.helper.reader import Readers logger = logging.getLogger(__name__) diff --git a/converter_app/readers/xrdml.py b/converter_app/readers/xrdml.py index e4364b4e..8d2d346f 100644 --- a/converter_app/readers/xrdml.py +++ b/converter_app/readers/xrdml.py @@ -1,8 +1,8 @@ import xml.etree.ElementTree as ET from converter_app.models import File -from converter_app.readers.xml_reader import XMLReader from converter_app.readers.helper.reader import Readers +from converter_app.readers.xml_reader import XMLReader class XRDMLReader(XMLReader): diff --git a/converter_app/router.py b/converter_app/router.py index 693d5a8f..e403e679 100644 --- a/converter_app/router.py +++ b/converter_app/router.py @@ -8,11 +8,11 @@ """ import json import os - from pathlib import Path + from flask import Flask, Response, abort, jsonify, make_response, request -from flask_httpauth import HTTPBasicAuth from flask_cors import CORS +from flask_httpauth import HTTPBasicAuth from converter_app.converters import Converter from converter_app.datasets import Dataset diff --git a/requirements/common.txt b/requirements/common.txt index e2155c92..123f9c07 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,15 +1,4 @@ -defusedxml~=0.7.1 -Flask~=2.0.2 -Flask-Cors~=3.0.10 -Flask-HTTPAuth~=4.5.0 -gunicorn~=20.1.0 -openpyxl~=3.0.9 -python-dotenv~=0.19.2 -python-magic~=0.4.24 -gemmi~=0.6.6 -xlrd~=2.0.1 -Werkzeug~=2.2.2 -jcamp~=1.2.2 -PyMuPDF==1.23.7 -pylint==3.0.3 -str2bool~=1.1 \ No newline at end of file +-r default.txt + +binaryparser @ https://github.com/ComPlat/BinaryParser/releases/download/v1.0.0/BinaryParser-0.0.1-cp312-cp312-linux_x86_64.whl ; sys_platform == "linux" and python_version == "3.12" + diff --git a/requirements/default.txt b/requirements/default.txt new file mode 100644 index 00000000..e2155c92 --- /dev/null +++ b/requirements/default.txt @@ -0,0 +1,15 @@ +defusedxml~=0.7.1 +Flask~=2.0.2 +Flask-Cors~=3.0.10 +Flask-HTTPAuth~=4.5.0 +gunicorn~=20.1.0 +openpyxl~=3.0.9 +python-dotenv~=0.19.2 +python-magic~=0.4.24 +gemmi~=0.6.6 +xlrd~=2.0.1 +Werkzeug~=2.2.2 +jcamp~=1.2.2 +PyMuPDF==1.23.7 +pylint==3.0.3 +str2bool~=1.1 \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index ed86f3fd..d06c55bb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ --r common.txt +-r default.txt coverage pytest @@ -6,3 +6,7 @@ pytest-dotenv pytest-cov pytest-flask GitPython +wheel +setuptools +pybind11 +binaryparser @ git+https://github.com/ComPlat/BinaryParser@main diff --git a/test_manager/__init__.py b/test_manager/__init__.py index 7fcaf84f..d103b02e 100644 --- a/test_manager/__init__.py +++ b/test_manager/__init__.py @@ -10,7 +10,6 @@ from werkzeug.datastructures import FileStorage - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test_manager.profile_test_generator import generate_profile_tests, generate_expected_profiles_results diff --git a/test_manager/profile_test_generator.py b/test_manager/profile_test_generator.py index 3bbde11f..3fa0b68f 100644 --- a/test_manager/profile_test_generator.py +++ b/test_manager/profile_test_generator.py @@ -2,19 +2,18 @@ import json import os import re -import shutil import traceback import zipfile from json import JSONDecodeError -from converter_app.writers.jcampzip import JcampZipWriter -from test_manager.test_file_manager import CURRENT_DIR, RES_PROFILE_PATH -from test_manager.utils import basic_walk from werkzeug.datastructures import FileStorage +from converter_app.converters import Converter from converter_app.models import File from converter_app.readers import READERS as registry -from converter_app.converters import Converter +from converter_app.writers.jcampzip import JcampZipWriter +from test_manager.test_file_manager import CURRENT_DIR +from test_manager.utils import basic_walk from test_manager.utils_test import set_flask_test_config TEST_IDX = 0 diff --git a/test_manager/test_file_manager.py b/test_manager/test_file_manager.py index dbd850e4..4e4fb3c6 100644 --- a/test_manager/test_file_manager.py +++ b/test_manager/test_file_manager.py @@ -1,6 +1,7 @@ import os import shutil import tempfile + import git # Get the current script's directory