Default profile added so developers can ensure a rudimentary workflow…

.github/workflows/pylint.yml

@@ -16,7 +16,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        pip install wheel setuptools pip pybind11 --upgrade
         pip install -r ./requirements/dev.txt
         pip install pylint
     - name: Analysing the code with pylint

.github/workflows/pytest.yml

@@ -9,20 +9,19 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r ./requirements/dev.txt
-        pip install pytest
-    - name: Build tests
-      run: |
-        python test_manager/__init__.py -t -g -tp
-    - name: Test the code with pytest
-      run: |
-        pytest ./test_manager/test_profiles.py
-        pytest ./test_manager/test_readers.py
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install wheel setuptools pip pybind11 --upgrade
+          pip install -r ./requirements/dev.txt
+          pip install pytest
+      - name: Build tests
+        run: |
+          python test_manager/__init__.py -t -g -tp
+      - name: Test the code with pytest
+        run: |
+          pytest .

.gitignore

@@ -32,3 +32,6 @@ __pycache__/
 test_manager/test_readers.py
 /test_manager/test_profiles.py
 /test_files/
+/.venv/
+logs/
+

README.md

@@ -32,6 +32,15 @@ The Flask development server can be started now:
 flask run
 ```
 
+Troubleshooting (when unsing Windows, updated 25.07.2024)
+---------------
+1. Please make sure that you are using a fresh Python3.10 virtual environment (without any packages installed, 3.12 and 3.9 are not supported)
+2. Before installing the requirements ```pip install wheel setuptools pip pybind11 # always using pip of your virtual environment ```
+3. After installing all requirements via ```pip install -e .``` and ```pip install -r requirements/dev.txt``` check if you have ```python-magic-bin``` and / or ```python-magic``` installed. If not ```pip install python-magic-bin```
+4. Now try starting it with the following env variables ```PYTHONUNBUFFERED=1;FLASK_APP=converter_app/app.py;FLASK_ENV=development;FLASK_DEBUG=1```
+5. If you are still getting ERRORS containing "libmagic" try uninstalling ```python-magic``` but keeping ```python-magic-bin```  
+
+
 Production setup
 ----------------
 

converter_app/converters.py

@@ -1,4 +1,5 @@
 import copy
+import datetime
 import logging
 import os
 import re
@@ -209,29 +210,9 @@ def process(self):
         :return:
         """
         for output_table_index, output_table in enumerate(self.output_tables):
-            header = {}
-            for key, value in output_table.get('header', {}).items():
-                if isinstance(value, dict):
-                    # this is a table identifier, e.g. FIRSTX
-                    match = self.match_identifier(value)
-                    if match:
-                        header[key] = match['value']
-                else:
-                    header[key] = value
-
-            # merge the metadata from the profile (header) with the metadata
-            # extracted using the identifiers (see self.match)
-            for match in self.matches:
-                match_result = match.get('result')
-                if match_result:
-                    match_output_key = match.get('identifier', {}).get('outputKey')
-                    match_output_table_index = match.get('identifier', {}).get('outputTableIndex')
-                    match_value = match_result.get('value')
-                    if match_output_key and (
-                            output_table_index == match_output_table_index or
-                            match_output_table_index is None
-                    ):
-                        header[match_output_key] = match_value
+            header = self._process_prepare_header(output_table)
+
+            self._process_prepare_metadata(header, output_table_index)
 
             x_column = output_table.get('table', {}).get('xColumn')
             y_column = output_table.get('table', {}).get('yColumn')
@@ -248,30 +229,7 @@ def process(self):
                 if operation.get('type') == 'column':
                     operation['rows'] = []
 
-            for table_index, table in enumerate(self.input_tables):
-                for _, row in enumerate(table['rows']):
-                    for column_index, _ in enumerate(table['columns']):
-                        if x_column and \
-                                table_index == x_column.get('tableIndex') and \
-                                column_index == x_column.get('columnIndex'):
-                            x_rows.append(self.get_value(row, column_index))
-
-                        if y_column and \
-                                table_index == y_column.get('tableIndex') and \
-                                column_index == y_column.get('columnIndex'):
-                            y_rows.append(self.get_value(row, column_index))
-
-                        for operation in x_operations:
-                            if operation.get('type') == 'column' and \
-                                    table_index == operation.get('column', {}).get('tableIndex') and \
-                                    column_index == operation.get('column', {}).get('columnIndex'):
-                                operation['rows'].append(self.get_value(row, column_index))
-
-                        for operation in y_operations:
-                            if operation.get('type') == 'column' and \
-                                    table_index == operation.get('column', {}).get('tableIndex') and \
-                                    column_index == operation.get('column', {}).get('columnIndex'):
-                                operation['rows'].append(self.get_value(row, column_index))
+            self._process_prepare_data(x_column, x_operations, x_rows, y_column, y_operations, y_rows)
 
             for operation in x_operations:
                 x_rows = self._run_operation(x_rows, operation)
@@ -284,6 +242,59 @@ def process(self):
                 'y': y_rows
             })
 
+    def _process_prepare_data(self, x_column, x_operations, x_rows, y_column, y_operations, y_rows):
+        for table_index, table in enumerate(self.input_tables):
+            for _, row in enumerate(table['rows']):
+                for column_index, _ in enumerate(table['columns']):
+                    if x_column and \
+                            table_index == x_column.get('tableIndex') and \
+                            column_index == x_column.get('columnIndex'):
+                        x_rows.append(self.get_value(row, column_index))
+
+                    if y_column and \
+                            table_index == y_column.get('tableIndex') and \
+                            column_index == y_column.get('columnIndex'):
+                        y_rows.append(self.get_value(row, column_index))
+
+                    for operation in x_operations:
+                        if operation.get('type') == 'column' and \
+                                table_index == operation.get('column', {}).get('tableIndex') and \
+                                column_index == operation.get('column', {}).get('columnIndex'):
+                            operation['rows'].append(self.get_value(row, column_index))
+
+                    for operation in y_operations:
+                        if operation.get('type') == 'column' and \
+                                table_index == operation.get('column', {}).get('tableIndex') and \
+                                column_index == operation.get('column', {}).get('columnIndex'):
+                            operation['rows'].append(self.get_value(row, column_index))
+
+    def _process_prepare_metadata(self, header, output_table_index):
+        # merge the metadata from the profile (header) with the metadata
+        # extracted using the identifiers (see self.match)
+        for match in self.matches:
+            match_result = match.get('result')
+            if match_result:
+                match_output_key = match.get('identifier', {}).get('outputKey')
+                match_output_table_index = match.get('identifier', {}).get('outputTableIndex')
+                match_value = match_result.get('value')
+                if match_output_key and (
+                        output_table_index == match_output_table_index or
+                        match_output_table_index is None
+                ):
+                    header[match_output_key] = match_value
+
+    def _process_prepare_header(self, output_table):
+        header = {}
+        for key, value in output_table.get('header', {}).items():
+            if isinstance(value, dict):
+                # this is a table identifier, e.g. FIRSTX
+                match = self.match_identifier(value)
+                if match:
+                    header[key] = match['value']
+            else:
+                header[key] = value
+        return header
+
     def _run_operation(self, rows, operation):
         for i, row in enumerate(rows):
             op_value = None
@@ -304,10 +315,16 @@ def _run_identifier_operation(self, value, operation):
         op_value = operation.get('value')
         if op_value:
             return self.apply_operation(value, op_value, operation.get('operator'))
-        else:
-            return value
+        return value
 
     def apply_operation(self, value, op_value, op_operator):
+        """
+        Apply mathematical operator
+        :param value: numeric value left
+        :param op_value: numeric value right
+        :param op_operator: mathematical operator
+        :return:
+        """
         try:
             float_value = float(self._fix_float(value))
 
@@ -328,8 +345,7 @@ def get_input_table(self, index, input_tables):
             try:
                 if int(index) >= len(input_tables):
                     return None
-                else:
-                    return input_tables[int(index)]
+                return input_tables[int(index)]
             except KeyError:
                 return None
 
@@ -364,15 +380,21 @@ def match_profile(cls, client_id, file_data):
         """
         converter = None
         matches = 0
-
-        for profile in Profile.list(client_id):
+        latest_profile_uploaded = 0
+        for profile in Profile.list_including_default(client_id):
             current_converter = cls(profile, file_data)
             current_matches = current_converter.match()
-
+            try:
+                profile_uploaded = datetime.datetime.fromisoformat(
+                    profile.as_dict['data']['metadata'].get('uploaded')).timestamp()
+            except (ValueError, TypeError):
+                profile_uploaded = 1
             logger.info('profile=%s matches=%s', profile.id, current_matches)
-
-            if current_matches is not False and current_matches > matches:
+            if (current_matches is not False and
+                    (current_matches > matches or current_matches == matches and
+                    profile_uploaded > latest_profile_uploaded)):
+                matches = max(matches, current_matches)
+                latest_profile_uploaded = profile_uploaded
                 converter = current_converter
-                matches = current_matches
 
         return converter