Skip to content

Commit

Permalink
change upload process
Browse files Browse the repository at this point in the history
  • Loading branch information
@Gahhh
Gahhh committed Nov 27, 2023
1 parent 8bfadf9 commit 28f199b
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 78 deletions.
62 changes: 62 additions & 0 deletions scripts/db/update_db_schema_v1.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
ALTER TABLE prescription
RENAME COLUMN rt_plan_path TO rt_plan_pres;

ALTER TABLE prescription
RENAME COLUMN rt_ct_path TO rt_ct_pres;

ALTER TABLE prescription
RENAME COLUMN rt_structure_path TO rt_structure_pres;

ALTER TABLE prescription
ALTER COLUMN rt_dose_path TO rt_dose_pres;

ALTER TABLE prescription
ALTER COLUMN rt_mri_path TO rt_mri_pres;

ALTER TABLE prescription
ALTER COLUMN RT_DVH_original_path TO planned_dvh_pres;

ALTER TABLE prescription
ADD COLUMN centroid_pres VARCHAR;

ALTER TABLE prescription
ADD COLUMN planned_dicom_pres VARCHAR;

ALTER TABLE images
ADD COLUMN rt_ct_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN rt_dose_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN rt_plan_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN rt_structure_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN centroid_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN kim_threshold VARCHAR;

ALTER TABLE images
ADD COLUMN planned_dvh_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN planned_dicom_fraction VARCHAR;

ALTER TABLE images
ADD COLUMN rpm_path VARCHAR;

-- ALTER TABLE images
-- ADD COLUMN fluoro_images_path VARCHAR;

-- ALTER TABLE images
-- ADD COLUMN cbct_images_path VARCHAR;

-- ALTER TABLE images
-- ADD COLUMN timepoint_ct_path VARCHAR;

-- ALTER TABLE images
-- ADD COLUMN pet_iamges_path VARCHAR;
5 changes: 5 additions & 0 deletions src/admin_console/CustomTypes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,8 @@ class DBUpdateResult(NamedTuple):
success: bool
rowsUpdated: int
message: str

class DBFindResult(NamedTuple):
success: bool
result : list
message: str
113 changes: 69 additions & 44 deletions src/admin_console/DataImporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,59 +118,84 @@ def insertMetadataIntoDatabase(self, dbProgressCallback:Callable[[str], None]=No
if not self.contentCopied and self.metadata["upload_type"] == "files":
return False, "Please ensure that the uploaded files are copied first before inserting in DB"

try:
with open("filetype_db_mapping.json", 'r') as filetypeMappingFile:
filetypeMapping = json.load(filetypeMappingFile)
except FileNotFoundError as err:
return False, "The filetype mapping JSON cannot be loaded"
# try:
# with open("filetype_db_mapping.json", 'r') as filetypeMappingFile:
# filetypeMapping = json.load(filetypeMappingFile)
# except FileNotFoundError as err:
# return False, "The filetype mapping JSON cannot be loaded"

for UploadDetails in self.metadata["uploaded_files"]:
if UploadDetails["file_type"] in filetypeMapping["mapping"].keys():
paths = []
multiValues = filetypeMapping["mapping"][UploadDetails["file_type"]]["multivalues"]
granularity = filetypeMapping["mapping"][UploadDetails["file_type"]]["granularity"]
if multiValues:
seperator = filetypeMapping["mapping"][UploadDetails["file_type"]]["delimiter"]
for filePath in UploadDetails["Files"]:
parentPath, sep, fileName = filePath.rpartition('/')
if granularity == "folder":
if parentPath not in paths:
paths.append(parentPath)
else:
paths.append(filePath)
if not multiValues:
break
fieldContent = ""
if multiValues and len(paths) > 1:
for path in paths:
if fieldContent != "":
fieldContent += seperator
fieldContent += path[len(self.currentContextId):]
else:
fieldContent += paths[0][len(self.currentContextId):]
tableName = filetypeMapping["mapping"][UploadDetails["file_type"]]["table"]
fieldName = filetypeMapping["mapping"][UploadDetails["file_type"]]["field"]
insertStmt = f"UPDATE {tableName} SET {fieldName} = \'{fieldContent}\' "
if tableName == "prescription":
insertStmt += "FROM patient " \
findTableNameSql = f"SELECT table_name from information_schema.columns where column_name=\'{UploadDetails['file_type']}\'"
result = self.dbAdapter.executeFindOnImageDB(findTableNameSql)
if not result.success:
return result.success, result.message
tableName = result.result[0][0]
insertStmt = ""
if tableName == "prescription":
insertStmt = f"UPDATE {tableName} SET {UploadDetails['file_type']} = \'{UploadDetails['folder_path'][0]}\' "
insertStmt += "FROM patient " \
"WHERE patient.id=prescription.patient_id " \
+ f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
+ f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
+ f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
elif tableName == "images":
insertStmt += "FROM patient, prescription, fraction " \
+ "WHERE patient.id=prescription.patient_id " \
+ "AND prescription.prescription_id=fraction.prescription_id " \
+ "AND images.fraction_id=fraction.fraction_id " \
+ f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
+ f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
+ f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
if not result.success:
return result.success, result.message
elif dbProgressCallback:
dbProgressCallback(f"updated {tableName}.{fieldName} = {fieldContent}")
self.markPacketAsImported()
else:
for fraction in UploadDetails["fraction"]:
fractionDetail = self.dbAdapter.getFractionIdAndName(self.metadata["patient_trial_id"], fraction)
if fractionDetail:
fractionId = fractionDetail[0][0]
insertStmt = f"UPDATE {tableName} SET {UploadDetails['file_type']} = \'{UploadDetails['folder_path'][fraction]}\' WHERE fraction_id = \'{fractionId}\'"
result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
if not result.success:
return result.success, result.message
# if UploadDetails["file_type"] in filetypeMapping["mapping"].keys():
# paths = []
# multiValues = filetypeMapping["mapping"][UploadDetails["file_type"]]["multivalues"]
# granularity = filetypeMapping["mapping"][UploadDetails["file_type"]]["granularity"]
# if multiValues:
# seperator = filetypeMapping["mapping"][UploadDetails["file_type"]]["delimiter"]
# for filePath in UploadDetails["Files"]:
# parentPath, sep, fileName = filePath.rpartition('/')
# if granularity == "folder":
# if parentPath not in paths:
# paths.append(parentPath)
# else:
# paths.append(filePath)
# if not multiValues:
# break
# fieldContent = ""
# if multiValues and len(paths) > 1:
# for path in paths:
# if fieldContent != "":
# fieldContent += seperator
# fieldContent += path[len(self.currentContextId):]
# else:
# fieldContent += paths[0][len(self.currentContextId):]
# tableName = filetypeMapping["mapping"][UploadDetails["file_type"]]["table"]
# fieldName = filetypeMapping["mapping"][UploadDetails["file_type"]]["field"]
# insertStmt = f"UPDATE {tableName} SET {fieldName} = \'{fieldContent}\' "
# if tableName == "prescription":
# insertStmt += "FROM patient " \
# "WHERE patient.id=prescription.patient_id " \
# + f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
# + f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
# + f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
# elif tableName == "images":
# insertStmt += "FROM patient, prescription, fraction " \
# + "WHERE patient.id=prescription.patient_id " \
# + "AND prescription.prescription_id=fraction.prescription_id " \
# + "AND images.fraction_id=fraction.fraction_id " \
# + f"AND patient.patient_trial_id=\'{self.metadata['patient_trial_id']}\' " \
# + f"AND patient.clinical_trial=\'{self.metadata['clinical_trial']}\' " \
# + f"AND patient.test_centre=\'{self.metadata['test_centre']}\'"
# result = self.dbAdapter.executeUpdateOnImageDB(insertStmt)
# if not result.success:
# return result.success, result.message
# elif dbProgressCallback:
# dbProgressCallback(f"updated {tableName}.{fieldName} = {fieldContent}")
# self.markPacketAsImported()
return True, "Success"

def rejectUploadPacket(self):
Expand Down
16 changes: 15 additions & 1 deletion src/admin_console/DatabaseAdapter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import psycopg2 as pg
import config
from typing import Dict, List, Tuple
from CustomTypes import SiteDetails, TrialDetails, DBUpdateResult
from CustomTypes import SiteDetails, TrialDetails, DBUpdateResult, DBFindResult
import sys


Expand Down Expand Up @@ -196,6 +196,20 @@ def executeUpdateOnImageDB(self, stmt:str) -> DBUpdateResult:
return DBUpdateResult(success=False, rowsUpdated=0, message=str(error))
return DBUpdateResult(success=True, rowsUpdated=rowsUpdated, message="Success")

def executeFindOnImageDB(self, stmt:str) -> List[Dict]:
print("Executing SELECT Statement:", stmt)
result = []
try:
conn = self.getImageDBConnection()
cur = conn.cursor()
cur.execute(stmt)
result = cur.fetchall()
cur.close()
except (Exception, pg.DatabaseError) as error:
print(error, file=sys.stderr)
return DBFindResult(success=False, result=[], message=str(error))
return DBFindResult(success=True, result=result, message="Success")

def getFractionIdAndDate(self, patientTrialId:str, fractionNumber:int) -> tuple:
strQuery = "SELECT fraction_id, fraction_date FROM fraction, patient, prescription " \
+ "WHERE patient.patient_trial_id = '" + patientTrialId + "' " \
Expand Down
51 changes: 25 additions & 26 deletions src/admin_console/application.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,32 +98,31 @@ def importUploadPacket(upload_id):
if config.APP_DEBUG_MODE:
print("Copied files into storage", result[1])
fileInfo = di.getUploadFileInfo()
if fileInfo['clinical_trial'] == "CHIRP":
# Only design for CHIRP data import
result = di.insertCHIRPDataIntoDatabase()
elif fileInfo['file_type'] == "fraction_folder":
result = di.insertFractionDataIntoDatabase()
result2 = di.insertImagePathIntoDatabase()
if config.APP_DEBUG_MODE:
print("Inserted fraction data into database", result[1])
print("Inserted image path into database", result2[1])
elif fileInfo['file_type'] == "image_folder":
result = di.checkAndInsertFractionDataIntoDatabase()
result2 = di.insertPatientLevelImagePathIntoDatabase()
elif fileInfo['file_type'] == "trajectory_log_folder":
result = di.insertTrajectoryLogIntoDatabase()
elif fileInfo['file_type'] == "DVH_folder" or fileInfo['file_type'] == "DICOM_folder":
result = di.insertDoseReconstrcutionFileIntoDatabase()
elif fileInfo['file_type'] == "triangulation_folder" or fileInfo['file_type'] == "kim_logs":
result = di.checkAndInsertFractionDataIntoDatabase()
result2 = di.insertFractionFilePathIntoDatabase()
if config.APP_DEBUG_MODE:
print("Inserted fraction data into database", result[1])
print("Inserted image path into database", result2[1])
else:
result = di.insertMetadataIntoDatabase()
if config.APP_DEBUG_MODE:
print("Inserted metadata into database", result[1])
# if fileInfo['clinical_trial'] == "CHIRP":
# # Only design for CHIRP data import
# result = di.insertCHIRPDataIntoDatabase()
# elif fileInfo['file_type'] == "fraction_folder":
# result = di.insertFractionDataIntoDatabase()
# result2 = di.insertImagePathIntoDatabase()
# if config.APP_DEBUG_MODE:
# print("Inserted fraction data into database", result[1])
# print("Inserted image path into database", result2[1])
# elif fileInfo['file_type'] == "image_folder":
# result = di.checkAndInsertFractionDataIntoDatabase()
# result2 = di.insertPatientLevelImagePathIntoDatabase()
# elif fileInfo['file_type'] == "trajectory_log_folder":
# result = di.insertTrajectoryLogIntoDatabase()
# elif fileInfo['file_type'] == "DVH_folder" or fileInfo['file_type'] == "DICOM_folder":
# result = di.insertDoseReconstrcutionFileIntoDatabase()
# elif fileInfo['file_type'] == "triangulation_folder" or fileInfo['file_type'] == "kim_logs":
# result = di.checkAndInsertFractionDataIntoDatabase()
# result2 = di.insertFractionFilePathIntoDatabase()
# if config.APP_DEBUG_MODE:
# print("Inserted fraction data into database", result[1])
# print("Inserted image path into database", result2[1])
result = di.insertMetadataIntoDatabase()
if config.APP_DEBUG_MODE:
print("Inserted metadata into database", result[1])

if not result[0]:
print("Error importing data:", result[1])
Expand Down
33 changes: 26 additions & 7 deletions src/data_service/ContentManager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,6 @@ def acceptAndSaveFile(self, req:request):
# with open(fileTypeToPathMappingPath, "r") as pathMappingFile:
# fileTypeToPathMapping = json.load(pathMappingFile)
try:

trialDetails = getTrialStructure(metadata["clinical_trial"])
filePath = trialDetails[metadata["level"]][metadata["file_type"]]['path']
except KeyError as e:
Expand All @@ -711,7 +710,21 @@ def acceptAndSaveFile(self, req:request):
filesSaved = []
for fileFieldName in req.files.keys():
uploadedFile = req.files[fileFieldName]
formatedPath = ""
fractionName = ""
fractionNumber = ""
filename = secure_filename(uploadedFile.filename)
if metadata['level'] == "fraction":
formatedPath = os.path.basename(req.form["file_path"]).replace("\\", "/").replace(filename, "")
fractionNumber = re.search(r'(?i)fx(\d+)', formatedPath).group(1)
fractionName = ""
if formatedPath.count("/") == 3:
fractionName = re.search(r'\/([^\/]+)\/$', formatedPath).group(1)
if fractionName=="":
fractionName = fractionNumber
else:
formatedPath = re.sub(r'^([^\/]+)\/', "", formatedPath)

# formatedPath = os.path.basename(req.form["file_path"]).replace("\\", "/").replace(filename, "")
# if metadata["clinical_trial"] == "CHIRP":
# saveFolderPath = self._processCHIRP(metadata, filename, uploadMetaData, filesSaved, fileTypeToPathMapping, formatedPath)
Expand All @@ -731,10 +744,8 @@ def acceptAndSaveFile(self, req:request):
clinical_trial=metadata['clinical_trial'],
test_centre=metadata["test_centre"],
patient_trial_id=metadata["patient_trial_id"],
fraction_name=metadata["fraction"],
sub_fraction_name=metadata["sub_fraction"],
centre_patient_no=int(metadata["centre_patient_no"])
)
) + formatedPath
relativePath = relativeFolderPath + filename
saveFolderPath = config.UPLOAD_FOLDER + '/' + relativeFolderPath
filesSaved.append(relativePath)
Expand All @@ -743,6 +754,13 @@ def acceptAndSaveFile(self, req:request):
for uploadedFileRecord in uploadMetaData["uploaded_files"]:
if uploadedFileRecord["file_type"] == metadata["file_type"]:
uploadedFileRecord["Files"].append(relativePath)
if fractionNumber and fractionNumber not in uploadedFileRecord["fraction"]:
uploadedFileRecord["fraction"].append(fractionNumber)
uploadedFileRecord["sub_fraction"].append(fractionName)
if fractionName and fractionName not in uploadedFileRecord["sub_fraction"]:
uploadedFileRecord["sub_fraction"].append(fractionName)
if saveFolderPath not in uploadedFileRecord["folder_path"]:
uploadedFileRecord["folder_path"].append(saveFolderPath)
filePathAppended = True
break

Expand All @@ -751,9 +769,10 @@ def acceptAndSaveFile(self, req:request):
{
"file_type": metadata["file_type"],
"level": metadata["level"],
"fraction": metadata["fraction"],
"sub_fraction": metadata["sub_fraction"],
"Files": [relativePath]
"fraction": [fractionNumber] if fractionNumber else [],
"sub_fraction": [fractionName] if fractionName else [],
"Files": [relativePath],
"folder_path": [saveFolderPath]
}
)
print(f"saving {filename} in {saveFolderPath}")
Expand Down

0 comments on commit 28f199b

Please sign in to comment.