From 854fd886a52125bd2395ad343fbd5aa49f95e985 Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Tue, 20 Feb 2024 16:27:41 +0100 Subject: [PATCH 1/9] started coding jannoColumns-all --- src/Poseidon/CLI/List.hs | 21 ++++++++++++------- .../CLI/OptparseApplicativeParsers.hs | 3 ++- src/Poseidon/CLI/Serve.hs | 7 ++++--- src/Poseidon/Janno.hs | 2 +- src/Poseidon/Package.hs | 13 +++++++----- 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index 3790e98af..1d62016f4 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -30,7 +30,7 @@ data ListOptions = ListOptions { _listRepoLocation :: RepoLocationSpec -- ^ the list of base directories to search for packages , _listListEntity :: ListEntity -- ^ what to list , _listRawOutput :: Bool -- ^ whether to output raw TSV instead of a nicely formatted table - , _listOnlyLatest :: Bool + , _listOnlyLatest :: Bool -- ^ whether to show only latest versions of packages } data RepoLocationSpec = RepoLocal [FilePath] | RepoRemote ArchiveEndpoint @@ -38,7 +38,7 @@ data RepoLocationSpec = RepoLocal [FilePath] | RepoRemote ArchiveEndpoint -- | A datatype to represent the options what to list data ListEntity = ListPackages | ListGroups - | ListIndividuals [String] + | ListIndividuals (Maybe [String]) -- Nothing means all Janno columns. Just [] means none. -- | The main function running the list command runList :: ListOptions -> PoseidonIO () @@ -89,25 +89,32 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do True <- return (not onlyLatest || isLatest) return [groupName, getPacName gi, showMaybeVersion (getPacVersion gi), show isLatest, show nrInds] return (tableH, tableB) - ListIndividuals moreJannoColumns -> do + ListIndividuals maybeMoreJannoColumns -> do extIndInfos <- case repoLocation of RepoRemote (ArchiveEndpoint remoteURL archive) -> do logInfo "Downloading individual data from server" - apiReturn <- processApiResponse (remoteURL ++ "/individuals" ++ qDefault archive ++ "&additionalJannoColumns=" ++ intercalate "," moreJannoColumns) False + let addJannoColFlag = case maybeMoreJannoColumns of + Nothing -> "&additionalJannoColumns=ALL" + Just moreJannoColumns -> "&additionalJannoColumns=" ++ intercalate "," moreJannoColumns + apiReturn <- processApiResponse (remoteURL ++ "/individuals" ++ qDefault archive ++ addJannoColFlag) False case apiReturn of ApiReturnExtIndividualInfo indInfo -> return indInfo _ -> error "should not happen" RepoLocal baseDirs -> do pacCollection <- readPoseidonPackageCollection pacReadOpts baseDirs - getExtendedIndividualInfo pacCollection moreJannoColumns + getExtendedIndividualInfo pacCollection maybeMoreJannoColumns -- warning in case the additional Columns do not exist in the entire janno dataset - forM_ (zip [0..] moreJannoColumns) $ \(i, columnKey) -> do + let addJannoCols = case extIndInfos of + [] -> [] + (e:es) -> map fst . extIndInfoAddCols $ e + forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do -- check entries in all individuals for that key let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] + logInfo $ columnKey ++ ": " ++ show nonEmptyEntries when (null nonEmptyEntries) . logWarning $ "Column Name " ++ columnKey ++ " not present in any individual" - let tableH = ["Individual", "Group", "Package", "PackageVersion", "Is Latest"] ++ moreJannoColumns + let tableH = ["Individual", "Group", "Package", "PackageVersion", "Is Latest"] ++ addJannoCols tableB = do i@(ExtendedIndividualInfo name groups _ isLatest addColumnEntries) <- extIndInfos True <- return (not onlyLatest || isLatest) diff --git a/src/Poseidon/CLI/OptparseApplicativeParsers.hs b/src/Poseidon/CLI/OptparseApplicativeParsers.hs index 50176dc8f..f644ef4ba 100644 --- a/src/Poseidon/CLI/OptparseApplicativeParsers.hs +++ b/src/Poseidon/CLI/OptparseApplicativeParsers.hs @@ -594,7 +594,8 @@ parseListEntity = parseListPackages <|> parseListGroups <|> (parseListIndividual parseListIndividualsDummy = OP.flag' () ( OP.long "individuals" <> OP.help "List all individuals/samples.") - parseListIndividualsExtraCols = ListIndividuals <$> OP.many parseExtraCol + parseListIndividualsExtraCols = ListIndividuals <$> (parseAllJannoCols <|> (Just <$> OP.many parseExtraCol)) + parseAllJannoCols = OP.flag' Nothing (OP.long "fullJanno" <> OP.help "output all Janno Columns") parseExtraCol = OP.strOption ( OP.short 'j' <> OP.long "jannoColumn" <> diff --git a/src/Poseidon/CLI/Serve.hs b/src/Poseidon/CLI/Serve.hs index 534a98536..3a21cdfca 100644 --- a/src/Poseidon/CLI/Serve.hs +++ b/src/Poseidon/CLI/Serve.hs @@ -27,7 +27,7 @@ import Control.Concurrent.MVar (MVar, newEmptyMVar, putMVar) import Control.Monad (forM, when) import Control.Monad.IO.Class (liftIO) import qualified Data.ByteString.Lazy as B -import Data.List (nub, sortOn) +import Data.List (nub, sortOn, (\\)) import Data.List.Split (splitOn) import Data.Maybe (isJust) import Data.Ord (Down (..)) @@ -121,10 +121,11 @@ runServer (ServeOptions archBaseDirs maybeZipPath port ignoreChecksums certFiles pacs <- getItemFromArchiveStore archiveStore maybeAdditionalColumnsString <- (Just <$> param "additionalJannoColumns") `rescue` (\_ -> return Nothing) indInfo <- case maybeAdditionalColumnsString of + Just "ALL" -> getExtendedIndividualInfo pacs Nothing -- Nothing means all Janno Columns Just additionalColumnsString -> let additionalColumnNames = splitOn "," additionalColumnsString - in getExtendedIndividualInfo pacs additionalColumnNames - Nothing -> getExtendedIndividualInfo pacs [] + in getExtendedIndividualInfo pacs (Just additionalColumnNames) + Nothing -> getExtendedIndividualInfo pacs (Just []) let retData = ApiReturnExtIndividualInfo indInfo return $ ServerApiReturnType [] (Just retData) diff --git a/src/Poseidon/Janno.hs b/src/Poseidon/Janno.hs index 352a651fa..3cff9f68b 100644 --- a/src/Poseidon/Janno.hs +++ b/src/Poseidon/Janno.hs @@ -808,7 +808,7 @@ cleanInput (Just rawInputBS) = transNA $ trimWS . removeNoBreakSpace $ rawInputB transNA x = Just x instance Csv.ToNamedRecord JannoRow where - toNamedRecord j = Csv.namedRecord [ + toNamedRecord j = Csv.namedRecord . filter ((/= "") . snd) $ [ "Poseidon_ID" Csv..= jPoseidonID j , "Genetic_Sex" Csv..= jGeneticSex j , "Group_Name" Csv..= jGroupName j diff --git a/src/Poseidon/Package.hs b/src/Poseidon/Package.hs index 714c1bc19..960d83b48 100644 --- a/src/Poseidon/Package.hs +++ b/src/Poseidon/Package.hs @@ -47,6 +47,7 @@ import Poseidon.Janno (JannoLibraryBuilt (..), JannoList (..), JannoRow (..), JannoRows (..), JannoSex (..), JannoUDG (..), createMinimalJanno, + jannoHeaderString, getMaybeJannoList, readJannoFile) import Poseidon.PoseidonVersion (asVersion, latestPoseidonVersion, showPoseidonVersion, @@ -819,15 +820,17 @@ getJointIndividualInfo packages = do return (map fst . concat $ indInfoLatestPairs, map snd . concat $ indInfoLatestPairs) -getExtendedIndividualInfo :: (MonadThrow m) => [PoseidonPackage] -> [String] -> m [ExtendedIndividualInfo] -getExtendedIndividualInfo allPackages additionalJannoColumns = sequence $ do -- list monad +getExtendedIndividualInfo :: (MonadThrow m) => [PoseidonPackage] -> Maybe [String] -> m [ExtendedIndividualInfo] +getExtendedIndividualInfo allPackages maybeAdditionalJannoColumns = sequence $ do -- list monad pac <- allPackages -- outer loop (automatically concatenating over inner loops) jannoRow <- getJannoRowsFromPac pac -- inner loop let name = jPoseidonID jannoRow groups = getJannoList . jGroupName $ jannoRow - additionalColumnEntries = case additionalJannoColumns of - [] -> [] - colNames -> [(k, BSC.unpack <$> toNamedRecord jannoRow HM.!? BSC.pack k) | k <- colNames] + colNames = case maybeAdditionalJannoColumns of + Nothing -> jannoHeaderString \\ ["Poseidon_ID", "Group_Name"] -- Nothing means all Janno columns + -- except for these two which are already explicit + Just c -> c + additionalColumnEntries = [(k, BSC.unpack <$> toNamedRecord jannoRow HM.!? BSC.pack k) | k <- colNames] isLatest <- isLatestInCollection allPackages pac -- this lives in monad m -- double-return for m and then list. return . return $ ExtendedIndividualInfo name groups (makePacNameAndVersion pac) isLatest additionalColumnEntries From 2ca3d4a558c8c3a0fd4ba25c23ed38b39ca8678a Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Sun, 12 May 2024 11:38:36 +0200 Subject: [PATCH 2/9] some more edits to ALL option --- src/Poseidon/CLI/List.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index 1d62016f4..e7bd7bd55 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -105,13 +105,12 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do getExtendedIndividualInfo pacCollection maybeMoreJannoColumns -- warning in case the additional Columns do not exist in the entire janno dataset - let addJannoCols = case extIndInfos of + let addJannoCols = case extIndInfos of -- get all add-column names from first extIndInfo [] -> [] - (e:es) -> map fst . extIndInfoAddCols $ e + (e:_) -> map fst . extIndInfoAddCols $ e forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do -- check entries in all individuals for that key let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] - logInfo $ columnKey ++ ": " ++ show nonEmptyEntries when (null nonEmptyEntries) . logWarning $ "Column Name " ++ columnKey ++ " not present in any individual" let tableH = ["Individual", "Group", "Package", "PackageVersion", "Is Latest"] ++ addJannoCols From 634662a1bb8f7662ebce3225b07527a6c9c0c6fb Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Sun, 12 May 2024 12:10:29 +0200 Subject: [PATCH 3/9] refined warnings for non-present columns --- src/Poseidon/CLI/List.hs | 16 +++++++++++----- src/Poseidon/Janno.hs | 4 ++-- .../GoldenTestsRunCommands.hs | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index e7bd7bd55..224aa39a7 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -104,14 +104,20 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do pacCollection <- readPoseidonPackageCollection pacReadOpts baseDirs getExtendedIndividualInfo pacCollection maybeMoreJannoColumns - -- warning in case the additional Columns do not exist in the entire janno dataset let addJannoCols = case extIndInfos of -- get all add-column names from first extIndInfo [] -> [] (e:_) -> map fst . extIndInfoAddCols $ e - forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do - -- check entries in all individuals for that key - let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] - when (null nonEmptyEntries) . logWarning $ "Column Name " ++ columnKey ++ " not present in any individual" + + -- warning in case the additional Columns do not exist in the entire janno dataset, + -- we only output this warning if the columns were requested explicitly. Not if + -- all columns were requested. We consider such a request to mean "all columns that are present". + case maybeMoreJannoColumns of + Just (e:_) -> do + forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do + -- check entries in all individuals for that key + let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] + when (null nonEmptyEntries) . logWarning $ "Column Name " ++ columnKey ++ " not present in any individual" + Nothing -> return () let tableH = ["Individual", "Group", "Package", "PackageVersion", "Is Latest"] ++ addJannoCols tableB = do diff --git a/src/Poseidon/Janno.hs b/src/Poseidon/Janno.hs index 3cff9f68b..cb3f1ee11 100644 --- a/src/Poseidon/Janno.hs +++ b/src/Poseidon/Janno.hs @@ -808,7 +808,7 @@ cleanInput (Just rawInputBS) = transNA $ trimWS . removeNoBreakSpace $ rawInputB transNA x = Just x instance Csv.ToNamedRecord JannoRow where - toNamedRecord j = Csv.namedRecord . filter ((/= "") . snd) $ [ + toNamedRecord j = (Csv.namedRecord . filter ((/= "") . snd) $ [ "Poseidon_ID" Csv..= jPoseidonID j , "Genetic_Sex" Csv..= jGeneticSex j , "Group_Name" Csv..= jGroupName j @@ -856,7 +856,7 @@ instance Csv.ToNamedRecord JannoRow where , "Note" Csv..= jComments j , "Keywords" Csv..= jKeywords j -- beyond that add what is in the hashmap of additional columns - ] `HM.union` (getCsvNR $ jAdditionalColumns j) + ]) `HM.union` (getCsvNR $ jAdditionalColumns j) -- | A function to create empty janno rows for a set of individuals createMinimalJanno :: [EigenstratIndEntry] -> JannoRows diff --git a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs index 9613413ff..e6a36e02a 100644 --- a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs +++ b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs @@ -322,7 +322,7 @@ testPipelineList testDir checkFilePath = do } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts2) "list" 2 let listOpts3 = listOpts1 { - _listListEntity = ListIndividuals ["Country", "Nr_SNPs"] + _listListEntity = ListIndividuals (Just ["Country", "Nr_SNPs"]) } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts3) "list" 3 let listOpts4 = listOpts3 { @@ -1060,7 +1060,7 @@ testPipelineListRemote testDir checkFilePath = do } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts2) "listRemote" 2 let listOpts3 = listOpts1 { - _listListEntity = ListIndividuals ["Publication"] + _listListEntity = ListIndividuals (Just ["Publication"]) , _listRawOutput = True } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts3) "listRemote" 3 From 6dc8aa3a4bfc5428e770b49909e6c8b8a93a467a Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Sun, 12 May 2024 13:13:57 +0200 Subject: [PATCH 4/9] updated golden tests --- src/Poseidon/CLI/List.hs | 6 ++--- src/Poseidon/CLI/Serve.hs | 2 +- src/Poseidon/Janno.hs | 4 +-- .../GoldenTestCheckSumFile.txt | 4 ++- .../GoldenTestData/chronicle/chronicle2.yml | 14 +++++----- .../GoldenTestData/list/list6 | 14 ++++++++++ .../GoldenTestData/listRemote/listRemote5 | 26 +++++++++++++++++++ .../GoldenTestsRunCommands.hs | 10 +++++++ 8 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 test/PoseidonGoldenTests/GoldenTestData/list/list6 create mode 100644 test/PoseidonGoldenTests/GoldenTestData/listRemote/listRemote5 diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index 224aa39a7..0bca419c1 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -110,14 +110,14 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do -- warning in case the additional Columns do not exist in the entire janno dataset, -- we only output this warning if the columns were requested explicitly. Not if - -- all columns were requested. We consider such a request to mean "all columns that are present". + -- all columns were requested. We consider such an "all" request to mean "all columns that are present". case maybeMoreJannoColumns of - Just (e:_) -> do + Just (_:_) -> do forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do -- check entries in all individuals for that key let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] when (null nonEmptyEntries) . logWarning $ "Column Name " ++ columnKey ++ " not present in any individual" - Nothing -> return () + _ -> return () let tableH = ["Individual", "Group", "Package", "PackageVersion", "Is Latest"] ++ addJannoCols tableB = do diff --git a/src/Poseidon/CLI/Serve.hs b/src/Poseidon/CLI/Serve.hs index 3a21cdfca..c91d8afd5 100644 --- a/src/Poseidon/CLI/Serve.hs +++ b/src/Poseidon/CLI/Serve.hs @@ -27,7 +27,7 @@ import Control.Concurrent.MVar (MVar, newEmptyMVar, putMVar) import Control.Monad (forM, when) import Control.Monad.IO.Class (liftIO) import qualified Data.ByteString.Lazy as B -import Data.List (nub, sortOn, (\\)) +import Data.List (nub, sortOn) import Data.List.Split (splitOn) import Data.Maybe (isJust) import Data.Ord (Down (..)) diff --git a/src/Poseidon/Janno.hs b/src/Poseidon/Janno.hs index cb3f1ee11..352a651fa 100644 --- a/src/Poseidon/Janno.hs +++ b/src/Poseidon/Janno.hs @@ -808,7 +808,7 @@ cleanInput (Just rawInputBS) = transNA $ trimWS . removeNoBreakSpace $ rawInputB transNA x = Just x instance Csv.ToNamedRecord JannoRow where - toNamedRecord j = (Csv.namedRecord . filter ((/= "") . snd) $ [ + toNamedRecord j = Csv.namedRecord [ "Poseidon_ID" Csv..= jPoseidonID j , "Genetic_Sex" Csv..= jGeneticSex j , "Group_Name" Csv..= jGroupName j @@ -856,7 +856,7 @@ instance Csv.ToNamedRecord JannoRow where , "Note" Csv..= jComments j , "Keywords" Csv..= jKeywords j -- beyond that add what is in the hashmap of additional columns - ]) `HM.union` (getCsvNR $ jAdditionalColumns j) + ] `HM.union` (getCsvNR $ jAdditionalColumns j) -- | A function to create empty janno rows for a set of individuals createMinimalJanno :: [EigenstratIndEntry] -> JannoRows diff --git a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt index 6c6899b47..4b9280c64 100644 --- a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt +++ b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt @@ -21,6 +21,7 @@ b18847f5498ae55882689b75916fdf64 list list/list2 63ef5f277f6f29163192382234211224 list list/list3 1c1f24de305405ece44393d378c0e15a list list/list4 bc636b9c03ea9359acd254a9911e5af3 list list/list5 +ad5590b0ad65e64d6b2c8d874571c9f8 list list/list6 b197fb8dd883c7469a4791e4a677f1c0 summarise summarise/summarise1 d9e4b3f15d4e129a365d2064198d95b6 summarise summarise/summarise2 a1186fdad9ed555dff4dd61dc9838645 survey survey/survey1 @@ -112,4 +113,5 @@ b43da4d5734371c0648553120f812466 fetch fetch/multi_packages_2/Lamnidis_2018-1.0. 1d2a588b88e6d1017147c01f19d0b878 listRemote listRemote/listRemote1 0ddad9ea097bca0253e0c3c6157efa68 listRemote listRemote/listRemote2 b2286cf9af7c6c8757b8109a1f58e2d9 listRemote listRemote/listRemote3 -0433b2a80ee5a2eb5bf8c6404130e562 listRemote listRemote/listRemote4 \ No newline at end of file +0433b2a80ee5a2eb5bf8c6404130e562 listRemote listRemote/listRemote4 +8a13e5b31acabca6839100f411c38453 listRemote listRemote/listRemote5 \ No newline at end of file diff --git a/test/PoseidonGoldenTests/GoldenTestData/chronicle/chronicle2.yml b/test/PoseidonGoldenTests/GoldenTestData/chronicle/chronicle2.yml index 998650990..014986adf 100644 --- a/test/PoseidonGoldenTests/GoldenTestData/chronicle/chronicle2.yml +++ b/test/PoseidonGoldenTests/GoldenTestData/chronicle/chronicle2.yml @@ -1,29 +1,29 @@ title: Chronicle title description: Chronicle description chronicleVersion: 0.2.0 -lastModified: 2023-09-22 +lastModified: 2024-05-12 packages: - title: Lamnidis_2018 version: 1.0.0 - commit: eb2e7c2af61b6738f0ad8862645c23dd57bf0bd1 + commit: 29acedb3a2c5dd89d0817176bccb70d26c611aaf path: Lamnidis_2018 - title: Lamnidis_2018 version: 1.0.1 - commit: eb2e7c2af61b6738f0ad8862645c23dd57bf0bd1 + commit: 29acedb3a2c5dd89d0817176bccb70d26c611aaf path: Lamnidis_2018_newVersion - title: Schiffels version: 1.1.1 - commit: fa2e92af97376489b32ce8b6874428c958d55f3f + commit: 64808be89e33423a35623ed09ae407b4e2311664 path: Schiffels - title: Schiffels_2016 version: 1.0.1 - commit: eb2e7c2af61b6738f0ad8862645c23dd57bf0bd1 + commit: 29acedb3a2c5dd89d0817176bccb70d26c611aaf path: Schiffels_2016 - title: Schmid_2028 version: 1.0.0 - commit: eb2e7c2af61b6738f0ad8862645c23dd57bf0bd1 + commit: 29acedb3a2c5dd89d0817176bccb70d26c611aaf path: Schmid_2028 - title: Wang_2020 version: 0.1.0 - commit: eb2e7c2af61b6738f0ad8862645c23dd57bf0bd1 + commit: 29acedb3a2c5dd89d0817176bccb70d26c611aaf path: Wang_2020 diff --git a/test/PoseidonGoldenTests/GoldenTestData/list/list6 b/test/PoseidonGoldenTests/GoldenTestData/list/list6 new file mode 100644 index 000000000..dc63d84c4 --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/list/list6 @@ -0,0 +1,14 @@ +.------------.-------.----------------.----------------.-----------.-------------.-----------------.-------------.-----------------.---------------.---------------.---------------.---------.-------------.----------.------.----------.-----------.-----------.----------------.-------------------.-----------------------.------------------.-------------------.-----------------.-----------.---------------.--------------.---------------.--------------.---------------.--------------.-----.---------------.-----------------.-------------------------------.------------.---------.-------------------------.--------.---------------.-------------------.--------------------.--------------------.------------------------------.-----------------.------------------------------------.------.----------. +| Individual | Group | Package | PackageVersion | Is Latest | Genetic_Sex | Alternative_IDs | Relation_To | Relation_Degree | Relation_Type | Relation_Note | Collection_ID | Country | Country_ISO | Location | Site | Latitude | Longitude | Date_Type | Date_C14_Labnr | Date_C14_Uncal_BP | Date_C14_Uncal_BP_Err | Date_BC_AD_Start | Date_BC_AD_Median | Date_BC_AD_Stop | Date_Note | MT_Haplogroup | Y_Haplogroup | Source_Tissue | Nr_Libraries | Library_Names | Capture_Type | UDG | Library_Built | Genotype_Ploidy | Data_Preparation_Pipeline_URL | Endogenous | Nr_SNPs | Coverage_on_Target_SNPs | Damage | Contamination | Contamination_Err | Contamination_Meas | Contamination_Note | Genetic_Source_Accession_IDs | Primary_Contact | Publication | Note | Keywords | +:============:=======:================:================:===========:=============:=================:=============:=================:===============:===============:===============:=========:=============:==========:======:==========:===========:===========:================:===================:=======================:==================:===================:=================:===========:===============:==============:===============:==============:===============:==============:=====:===============:=================:===============================:============:=========:=========================:========:===============:===================:====================:====================:==============================:=================:====================================:======:==========: +| XXX001 | POP1 | Schiffels_2016 | 1.0.1 | True | M | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016 | | | +| XXX002 | POP2 | Schiffels_2016 | 1.0.1 | True | F | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016 | | | +| XXX003 | POP1 | Schiffels_2016 | 1.0.1 | True | M | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016 | | | +| XXX004 | POP2 | Schiffels_2016 | 1.0.1 | True | F | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016 | | | +| XXX005 | POP2 | Schiffels_2016 | 1.0.1 | True | M | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestPaper1 | | | +| XXX006 | POP2 | Schiffels_2016 | 1.0.1 | True | F | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestPaper1 | | | +| XXX007 | POP1 | Schiffels_2016 | 1.0.1 | True | M | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestBook1 | | | +| XXX008 | POP3 | Schiffels_2016 | 1.0.1 | True | F | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestBook1 | | | +| XXX009 | POP1 | Schiffels_2016 | 1.0.1 | True | F | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestPaper1;TestBook1 | | | +| XXX010 | POP3 | Schiffels_2016 | 1.0.1 | True | M | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | Schiffels2016;TestPaper1;TestBook1 | | | +'------------'-------'----------------'----------------'-----------'-------------'-----------------'-------------'-----------------'---------------'---------------'---------------'---------'-------------'----------'------'----------'-----------'-----------'----------------'-------------------'-----------------------'------------------'-------------------'-----------------'-----------'---------------'--------------'---------------'--------------'---------------'--------------'-----'---------------'-----------------'-------------------------------'------------'---------'-------------------------'--------'---------------'-------------------'--------------------'--------------------'------------------------------'-----------------'------------------------------------'------'----------' diff --git a/test/PoseidonGoldenTests/GoldenTestData/listRemote/listRemote5 b/test/PoseidonGoldenTests/GoldenTestData/listRemote/listRemote5 new file mode 100644 index 000000000..66f95a8ea --- /dev/null +++ b/test/PoseidonGoldenTests/GoldenTestData/listRemote/listRemote5 @@ -0,0 +1,26 @@ +Individual Group Package PackageVersion Is Latest Genetic_Sex Alternative_IDs Relation_To Relation_Degree Relation_Type Relation_Note Collection_ID Country Country_ISO Location Site Latitude Longitude Date_Type Date_C14_Labnr Date_C14_Uncal_BP Date_C14_Uncal_BP_Err Date_BC_AD_Start Date_BC_AD_Median Date_BC_AD_Stop Date_Note MT_Haplogroup Y_Haplogroup Source_Tissue Nr_Libraries Library_Names Capture_Type UDG Library_Built Genotype_Ploidy Data_Preparation_Pipeline_URL Endogenous Nr_SNPs Coverage_on_Target_SNPs Damage Contamination Contamination_Err Contamination_Meas Contamination_Note Genetic_Source_Accession_IDs Primary_Contact Publication Note Keywords +XXX011 POP1 Lamnidis_2018 1.0.0 False M Lamnidis2018 +XXX012 POP2 Lamnidis_2018 1.0.0 False F Lamnidis2018 +XXX013 POP1 Lamnidis_2018 1.0.0 False M Lamnidis2018 +XXX014 POP2 Lamnidis_2018 1.0.0 False F Lamnidis2018 +XXX015 POP2 Lamnidis_2018 1.0.0 False M Lamnidis2018 +XXX016 POP2 Lamnidis_2018 1.0.0 False F Lamnidis2018 +XXX017 POP1 Lamnidis_2018 1.0.0 False M Lamnidis2018 +XXX018 POP3 Lamnidis_2018 1.0.0 False F Lamnidis2018 +XXX019 POP1 Lamnidis_2018 1.0.0 False F Lamnidis2018 +XXX099 POP3 Lamnidis_2018 1.0.0 False M Lamnidis2018 +XXX011 POP1 Lamnidis_2018 1.0.1 True M Lamnidis2018 +XXX012 POP2 Lamnidis_2018 1.0.1 True F Lamnidis2018 +XXX013 POP1 Lamnidis_2018 1.0.1 True M Lamnidis2018 +XXX014 POP2 Lamnidis_2018 1.0.1 True F Lamnidis2018 +XXX015 POP2 Lamnidis_2018 1.0.1 True M Lamnidis2018 +XXX016 POP2 Lamnidis_2018 1.0.1 True F Lamnidis2018 +XXX017 POP1 Lamnidis_2018 1.0.1 True M Lamnidis2018 +XXX018 POP3 Lamnidis_2018 1.0.1 True F Lamnidis2018 +XXX019 POP1 Lamnidis_2018 1.0.1 True F Lamnidis2018 +XXX020 POP3 Lamnidis_2018 1.0.1 True M Lamnidis2018 +SAMPLE0 1 Wang_2020 0.1.0 True F +SAMPLE1 2 Wang_2020 0.1.0 True M TestPaper1 +SAMPLE2 3 Wang_2020 0.1.0 True F Wang2020;TestPaper1 +SAMPLE3 4 Wang_2020 0.1.0 True M Wang2020;TestBook2 +SAMPLE4 5 Wang_2020 0.1.0 True F Wang2020;TestPaper1;TestBook2 diff --git a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs index e6a36e02a..4fa34ebf3 100644 --- a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs +++ b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs @@ -333,6 +333,10 @@ testPipelineList testDir checkFilePath = do _listOnlyLatest = True } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts5) "list" 5 + let listOpts6 = listOpts1 { + _listListEntity = ListIndividuals Nothing + } + runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts6) "list" 6 testPipelineSummarise :: FilePath -> FilePath -> IO () testPipelineSummarise testDir checkFilePath = do @@ -1073,6 +1077,12 @@ testPipelineListRemote testDir checkFilePath = do , _listOnlyLatest = False } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts4) "listRemote" 4 + + let listOpts5 = listOpts1 { + _listListEntity = ListIndividuals Nothing + , _listRawOutput = True + } + runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts5) "listRemote" 5 ) ( killThread threadID ) From 7dfefaa5b4c2cf5f60a52a291f5033daaa832984 Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Mon, 13 May 2024 22:29:42 +0200 Subject: [PATCH 5/9] bumped version nr. --- CHANGELOG.md | 3 +++ poseidon-hs.cabal | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf95e9a1f..a0ae69e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +- V 1.4.1.0: + - A new option `list --individuals --fullJanno` adds all standard columns from the Janno to the per-individual output. + - A new API option `/individuals?additionalJannoColumns=ALL` triggers the same behaviour for the Web API. - V 1.4.0.3: - Fixed a severe performance leak in code around `resolveEntityIndices`, which was called in various functions and wastefully recomputed `isLatestInCollection` way too often. This affected simple commands, like fetching a few packages from the server, forging, and has effects also in xerxes. - Bumped to a newer Compiler (GHC 9.4.7) and new Stackage Snapshot (LTS-21.17) diff --git a/poseidon-hs.cabal b/poseidon-hs.cabal index 9f96073b2..a3e31dcde 100644 --- a/poseidon-hs.cabal +++ b/poseidon-hs.cabal @@ -1,5 +1,5 @@ name: poseidon-hs -version: 1.4.0.3 +version: 1.4.1.0 synopsis: A package with tools for working with Poseidon Genotype Data description: The tools in this package read and analyse Poseidon-formatted genotype databases, a modular system for storing genotype data from thousands of individuals. license: MIT From 84fccde2d013a5f829c7ab64bcb490a4ed8f0b9f Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Mon, 13 May 2024 22:36:16 +0200 Subject: [PATCH 6/9] removed empty line from CheckumsFile --- test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt index 621ddc31c..8970c8779 100644 --- a/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt +++ b/test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt @@ -117,4 +117,4 @@ b2286cf9af7c6c8757b8109a1f58e2d9 listRemote listRemote/listRemote3 8a13e5b31acabca6839100f411c38453 listRemote listRemote/listRemote5 282cedf121f37e81c1e45ec0dfb97560 jannocoalesce jannocoalesce/target1.janno df34d0542c0a94cf9556619bff2e301d jannocoalesce jannocoalesce/target2.janno -a202f0c1636d55258454ad0a0dfea977 jannocoalesce jannocoalesce/target3.janno +a202f0c1636d55258454ad0a0dfea977 jannocoalesce jannocoalesce/target3.janno \ No newline at end of file From 6d1e214990c2d44e93fabf865fe01b564c848c21 Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Mon, 13 May 2024 22:37:02 +0200 Subject: [PATCH 7/9] stylish-haskell --- src/Poseidon/CLI/List.hs | 4 ++-- src/Poseidon/Package.hs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index 0bca419c1..e3939a261 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -105,11 +105,11 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do getExtendedIndividualInfo pacCollection maybeMoreJannoColumns let addJannoCols = case extIndInfos of -- get all add-column names from first extIndInfo - [] -> [] + [] -> [] (e:_) -> map fst . extIndInfoAddCols $ e -- warning in case the additional Columns do not exist in the entire janno dataset, - -- we only output this warning if the columns were requested explicitly. Not if + -- we only output this warning if the columns were requested explicitly. Not if -- all columns were requested. We consider such an "all" request to mean "all columns that are present". case maybeMoreJannoColumns of Just (_:_) -> do diff --git a/src/Poseidon/Package.hs b/src/Poseidon/Package.hs index 2461f082b..01241d215 100644 --- a/src/Poseidon/Package.hs +++ b/src/Poseidon/Package.hs @@ -47,8 +47,8 @@ import Poseidon.Janno (JannoLibraryBuilt (..), JannoList (..), JannoRow (..), JannoRows (..), JannoSex (..), JannoUDG (..), createMinimalJanno, - jannoHeaderString, - getMaybeJannoList, readJannoFile) + getMaybeJannoList, + jannoHeaderString, readJannoFile) import Poseidon.PoseidonVersion (asVersion, latestPoseidonVersion, showPoseidonVersion, validPoseidonVersions) @@ -833,7 +833,7 @@ getExtendedIndividualInfo allPackages maybeAdditionalJannoColumns = sequence $ d colNames = case maybeAdditionalJannoColumns of Nothing -> jannoHeaderString \\ ["Poseidon_ID", "Group_Name"] -- Nothing means all Janno columns -- except for these two which are already explicit - Just c -> c + Just c -> c additionalColumnEntries = [(k, BSC.unpack <$> toNamedRecord jannoRow HM.!? BSC.pack k) | k <- colNames] isLatest <- isLatestInCollection allPackages pac -- this lives in monad m -- double-return for m and then list. From 0cf594400fad304f8d418d9d32affe8797d185d9 Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Fri, 24 May 2024 08:50:34 +0200 Subject: [PATCH 8/9] added new type for janno column spec --- src/Poseidon/CLI/List.hs | 20 ++++++++++--------- .../CLI/OptparseApplicativeParsers.hs | 6 +++--- src/Poseidon/CLI/Serve.hs | 9 +++++---- src/Poseidon/Package.hs | 12 +++++------ src/Poseidon/ServerClient.hs | 7 ++++++- .../GoldenTestsRunCommands.hs | 10 +++++----- 6 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index e3939a261..2b9679054 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -13,7 +13,8 @@ import Poseidon.ServerClient (ApiReturnData (..), ArchiveEndpoint (..), ExtendedIndividualInfo (..), GroupInfo (..), PackageInfo (..), - processApiResponse, qDefault) + processApiResponse, qDefault, + AddJannoColSpec(..)) import Poseidon.Utils (PoseidonIO, logInfo, logWarning) import Control.Monad (forM_, when) @@ -38,7 +39,7 @@ data RepoLocationSpec = RepoLocal [FilePath] | RepoRemote ArchiveEndpoint -- | A datatype to represent the options what to list data ListEntity = ListPackages | ListGroups - | ListIndividuals (Maybe [String]) -- Nothing means all Janno columns. Just [] means none. + | ListIndividuals AddJannoColSpec -- | The main function running the list command runList :: ListOptions -> PoseidonIO () @@ -89,20 +90,21 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do True <- return (not onlyLatest || isLatest) return [groupName, getPacName gi, showMaybeVersion (getPacVersion gi), show isLatest, show nrInds] return (tableH, tableB) - ListIndividuals maybeMoreJannoColumns -> do + ListIndividuals addJannoColSpec -> do extIndInfos <- case repoLocation of RepoRemote (ArchiveEndpoint remoteURL archive) -> do logInfo "Downloading individual data from server" - let addJannoColFlag = case maybeMoreJannoColumns of - Nothing -> "&additionalJannoColumns=ALL" - Just moreJannoColumns -> "&additionalJannoColumns=" ++ intercalate "," moreJannoColumns + let addJannoColFlag = case addJannoColSpec of + AddJannoColAll -> "&additionalJannoColumns=ALL" + AddJannoColList [] -> "" + AddJannoColList moreJannoColumns -> "&additionalJannoColumns=" ++ intercalate "," moreJannoColumns apiReturn <- processApiResponse (remoteURL ++ "/individuals" ++ qDefault archive ++ addJannoColFlag) False case apiReturn of ApiReturnExtIndividualInfo indInfo -> return indInfo _ -> error "should not happen" RepoLocal baseDirs -> do pacCollection <- readPoseidonPackageCollection pacReadOpts baseDirs - getExtendedIndividualInfo pacCollection maybeMoreJannoColumns + getExtendedIndividualInfo pacCollection addJannoColSpec let addJannoCols = case extIndInfos of -- get all add-column names from first extIndInfo [] -> [] @@ -111,8 +113,8 @@ runList (ListOptions repoLocation listEntity rawOutput onlyLatest) = do -- warning in case the additional Columns do not exist in the entire janno dataset, -- we only output this warning if the columns were requested explicitly. Not if -- all columns were requested. We consider such an "all" request to mean "all columns that are present". - case maybeMoreJannoColumns of - Just (_:_) -> do + case addJannoColSpec of + AddJannoColList (_:_) -> do forM_ (zip [0..] addJannoCols) $ \(i, columnKey) -> do -- check entries in all individuals for that key let nonEmptyEntries = catMaybes [snd (entries !! i) | ExtendedIndividualInfo _ _ _ _ entries <- extIndInfos] diff --git a/src/Poseidon/CLI/OptparseApplicativeParsers.hs b/src/Poseidon/CLI/OptparseApplicativeParsers.hs index 66804c151..2e7f528e5 100644 --- a/src/Poseidon/CLI/OptparseApplicativeParsers.hs +++ b/src/Poseidon/CLI/OptparseApplicativeParsers.hs @@ -20,7 +20,7 @@ import Poseidon.GenotypeData (GenoDataSource (..), GenotypeDataSpec (..), GenotypeFormatSpec (..), SNPSetSpec (..)) -import Poseidon.ServerClient (ArchiveEndpoint (..)) +import Poseidon.ServerClient (ArchiveEndpoint (..), AddJannoColSpec(..)) import Poseidon.Utils (LogMode (..), TestMode (..)) import Poseidon.Version (VersionComponent (..), parseVersion) @@ -600,8 +600,8 @@ parseListEntity = parseListPackages <|> parseListGroups <|> (parseListIndividual parseListIndividualsDummy = OP.flag' () ( OP.long "individuals" <> OP.help "List all individuals/samples.") - parseListIndividualsExtraCols = ListIndividuals <$> (parseAllJannoCols <|> (Just <$> OP.many parseExtraCol)) - parseAllJannoCols = OP.flag' Nothing (OP.long "fullJanno" <> OP.help "output all Janno Columns") + parseListIndividualsExtraCols = ListIndividuals <$> (parseAllJannoCols <|> (AddJannoColList <$> OP.many parseExtraCol)) + parseAllJannoCols = OP.flag' AddJannoColAll (OP.long "fullJanno" <> OP.help "output all Janno Columns") parseExtraCol = OP.strOption ( OP.short 'j' <> OP.long "jannoColumn" <> diff --git a/src/Poseidon/CLI/Serve.hs b/src/Poseidon/CLI/Serve.hs index c91d8afd5..e08bd0945 100644 --- a/src/Poseidon/CLI/Serve.hs +++ b/src/Poseidon/CLI/Serve.hs @@ -16,7 +16,8 @@ import Poseidon.Package (PackageReadOptions (..), readPoseidonPackageCollection) import Poseidon.PoseidonVersion (minimalRequiredClientVersion) import Poseidon.ServerClient (ApiReturnData (..), - ServerApiReturnType (..)) + ServerApiReturnType (..), + AddJannoColSpec(..)) import Poseidon.Utils (LogA, PoseidonIO, envLogAction, logDebug, logInfo, logWithEnv) @@ -121,11 +122,11 @@ runServer (ServeOptions archBaseDirs maybeZipPath port ignoreChecksums certFiles pacs <- getItemFromArchiveStore archiveStore maybeAdditionalColumnsString <- (Just <$> param "additionalJannoColumns") `rescue` (\_ -> return Nothing) indInfo <- case maybeAdditionalColumnsString of - Just "ALL" -> getExtendedIndividualInfo pacs Nothing -- Nothing means all Janno Columns + Just "ALL" -> getExtendedIndividualInfo pacs AddJannoColAll -- Nothing means all Janno Columns Just additionalColumnsString -> let additionalColumnNames = splitOn "," additionalColumnsString - in getExtendedIndividualInfo pacs (Just additionalColumnNames) - Nothing -> getExtendedIndividualInfo pacs (Just []) + in getExtendedIndividualInfo pacs (AddJannoColList additionalColumnNames) + Nothing -> getExtendedIndividualInfo pacs (AddJannoColList []) let retData = ApiReturnExtIndividualInfo indInfo return $ ServerApiReturnType [] (Just retData) diff --git a/src/Poseidon/Package.hs b/src/Poseidon/Package.hs index 01241d215..7abb81e0c 100644 --- a/src/Poseidon/Package.hs +++ b/src/Poseidon/Package.hs @@ -57,7 +57,7 @@ import Poseidon.SequencingSource (SSFLibraryBuilt (..), SSFUDG (..), SeqSourceRows (..), readSeqSourceFile) import Poseidon.ServerClient (ExtendedIndividualInfo (..), - GroupInfo (..), PackageInfo (..)) + GroupInfo (..), PackageInfo (..), AddJannoColSpec(..)) import Poseidon.Utils (LogA, PoseidonException (..), PoseidonIO, checkFile, envInputPlinkMode, envLogAction, @@ -824,16 +824,16 @@ getJointIndividualInfo packages = do return (map fst . concat $ indInfoLatestPairs, map snd . concat $ indInfoLatestPairs) -getExtendedIndividualInfo :: (MonadThrow m) => [PoseidonPackage] -> Maybe [String] -> m [ExtendedIndividualInfo] -getExtendedIndividualInfo allPackages maybeAdditionalJannoColumns = sequence $ do -- list monad +getExtendedIndividualInfo :: (MonadThrow m) => [PoseidonPackage] -> AddJannoColSpec -> m [ExtendedIndividualInfo] +getExtendedIndividualInfo allPackages addJannoColSpec = sequence $ do -- list monad pac <- allPackages -- outer loop (automatically concatenating over inner loops) jannoRow <- getJannoRowsFromPac pac -- inner loop let name = jPoseidonID jannoRow groups = getJannoList . jGroupName $ jannoRow - colNames = case maybeAdditionalJannoColumns of - Nothing -> jannoHeaderString \\ ["Poseidon_ID", "Group_Name"] -- Nothing means all Janno columns + colNames = case addJannoColSpec of + AddJannoColAll -> jannoHeaderString \\ ["Poseidon_ID", "Group_Name"] -- Nothing means all Janno columns -- except for these two which are already explicit - Just c -> c + AddJannoColList c -> c additionalColumnEntries = [(k, BSC.unpack <$> toNamedRecord jannoRow HM.!? BSC.pack k) | k <- colNames] isLatest <- isLatestInCollection allPackages pac -- this lives in monad m -- double-return for m and then list. diff --git a/src/Poseidon/ServerClient.hs b/src/Poseidon/ServerClient.hs index 8a580bd1d..4d12452a9 100644 --- a/src/Poseidon/ServerClient.hs +++ b/src/Poseidon/ServerClient.hs @@ -8,7 +8,8 @@ module Poseidon.ServerClient ( ArchiveEndpoint(..), PackageInfo (..), GroupInfo (..), ExtendedIndividualInfo(..), extIndInfo2IndInfoCollection, - qDefault, qArchive, qPacVersion, (+&+) + qDefault, qArchive, qPacVersion, (+&+), + AddJannoColSpec(..) ) where import Paths_poseidon_hs (version) @@ -225,3 +226,7 @@ extIndInfo2IndInfoCollection extIndInfos = let indInfos = [IndividualInfo n g p | ExtendedIndividualInfo n g p _ _ <- extIndInfos] areLatest = map extIndInfoIsLatest extIndInfos in (indInfos, areLatest) + +-- type needed to specify additional Janno Columns to be queried from packages +data AddJannoColSpec = AddJannoColList [String] | AddJannoColAll + diff --git a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs index aba66235f..2c85089e7 100644 --- a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs +++ b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs @@ -34,7 +34,7 @@ import Poseidon.GenotypeData (GenoDataSource (..), GenotypeDataSpec (..), GenotypeFormatSpec (..), SNPSetSpec (..)) -import Poseidon.ServerClient (ArchiveEndpoint (..)) +import Poseidon.ServerClient (ArchiveEndpoint (..), AddJannoColSpec(..)) import Poseidon.Utils (LogMode (..), TestMode (..), getChecksum, testLog, usePoseidonLogger) @@ -331,7 +331,7 @@ testPipelineList testDir checkFilePath = do } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts2) "list" 2 let listOpts3 = listOpts1 { - _listListEntity = ListIndividuals (Just ["Country", "Nr_SNPs"]) + _listListEntity = ListIndividuals (AddJannoColList ["Country", "Nr_SNPs"]) } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts3) "list" 3 let listOpts4 = listOpts3 { @@ -343,7 +343,7 @@ testPipelineList testDir checkFilePath = do } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts5) "list" 5 let listOpts6 = listOpts1 { - _listListEntity = ListIndividuals Nothing + _listListEntity = ListIndividuals AddJannoColAll } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts6) "list" 6 @@ -1073,7 +1073,7 @@ testPipelineListRemote testDir checkFilePath = do } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts2) "listRemote" 2 let listOpts3 = listOpts1 { - _listListEntity = ListIndividuals (Just ["Publication"]) + _listListEntity = ListIndividuals (AddJannoColList ["Publication"]) , _listRawOutput = True } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts3) "listRemote" 3 @@ -1088,7 +1088,7 @@ testPipelineListRemote testDir checkFilePath = do runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts4) "listRemote" 4 let listOpts5 = listOpts1 { - _listListEntity = ListIndividuals Nothing + _listListEntity = ListIndividuals AddJannoColAll , _listRawOutput = True } runAndChecksumStdOut checkFilePath testDir (testLog $ runList listOpts5) "listRemote" 5 From fc9cbdc622b599aecfeed6a56c0bbe34627a783f Mon Sep 17 00:00:00 2001 From: Stephan Schiffels Date: Fri, 24 May 2024 08:53:51 +0200 Subject: [PATCH 9/9] stylish haskell --- src/Poseidon/CLI/List.hs | 6 +++--- src/Poseidon/CLI/OptparseApplicativeParsers.hs | 3 ++- src/Poseidon/CLI/Serve.hs | 6 +++--- src/Poseidon/Package.hs | 5 +++-- test/PoseidonGoldenTests/GoldenTestsRunCommands.hs | 3 ++- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Poseidon/CLI/List.hs b/src/Poseidon/CLI/List.hs index 2b9679054..9bcc22e64 100644 --- a/src/Poseidon/CLI/List.hs +++ b/src/Poseidon/CLI/List.hs @@ -9,12 +9,12 @@ import Poseidon.Package (PackageReadOptions (..), getExtendedIndividualInfo, packagesToPackageInfos, readPoseidonPackageCollection) -import Poseidon.ServerClient (ApiReturnData (..), +import Poseidon.ServerClient (AddJannoColSpec (..), + ApiReturnData (..), ArchiveEndpoint (..), ExtendedIndividualInfo (..), GroupInfo (..), PackageInfo (..), - processApiResponse, qDefault, - AddJannoColSpec(..)) + processApiResponse, qDefault) import Poseidon.Utils (PoseidonIO, logInfo, logWarning) import Control.Monad (forM_, when) diff --git a/src/Poseidon/CLI/OptparseApplicativeParsers.hs b/src/Poseidon/CLI/OptparseApplicativeParsers.hs index 2e7f528e5..7837ef547 100644 --- a/src/Poseidon/CLI/OptparseApplicativeParsers.hs +++ b/src/Poseidon/CLI/OptparseApplicativeParsers.hs @@ -20,7 +20,8 @@ import Poseidon.GenotypeData (GenoDataSource (..), GenotypeDataSpec (..), GenotypeFormatSpec (..), SNPSetSpec (..)) -import Poseidon.ServerClient (ArchiveEndpoint (..), AddJannoColSpec(..)) +import Poseidon.ServerClient (AddJannoColSpec (..), + ArchiveEndpoint (..)) import Poseidon.Utils (LogMode (..), TestMode (..)) import Poseidon.Version (VersionComponent (..), parseVersion) diff --git a/src/Poseidon/CLI/Serve.hs b/src/Poseidon/CLI/Serve.hs index e08bd0945..9377df8f3 100644 --- a/src/Poseidon/CLI/Serve.hs +++ b/src/Poseidon/CLI/Serve.hs @@ -15,9 +15,9 @@ import Poseidon.Package (PackageReadOptions (..), packagesToPackageInfos, readPoseidonPackageCollection) import Poseidon.PoseidonVersion (minimalRequiredClientVersion) -import Poseidon.ServerClient (ApiReturnData (..), - ServerApiReturnType (..), - AddJannoColSpec(..)) +import Poseidon.ServerClient (AddJannoColSpec (..), + ApiReturnData (..), + ServerApiReturnType (..)) import Poseidon.Utils (LogA, PoseidonIO, envLogAction, logDebug, logInfo, logWithEnv) diff --git a/src/Poseidon/Package.hs b/src/Poseidon/Package.hs index 7abb81e0c..f95cb2f95 100644 --- a/src/Poseidon/Package.hs +++ b/src/Poseidon/Package.hs @@ -56,8 +56,9 @@ import Poseidon.SequencingSource (SSFLibraryBuilt (..), SSFUDG (..), SeqSourceRow (..), SeqSourceRows (..), readSeqSourceFile) -import Poseidon.ServerClient (ExtendedIndividualInfo (..), - GroupInfo (..), PackageInfo (..), AddJannoColSpec(..)) +import Poseidon.ServerClient (AddJannoColSpec (..), + ExtendedIndividualInfo (..), + GroupInfo (..), PackageInfo (..)) import Poseidon.Utils (LogA, PoseidonException (..), PoseidonIO, checkFile, envInputPlinkMode, envLogAction, diff --git a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs index 2c85089e7..de3cdb49c 100644 --- a/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs +++ b/test/PoseidonGoldenTests/GoldenTestsRunCommands.hs @@ -34,7 +34,8 @@ import Poseidon.GenotypeData (GenoDataSource (..), GenotypeDataSpec (..), GenotypeFormatSpec (..), SNPSetSpec (..)) -import Poseidon.ServerClient (ArchiveEndpoint (..), AddJannoColSpec(..)) +import Poseidon.ServerClient (AddJannoColSpec (..), + ArchiveEndpoint (..)) import Poseidon.Utils (LogMode (..), TestMode (..), getChecksum, testLog, usePoseidonLogger)