Merge pull request #18 from jluethi/dask

Lazy Loading of data to improve memory efficiency

Author: fstur

examples/run_zmb.py

@@ -44,7 +44,8 @@
 # ]
 
 mode = "zmb"
-query = "well=='D05' and field==['s1','s2','s3']"
+#query = "well=='D05' and field==['s1','s2','s3']"
+query = ""
 grid_montage = False
 output_name = "Test_ZMB_3D_new"
 

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "fractal-tasks-core==0.10.0",
+    "fractal-tasks-core==0.11.0",
     "pydantic",
     "zarr",
     "faim-hcs @ git+https://github.com/jluethi/faim-hcs.git@generalized_wavelength_extraction",
@@ -101,6 +101,9 @@ extend-ignore = [
     "D416", # Section name should end with a colon
 ]
 
+[tool.ruff.pydocstyle]
+convention = "google"
+
 [tool.ruff.per-file-ignores]
 "tests/*.py" = ["D", "S"]
 "setup.py" = ["D"]

resources/zmb_test_data/40x-1time-26z-2well-6site-4channel.HTD

@@ -0,0 +1,35 @@
+"HTSInfoFile", Version 1.0
+"Description", "test"
+"PlateType", 2
+"TimePoints", 1
+"ZSeries", TRUE
+"ZSteps", 26
+"ZProjection", FALSE
+"XWells", 12
+"YWells", 8
+"WellsSelection1", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection2", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection3", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection4", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection5", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection6", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection7", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection8", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"Sites", TRUE
+"XSites", 3
+"YSites", 3
+"SiteSelection1", TRUE, TRUE, TRUE
+"SiteSelection2", FALSE, TRUE, TRUE
+"SiteSelection3", FALSE, FALSE, TRUE
+"Waves", TRUE
+"NWavelengths", 4
+"WaveName1", "DAPI"
+"WaveName2", "FITC"
+"WaveName3", "Texas Red"
+"WaveName4", "Cy5"
+"WaveCollect1", 1
+"WaveCollect2", 1
+"WaveCollect3", 1
+"WaveCollect4", 1
+"UniquePlateIdentifier", "c66340dd-269c-45dc-a7bb-071246f177d0"
+"EndFile"

resources/zmb_test_data/TimePoint_1/40x-1time-26z-2well-6site-4channel.HTD

@@ -0,0 +1,35 @@
+"HTSInfoFile", Version 1.0
+"Description", "test"
+"PlateType", 2
+"TimePoints", 1
+"ZSeries", TRUE
+"ZSteps", 26
+"ZProjection", FALSE
+"XWells", 12
+"YWells", 8
+"WellsSelection1", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection2", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection3", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection4", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection5", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection6", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection7", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection8", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"Sites", TRUE
+"XSites", 3
+"YSites", 3
+"SiteSelection1", TRUE, TRUE, TRUE
+"SiteSelection2", FALSE, TRUE, TRUE
+"SiteSelection3", FALSE, FALSE, TRUE
+"Waves", TRUE
+"NWavelengths", 4
+"WaveName1", "DAPI"
+"WaveName2", "FITC"
+"WaveName3", "Texas Red"
+"WaveName4", "Cy5"
+"WaveCollect1", 1
+"WaveCollect2", 1
+"WaveCollect3", 1
+"WaveCollect4", 1
+"UniquePlateIdentifier", "c66340dd-269c-45dc-a7bb-071246f177d0"
+"EndFile"

resources/zmb_test_data/TimePoint_1/ZStep_1/40x-1time-26z-2well-6site-4channel.HTD

@@ -0,0 +1,35 @@
+"HTSInfoFile", Version 1.0
+"Description", "test"
+"PlateType", 2
+"TimePoints", 1
+"ZSeries", TRUE
+"ZSteps", 26
+"ZProjection", FALSE
+"XWells", 12
+"YWells", 8
+"WellsSelection1", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection2", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection3", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection4", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection5", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection6", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection7", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection8", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"Sites", TRUE
+"XSites", 3
+"YSites", 3
+"SiteSelection1", TRUE, TRUE, TRUE
+"SiteSelection2", FALSE, TRUE, TRUE
+"SiteSelection3", FALSE, FALSE, TRUE
+"Waves", TRUE
+"NWavelengths", 4
+"WaveName1", "DAPI"
+"WaveName2", "FITC"
+"WaveName3", "Texas Red"
+"WaveName4", "Cy5"
+"WaveCollect1", 1
+"WaveCollect2", 1
+"WaveCollect3", 1
+"WaveCollect4", 1
+"UniquePlateIdentifier", "c66340dd-269c-45dc-a7bb-071246f177d0"
+"EndFile"

resources/zmb_test_data/TimePoint_1/ZStep_1/40x-1time-26z-2well-6site-4channel_C03_s1_w1.TIF

@@ -0,0 +1,35 @@
+"HTSInfoFile", Version 1.0
+"Description", "test"
+"PlateType", 2
+"TimePoints", 1
+"ZSeries", TRUE
+"ZSteps", 26
+"ZProjection", FALSE
+"XWells", 12
+"YWells", 8
+"WellsSelection1", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection2", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection3", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection4", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection5", FALSE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection6", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection7", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"WellsSelection8", FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE
+"Sites", TRUE
+"XSites", 3
+"YSites", 3
+"SiteSelection1", TRUE, TRUE, TRUE
+"SiteSelection2", FALSE, TRUE, TRUE
+"SiteSelection3", FALSE, FALSE, TRUE
+"Waves", TRUE
+"NWavelengths", 4
+"WaveName1", "DAPI"
+"WaveName2", "FITC"
+"WaveName3", "Texas Red"
+"WaveName4", "Cy5"
+"WaveCollect1", 1
+"WaveCollect2", 1
+"WaveCollect3", 1
+"WaveCollect4", 1
+"UniquePlateIdentifier", "c66340dd-269c-45dc-a7bb-071246f177d0"
+"EndFile"