Skip to content

Commit 0b0d0ef

Browse files
FrancescAltedFrancescAlted
committed
Updated example for cat2cloud and hdf5 proxies
1 parent c208966 commit 0b0d0ef

File tree

1 file changed

+53
-104
lines changed

1 file changed

+53
-104
lines changed

‎examples/Video6-Cat2Cloud_and_HDF5.ipynb

Lines changed: 53 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -21,46 +21,42 @@
2121
"execution_count": null,
2222
"id": "71d3ef232dae45bc",
2323
"metadata": {
24-
"ExecuteTime": {
25-
"end_time": "2025-05-22T09:19:33.466272Z",
26-
"start_time": "2025-05-22T09:19:32.950545Z"
27-
},
2824
"collapsed": true,
2925
"jupyter": {
3026
"outputs_hidden": true
3127
}
3228
},
3329
"outputs": [],
3430
"source": [
35-
"#Install necessary libraries\n",
31+
"# Install necessary libraries\n",
3632
"# !pip install caterva2 blosc2 matplotlib requests aiohttp fsspec\n",
37-
"#Imports\n",
38-
"import blosc2\n",
33+
"# Imports\n",
3934
"import requests\n",
35+
"\n",
4036
"import caterva2 as cat2\n",
41-
"import numpy as np\n",
37+
"\n",
4238
"%matplotlib inline\n",
43-
"import matplotlib.pyplot as plt\n",
44-
"import time\n",
4539
"import os.path\n",
4640
"\n",
47-
"plt.style.use('default')\n",
41+
"import matplotlib.pyplot as plt\n",
42+
"\n",
43+
"plt.style.use(\"default\")\n",
4844
"plt.rcParams[\"animation.html\"] = \"jshtml\"\n",
4945
"# plt.rcParams.update({'text.usetex': False, 'font.serif': ['cm'], 'font.size': 16})\n",
50-
"plt.rcParams['figure.dpi'] = 100\n",
51-
"plt.rcParams['savefig.dpi'] = 100\n",
52-
"plt.style.use('seaborn-v0_8-paper')\n",
46+
"plt.rcParams[\"figure.dpi\"] = 100\n",
47+
"plt.rcParams[\"savefig.dpi\"] = 100\n",
48+
"plt.style.use(\"seaborn-v0_8-paper\")\n",
5349
"SMALL_SIZE = 1\n",
5450
"MEDIUM_SIZE = 2\n",
5551
"BIGGER_SIZE = 3\n",
5652
"\n",
57-
"plt.rc('font', size=SMALL_SIZE) # controls default text sizes\n",
58-
"plt.rc('axes', titlesize=SMALL_SIZE) # fontsize of the axes title\n",
59-
"plt.rc('axes', labelsize=MEDIUM_SIZE) # fontsize of the x and y labels\n",
60-
"plt.rc('xtick', labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
61-
"plt.rc('ytick', labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
62-
"plt.rc('legend', fontsize=SMALL_SIZE) # legend fontsize\n",
63-
"plt.rc('figure', titlesize=BIGGER_SIZE) # fontsize of the figure title"
53+
"plt.rc(\"font\", size=SMALL_SIZE) # controls default text sizes\n",
54+
"plt.rc(\"axes\", titlesize=SMALL_SIZE) # fontsize of the axes title\n",
55+
"plt.rc(\"axes\", labelsize=MEDIUM_SIZE) # fontsize of the x and y labels\n",
56+
"plt.rc(\"xtick\", labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
57+
"plt.rc(\"ytick\", labelsize=SMALL_SIZE) # fontsize of the tick labels\n",
58+
"plt.rc(\"legend\", fontsize=SMALL_SIZE) # legend fontsize\n",
59+
"plt.rc(\"figure\", titlesize=BIGGER_SIZE) # fontsize of the figure title"
6460
]
6561
},
6662
{
@@ -75,18 +71,15 @@
7571
"cell_type": "code",
7672
"execution_count": null,
7773
"id": "ed42711c192a3f08",
78-
"metadata": {
79-
"ExecuteTime": {
80-
"end_time": "2025-05-22T09:19:51.282568Z",
81-
"start_time": "2025-05-22T09:19:33.603794Z"
82-
}
83-
},
74+
"metadata": {},
8475
"outputs": [],
8576
"source": [
77+
"# This is a large file (~ 1 GB), so it may take a while to download.\n",
8678
"dir_path = \"kevlar\"\n",
87-
"response = requests.get(f\"http://www.silx.org/pub/pyFAI/pyFAI_UM_2020/data_ID13/{dir_path}.h5\")\n",
88-
"with open(f\"{dir_path}.h5\", mode=\"wb\") as file:\n",
89-
" file.write(response.content)"
79+
"if not os.path.exists(f\"{dir_path}.h5\"):\n",
80+
" response = requests.get(f\"http://www.silx.org/pub/pyFAI/pyFAI_UM_2020/data_ID13/{dir_path}.h5\")\n",
81+
" with open(f\"{dir_path}.h5\", mode=\"wb\") as file:\n",
82+
" file.write(response.content)"
9083
]
9184
},
9285
{
@@ -101,18 +94,13 @@
10194
"cell_type": "code",
10295
"execution_count": null,
10396
"id": "69e18e8f33657d7b",
104-
"metadata": {
105-
"ExecuteTime": {
106-
"end_time": "2025-05-22T09:19:51.455017Z",
107-
"start_time": "2025-05-22T09:19:51.294052Z"
108-
}
109-
},
97+
"metadata": {},
11098
"outputs": [],
11199
"source": [
112100
"# url = \"https://cat2.cloud/demo\"\n",
113101
"url = \"http://localhost:8002\"\n",
114-
"client = cat2.Client(url, ('user@example.com', 'foobar11'))\n",
115-
"myroot = client.get('@shared') # shared so all project users have access\n",
102+
"client = cat2.Client(url, (\"user@example.com\", \"foobar11\"))\n",
103+
"myroot = client.get(\"@shared\") # shared so all project users have access\n",
116104
"print(f\"Before uploading and unfolding: {myroot.file_list}\")"
117105
]
118106
},
@@ -128,16 +116,11 @@
128116
"cell_type": "code",
129117
"execution_count": null,
130118
"id": "6896f3c475bda18d",
131-
"metadata": {
132-
"ExecuteTime": {
133-
"end_time": "2025-05-22T09:19:57.896970Z",
134-
"start_time": "2025-05-22T09:19:51.476007Z"
135-
}
136-
},
119+
"metadata": {},
137120
"outputs": [],
138121
"source": [
139122
"local_address = f\"{dir_path}.h5\"\n",
140-
"remote_address = myroot.name + '/' + local_address\n",
123+
"remote_address = myroot.name + \"/\" + local_address\n",
141124
"apath = client.upload(local_address, remote_address)\n",
142125
"bloscpath = client.unfold(apath)\n",
143126
"print(f\"After uploading and unfolding: {myroot.file_list}\")"
@@ -155,38 +138,27 @@
155138
"cell_type": "code",
156139
"execution_count": null,
157140
"id": "a9f85a63729f28c0",
158-
"metadata": {
159-
"ExecuteTime": {
160-
"end_time": "2025-05-22T09:19:59.472883Z",
161-
"start_time": "2025-05-22T09:19:57.982622Z"
162-
}
163-
},
141+
"metadata": {},
164142
"outputs": [],
165143
"source": [
166144
"# Examine random frame of data\n",
167145
"proxy = myroot[\"kevlar/entry/data/data.b2nd\"]\n",
168146
"cmap = plt.cm.viridis\n",
169147
"example_image = proxy[5]\n",
170148
"fig = plt.figure()\n",
171-
"plt.imshow(example_image/65535, figure=fig, cmap=cmap, vmax=1, vmin=0)\n",
172-
"# plt.show()"
149+
"plt.imshow(example_image / 65535, figure=fig, cmap=cmap, vmax=1, vmin=0)"
173150
]
174151
},
175152
{
176153
"cell_type": "code",
177154
"execution_count": null,
178155
"id": "d43be1933e9794c3",
179-
"metadata": {
180-
"ExecuteTime": {
181-
"end_time": "2025-05-22T09:20:01.177656Z",
182-
"start_time": "2025-05-22T09:19:59.486438Z"
183-
}
184-
},
156+
"metadata": {},
185157
"outputs": [],
186158
"source": [
187-
"#Get an altered image that is more legible\n",
188-
"remastered_image = client.lazyexpr(\"proxy_where\", \"where(a[5] < 10, a[5] * 32000, a[5])\", {\"a\": proxy.path}, compute=True)\n",
189-
"example_image = client.get(remastered_image)[:]/65535 # decompress locally\n",
159+
"# Get an altered image that is more legible\n",
160+
"remastered_image = client.lazyexpr(\"proxy_where\", \"where(a < 10, a * 32000, a)\", {\"a\": proxy.path})\n",
161+
"example_image = client.get(remastered_image)[5] / 65535 # decompress locally\n",
190162
"fig = plt.figure()\n",
191163
"plt.imshow(example_image, figure=fig, cmap=cmap, vmax=1, vmin=0)"
192164
]
@@ -195,24 +167,18 @@
195167
"cell_type": "code",
196168
"execution_count": null,
197169
"id": "edf8aa0aa17bbd",
198-
"metadata": {
199-
"ExecuteTime": {
200-
"end_time": "2025-05-22T09:20:10.322942Z",
201-
"start_time": "2025-05-22T09:20:01.222719Z"
202-
}
203-
},
170+
"metadata": {},
204171
"outputs": [],
205172
"source": [
206-
"#Make a wee video - takes around 7s to load\n",
173+
"# Make a wee video - takes around 7s to load\n",
207174
"import matplotlib.animation as animation\n",
175+
"\n",
208176
"fig = plt.figure()\n",
209177
"ax = fig.gca()\n",
210-
"artists=[]\n",
211-
"nframes=10\n",
212-
"remastered_image = client.lazyexpr(\"proxy_where\", f\"where(a[:{nframes}] < 10, a[:{nframes}] * 20000, a[:{nframes}])\", {\"a\": proxy.path}, compute=True)\n",
213-
"\n",
178+
"artists = []\n",
179+
"nframes = 10\n",
214180
"for frame in range(nframes):\n",
215-
" example_image = client.get(remastered_image)[frame]/65535 # decompress locally\n",
181+
" example_image = client.get(remastered_image)[frame] / 65535 # decompress locally\n",
216182
" im = ax.imshow(example_image, figure=fig, cmap=cmap, vmax=1, vmin=0)\n",
217183
" artists.append((im,))\n",
218184
"\n",
@@ -233,48 +199,36 @@
233199
"cell_type": "code",
234200
"execution_count": null,
235201
"id": "71244890b29df63c",
236-
"metadata": {
237-
"ExecuteTime": {
238-
"end_time": "2025-05-22T09:20:11.446115Z",
239-
"start_time": "2025-05-22T09:20:10.363873Z"
240-
}
241-
},
202+
"metadata": {},
242203
"outputs": [],
243204
"source": [
244-
" # copy 20 frames of data to blosc2 array on server\n",
245-
"bloscfile = client.lazyexpr(\"kevlar20\", f\"a[:20]\", {\"a\": proxy.path}, compute=True) #automatically stored in @personal root as @personal/kevlar20.b2nd"
205+
"# copy 20 frames of data to blosc2 array on server\n",
206+
"# automatically stored in @personal root as @personal/kevlar20.b2nd\n",
207+
"bloscfile = client.lazyexpr(\"kevlar20\", \"a[:20]\", {\"a\": proxy.path}, compute=True)"
246208
]
247209
},
248210
{
249211
"cell_type": "code",
250212
"execution_count": null,
251213
"id": "8094982ad7c22fe5",
252-
"metadata": {
253-
"ExecuteTime": {
254-
"end_time": "2025-05-22T09:20:11.489084Z",
255-
"start_time": "2025-05-22T09:20:11.466927Z"
256-
}
257-
},
214+
"metadata": {},
258215
"outputs": [],
259216
"source": [
260-
"#Define lazyexpr that is saved but not executed - very quick\n",
261-
"remastered_image = client.lazyexpr(\"blosc_where\", \"where(a < 10, a * 32000, a)\", {\"a\": bloscfile}, compute=False)"
217+
"# Define lazyexpr that is saved but not executed - very quick\n",
218+
"remastered_image = client.lazyexpr(\n",
219+
" \"blosc_where\", \"where(a < 10, a * 32000, a)\", {\"a\": bloscfile}, compute=False\n",
220+
")"
262221
]
263222
},
264223
{
265224
"cell_type": "code",
266225
"execution_count": null,
267226
"id": "51625f84608f8385",
268-
"metadata": {
269-
"ExecuteTime": {
270-
"end_time": "2025-05-22T09:20:12.490988Z",
271-
"start_time": "2025-05-22T09:20:11.510004Z"
272-
}
273-
},
227+
"metadata": {},
274228
"outputs": [],
275229
"source": [
276-
"#Query a single image, execute lazyexpr on the fly\n",
277-
"example_image = client.get(remastered_image)[5]/65535 # decompress locally\n",
230+
"# Query a single image, execute lazyexpr on the fly\n",
231+
"example_image = client.get(remastered_image)[5] / 65535 # decompress locally\n",
278232
"fig = plt.figure()\n",
279233
"plt.imshow(example_image, figure=fig, cmap=cmap, vmax=1, vmin=0)"
280234
]
@@ -283,12 +237,7 @@
283237
"cell_type": "code",
284238
"execution_count": null,
285239
"id": "d22872a0a54de791",
286-
"metadata": {
287-
"ExecuteTime": {
288-
"end_time": "2025-05-22T09:20:20.731435Z",
289-
"start_time": "2025-05-22T09:20:12.511338Z"
290-
}
291-
},
240+
"metadata": {},
292241
"outputs": [],
293242
"source": [
294243
"# Make the same video as before, but using the blosc2 array. Takes about 8s.\n",

0 commit comments

Comments
 (0)