Skip to content

Commit 35c5a42

Browse files
fmigneaultfmigneault
committed
force caching off during test-thredds requests to allow subsequent permission-update/access-requests
1 parent 06c9d45 commit 35c5a42

File tree

1 file changed

+56
-46
lines changed

1 file changed

+56
-46
lines changed

Diff for: ‎notebooks-auth/test_thredds.ipynb

+56-46
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
},
3030
{
3131
"cell_type": "code",
32-
"execution_count": 10,
32+
"execution_count": 1,
3333
"metadata": {
3434
"pycharm": {
3535
"name": "#%%\n"
@@ -56,15 +56,15 @@
5656
"import requests\n",
5757
"print(\"Setup configuration parameters...\")\n",
5858
"\n",
59-
"PAVICS_HOST = os.getenv(\"PAVICS_HOST\", \"pavics.ouranos.ca\").rstrip(\"/\")\n",
59+
"PAVICS_HOST = (os.getenv(\"PAVICS_HOST\") or \"pavics.ouranos.ca\").rstrip(\"/\")\n",
6060
"if not PAVICS_HOST:\n",
6161
" raise ValueError(\"Cannot run test without a PAVICS_HOST value.\")\n",
6262
"\n",
6363
"PAVICS_URL = \"https://{}\".format(PAVICS_HOST)\n",
6464
"VERIFY_SSL = True if \"DISABLE_VERIFY_SSL\" not in os.environ else False\n",
65-
"MAGPIE_URL = os.getenv(\"MAGPIE_URL\", PAVICS_URL + \"/magpie\")\n",
65+
"MAGPIE_URL = os.getenv(\"MAGPIE_URL\") or (PAVICS_URL + \"/magpie\")\n",
6666
"TWITCHER_PROXY = \"/twitcher/ows/proxy\"\n",
67-
"TWITCHER_URL = os.getenv(\"TWITCHER_URL\", PAVICS_URL + TWITCHER_PROXY)\n",
67+
"TWITCHER_URL = os.getenv(\"TWITCHER_URL\") or (PAVICS_URL + TWITCHER_PROXY)\n",
6868
"THREDDS_SERVICE = \"thredds\"\n",
6969
"THREDDS_URL = \"{}/{}\".format(TWITCHER_URL, THREDDS_SERVICE)\n",
7070
"\n",
@@ -76,6 +76,13 @@
7676
"TEST_MAGPIE_ADMIN_PASSWORD = os.getenv(\"TEST_MAGPIE_ADMIN_PASSWORD\")\n",
7777
"if not TEST_MAGPIE_ADMIN_USERNAME or not TEST_MAGPIE_ADMIN_PASSWORD:\n",
7878
" raise ValueError(\"Missing test admin credentials to run tests.\")\n",
79+
" \n",
80+
"TEST_GROUP_NAME = os.getenv(\"TEST_MAGPIE_THREDDS_GROUP\") or \"test-auth-{!s}\".format(uuid.uuid4())\n",
81+
"TEST_USER_NAME = os.getenv(\"TEST_MAGPIE_THREDDS_USERNAME\") or \"test-user-{!s}\".format(uuid.uuid4())\n",
82+
"# password must not be displayed to keep minimal obfuscation\n",
83+
"# since test user name will be logged, and eventually receive slightly more access than 'anonymous',\n",
84+
"# don't give a chance for anyone to guess the credentials\n",
85+
"TEST_USER_PWD = os.getenv(\"TEST_MAGPIE_THREDDS_PASSWORD\") or str(uuid.uuid4())\n",
7986
"\n",
8087
"print(\" Will use Magpie URL: [{}]\".format(MAGPIE_URL))\n",
8188
"print(\" Will use Twitcher URL: [{}]\".format(TWITCHER_URL))\n",
@@ -118,7 +125,7 @@
118125
},
119126
{
120127
"cell_type": "code",
121-
"execution_count": 11,
128+
"execution_count": 2,
122129
"metadata": {
123130
"pycharm": {
124131
"name": "#%%\n"
@@ -180,7 +187,7 @@
180187
},
181188
{
182189
"cell_type": "code",
183-
"execution_count": 12,
190+
"execution_count": 3,
184191
"metadata": {
185192
"collapsed": false,
186193
"jupyter": {
@@ -194,12 +201,6 @@
194201
"source": [
195202
"# NBVAL_IGNORE_OUTPUT\n",
196203
"\n",
197-
"TEST_GROUP_NAME = os.getenv(\"TEST_MAGPIE_THREDDS_GROUP\", \"test-auth-{!s}\".format(uuid.uuid4()))\n",
198-
"TEST_USER_NAME = os.getenv(\"TEST_MAGPIE_THREDDS_USERNAME\", \"test-user-{!s}\".format(uuid.uuid4()))\n",
199-
"# password must not be displayed to keep minimal obfuscation\n",
200-
"# since test user name will be logged, and eventually receive slightly more access than 'anonymous',\n",
201-
"# don't give a chance for anyone to guess the credentials\n",
202-
"TEST_USER_PWD = os.getenv(\"TEST_MAGPIE_THREDDS_PASSWORD\", str(uuid.uuid4()))\n",
203204
"CLEANUP_CALLED = False\n",
204205
"\n",
205206
"def cleanup_test_data(skip_fail=True):\n",
@@ -251,7 +252,7 @@
251252
},
252253
{
253254
"cell_type": "code",
254-
"execution_count": 13,
255+
"execution_count": 4,
255256
"metadata": {
256257
"pycharm": {
257258
"name": "#%%\n"
@@ -263,8 +264,8 @@
263264
"output_type": "stream",
264265
"text": [
265266
"Create test group and test user... \n",
266-
" Will use TEST_USER_NAME: [test-birdhouse-thredds]\n",
267-
" Will use TEST_GROUP_NAME: [test-auth-300e787d-b83d-4e0b-ab20-1dea33062f78]\n",
267+
" Will use TEST_USER_NAME: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
268+
" Will use TEST_GROUP_NAME: [test-auth-113a7553-cc63-4309-9419-5d8f1ac3fb2e]\n",
268269
"Creation: OK\n"
269270
]
270271
}
@@ -312,7 +313,7 @@
312313
},
313314
{
314315
"cell_type": "code",
315-
"execution_count": 14,
316+
"execution_count": 5,
316317
"metadata": {
317318
"pycharm": {
318319
"name": "#%%\n"
@@ -412,7 +413,7 @@
412413
},
413414
{
414415
"cell_type": "code",
415-
"execution_count": 15,
416+
"execution_count": 6,
416417
"metadata": {
417418
"pycharm": {
418419
"name": "#%%\n"
@@ -466,15 +467,19 @@
466467
"\n",
467468
"Only public resources will be accessible by anyone. Therefore, anything under `secure` directory will be blocked to both the anonymous identify (no user logged in), and the specific test user. Being logged in as the test user does not provide any more access at this point than simply not being logged in at all.\n",
468469
"\n",
469-
"#### **Note**\n",
470+
"#### **Note 1**\n",
470471
"\n",
471472
"Resources allowed/denied access result could differ if `thredds` is not correctly configured since custom configuration is effective.\n",
472-
"If errors occur, make sure that `configuration` settings from `GET {MAGPIE_URL}/services/thredds` response correspond to `configuration` definition under the corresponding service applied onto the server from: https://github.com/bird-house/birdhouse-deploy/blob/master/birdhouse/config/magpie/providers.cfg.template\n"
473+
"If errors occur, make sure that `configuration` settings from `GET {MAGPIE_URL}/services/thredds` response correspond to `configuration` definition under the corresponding service applied onto the server from: https://github.com/bird-house/birdhouse-deploy/blob/master/birdhouse/config/magpie/providers.cfg.template\n",
474+
"\n",
475+
"#### **Note 2**\n",
476+
"\n",
477+
"Because permissions are applied onto Magpie and are resolved for access by Twitcher instead, any active caching of older requests is not yet synchronized on Twitcher side right after the permission update on Magpie side (caches are not shared, and therefore not invalidated on permission update). Following requests to the same resources (within the caching expiration delay) will hit the cached response triggered by the access during the previous requests. New permissions resolution will not be effective until the cache expires. For this reason, we explicitly ask (via request headers) to ignore caches during those resource access requests to bypass the synchronization problem. Caching is not a critical component performance-wise for one-shot requests, and can be safely disabled for this test. \n"
473478
]
474479
},
475480
{
476481
"cell_type": "code",
477-
"execution_count": 16,
482+
"execution_count": 7,
478483
"metadata": {
479484
"pycharm": {
480485
"name": "#%%\n"
@@ -518,32 +523,32 @@
518523
" Access: [Allowed]\n",
519524
"Detail:\n",
520525
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/catalog.html]\n",
521-
" User: [test-birdhouse-thredds]\n",
526+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
522527
" Code: [200]\n",
523528
" Access: [Allowed]\n",
524529
"Detail:\n",
525530
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/catalog.html]\n",
526-
" User: [test-birdhouse-thredds]\n",
531+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
527532
" Code: [200]\n",
528533
" Access: [Allowed]\n",
529534
"Detail:\n",
530535
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/secure/catalog.html]\n",
531-
" User: [test-birdhouse-thredds]\n",
536+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
532537
" Code: [403]\n",
533538
" Access: [Denied]\n",
534539
"Detail:\n",
535540
" Resource: [http://localhost:8001/ows/proxy/thredds/fileServer/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
536-
" User: [test-birdhouse-thredds]\n",
541+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
537542
" Code: [403]\n",
538543
" Access: [Denied]\n",
539544
"Detail:\n",
540545
" Resource: [http://localhost:8001/ows/proxy/thredds/ncml/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
541-
" User: [test-birdhouse-thredds]\n",
546+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
542547
" Code: [403]\n",
543548
" Access: [Denied]\n",
544549
"Detail:\n",
545550
" Resource: [http://localhost:8001/ows/proxy/thredds/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc.html]\n",
546-
" User: [test-birdhouse-thredds]\n",
551+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
547552
" Code: [403]\n",
548553
" Access: [Denied]\n",
549554
"Detail:\n",
@@ -594,8 +599,9 @@
594599
"\n",
595600
"\n",
596601
"def has_access(resource_path, user_cookies, user_name):\n",
602+
" _header_ignore_cache = {\"Cache-Control\": \"no-cache\"}\n",
597603
" _path = \"{}/{}\".format(THREDDS_URL, resource_path)\n",
598-
" _resp = requests.get(_path, verify=VERIFY_SSL, cookies=user_cookies)\n",
604+
" _resp = requests.get(_path, verify=VERIFY_SSL, cookies=user_cookies, headers=_header_ignore_cache)\n",
599605
" _code = _resp.status_code\n",
600606
" if _code in [200, 401, 403]:\n",
601607
" is_allowed = _code == 200\n",
@@ -654,13 +660,14 @@
654660
"\n",
655661
"Anonymous user will still not have access to resources under `secure` as its is not a member of the test group.\n",
656662
"\n",
657-
"**Note**:\n",
658-
"What defines an endpoint to require `browse` vs `read` access is according to THREDDS service configuration in Magpie (see previous cell Note)."
663+
"#### **Note**\n",
664+
"\n",
665+
"What defines an endpoint to require `browse` vs `read` access is according to THREDDS service configuration in Magpie (see previous cell Note).\n"
659666
]
660667
},
661668
{
662669
"cell_type": "code",
663-
"execution_count": 18,
670+
"execution_count": 8,
664671
"metadata": {
665672
"pycharm": {
666673
"name": "#%%\n"
@@ -673,32 +680,32 @@
673680
"text": [
674681
"Detail:\n",
675682
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/catalog.html]\n",
676-
" User: [test-birdhouse-thredds]\n",
683+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
677684
" Code: [200]\n",
678685
" Access: [Allowed]\n",
679686
"Detail:\n",
680687
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/catalog.html]\n",
681-
" User: [test-birdhouse-thredds]\n",
688+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
682689
" Code: [200]\n",
683690
" Access: [Allowed]\n",
684691
"Detail:\n",
685692
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/secure/catalog.html]\n",
686-
" User: [test-birdhouse-thredds]\n",
693+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
687694
" Code: [200]\n",
688695
" Access: [Allowed]\n",
689696
"Detail:\n",
690697
" Resource: [http://localhost:8001/ows/proxy/thredds/ncml/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
691-
" User: [test-birdhouse-thredds]\n",
698+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
692699
" Code: [200]\n",
693700
" Access: [Allowed]\n",
694701
"Detail:\n",
695702
" Resource: [http://localhost:8001/ows/proxy/thredds/fileServer/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
696-
" User: [test-birdhouse-thredds]\n",
703+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
697704
" Code: [403]\n",
698705
" Access: [Denied]\n",
699706
"Detail:\n",
700707
" Resource: [http://localhost:8001/ows/proxy/thredds/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc.html]\n",
701-
" User: [test-birdhouse-thredds]\n",
708+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
702709
" Code: [403]\n",
703710
" Access: [Denied]\n",
704711
"Detail:\n",
@@ -769,7 +776,7 @@
769776
},
770777
{
771778
"cell_type": "code",
772-
"execution_count": 42,
779+
"execution_count": 9,
773780
"metadata": {
774781
"pycharm": {
775782
"name": "#%%\n"
@@ -782,32 +789,32 @@
782789
"text": [
783790
"Detail:\n",
784791
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/catalog.html]\n",
785-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
792+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
786793
" Code: [200]\n",
787794
" Access: [Allowed]\n",
788795
"Detail:\n",
789796
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/catalog.html]\n",
790-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
797+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
791798
" Code: [200]\n",
792799
" Access: [Allowed]\n",
793800
"Detail:\n",
794801
" Resource: [http://localhost:8001/ows/proxy/thredds/catalog/birdhouse/testdata/secure/catalog.html]\n",
795-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
802+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
796803
" Code: [200]\n",
797804
" Access: [Allowed]\n",
798805
"Detail:\n",
799806
" Resource: [http://localhost:8001/ows/proxy/thredds/ncml/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
800-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
807+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
801808
" Code: [200]\n",
802809
" Access: [Allowed]\n",
803810
"Detail:\n",
804811
" Resource: [http://localhost:8001/ows/proxy/thredds/fileServer/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc]\n",
805-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
812+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
806813
" Code: [200]\n",
807814
" Access: [Allowed]\n",
808815
"Detail:\n",
809816
" Resource: [http://localhost:8001/ows/proxy/thredds/dodsC/birdhouse/testdata/secure/tasmax_Amon_MPI-ESM-MR_rcp45_r2i1p1_200601-200612.nc.html]\n",
810-
" User: [test-user-7f51bcf6-81d4-4931-aa37-ec2b15af5141]\n",
817+
" User: [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]\n",
811818
" Code: [200]\n",
812819
" Access: [Allowed]\n",
813820
"Detail:\n",
@@ -871,7 +878,7 @@
871878
},
872879
{
873880
"cell_type": "code",
874-
"execution_count": 19,
881+
"execution_count": 10,
875882
"metadata": {
876883
"collapsed": false,
877884
"jupyter": {
@@ -886,7 +893,8 @@
886893
"name": "stdout",
887894
"output_type": "stream",
888895
"text": [
889-
"WARNING - Skipping test data cleanup because of TEST_MAGPIE_DISABLE_CLEANUP option!\n",
896+
"Cleanup of TEST_USER_NAME [test-user-58ec4722-afcd-4f90-95a9-3ccee8dc162b]: OK\n",
897+
"Cleanup of TEST_GROUP_NAME [test-auth-113a7553-cc63-4309-9419-5d8f1ac3fb2e]: OK\n",
890898
"All tests: OK\n"
891899
]
892900
}
@@ -901,8 +909,10 @@
901909
]
902910
},
903911
{
904-
"cell_type": "markdown",
912+
"cell_type": "code",
913+
"execution_count": null,
905914
"metadata": {},
915+
"outputs": [],
906916
"source": []
907917
}
908918
],

0 commit comments

Comments
 (0)