Enable SSL verification for server-side operations rucio#6632

This affects Virtual Placement, replica sorting with the ‘custom’
implementation, and interactions with FTS. The recommended way to
specify a trusted CA bundle is through the ‘REQUESTS_CA_BUNDLE’
environmental variable [1].

For now, specifically for FTS, it remains possible to disable the
verification via the ‘core.fts_verify_tls’ configuration option.

[1] https://requests.readthedocs.io/en/latest/user/advanced/#ssl-cert-verification

lib/rucio/core/replica.py

@@ -859,7 +859,7 @@ def get_multi_cache_prefix(
     x_caches = REGION.get('CacheSites')
     if x_caches is NO_VALUE:
         try:
-            response = requests.get('{}/serverRanges'.format(vp_endpoint), timeout=1, verify=False)
+            response = requests.get('{}/serverRanges'.format(vp_endpoint), timeout=1)
             if response.ok:
                 x_caches = response.json()
                 REGION.set('CacheSites', x_caches)
@@ -3490,7 +3490,6 @@ def list_dataset_replicas_vp(
 
     try:
         vp_replies = requests.get('{}/ds/{}/{}:{}'.format(vp_endpoint, nr_replies, scope, name),
-                                  verify=False,
                                   timeout=1)
         if vp_replies.status_code == 200:
             vp_replies = vp_replies.json()

lib/rucio/core/replica_sorter.py

@@ -177,7 +177,7 @@ def __download_custom_distance_table() -> None:
         download_url = config_get('core', 'custom_distance_download_url', raise_exception=False, default=None)
         if download_url is None:
             raise Exception('Cannot download custom distance table: no URL provided')
-        result = requests.get(download_url, stream=True, verify=False)
+        result = requests.get(download_url, stream=True)
         if result and result.status_code in [200, ]:
             with open(db_path, mode='w') as file_obj:
                 file_obj.write(result.text)

lib/rucio/transfertool/fts3.py

@@ -879,6 +879,7 @@ def __init__(self,
                  bring_online: Optional[int] = 43200,
                  default_lifetime: Optional[int] = 172800,
                  archive_timeout_override: Optional[int] = None,
+                 verify_tls: Optional[bool] = None
                  logger: "LoggerFunction" = logging.log
                  ):
         """
@@ -919,12 +920,12 @@ def __init__(self,
         if self.external_host.startswith('https://'):
             if self.token:
                 self.cert = None
-                self.verify = False
                 self.headers['Authorization'] = 'Bearer ' + self.token
             else:
                 cert = _pick_cert_file(vo=vo)
                 self.cert = (cert, cert)
-                self.verify = False
+            self.verify = config_get_bool('core', 'fts_verify_tls', raise_exception=False,
+                                          default=verify_tls if verify_tls is not None else True))
         else:
             self.cert = None
             self.verify = True  # True is the default setting of a requests.* method