Skip to content

Commit c3a0164

Browse files
maskarbmaskarb
authored
upgrade watchtower (#5534)
1 parent 68a455e commit c3a0164

File tree

3 files changed

+113
-66
lines changed

3 files changed

+113
-66
lines changed

‎Pipfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,14 +52,14 @@ sentry-sdk = ">=0.13"
5252
statsmodels = ">=0.12"
5353
trino = "*"
5454
unleashclient = "<5" # new version breaks for unknown reason
55-
watchtower = "==1.0.6" # new version breaks for unknown reason
5655
whitenoise = ">=5.0"
5756
pydantic = ">=2"
5857
sqlparse = "*"
5958
polars = "*"
6059
adbc-driver-postgresql = "*"
6160
packaging = "*"
6261
psycopg2-binary = "*"
62+
watchtower = "*"
6363

6464
[dev-packages]
6565
argh = ">=0.26.2"

‎Pipfile.lock

+6-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎koku/koku/settings.py

+106-59
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@
2020
from json import JSONDecodeError
2121
from zoneinfo import ZoneInfo
2222

23-
from boto3.session import Session
24-
from botocore.exceptions import ClientError
23+
import boto3
2524
from corsheaders.defaults import default_headers
2625
from oci import config
2726
from oci.exceptions import ConfigFileNotFound
@@ -361,8 +360,13 @@
361360
}
362361

363362
CW_AWS_ACCESS_KEY_ID = CONFIGURATOR.get_cloudwatch_access_id()
364-
CW_AWS_SECRET_ACCESS_KEY = CONFIGURATOR.get_cloudwatch_access_key()
365-
CW_AWS_REGION = CONFIGURATOR.get_cloudwatch_region()
363+
364+
CLOUDWATCH_CREDENTIALS = {
365+
"aws_access_key_id": CW_AWS_ACCESS_KEY_ID,
366+
"aws_secret_access_key": CONFIGURATOR.get_cloudwatch_access_key(),
367+
"region_name": CONFIGURATOR.get_cloudwatch_region(),
368+
}
369+
366370
CW_LOG_GROUP = CONFIGURATOR.get_cloudwatch_log_group()
367371

368372
LOGGING_FORMATTER = ENVIRONMENT.get_value("DJANGO_LOG_FORMATTER", default="simple")
@@ -384,52 +388,45 @@
384388
LOGGING_FILE = ENVIRONMENT.get_value("DJANGO_LOG_FILE", default=DEFAULT_LOG_FILE)
385389

386390
if CW_AWS_ACCESS_KEY_ID:
387-
try:
388-
POD_NAME = ENVIRONMENT.get_value("APP_POD_NAME", default="local")
389-
BOTO3_SESSION = Session(
390-
aws_access_key_id=CW_AWS_ACCESS_KEY_ID,
391-
aws_secret_access_key=CW_AWS_SECRET_ACCESS_KEY,
392-
region_name=CW_AWS_REGION,
393-
)
394-
watchtower = BOTO3_SESSION.client("logs")
395-
watchtower.create_log_stream(logGroupName=CW_LOG_GROUP, logStreamName=POD_NAME)
396-
LOGGING_HANDLERS += ["watchtower"]
397-
WATCHTOWER_HANDLER = {
398-
"level": KOKU_LOGGING_LEVEL,
399-
"class": "watchtower.CloudWatchLogHandler",
400-
"boto3_session": BOTO3_SESSION,
401-
"log_group": CW_LOG_GROUP,
402-
"stream_name": POD_NAME,
403-
"formatter": LOGGING_FORMATTER,
404-
"use_queues": False,
405-
"create_log_group": False,
406-
}
407-
except ClientError as e:
408-
if e.response.get("Error", {}).get("Code") == "ResourceAlreadyExistsException":
409-
LOGGING_HANDLERS += ["watchtower"]
410-
WATCHTOWER_HANDLER = {
411-
"level": KOKU_LOGGING_LEVEL,
412-
"class": "watchtower.CloudWatchLogHandler",
413-
"boto3_session": BOTO3_SESSION,
414-
"log_group": CW_LOG_GROUP,
415-
"stream_name": POD_NAME,
416-
"formatter": LOGGING_FORMATTER,
417-
"use_queues": False,
418-
"create_log_group": False,
419-
}
420-
else:
421-
print("CloudWatch not configured.")
391+
cw_client = boto3.client("logs", **CLOUDWATCH_CREDENTIALS)
392+
POD_NAME = ENVIRONMENT.get_value("APP_POD_NAME", default="local")
393+
LOGGING_HANDLERS += ["watchtower"]
394+
WATCHTOWER_HANDLER = {
395+
"level": KOKU_LOGGING_LEVEL,
396+
"formatter": LOGGING_FORMATTER,
397+
"class": "watchtower.CloudWatchLogHandler",
398+
"log_group_name": CW_LOG_GROUP,
399+
"log_stream_name": POD_NAME,
400+
"use_queues": False,
401+
"boto3_client": cw_client,
402+
"create_log_group": False,
403+
"create_log_stream": True, # will create stream if it does not exist
404+
}
405+
else:
406+
print("CloudWatch not configured.")
422407

423408
LOGGING = {
424409
"version": 1,
425410
"disable_existing_loggers": False,
426411
"formatters": {
427-
"verbose": {"()": "koku.log.TaskFormatter", "format": VERBOSE_FORMATTING},
428-
"simple": {"()": "koku.log.TaskFormatter", "format": SIMPLE_FORMATTING},
412+
"verbose": {
413+
"()": "koku.log.TaskFormatter",
414+
"format": VERBOSE_FORMATTING,
415+
},
416+
"simple": {
417+
"()": "koku.log.TaskFormatter",
418+
"format": SIMPLE_FORMATTING,
419+
},
429420
},
430421
"handlers": {
431-
"celery": {"class": "logging.StreamHandler", "formatter": LOGGING_FORMATTER},
432-
"console": {"class": "logging.StreamHandler", "formatter": LOGGING_FORMATTER},
422+
"celery": {
423+
"class": "logging.StreamHandler",
424+
"formatter": LOGGING_FORMATTER,
425+
},
426+
"console": {
427+
"class": "logging.StreamHandler",
428+
"formatter": LOGGING_FORMATTER,
429+
},
433430
"file": {
434431
"level": KOKU_LOGGING_LEVEL,
435432
"class": "logging.FileHandler",
@@ -446,24 +443,74 @@
446443
"handlers": LOGGING_HANDLERS,
447444
"level": ENVIRONMENT.get_value("GUNICORN_LOG_LEVEL", default="DEBUG"),
448445
},
449-
"django": {"handlers": LOGGING_HANDLERS, "level": DJANGO_LOGGING_LEVEL},
450-
"api": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
451-
"celery": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL, "propagate": False},
452-
"cost_models": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
453-
"forecast": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
454-
"hcs": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
455-
"kafka_utils": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
456-
"koku": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
457-
"providers": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
458-
"reporting": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
459-
"reporting_common": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
460-
"masu": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL, "propagate": False},
461-
"sources": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
462-
"subs": {"handlers": LOGGING_HANDLERS, "level": KOKU_LOGGING_LEVEL},
446+
"django": {
447+
"handlers": LOGGING_HANDLERS,
448+
"level": DJANGO_LOGGING_LEVEL,
449+
},
450+
"api": {
451+
"handlers": LOGGING_HANDLERS,
452+
"level": KOKU_LOGGING_LEVEL,
453+
},
454+
"celery": {
455+
"handlers": LOGGING_HANDLERS,
456+
"level": KOKU_LOGGING_LEVEL,
457+
"propagate": False,
458+
},
459+
"cost_models": {
460+
"handlers": LOGGING_HANDLERS,
461+
"level": KOKU_LOGGING_LEVEL,
462+
},
463+
"forecast": {
464+
"handlers": LOGGING_HANDLERS,
465+
"level": KOKU_LOGGING_LEVEL,
466+
},
467+
"hcs": {
468+
"handlers": LOGGING_HANDLERS,
469+
"level": KOKU_LOGGING_LEVEL,
470+
},
471+
"kafka_utils": {
472+
"handlers": LOGGING_HANDLERS,
473+
"level": KOKU_LOGGING_LEVEL,
474+
},
475+
"koku": {
476+
"handlers": LOGGING_HANDLERS,
477+
"level": KOKU_LOGGING_LEVEL,
478+
},
479+
"providers": {
480+
"handlers": LOGGING_HANDLERS,
481+
"level": KOKU_LOGGING_LEVEL,
482+
},
483+
"reporting": {
484+
"handlers": LOGGING_HANDLERS,
485+
"level": KOKU_LOGGING_LEVEL,
486+
},
487+
"reporting_common": {
488+
"handlers": LOGGING_HANDLERS,
489+
"level": KOKU_LOGGING_LEVEL,
490+
},
491+
"masu": {
492+
"handlers": LOGGING_HANDLERS,
493+
"level": KOKU_LOGGING_LEVEL,
494+
"propagate": False,
495+
},
496+
"sources": {
497+
"handlers": LOGGING_HANDLERS,
498+
"level": KOKU_LOGGING_LEVEL,
499+
},
500+
"subs": {
501+
"handlers": LOGGING_HANDLERS,
502+
"level": KOKU_LOGGING_LEVEL,
503+
},
463504
# The following set the log level for the UnleashClient and Unleash cache refresh jobs.
464505
# Setting to WARNING will prevent the INFO level spam.
465-
"UnleashClient": {"handlers": LOGGING_HANDLERS, "level": UNLEASH_LOGGING_LEVEL},
466-
"apscheduler": {"handlers": LOGGING_HANDLERS, "level": UNLEASH_LOGGING_LEVEL},
506+
"UnleashClient": {
507+
"handlers": LOGGING_HANDLERS,
508+
"level": UNLEASH_LOGGING_LEVEL,
509+
},
510+
"apscheduler": {
511+
"handlers": LOGGING_HANDLERS,
512+
"level": UNLEASH_LOGGING_LEVEL,
513+
},
467514
},
468515
}
469516

0 commit comments

Comments
 (0)