forked from apache/airflow
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path_utils.sh
1728 lines (1600 loc) · 69.7 KB
/
_utils.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Assume all the scripts are sourcing the _utils.sh from the scripts/ci directory
# and MY_DIR variable is set to this directory. It can be overridden however
# extra flags passed to the docker run for CI image commands (as Bash array)
declare -a EXTRA_DOCKER_FLAGS
# extra flags passed to the docker run for PROD image commands(as Bash array)
declare -a EXTRA_DOCKER_PROD_BUILD_FLAGS
export EXTRA_DOCKER_FLAGS
export EXTRA_DOCKER_PROD_BUILD_FLAGS
# In case "VERBOSE_COMMANDS" is set to "true" set -x is used to enable debugging
function check_verbose_setup {
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
set -x
else
set +x
fi
}
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be
# printed before execution
function verbose_docker {
if [[ ${VERBOSE:="false"} == "true" ]]; then
echo "docker" "${@}"
fi
docker "${@}"
}
# Common environment that is initialized by both Breeze and CI scripts
function initialize_common_environment {
# default python Major/Minor version
PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
# Sets to where airflow sources are located
AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${MY_DIR}/../../" && pwd)}
export AIRFLOW_SOURCES
# Sets to the build cache directory - status of build and convenience scripts are stored there
BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
export BUILD_CACHE_DIR
# File to keep the last forced answer. This is useful for pre-commits where you need to
# only answer once if the image should be rebuilt or not and your answer is used for
# All the subsequent questions
LAST_FORCE_ANSWER_FILE="${BUILD_CACHE_DIR}/last_force_answer.sh"
# Create useful directories if not yet created
mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
mkdir -p "${AIRFLOW_SOURCES}/logs"
mkdir -p "${AIRFLOW_SOURCES}/tmp"
mkdir -p "${AIRFLOW_SOURCES}/files"
mkdir -p "${AIRFLOW_SOURCES}/dist"
# Read common values used across Breeze and CI scripts
# shellcheck source=common/_common_values.sh
. "${AIRFLOW_SOURCES}/common/_common_values.sh"
# Read image-specific values used across Breeze and CI scripts
# shellcheck source=common/_image_variables.sh
. "${AIRFLOW_SOURCES}/common/_image_variables.sh"
# Read information about files that are checked if image should be rebuilt
# shellcheck source=common/_files_for_rebuild_check.sh
. "${AIRFLOW_SOURCES}/common/_files_for_rebuild_check.sh"
# Default branch name for triggered builds is the one configured in default branch
export BRANCH_NAME=${BRANCH_NAME:=${DEFAULT_BRANCH}}
export GITHUB_ORGANISATION=${GITHUB_ORGANISATION:="apache"}
export GITHUB_REPO=${GITHUB_REPO:="airflow"}
export CACHE_REGISTRY=${CACHE_REGISTRY:="docker.pkg.github.com"}
export ENABLE_REGISTRY_CACHE=${ENABLE_REGISTRY_CACHE:="false"}
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
export MYSQL_HOST_PORT=${MYSQL_HOST_PORT:="23306"}
# Default MySQL/Postgres versions
export POSTGRES_VERSION=${POSTGRES_VERSION:="9.6"}
export MYSQL_VERSION=${MYSQL_VERSION:="5.7"}
# Whether base python images should be pulled from cache
export PULL_PYTHON_BASE_IMAGES_FROM_CACHE=${PULL_PYTHON_BASE_IMAGES_FROM_CACHE:="true"}
# Disable writing .pyc files - slightly slower imports but not messing around when switching
# Python version and avoids problems with root-owned .pyc files in host
export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"}
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
#
# Sets mounting of host volumes to container for static checks
# unless MOUNT_HOST_AIRFLOW_VOLUME is not true
#
MOUNT_HOST_AIRFLOW_VOLUME=${MOUNT_HOST_AIRFLOW_VOLUME:="true"}
export MOUNT_HOST_AIRFLOW_VOLUME
# If this variable is set, we mount the whole sources directory to the host rather than
# selected volumes. This is needed to check ALL source files during licence check
# for example
MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS=${MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS="false"}
export MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS
# Set host user id to current user. This is used to set the ownership properly when exiting
# The container on Linux - all files created inside docker are created with root user
# but they should be restored back to the host user
HOST_USER_ID="$(id -ur)"
export HOST_USER_ID
# Set host group id to current group This is used to set the ownership properly when exiting
# The container on Linux - all files created inside docker are created with root user
# but they should be restored back to the host user
HOST_GROUP_ID="$(id -gr)"
export HOST_GROUP_ID
# Add the right volume mount for sources, depending which mount strategy is used
if [[ ${MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
print_info "Mount whole airflow source directory for static checks (make sure all files are in container)"
print_info
EXTRA_DOCKER_FLAGS=( \
"-v" "${AIRFLOW_SOURCES}:/opt/airflow" \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
elif [[ ${MOUNT_HOST_AIRFLOW_VOLUME} == "true" ]]; then
print_info
print_info "Mounting necessary host volumes to Docker"
print_info
EXTRA_DOCKER_FLAGS=()
while IFS= read -r LINE; do
EXTRA_DOCKER_FLAGS+=( "${LINE}")
done < <(convert_local_mounts_to_docker_params)
else
print_info
print_info "Skip mounting host volumes to Docker"
print_info
EXTRA_DOCKER_FLAGS=( \
"--env" "PYTHONDONTWRITEBYTECODE" \
)
fi
# In case of the CI build get environment variables from codecov.io and
# set it as the extra docker flags. As described in https://docs.codecov.io/docs/testing-with-docker
if [[ ${CI:=} == "true" ]]; then
CI_CODECOV_ENV="$(bash <(curl -s https://codecov.io/env))"
for ENV_PARAM in ${CI_CODECOV_ENV}
do
EXTRA_DOCKER_FLAGS+=("${ENV_PARAM}")
done
fi
EXTRA_DOCKER_PROD_BUILD_FLAGS=()
# We use pulled docker image cache by default to speed up the builds
# but we can also set different docker caching strategy (for example we can use local cache
# to build the images in case we iterate with the image
export DOCKER_CACHE=${DOCKER_CACHE:="pulled"}
# By default we are not upgrading to latest requirements when building Docker CI image
# This will only be done in cron jobs
export UPGRADE_TO_LATEST_REQUIREMENTS=${UPGRADE_TO_LATEST_REQUIREMENTS:="false"}
# In case of MacOS we need to use gstat - gnu version of the stats
STAT_BIN=stat
if [[ "${OSTYPE}" == "darwin"* ]]; then
STAT_BIN=gstat
fi
# Read airflow version from the version.py
AIRFLOW_VERSION=$(grep version "airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
export AIRFLOW_VERSION
# default version of python used to tag the "master" and "latest" images in DockerHub
export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
# In case we are not in CI - we assume we run locally. There are subtle changes if you run
# CI scripts locally - for example requirements are eagerly updated if you do local run
# in generate requirements
if [[ ${CI:="false"} == "true" ]]; then
export LOCAL_RUN="false"
else
export LOCAL_RUN="true"
fi
# eager upgrade while generating requirements should only happen in locally run
# pre-commits or in cron job
if [[ ${LOCAL_RUN} == "true" ]]; then
export UPGRADE_WHILE_GENERATING_REQUIREMENTS="true"
else
export UPGRADE_WHILE_GENERATING_REQUIREMENTS=${UPGRADE_WHILE_GENERATING_REQUIREMENTS:="false"}
fi
# Default extras used for building CI image
export DEFAULT_CI_EXTRAS="devel_ci"
# Default extras used for building Production image. The master of this information is in the Dockerfile
DEFAULT_PROD_EXTRAS=$(grep "ARG AIRFLOW_EXTRAS=" "${AIRFLOW_SOURCES}/Dockerfile"|
awk 'BEGIN { FS="=" } { print $2 }' | tr -d '"')
# By default we build CI images but when we specify `--producton-image` we switch to production image
export PRODUCTION_IMAGE="false"
# The SQLlite URL used for sqlite runs
export SQLITE_URL="sqlite:////root/airflow/airflow.db"
# Determines if airflow should be installed from a specified reference in GitHub
export INSTALL_AIRFLOW_REFERENCE=""
# Version suffix for PyPI packaging
export VERSION_SUFFIX_FOR_PYPI=""
# Artifact name suffix for SVN packaging
export VERSION_SUFFIX_FOR_SVN=""
}
# Prints verbose information in case VERBOSE variable is set
function print_info() {
if [[ ${VERBOSE:="false"} == "true" ]]; then
echo "$@"
fi
}
# Those are files that are mounted locally when mounting local sources is requested
# By default not the whole airflow sources directory is mounted because there are often
# artifacts created there (for example .egg-info files) that are breaking the capability
# of running different python versions in Breeze. So we only mount what is needed by default.
LOCAL_MOUNTS="
.bash_aliases /root/
.bash_history /root/
.coveragerc /opt/airflow/
.dockerignore /opt/airflow/
.flake8 /opt/airflow/
.github /opt/airflow/
.inputrc /root/
.kube /root/
.rat-excludes /opt/airflow/
CHANGELOG.txt /opt/airflow/
Dockerfile.ci /opt/airflow/
LICENSE /opt/airflow/
MANIFEST.in /opt/airflow/
NOTICE /opt/airflow/
airflow /opt/airflow/
backport_packages /opt/airflow/
common /opt/airflow/
dags /opt/airflow/
dev /opt/airflow/
docs /opt/airflow/
files /
dist /
hooks /opt/airflow/
logs /root/airflow/
pylintrc /opt/airflow/
pytest.ini /opt/airflow/
requirements /opt/airflow/
scripts /opt/airflow/
scripts/ci/in_container/entrypoint_ci.sh /
setup.cfg /opt/airflow/
setup.py /opt/airflow/
tests /opt/airflow/
tmp /opt/airflow/
"
# Converts the local mounts that we defined above to the right set of -v
# volume mappings in docker-compose file. This is needed so that we only
# maintain the volumes in one place (above)
function convert_local_mounts_to_docker_params() {
echo "${LOCAL_MOUNTS}" |sed '/^$/d' | awk -v AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
'
function basename(file) {
sub(".*/", "", file)
return file
}
{ print "-v"; print AIRFLOW_SOURCES "/" $1 ":" $2 basename($1) ":cached" }'
}
# Fixes a file that is expected to be a file. If - for whatever reason - the local file is not created
# When mounting it to container, docker assumes it is a missing directory and creates it. Such mistakenly
# Created directories should be removed and replaced with files
function sanitize_file() {
if [[ -d "${1}" ]]; then
rm -rf "${1}"
fi
touch "${1}"
}
# Those files are mounted into container when run locally
# .bash_history is preserved and you can modify .bash_aliases and .inputrc
# according to your liking
function sanitize_mounted_files() {
sanitize_file "${AIRFLOW_SOURCES}/.bash_history"
sanitize_file "${AIRFLOW_SOURCES}/.bash_aliases"
sanitize_file "${AIRFLOW_SOURCES}/.inputrc"
# When KinD cluster is created, the folder keeps authentication information
# across sessions
mkdir -p "${AIRFLOW_SOURCES}/.kube" >/dev/null 2>&1
}
#
# Creates cache directory where we will keep temporary files needed for the docker build
#
# This directory will be automatically deleted when the script is killed or exists (via trap)
# Unless SKIP_CACHE_DELETION variable is set. You can set this variable and then see
# the output/files generated by the scripts in this directory.
#
# Most useful is out.log file in this directory storing verbose output of the scripts.
#
function create_cache_directory() {
CACHE_TMP_FILE_DIR=$(mktemp -d)
export CACHE_TMP_FILE_DIR
if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then
trap 'rm -rf -- "${CACHE_TMP_FILE_DIR}"' INT TERM HUP
fi
OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
export OUTPUT_LOG
}
# Removes the cache temporary directory
function remove_cache_directory() {
if [[ -d ${CACHE_TMP_FILE_DIR} ]]; then
rm -rf "${CACHE_TMP_FILE_DIR}"
fi
}
#
# Verifies if stored md5sum of the file changed since the last tme ot was checked
# The md5sum files are stored in .build directory - you can delete this directory
# If you want to rebuild everything from the scratch
#
function check_file_md5sum {
local FILE="${1}"
local MD5SUM
local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
mkdir -pv "${MD5SUM_CACHE_DIR}"
MD5SUM=$(md5sum "${FILE}")
local MD5SUM_FILE
MD5SUM_FILE="${MD5SUM_CACHE_DIR}"/$(basename "${FILE}").md5sum
local MD5SUM_FILE_NEW
MD5SUM_FILE_NEW=${CACHE_TMP_FILE_DIR}/$(basename "${FILE}").md5sum.new
echo "${MD5SUM}" > "${MD5SUM_FILE_NEW}"
local RET_CODE=0
if [[ ! -f "${MD5SUM_FILE}" ]]; then
print_info "Missing md5sum for ${FILE#${AIRFLOW_SOURCES}} (${MD5SUM_FILE#${AIRFLOW_SOURCES}})"
RET_CODE=1
else
diff "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}" >/dev/null
RES=$?
if [[ "${RES}" != "0" ]]; then
print_info "The md5sum changed for ${FILE}"
RET_CODE=1
fi
fi
return ${RET_CODE}
}
#
# Moves md5sum file from it's temporary location in CACHE_TMP_FILE_DIR to
# BUILD_CACHE_DIR - thus updating stored MD5 sum fo the file
#
function move_file_md5sum {
local FILE="${1}"
local MD5SUM_FILE
local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${DEFAULT_BRANCH}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
mkdir -pv "${MD5SUM_CACHE_DIR}"
MD5SUM_FILE="${MD5SUM_CACHE_DIR}"/$(basename "${FILE}").md5sum
local MD5SUM_FILE_NEW
MD5SUM_FILE_NEW=${CACHE_TMP_FILE_DIR}/$(basename "${FILE}").md5sum.new
if [[ -f "${MD5SUM_FILE_NEW}" ]]; then
mv "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}"
print_info "Updated md5sum file ${MD5SUM_FILE} for ${FILE}."
fi
}
#
# Stores md5sum files for all important files and
# records that we built the images locally so that next time we use
# it from the local docker cache rather than pull (unless forced)
#
function update_all_md5_files() {
print_info
print_info "Updating md5sum files"
print_info
for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
do
move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
done
mkdir -pv "${BUILD_CACHE_DIR}/${BRANCH_NAME}"
touch "${BUILT_IMAGE_FLAG_FILE}"
}
#
# Checks md5sum of all important files in order to optimise speed of running various operations
# That mount sources of Airflow to container and require docker image built with latest dependencies.
# the Docker image will only be marked for rebuilding only in case any of the important files change:
# * setup.py
# * setup.cfg
# * Dockerfile.ci
# * airflow/version.py
#
# This is needed because we want to skip rebuilding of the image when only airflow sources change but
# Trigger rebuild in case we need to change dependencies (setup.py, setup.cfg, change version of Airflow
# or the Dockerfile.ci itself changes.
#
# Another reason to skip rebuilding Docker is thar currently it takes a bit longer time than simple Docker
# We need to fix group permissions of files in Docker because different linux build services have
# different default umask and Docker uses group permissions in checking for cache invalidation.
#
# As result of this check - most of the static checks will start pretty much immediately.
#
function check_if_docker_build_is_needed() {
print_info
print_info "Checking if image build is needed for ${THE_IMAGE_TYPE} image."
print_info
if [[ ${FORCE_BUILD_IMAGES:=""} == "true" ]]; then
echo "Docker image build is forced for ${THE_IMAGE_TYPE} image"
set +e
for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
do
# Just store md5sum for all files in md5sum.new - do not check if it is different
check_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
done
set -e
export NEEDS_DOCKER_BUILD="true"
else
set +e
for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
do
if ! check_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"; then
export NEEDS_DOCKER_BUILD="true"
fi
done
set -e
if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
echo "Docker image build is needed for ${THE_IMAGE_TYPE} image!"
else
print_info "Docker image build is not needed for ${THE_IMAGE_TYPE} image!"
fi
fi
print_info
}
#
# Checks if core utils required in the host system are installed and explain what needs to be done if not
#
function check_if_coreutils_installed() {
set +e
getopt -T >/dev/null
GETOPT_RETVAL=$?
if [[ $(uname -s) == 'Darwin' ]] ; then
command -v gstat >/dev/null
STAT_PRESENT=$?
else
command -v stat >/dev/null
STAT_PRESENT=$?
fi
command -v md5sum >/dev/null
MD5SUM_PRESENT=$?
set -e
CMDNAME="$(basename -- "$0")"
#################### Parsing options/arguments
if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then
print_info
if [[ $(uname -s) == 'Darwin' ]] ; then
echo >&2 "You are running ${CMDNAME} in OSX environment"
echo >&2 "And you need to install gnu commands"
echo >&2
echo >&2 "Run 'brew install gnu-getopt coreutils'"
echo >&2
echo >&2 "Then link the gnu-getopt to become default as suggested by brew."
echo >&2
echo >&2 "If you use bash, you should run this command:"
echo >&2
echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.bash_profile"
echo >&2 ". ~/.bash_profile"
echo >&2
echo >&2 "If you use zsh, you should run this command:"
echo >&2
echo >&2 "echo 'export PATH=\"/usr/local/opt/gnu-getopt/bin:\$PATH\"' >> ~/.zprofile"
echo >&2 ". ~/.zprofile"
echo >&2
echo >&2 "Login and logout afterwards !!"
echo >&2
echo >&2 "After re-login, your PATH variable should start with \"/usr/local/opt/gnu-getopt/bin\""
echo >&2 "Your current path is ${PATH}"
echo >&2
else
echo >&2 "You do not have necessary tools in your path (getopt, stat, md5sum)."
echo >&2 "Please install latest/GNU version of getopt and coreutils."
echo >&2 "This can usually be done with 'apt install util-linux coreutils'"
fi
print_info
exit 1
fi
}
#
# Asserts that we are not inside of the container
#
function assert_not_in_container() {
if [[ ${SKIP_IN_CONTAINER_CHECK:=} == "true" ]]; then
return
fi
if [[ -f /.dockerenv ]]; then
echo >&2
echo >&2 "You are inside the Airflow docker container!"
echo >&2 "You should only run this script from the host."
echo >&2 "Learn more about how we develop and test airflow in:"
echo >&2 "https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst"
echo >&2
exit 1
fi
}
# Removes the "Forced answer" (yes/no/quit) given previously, unles you specifically want to remember it.
#
# This is the default behaviour of all rebuild scripts to ask independently whether you want to
# rebuild the image or not. Sometimes however we want to remember answer previously given. For
# example if you answered "no" to rebuild the image, the assumption is that you do not
# want to rebuild image also for other rebuilds in the same pre-commit execution.
#
# All the pre-commit checks therefore have `export REMEMBER_LAST_ANSWER="true"` set
# So that in case they are run in a sequence of commits they will not rebuild. Similarly if your most
# recent answer was "no" and you run `pre-commit run mypy` (for example) it will also reuse the
# "no" answer given previously. This happens until you run any of the breeze commands or run all
# precommits `pre-commit run` - then the "LAST_FORCE_ANSWER_FILE" will be removed and you will
# be asked again.
function forget_last_answer() {
if [[ ${REMEMBER_LAST_ANSWER:="false"} != "true" ]]; then
print_info
print_info "Forgetting last answer from ${LAST_FORCE_ANSWER_FILE}:"
print_info
rm -f "${LAST_FORCE_ANSWER_FILE}"
else
if [[ -f "${LAST_FORCE_ANSWER_FILE}" ]]; then
print_info
print_info "Still remember last answer from ${LAST_FORCE_ANSWER_FILE}:"
print_info "$(cat "${LAST_FORCE_ANSWER_FILE}")"
print_info
fi
fi
}
# Confirms if hte image should be rebuild and interactively checks it with the user.
# In case iit needs to be rebuild. It only ask the user if it determines that the rebuild
# is needed and that the rebuild is not already forced. It asks the user using available terminals
# So that the script works also from within pre-commit run via git hooks - where stdin is not
# available - it tries to find usable terminal and ask the user via this terminal.
function confirm_image_rebuild() {
ACTION="rebuild"
if [[ ${FORCE_PULL_IMAGES:=} == "true" ]]; then
ACTION="pull and rebuild"
fi
if [[ -f "${LAST_FORCE_ANSWER_FILE}" ]]; then
# set variable from last answered response given in the same pre-commit run - so that it can be
# answered in teh first pre-commit check (build) and then used in another (pylint/mypy/flake8 etc).
# shellcheck disable=SC1090
source "${LAST_FORCE_ANSWER_FILE}"
fi
set +e
local RES
if [[ ${CI:="false"} == "true" ]]; then
print_info
print_info "CI environment - forcing rebuild for image ${THE_IMAGE_TYPE}."
print_info
RES="0"
elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS:=""}" ]]; then
print_info
print_info "Forcing answer '${FORCE_ANSWER_TO_QUESTIONS}'"
print_info
case "${FORCE_ANSWER_TO_QUESTIONS}" in
[yY][eE][sS]|[yY])
RES="0" ;;
[qQ][uU][iI][tT]|[qQ])
RES="2" ;;
*)
RES="1" ;;
esac
elif [[ -t 0 ]]; then
# Check if this script is run interactively with stdin open and terminal attached
"${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}"
RES=$?
elif [[ ${DETECTED_TERMINAL:=$(tty)} != "not a tty" ]]; then
# Make sure to use output of tty rather than stdin/stdout when available - this way confirm
# will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks)
# shellcheck disable=SC2094
"${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \
<"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
RES=$?
export DETECTED_TERMINAL
elif [[ -c /dev/tty ]]; then
export DETECTED_TERMINAL=/dev/tty
# Make sure to use /dev/tty first rather than stdin/stdout when available - this way confirm
# will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks)
# shellcheck disable=SC2094
"${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \
<"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
RES=$?
else
print_info
print_info "No terminal, no stdin - quitting"
print_info
# No terminal, no stdin, no force answer - quitting!
RES="2"
fi
set -e
if [[ ${RES} == "1" ]]; then
print_info
print_info "Skipping rebuilding the image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}"
print_info
SKIP_REBUILD="true"
# Force "no" also to subsequent questions so that if you answer it once, you are not asked
# For all other pre-commits and you will continue using the images you already have
export FORCE_ANSWER_TO_QUESTIONS="no"
echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' > "${LAST_FORCE_ANSWER_FILE}"
elif [[ ${RES} == "2" ]]; then
echo >&2
echo >&2 "ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. "
echo >&2 " Make sure you build the images bu running"
echo >&2
echo >&2 " ./breeze --python ${PYTHON_MAJOR_MINOR_VERSION}" build-image
echo >&2
echo >&2 " If you run it via pre-commit as individual hook, you can run 'pre-commit run build'."
echo >&2
exit 1
else
# Force "yes" also to subsequent questions
export FORCE_ANSWER_TO_QUESTIONS="yes"
fi
}
# Builds local image manifest
# It contains only one .json file - result of docker inspect - describing the image
# We cannot use docker registry APIs as they are available only with authorisation
# But this image can be pulled without authentication
function build_ci_image_manifest() {
verbose_docker inspect "${AIRFLOW_CI_IMAGE}" > "manifests/${AIRFLOW_CI_BASE_TAG}.json"
verbose_docker build \
--build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
--tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
-f- . <<EOF
ARG AIRFLOW_CI_BASE_TAG
FROM scratch
COPY "manifests/${AIRFLOW_CI_BASE_TAG}.json" .
CMD ""
EOF
}
#
# Retrieves information about layers in the local IMAGE
# it stores list of SHAs of image layers in the file pointed at by TMP_MANIFEST_LOCAL_SHA
#
function get_local_image_info() {
TMP_MANIFEST_LOCAL_JSON=$(mktemp)
TMP_MANIFEST_LOCAL_SHA=$(mktemp)
set +e
# Remove the container just in case
verbose_docker rm --force "local-airflow-manifest" >/dev/null 2>&1
# Create manifest from the local manifest image
if ! verbose_docker create --name "local-airflow-manifest" \
"${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" >/dev/null 2>&1 ; then
echo
echo "Local manifest image not available"
echo
LOCAL_MANIFEST_IMAGE_UNAVAILABLE="true"
return
fi
set -e
# Create manifest from the local manifest image
verbose_docker cp "local-airflow-manifest:${AIRFLOW_CI_BASE_TAG}.json" "${TMP_MANIFEST_LOCAL_JSON}" >/dev/null 2>&1
sed 's/ *//g' "${TMP_MANIFEST_LOCAL_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_LOCAL_SHA}"
verbose_docker rm --force "local-airflow-manifest" >/dev/null 2>&1
}
#
# Retrieves information about layers in the remote IMAGE
# it stores list of SHAs of image layers in the file pointed at by TMP_MANIFEST_REMOTE_SHA
# This cannot be done easily with existing APIs of Dockerhub because they require additional authentication
# even for public images. Therefore instead we are downloading a specially prepared manifest image
# which is built together with the main image. This special manifest image is prepared during
# building of the main image and contains single JSON file being result of docker inspect on that image
# This image is from scratch so it is very tiny
function get_remote_image_info() {
set +e
# Pull remote manifest image
if ! verbose_docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" >/dev/null 2>&1 ; then
echo
echo "Remote docker registry unreachable"
echo
REMOTE_DOCKER_REGISTRY_UNREACHABLE="true"
return
fi
TMP_MANIFEST_REMOTE_JSON=$(mktemp)
TMP_MANIFEST_REMOTE_SHA=$(mktemp)
# delete container just in case
verbose_docker rm --force "remote-airflow-manifest" >/dev/null 2>&1
set -e
# Create container out of the manifest image without runnning it
verbose_docker create --name "remote-airflow-manifest" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" >/dev/null 2>&1
# Extract manifest and store it in local file
verbose_docker cp "remote-airflow-manifest:${AIRFLOW_CI_BASE_TAG}.json" "${TMP_MANIFEST_REMOTE_JSON}" >/dev/null 2>&1
# Filter everything except SHAs of image layers
sed 's/ *//g' "${TMP_MANIFEST_REMOTE_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_REMOTE_SHA}"
verbose_docker rm --force "remote-airflow-manifest" >/dev/null 2>&1
}
# The Number determines the cut-off between local building time and pull + build time.
# It is a bit experimental and it will have to be kept
# updated as we keep on changing layers. The cut-off point is at the moment when we do first
# pip install "https://github.com/apache/airflow/archive/${AIRFLOW_BRANCH}.tar...
# you can get it via this command:
# docker history --no-trunc apache/airflow:master-python3.6-ci | \
# grep ^sha256 | grep -n "pip uninstall" | awk 'BEGIN {FS=":"} {print $1 }'
#
# This command returns the number of layer in docker history where pip uninstall is called. This is the
# line that will take a lot of time to run and at this point it's worth to pull the image from repo
# if there are at least NN changed layers in your docker file, you should pull the image.
#
# Note that this only matters if you have any of the important files changed since the last build
# of your image such as Dockerfile.ci, setup.py etc.
#
MAGIC_CUT_OFF_NUMBER_OF_LAYERS=34
# Compares layers from both remote and local image and set FORCE_PULL_IMAGES to true in case
# More than the last NN layers are different.
function compare_layers() {
NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" | \
wc -l || true)
rm -f "${TMP_MANIFEST_REMOTE_JSON}" "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_JSON}" "${TMP_MANIFEST_LOCAL_SHA}"
echo
echo "Number of layers different between the local and remote image: ${NUM_DIFF}"
echo
# This is where setup py is rebuilt - it will usually take a looooot of time to build it, so it is
# Better to pull here
if (( NUM_DIFF >= MAGIC_CUT_OFF_NUMBER_OF_LAYERS )); then
echo
echo
echo "WARNING! Your image and the dockerhub image differ signifcantly"
echo
echo "Forcing pulling the images. It will be faster than rebuilding usually."
echo "You can avoid it by setting SKIP_CHECK_REMOTE_IMAGE to true"
echo
export FORCE_PULL_IMAGES="true"
else
echo
echo "No need to pull the image. Local rebuild will be faster"
echo
fi
}
# Only rebuilds CI image if needed. It checks if the docker image build is needed
# because any of the important source files (from common/_files_for_rebuild_check.sh) has
# changed or in any of the edge cases (docker image removed, .build cache removed etc.
# In case rebuild is needed, it determines (by comparing layers in local and remote image)
# Whether pull is needed before rebuild.
function rebuild_ci_image_if_needed() {
if [[ ${SKIP_CI_IMAGE_CHECK:="false"} == "true" ]]; then
echo
echo "Skip checking CI image"
echo
return
fi
if [[ -f "${BUILT_IMAGE_FLAG_FILE}" ]]; then
print_info
print_info "${THE_IMAGE_TYPE} image already built locally."
print_info
else
print_info
print_info "${THE_IMAGE_TYPE} image not built locally: pulling and building"
print_info
export FORCE_PULL_IMAGES="true"
export FORCE_BUILD_IMAGES="true"
fi
NEEDS_DOCKER_BUILD="false"
check_if_docker_build_is_needed
if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
if [[ ${SKIP_CHECK_REMOTE_IMAGE:=} != "true" && ${DOCKER_CACHE} == "pulled" ]]; then
# Check if remote image is different enough to force pull
# This is an optimisation pull vs. build time. When there
# are enough changes (specifically after setup.py changes) it is faster to pull
# and build the image rather than just build it
echo
echo "Checking if the remote image needs to be pulled"
echo
get_remote_image_info
if [[ ${REMOTE_DOCKER_REGISTRY_UNREACHABLE:=} != "true" ]]; then
get_local_image_info
if [[ ${LOCAL_MANIFEST_IMAGE_UNAVAILABLE:=} != "true" ]]; then
compare_layers
else
FORCE_PULL_IMAGES="true"
fi
fi
fi
SKIP_REBUILD="false"
if [[ ${CI:=} != "true" && "${FORCE_BUILD:=}" != "true" ]]; then
confirm_image_rebuild
fi
if [[ ${SKIP_REBUILD} != "true" ]]; then
SYSTEM=$(uname -s)
if [[ ${SYSTEM} != "Darwin" ]]; then
ROOT_FILES_COUNT=$(find "airflow" "tests" -user root | wc -l | xargs)
if [[ ${ROOT_FILES_COUNT} != "0" ]]; then
./scripts/ci/ci_fix_ownership.sh
fi
fi
print_info
print_info "Build start: ${THE_IMAGE_TYPE} image."
print_info
build_ci_image
update_all_md5_files
build_ci_image_manifest
print_info
print_info "Build completed: ${THE_IMAGE_TYPE} image."
print_info
fi
else
print_info
print_info "No need to build - none of the important files changed: ${FILES_FOR_REBUILD_CHECK[*]}"
print_info
fi
}
#
# Starts the script.
# If VERBOSE_COMMANDS variable is set to true, it enables verbose output of commands executed
# Also prints some useful diagnostics information at start of the script if VERBOSE is set to true
#
function script_start {
print_info
print_info "Running $(basename "$0")"
print_info
print_info "Log is redirected to ${OUTPUT_LOG}"
print_info
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
print_info
print_info "Variable VERBOSE_COMMANDS Set to \"true\""
print_info "You will see a lot of output"
print_info
set -x
else
print_info "You can increase verbosity by running 'export VERBOSE_COMMANDS=\"true\""
if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then
print_info "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\""
fi
print_info
set +x
fi
START_SCRIPT_TIME=$(date +%s)
}
#
# Trap function executed always at the end of the script. In case of verbose output it also
# Prints the exit code that the script exits with. Removes verbosity of commands in case it was run with
# command verbosity and in case the script was not run from Breeze (so via ci scripts) it displays
# total time spent in the script so that we can easily see it.
#
function script_end {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
print_info "###########################################################################################"
print_info " EXITING WITH STATUS CODE ${EXIT_CODE}"
print_info "###########################################################################################"
fi
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
set +x
fi
END_SCRIPT_TIME=$(date +%s)
RUN_SCRIPT_TIME=$((END_SCRIPT_TIME-START_SCRIPT_TIME))
if [[ ${BREEZE:=} != "true" ]]; then
print_info
print_info "Finished the script $(basename "$0")"
print_info "Elapsed time spent in the script: ${RUN_SCRIPT_TIME} seconds"
print_info "Exit code ${EXIT_CODE}"
print_info
fi
remove_cache_directory
}
# Changes directory to local sources
function go_to_airflow_sources {
print_info
pushd "${AIRFLOW_SOURCES}" &>/dev/null || exit 1
print_info
print_info "Running in host in $(pwd)"
print_info
}
#
# Performs basic sanity checks common for most of the scripts in this directory
#
function basic_sanity_checks() {
assert_not_in_container
go_to_airflow_sources
check_if_coreutils_installed
create_cache_directory
sanitize_mounted_files
}
# In case of the pylint checks we filter out some files which are still in pylint_todo.txt list
function filter_out_files_from_pylint_todo_list() {
FILTERED_FILES=()
set +e
for FILE in "$@"
do
if [[ ${FILE} == "airflow/migrations/versions/"* ]]; then
# Skip all generated migration scripts
continue
fi
if ! grep -x "./${FILE}" <"${AIRFLOW_SOURCES}/scripts/ci/pylint_todo.txt" >/dev/null; then
FILTERED_FILES+=("${FILE}")
fi
done
set -e
export FILTERED_FILES
}
# Interactive version of confirming the ci image that is used in pre-commits
# it displays additional information - what the user should do in order to bring the local images
# back to state that pre-commit will be happy with
function rebuild_ci_image_if_needed_and_confirmed() {
NEEDS_DOCKER_BUILD="false"
THE_IMAGE_TYPE="CI"
check_if_docker_build_is_needed
if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
print_info
print_info "Docker image build is needed!"
print_info
else
print_info
print_info "Docker image build is not needed!"
print_info
fi
if [[ "${NEEDS_DOCKER_BUILD}" == "true" ]]; then
echo
echo "Some of your images need to be rebuild because important files (like package list) has changed."
echo
echo "You have those options:"
echo " * Rebuild the images now by answering 'y' (this might take some time!)"
echo " * Skip rebuilding the images and hope changes are not big (you will be asked again)"
echo " * Quit and manually rebuild the images using one of the following commands"
echo " * ./breeze build-image"
echo " * ./breeze build-image --force-pull-images"
echo
echo " The first command works incrementally from your last local build."
echo " The second command you use if you want to completely refresh your images from dockerhub."
echo
SKIP_REBUILD="false"
confirm_image_rebuild
if [[ ${SKIP_REBUILD} != "true" ]]; then
rebuild_ci_image_if_needed
fi
fi
}
# Checks if any of the files match the regexp specified the parameters here should be
# match_files_regexp REGEXP FILE FILE ...
function match_files_regexp() {
FILE_MATCHES="false"
REGEXP=${1}
while (($#))
do
REGEXP=${1}
for FILE in ${CHANGED_FILE_NAMES}
do