Skip to content

Commit a13314c

Browse files
shahzebsiddiquishahzebsiddiqui
committed
adding logging support for application with Easybuild that needs to be
built from source.
1 parent 79ed5ba commit a13314c

File tree

3 files changed

+89
-56
lines changed

3 files changed

+89
-56
lines changed

‎framework/main.py

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -257,9 +257,12 @@ def main():
257257
systempkg = args.system
258258
if systempkg == "all":
259259

260+
logger.info("Generating all system package tests from YAML files in %s", os.path.join(BUILDTEST_SOURCEDIR,"system"))
261+
260262
os.environ["BUILDTEST_LOGDIR"] = os.path.join(BUILDTEST_ROOT,"log","system","all")
261263
systempkg_list = os.listdir(os.path.join(BUILDTEST_SOURCEDIR,"system"))
262-
BUILDTEST_LOGCONTENT.append("System Packages: \n")
264+
265+
logger.info("List of system packages to test: %s ", systempkg_list)
263266

264267
for pkg in systempkg_list:
265268
generate_binary_test(args_dict,verbose,pkg)
@@ -268,8 +271,19 @@ def main():
268271
#logcontent += systempkg_generate_binary_test(systempkg,verbose,logdir)
269272
generate_binary_test(args_dict,verbose,systempkg)
270273

271-
update_logfile(verbose)
272-
sys.exit(1)
274+
#update_logfile(verbose)
275+
276+
if not os.path.exists(os.environ["BUILDTEST_LOGDIR"]):
277+
cmd = "mkdir -p " + os.environ["BUILDTEST_LOGDIR"]
278+
os.system(cmd)
279+
logger.warning("Directory not found: %s", os.environ["BUILDTEST_LOGDIR"])
280+
logger.info("Executing Command: %s", cmd)
281+
282+
cmd = "mv " + logpath + " " + os.environ["BUILDTEST_LOGDIR"]
283+
os.system(cmd)
284+
285+
print "Writing Log file to:", os.path.join(os.environ["BUILDTEST_LOGDIR"],logfile)
286+
sys.exit(0)
273287
# when -s is specified
274288
if software != None:
275289
software=software.split("/")
@@ -285,6 +299,15 @@ def main():
285299
tcname = get_toolchain_name()
286300
tcversion = get_toolchain_version()
287301

302+
logger.debug("Generating Test from EB Application")
303+
304+
logger.debug("Software: %s", appname)
305+
logger.debug("Software Version: %s", appversion)
306+
logger.debug("Toolchain: %s", tcname)
307+
logger.debug("Toolchain Version: %s", tcversion)
308+
309+
logger.debug("Checking if software: %s/%s exists",appname,appversion)
310+
288311
# checking if software exists
289312
software_exists(software,verbose)
290313

@@ -298,11 +321,6 @@ def main():
298321
toolchain_exists(toolchain,verbose)
299322

300323

301-
302-
if verbose >= 1:
303-
text = "Toolchain:" + tcname + " " + tcversion + " found in system \n"
304-
print text
305-
306324
# check that the software,toolchain match the easyconfig.
307325
ret=check_software_version_in_easyconfig(BUILDTEST_EASYCONFIGDIR,verbose)
308326
# generate_binary_test(software,toolchain,verbose)
@@ -315,9 +333,9 @@ def main():
315333
logdir=os.environ["BUILDTEST_LOGDIR"]
316334

317335

318-
BUILDTEST_LOGCONTENT.append("Source App Directory:" + source_app_dir + "\n")
319-
BUILDTEST_LOGCONTENT.append("Config Directory: " + configdir + "\n")
320-
BUILDTEST_LOGCONTENT.append("Code Directory:" + codedir + "\n")
336+
logger.debug("Source App Directory: %s" + source_app_dir)
337+
logger.debug("Config Directory: %s " + configdir)
338+
logger.debug("Code Directory: %s" + codedir)
321339

322340
generate_binary_test(args_dict,verbose,None)
323341
# this generates all the compilation tests found in application directory ($BUILDTEST_SOURCEDIR/ebapps/<software>)
@@ -327,7 +345,17 @@ def main():
327345
if testset != None:
328346
run_testset(software,toolchain,testset,verbose)
329347

330-
update_logfile(verbose)
348+
if not os.path.isdir(logdir):
349+
cmd = "mkdir -p " + logdir
350+
os.system(cmd)
351+
logger.debug("Executing Command: %s", cmd)
352+
353+
354+
cmd = "mv " + logpath + " " + logdir
355+
os.system(cmd)
356+
logger.debug("Executing command: %s ", cmd)
357+
358+
print "Writing Log file: %s", os.path.join(logdir,logfile)
331359

332360
if __name__ == "__main__":
333361
main()

‎framework/modules.py

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,25 +222,30 @@ def software_exists(software,verbose):
222222

223223
success_msg = "Checking Software: " + software[0] + "/" + software[1] + " ... SUCCESS"
224224
fail_msg = "Checking Software: " + software[0] + "/" + software[1] + " ... FAILED"
225+
226+
logger = logging.getLogger(logID)
227+
228+
logger.debug("Checking argument list length for software, must be equal to 2")
225229
if len(software) != 2:
226230
print fail_msg
227231
msg = "Too many arguments, -s takes argument <software>,<version> \n"
228232
print msg
229-
BUILDTEST_LOGCONTENT.append(msg)
230-
update_logfile(verbose)
233+
logger.error("%s",msg)
231234
sys.exit(1)
232-
235+
233236
softwarecollection=get_unique_software_version(BUILDTEST_MODULE_EBROOT)
234237
software_name=software[0]+" "+software[1]
238+
239+
logger.debug("Checking %s is found in software version list", software_name)
240+
235241
if software_name not in softwarecollection:
236242
print fail_msg
237243
msg = "Can't find software: " + software_name + "\n"
238244
print msg
239-
BUILDTEST_LOGCONTENT.append(fail_msg + "\n")
240-
update_logfile(verbose)
245+
logger.error("%s",fail_msg)
241246
sys.exit(1)
242247

243-
BUILDTEST_LOGCONTENT.append(success_msg + "\n")
248+
logger.info("%s",success_msg)
244249

245250
print success_msg
246251

‎framework/testgen.py

Lines changed: 38 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ def generate_source_test(configmap,codedir,verbose,subdir):
120120
tcname=get_toolchain_name()
121121
tcver=get_toolchain_version()
122122

123+
logger = logging.getLogger(logID)
123124
# app_destdir is root of test directory
124125
app_destdir = os.path.join(BUILDTEST_TESTDIR,"ebapp",appname,appver,tcname,tcver)
125126

@@ -128,11 +129,6 @@ def generate_source_test(configmap,codedir,verbose,subdir):
128129
destdir=os.path.join(app_destdir,subdir)
129130
cmakelist=os.path.join(destdir,"CMakeLists.txt")
130131

131-
BUILDTEST_LOGCONTENT.append("\n")
132-
BUILDTEST_LOGCONTENT.append("------------------------------------------------ \n")
133-
BUILDTEST_LOGCONTENT.append("function: generate_source_test \n")
134-
BUILDTEST_LOGCONTENT.append("------------------------------------------------ \n")
135-
136132
# if subdirectory exists, create subdirectory in destdir so we can write test script
137133
if subdir != "":
138134
# if sub directory does not exist, then create all directories and its parents directories
@@ -151,11 +147,11 @@ def generate_source_test(configmap,codedir,verbose,subdir):
151147
if "buildopts" in configmap:
152148
flags=configmap["buildopts"]
153149

154-
BUILDTEST_LOGCONTENT.append("Test Name: " + testname + "\n")
155-
BUILDTEST_LOGCONTENT.append("Test Path: " + testpath + "\n")
156-
BUILDTEST_LOGCONTENT.append("Source File: " + sourcefilepath + "\n")
157-
BUILDTEST_LOGCONTENT.append("Executable Name: " + executable + "\n")
158-
BUILDTEST_LOGCONTENT.append("Build Flags: " + flags + "\n")
150+
logger.debug("Test Name: %s", testname)
151+
logger.debug("Test Path: %s", testpath)
152+
logger.debug("Source File: %s", sourcefilepath)
153+
logger.debug("Executable Name: %s", executable)
154+
logger.debug("Build Flags: %s", flags)
159155

160156

161157
# write the preamble to test-script to initialize app environment using module cmds
@@ -172,11 +168,14 @@ def generate_source_test(configmap,codedir,verbose,subdir):
172168
# used for parallel processing to specify # of procs with mpirun -np
173169
nproc = ""
174170

171+
172+
logger.debug("""Checking for YAML key "buildcmd" and "runcmd" """)
173+
175174
# if there is a buildcmd & runcmd in yaml file, place this directly in test script
176175
if "buildcmd" in configmap and "runcmd" in configmap:
177176

178-
BUILDTEST_LOGCONTENT.append("YAML file found buildcmd and runcmd. \n")
179-
BUILDTEST_LOGCONTENT.append("buildtest will generate explicit build/run commands from buildcmd and runcmd fields \n")
177+
logger.debug("YAML file found buildcmd and runcmd.")
178+
logger.debug("buildtest will generate explicit build/run commands from buildcmd and runcmd field")
180179

181180
# only process buildcmd if there is a value specified for buildcmd key
182181
if configmap["buildcmd"] != None:
@@ -186,7 +185,7 @@ def generate_source_test(configmap,codedir,verbose,subdir):
186185
buildcmd += cmd + "\n"
187186
else:
188187
msg = "buildcmd is declared but value is not specified \n"
189-
BUILDTEST_LOGCONTENT.append(msg)
188+
logger.debug("%s",msg)
190189

191190
if configmap["runcmd"] != None:
192191
# process the runcmd tag similar same as buildcmd and store in variable
@@ -197,10 +196,9 @@ def generate_source_test(configmap,codedir,verbose,subdir):
197196
else:
198197
msg = "runcmd is declared but value is not specified. Need runcmd to run executable \n"
199198
print msg
200-
BUILDTEST_LOGCONTENT.append(msg)
201-
BUILDTEST_LOGCONTENT.append("Program Terminating \n")
202-
update_logfile(verbose)
203-
sys.exit(1)
199+
logger.debug.append("%s",msg)
200+
logging.warning("Unable to create test from YAML config, skipping test generation")
201+
return
204202

205203

206204
if verbose >=1:
@@ -230,17 +228,15 @@ def generate_source_test(configmap,codedir,verbose,subdir):
230228
if "buildcmd" in configmap and "runcmd" not in configmap or "buildcmd" not in configmap and "runcmd" in configmap:
231229
print "Need to specify both key: buildcmd and runcmd"
232230

233-
BUILDTEST_LOGCONTENT.append("Need to declare both key: buildcmd and runcmd \n")
234-
BUILDTEST_LOGCONTENT.append("Program Terminating \n")
235-
update_logfile(verbose)
236-
sys.exit(1)
231+
logger.warning("Need to declare both key: buildcmd and runcmd. Skipping to next YAML config \n")
232+
return
237233

238234
# get the compiler tag and type based on application and toolchain
239235
compiler,compiler_type=get_compiler(configmap,appname,tcname)
240236

241-
BUILDTEST_LOGCONTENT.append("buildtest will auto-generate buildcmd & runcmd \n")
242-
BUILDTEST_LOGCONTENT.append("Compiler: " + compiler + "\n")
243-
BUILDTEST_LOGCONTENT.append("Compiler Type: " + compiler_type + "\n")
237+
logger.debug("buildtest will auto-generate buildcmd & runcmd")
238+
logger.debug("Compiler: %s", compiler)
239+
logger.debug("Compiler Type: %s", compiler_type)
244240

245241
# set buildcmd based on compiler_type. compiler is either nvcc,gcc,icc,mpicc, or mpiicc for intel
246242
if compiler_type == "gnu" or compiler_type == "intel" or compiler_type == "cuda":
@@ -254,13 +250,13 @@ def generate_source_test(configmap,codedir,verbose,subdir):
254250
if "nproc" in configmap:
255251
nproc = str(configmap["nproc"])
256252

257-
BUILDTEST_LOGCONTENT.append("nproc key found in YAML config file \n")
258-
BUILDTEST_LOGCONTENT.append("nproc: " + nproc + "\n")
253+
logger.debug("nproc key found in YAML config file")
254+
logger.debug("nproc: ", nproc)
259255
# if nproc is not specified set it to 1 when building mpi apps
260256
else:
261257
nproc = "1"
262258

263-
BUILDTEST_LOGCONTENT.append("nproc key not found in YAML config file, will set nproc = 1 \n")
259+
logger.debug("nproc key not found in YAML config file, will set nproc = 1")
264260
# add argument to runcmd in MPI jobs
265261
if "args" in configmap:
266262
arglist = configmap["args"]
@@ -329,8 +325,8 @@ def generate_source_test(configmap,codedir,verbose,subdir):
329325
for cmd in configmap["runextracmd"]:
330326
fd.write(cmd + "\n")
331327

332-
BUILDTEST_LOGCONTENT.append("runextracmd found in YAML config file \n")
333-
BUILDTEST_LOGCONTENT.append("runextracmd:" + str(configmap["runextracmd"]) + "\n")
328+
logger.debug("runextracmd found in YAML config file")
329+
logger.debug("runextracmd: %s", str(configmap["runextracmd"]))
334330
fd.close()
335331

336332
# by default run the commands below which will add the test to CMakeLists.txt and update the logfile
@@ -339,17 +335,17 @@ def generate_source_test(configmap,codedir,verbose,subdir):
339335

340336
# print "Creating Test: " + testpath
341337

342-
BUILDTEST_LOGCONTENT.append("Creating Test: " + testpath + "\n")
343-
BUILDTEST_LOGCONTENT.append("Content of Testfile: " + testpath + "\n")
344-
BUILDTEST_LOGCONTENT.append("----------------------- \n")
345-
338+
logger.debug("Creating Test: %s ", testpath)
339+
logger.debug("[TEST START-BLOCK]")
340+
346341
fd=open(testpath,'r')
347-
content=fd.read()
348-
BUILDTEST_LOGCONTENT.append(content)
342+
content=fd.read().splitlines()
343+
for line in content:
344+
logger.debug(line)
349345
fd.close()
350346

347+
logger.debug("[TEST END-BLOCK]")
351348

352-
BUILDTEST_LOGCONTENT.append("\n -------------------------------------------------- \n")
353349

354350
# if keyword iter is found in YAML, lets try to recreate N tests by renaming test such as
355351
# hello.sh to hello_1.sh and create N-1 copies with file names hello_2.sh, hello_3.sh, ...
@@ -603,13 +599,17 @@ def process_binary_file(filename,args_dict,test_type,verbose,pkg):
603599
add_test_str="add_test(NAME system-" + pkg + "-" + testname + "\t COMMAND sh " + testname + "\t WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) \n"
604600

605601

606-
logger.debug("Adding content: %s ", add_test_str,)
602+
logger.debug("Adding content: %s ", add_test_str)
607603
fd.write(add_test_str)
608604

609605
# print "Creating Test:", testpath
610606

611607
print
612-
print "Generating " + str(count) + " binary tests for package: " + pkg
608+
if test_type == "system":
609+
print "Generating " + str(count) + " binary tests for package: " + pkg
610+
else:
611+
print "Generating " + str(count) + " binary tests for Application: " + name + "/" + version
612+
613613
print "Binary Tests are written in " + test_destdir
614614

615615

0 commit comments

Comments
 (0)