Merge pull request #1227 from centricular/ninja-clean-recursive-dir-delete

run_project_tests.py: Also do ninja clean on tests

Author: jpakkane

mesonbuild/backend/backends.py

@@ -22,6 +22,15 @@
 import subprocess
 from ..mesonlib import MesonException, get_compiler_for_source, classify_unity_sources
 
+class CleanTrees():
+    '''
+    Directories outputted by custom targets that have to be manually cleaned
+    because on Linux `ninja clean` only deletes empty directories.
+    '''
+    def __init__(self, build_dir, trees):
+        self.build_dir = build_dir
+        self.trees = trees
+
 class InstallData():
     def __init__(self, source_dir, build_dir, prefix):
         self.source_dir = source_dir

mesonbuild/backend/ninjabackend.py

@@ -20,7 +20,7 @@
 from .. import dependencies
 from .. import compilers
 from ..mesonlib import File, MesonException, get_compiler_for_source, Popen_safe
-from .backends import InstallData
+from .backends import CleanTrees, InstallData
 from ..build import InvalidArguments
 import os, sys, pickle, re
 import subprocess, shutil
@@ -2109,6 +2109,22 @@ def generate_shlib_aliases(self, target, outdir):
             except OSError:
                 mlog.debug("Library versioning disabled because we do not have symlink creation privileges.")
 
+    def generate_custom_target_clean(self, outfile, trees):
+        e = NinjaBuildElement(self.all_outputs, 'clean-ctlist', 'CUSTOM_COMMAND', 'PHONY')
+        d = CleanTrees(self.environment.get_build_dir(), trees)
+        d_file = os.path.join(self.environment.get_scratch_dir(), 'cleantrees.dat')
+        script_root = self.environment.get_script_dir()
+        clean_script = os.path.join(script_root, 'cleantrees.py')
+        e.add_item('COMMAND', [sys.executable,
+                               self.environment.get_build_command(),
+                               '--internal', 'cleantrees', d_file])
+        e.add_item('description', 'Cleaning CustomTarget directories')
+        e.write(outfile)
+        # Write out the data file passed to the script
+        with open(d_file, 'wb') as ofile:
+            pickle.dump(d, ofile)
+        return 'clean-ctlist'
+
     def generate_gcov_clean(self, outfile):
             gcno_elem = NinjaBuildElement(self.all_outputs, 'clean-gcno', 'CUSTOM_COMMAND', 'PHONY')
             script_root = self.environment.get_script_dir()
@@ -2136,14 +2152,19 @@ def generate_utils(self, outfile):
 
     def generate_ending(self, outfile):
         targetlist = []
+        ctlist = []
         for t in self.build.get_targets().values():
             # RunTargets are meant to be invoked manually
             if isinstance(t, build.RunTarget):
                 continue
-            # CustomTargets that aren't installed should only be built if they
-            # are used by something else or are meant to be always built
-            if isinstance(t, build.CustomTarget) and not (t.install or t.build_always):
-                continue
+            if isinstance(t, build.CustomTarget):
+                # Create a list of all custom target outputs
+                for o in t.get_outputs():
+                    ctlist.append(os.path.join(self.get_target_dir(t), o))
+                # CustomTargets that aren't installed should only be built if
+                # they are used by something else or are to always be built
+                if not (t.install or t.build_always):
+                    continue
             # Add the first output of each target to the 'all' target so that
             # they are all built
             targetlist.append(os.path.join(self.get_target_dir(t), t.get_outputs()[0]))
@@ -2160,6 +2181,14 @@ def generate_ending(self, outfile):
         elem = NinjaBuildElement(self.all_outputs, 'clean', 'CUSTOM_COMMAND', 'PHONY')
         elem.add_item('COMMAND', [ninja_command, '-t', 'clean'])
         elem.add_item('description', 'Cleaning')
+        # If we have custom targets in this project, add all their outputs to
+        # the list that is passed to the `cleantrees.py` script. The script
+        # will manually delete all custom_target outputs that are directories
+        # instead of files. This is needed because on platforms other than
+        # Windows, Ninja only deletes directories while cleaning if they are
+        # empty. https://github.com/mesonbuild/meson/issues/1220
+        if ctlist:
+            elem.add_dep(self.generate_custom_target_clean(outfile, ctlist))
         if 'b_coverage' in self.environment.coredata.base_options and \
            self.environment.coredata.base_options['b_coverage'].value:
             self.generate_gcov_clean(outfile)

mesonbuild/coredata.py

@@ -236,6 +236,7 @@ def get_builtin_option_default(optname):
         }
 
 forbidden_target_names = {'clean': None,
+                          'clean-ctlist': None,
                           'clean-gcno': None,
                           'clean-gcda': None,
                           'coverage-text': None,

mesonbuild/mesonmain.py

@@ -198,6 +198,9 @@ def run_script_command(args):
     if cmdname == 'exe':
         import mesonbuild.scripts.meson_exe as abc
         cmdfunc = abc.run
+    elif cmdname == 'cleantrees':
+        import mesonbuild.scripts.cleantrees as abc
+        cmdfunc = abc.run
     elif cmdname == 'install':
         import mesonbuild.scripts.meson_install as abc
         cmdfunc = abc.run

mesonbuild/scripts/cleantrees.py

@@ -0,0 +1,43 @@
+# Copyright 2016 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import shutil
+import pickle
+
+def rmtrees(build_dir, trees):
+    for t in trees:
+        # Never delete trees outside of the builddir
+        if os.path.isabs(t):
+            print('Cannot delete dir with absolute path {!r}'.format(t))
+            continue
+        bt = os.path.join(build_dir, t)
+        # Skip if it doesn't exist, or if it is not a directory
+        if os.path.isdir(bt):
+            shutil.rmtree(bt, ignore_errors=True)
+
+def run(args):
+    if len(args) != 1:
+        print('Cleaner script for Meson. Do not run on your own please.')
+        print('cleantrees.py <data-file>')
+        return 1
+    with open(args[0], 'rb') as f:
+        data = pickle.load(f)
+    rmtrees(data.build_dir, data.trees)
+    # Never fail cleaning
+    return 0
+
+if __name__ == '__main__':
+    run(sys.argv[1:])

run_project_tests.py

@@ -24,7 +24,7 @@
 from mesonbuild import mesonlib
 from mesonbuild import mlog
 from mesonbuild import mesonmain
-from mesonbuild.mesonlib import stringlistify
+from mesonbuild.mesonlib import stringlistify, Popen_safe
 import argparse
 import xml.etree.ElementTree as ET
 import time
@@ -93,26 +93,20 @@ def stop_handler(signal, frame):
 backend_flags = None
 compile_commands = None
 test_commands = None
-install_commands = None
+install_commands = []
+clean_commands = []
 
 def setup_commands(backend):
-    global backend_flags, compile_commands, test_commands, install_commands
+    global backend_flags, compile_commands, test_commands, install_commands, clean_commands
     msbuild_exe = shutil.which('msbuild')
-    if backend == 'vs2010' or (backend is None and msbuild_exe is not None):
-        backend_flags = ['--backend=vs2010']
+    if (backend and backend.startswith('vs')) or (backend is None and msbuild_exe is not None):
+        backend_flags = ['--backend=' + backend]
         compile_commands = ['msbuild']
         test_commands = ['msbuild', 'RUN_TESTS.vcxproj']
-        install_commands = []
-    elif backend == 'vs2015':
-        backend_flags = ['--backend=vs2015']
-        compile_commands = ['msbuild']
-        test_commands = ['msbuild', 'RUN_TESTS.vcxproj']
-        install_commands = []
     elif backend == 'xcode' or (backend is None and mesonlib.is_osx()):
         backend_flags = ['--backend=xcode']
         compile_commands = ['xcodebuild']
         test_commands = ['xcodebuild', '-target', 'RUN_TESTS']
-        install_commands = []
     else:
         backend_flags = []
         ninja_command = environment.detect_ninja()
@@ -125,6 +119,7 @@ def setup_commands(backend):
         compile_commands += ['-w', 'dupbuild=err']
         test_commands = [ninja_command, 'test', 'benchmark']
         install_commands = [ninja_command, 'install']
+        clean_commands = [ninja_command, 'clean']
 
 def get_relative_files_list_from_dir(fromdir):
     paths = []
@@ -233,17 +228,18 @@ def parse_test_args(testdir):
         pass
     return args
 
-def run_test(skipped, testdir, extra_args, flags, compile_commands, install_commands, should_fail):
+def run_test(skipped, testdir, extra_args, flags, compile_commands, should_fail):
     if skipped:
         return None
     with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
         with AutoDeletedDir(tempfile.mkdtemp(prefix='i ', dir=os.getcwd())) as install_dir:
             try:
-                return _run_test(testdir, build_dir, install_dir, extra_args, flags, compile_commands, install_commands, should_fail)
+                return _run_test(testdir, build_dir, install_dir, extra_args, flags, compile_commands, should_fail)
             finally:
                 mlog.shutdown() # Close the log file because otherwise Windows wets itself.
 
-def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_commands, install_commands, should_fail):
+def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_commands, should_fail):
+    global install_commands, clean_commands
     test_args = parse_test_args(testdir)
     gen_start = time.time()
     gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\
@@ -268,12 +264,10 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_c
     else:
         comp = compile_commands
     build_start = time.time()
-    pc = subprocess.Popen(comp, cwd=test_build_dir,
-                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (o, e) = pc.communicate()
+    pc, o, e = Popen_safe(comp, cwd=test_build_dir)
     build_time = time.time() - build_start
-    stdo += o.decode(sys.stdout.encoding)
-    stde += e.decode(sys.stdout.encoding)
+    stdo += o
+    stde += e
     if should_fail == 'build':
         if pc.returncode != 0:
             return TestResult('', stdo, stde, mesonlog, gen_time)
@@ -294,19 +288,24 @@ def _run_test(testdir, test_build_dir, install_dir, extra_args, flags, compile_c
         return TestResult('Test that should have failed to run unit tests succeeded', stdo, stde, mesonlog, gen_time)
     if returncode != 0:
         return TestResult('Running unit tests failed.', stdo, stde, mesonlog, gen_time, build_time, test_time)
+    # Do installation
     if len(install_commands) == 0:
         return TestResult('', '', '', gen_time, build_time, test_time)
-    else:
+    env = os.environ.copy()
+    env['DESTDIR'] = install_dir
+    pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
+    stdo += o
+    stde += e
+    if pi.returncode != 0:
+        return TestResult('Running install failed.', stdo, stde, mesonlog, gen_time, build_time, test_time)
+    if len(clean_commands) != 0:
         env = os.environ.copy()
-        env['DESTDIR'] = install_dir
-        pi = subprocess.Popen(install_commands, cwd=test_build_dir, env=env,
-                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (o, e) = pi.communicate()
-        stdo += o.decode(sys.stdout.encoding)
-        stde += e.decode(sys.stdout.encoding)
+        pi, o, e = Popen_safe(clean_commands, cwd=test_build_dir, env=env)
+        stdo += o
+        stde += e
         if pi.returncode != 0:
-            return TestResult('Running install failed.', stdo, stde, mesonlog, gen_time, build_time, test_time)
-        return TestResult(validate_install(testdir, install_dir), stdo, stde, mesonlog, gen_time, build_time, test_time)
+            return TestResult('Running clean failed.', stdo, stde, mesonlog, gen_time, build_time, test_time)
+    return TestResult(validate_install(testdir, install_dir), stdo, stde, mesonlog, gen_time, build_time, test_time)
 
 def gather_tests(testdir):
     tests = [t.replace('\\', '/').split('/', 2)[2] for t in glob(os.path.join(testdir, '*'))]
@@ -372,7 +371,7 @@ def detect_tests_to_run():
     return all_tests
 
 def run_tests(extra_args):
-    global passing_tests, failing_tests, stop, executor, futures
+    global install_commands, passing_tests, failing_tests, stop, executor, futures
     all_tests = detect_tests_to_run()
     logfile = open('meson-test-run.txt', 'w', encoding="utf_8")
     junit_root = ET.Element('testsuites')
@@ -404,7 +403,7 @@ def run_tests(extra_args):
             should_fail = False
             if name.startswith('failing'):
                 should_fail = name.split('failing-')[1]
-            result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, install_commands, should_fail)
+            result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, should_fail)
             futures.append((testname, t, result))
         for (testname, t, result) in futures:
             result = result.result()