From 4806680e7b17a83c14ea2b64c8ab6277a8b380b1 Mon Sep 17 00:00:00 2001 From: David Robillard Date: Sun, 14 Apr 2019 18:03:16 +0200 Subject: Squashed 'waflib/' changes from d7a7ca4..27a69a7 27a69a7 Add option to filter tests by regular expression ac29b74 Fix configuring with root prefix 6134be6 Print test coverage summary 6d7dd59 Don't attempt to clear coverage when coverage is disabled cddbcb5 Pass file comparison test if difflib finds no changes 6e4e92b Seek to end before checking the size of stderr output e7a29b6 Upgrade to waf 2.0.15 8280f9d Add command for running executables from the build directory 8073c1a Make make_simple_dox() safe in case of exception 70d03b8 Avoid use of global counter hacks for configuration display b7d689a Rewrite test framework 94deadf Automatically add options and move add_flags() to options context f4259ee Reduce system include path noise 927b608 Automatically display configuration header c44b8f3 Set line justification from a constant in the wscript a48e26f Automatically detect if wscript has a test hook ef66724 Save runtime variables in the environment 63bcbcd Clean up TestContext b1d9505 Add ExecutionContext for setting runtime environment 387c1df Add show_diff() and test_file_equals() utilities 29d4d29 Fix in-tree library paths 9fde01f Add custom configuration context 6d3612f Add lib_path_name constant 915dcb1 Report test time correctly 26f0a2b Remove redundant verbose-tests option fc35c2c Fix MSVC CFLAGS 5ea8f99 Improve test output spacing 0e23b29 Raise exception when test suite fails to ensure non-zero exit status d6de073 Show run time of unit tests 5b65554 Add short configure option for ultra-strict flags 4687ba6 Use gtest-like test output 258903d Fix failure count in test group summaries da07e73 Fix verbose tests with Python 3 6e726eb Add support for suppressing warnings from system libraries 3e43d09 Add short forms for common options 5db477c Automatically define version for child libraries 982416b Fix version definition for projects with special characters 9eb0cf8 Remove blank line padding around configuration summary git-subtree-dir: waflib git-subtree-split: 27a69a76d1f625ad86b60b5dbd12368bc25cae25 --- extras/autowaf.py | 760 +++++++++++++++++++++++++++++++---------------- extras/buildcopy.py | 7 +- extras/cpplint.py | 6 +- extras/cython.py | 13 +- extras/distnet.py | 2 +- extras/erlang.py | 2 +- extras/fast_partial.py | 2 +- extras/fc_nfort.py | 52 ++++ extras/gccdeps.py | 6 +- extras/kde4.py | 2 +- extras/lv2.py | 32 +- extras/ocaml.py | 2 +- extras/parallel_debug.py | 9 +- extras/pgicc.py | 2 +- extras/protoc.py | 92 +++--- extras/pyqt5.py | 4 +- extras/qt4.py | 6 +- extras/remote.py | 2 +- extras/run_do_script.py | 2 +- extras/swig.py | 4 +- 20 files changed, 663 insertions(+), 344 deletions(-) create mode 100644 extras/fc_nfort.py (limited to 'extras') diff --git a/extras/autowaf.py b/extras/autowaf.py index a844bf5..5feef03 100644 --- a/extras/autowaf.py +++ b/extras/autowaf.py @@ -2,41 +2,48 @@ import glob import os import subprocess import sys +import time -from waflib import Build, Context, Logs, Options, Utils +from waflib import Configure, ConfigSet, Build, Context, Logs, Options, Utils from waflib.TaskGen import feature, before, after global g_is_child g_is_child = False -# Only run autowaf hooks once (even if sub projects call several times) -global g_step -g_step = 0 +NONEMPTY = -10 -global line_just -line_just = 40 +if sys.platform == 'win32': + lib_path_name = 'PATH' +elif sys.platform == 'darwin': + lib_path_name = 'DYLD_LIBRARY_PATH' +else: + lib_path_name = 'LD_LIBRARY_PATH' # Compute dependencies globally # import preproc # preproc.go_absolute = True -# Test context that inherits build context to make configuration available -class TestContext(Build.BuildContext): - "Run tests" - cmd = 'test' - fun = 'test' - @feature('c', 'cxx') @after('apply_incpaths') def include_config_h(self): self.env.append_value('INCPATHS', self.bld.bldnode.abspath()) -def set_options(opt, debug_by_default=False, test=False): - "Add standard autowaf options if they havn't been added yet" - global g_step - if g_step > 0: - return +class OptionsContext(Options.OptionsContext): + def __init__(self, **kwargs): + super(OptionsContext, self).__init__(**kwargs) + set_options(self) + def configuration_options(self): + return self.get_option_group('Configuration options') + + def add_flags(self, group, flags): + """Tersely add flags (a dictionary of longname:desc) to a group""" + for name, desc in flags.items(): + group.add_option('--' + name, action='store_true', + dest=name.replace('-', '_'), help=desc) + +def set_options(opt, debug_by_default=False): + "Add standard autowaf options" opts = opt.get_option_group('Configuration options') # Standard directory options @@ -60,42 +67,71 @@ def set_options(opt, debug_by_default=False, test=False): opts.add_option('--optimize', action='store_false', default=True, dest='debug', help="build optimized binaries") else: - opts.add_option('--debug', action='store_true', default=False, + opts.add_option('-d', '--debug', action='store_true', default=False, dest='debug', help="build debuggable binaries") opts.add_option('--pardebug', action='store_true', default=False, dest='pardebug', help="build debug libraries with D suffix") - opts.add_option('--strict', action='store_true', default=False, + opts.add_option('-s', '--strict', action='store_true', default=False, dest='strict', help="use strict compiler flags and show all warnings") - opts.add_option('--ultra-strict', action='store_true', default=False, + opts.add_option('-S', '--ultra-strict', action='store_true', default=False, dest='ultra_strict', help="use extremely strict compiler flags (likely noisy)") opts.add_option('--docs', action='store_true', default=False, dest='docs', help="build documentation (requires doxygen)") # Test options - if test: + if hasattr(Context.g_module, 'test'): test_opts = opt.add_option_group('Test options', '') - opts.add_option('--test', action='store_true', dest='build_tests', + opts.add_option('-T', '--test', action='store_true', dest='build_tests', help='build unit tests') opts.add_option('--no-coverage', action='store_true', dest='no_coverage', help='do not instrument code for test coverage') - test_opts.add_option('--test-wrapper', type='string', + test_opts.add_option('--wrapper', type='string', dest='test_wrapper', help='command prefix for tests (e.g. valgrind)') - test_opts.add_option('--verbose-tests', action='store_true', - default=False, dest='verbose_tests', - help='always show test output') - - g_step = 1 - -def add_flags(opt, flags): - for name, desc in flags.items(): - opt.add_option('--' + name, action='store_true', - dest=name.replace('-', '_'), help=desc) + test_opts.add_option('--test-filter', type='string', + dest='test_filter', + help='regular expression for tests to run') + + # Run options + run_opts = opt.add_option_group('Run options') + run_opts.add_option('--cmd', type='string', dest='cmd', + help='command to run from build directory') + +class ConfigureContext(Configure.ConfigurationContext): + """configures the project""" + + def __init__(self, **kwargs): + self.line_just = 45 + if hasattr(Context.g_module, 'line_just'): + self.line_just = Context.g_module.line_just + + super(ConfigureContext, self).__init__(**kwargs) + self.run_env = ConfigSet.ConfigSet() + self.system_include_paths = set() + + def pre_recurse(self, node): + if len(self.stack_path) == 1: + Logs.pprint('BOLD', 'Configuring %s' % node.parent.srcpath()) + super(ConfigureContext, self).pre_recurse(node) + + def store(self): + self.env.AUTOWAF_RUN_ENV = self.run_env.get_merged_dict() + for path in sorted(self.system_include_paths): + if 'COMPILER_CC' in self.env: + self.env.append_value('CFLAGS', ['-isystem', path]) + if 'COMPILER_CXX' in self.env: + self.env.append_value('CXXFLAGS', ['-isystem', path]) + + super(ConfigureContext, self).store() + + def build_path(self, path='.'): + """Return `path` within the build directory""" + return str(self.path.get_bld().find_node(path)) def get_check_func(conf, lang): if lang == 'c': @@ -180,6 +216,10 @@ def check_pkg(conf, name, **args): else: conf.env[var_name] = CheckType.OPTIONAL + if not conf.env.MSVC_COMPILER and 'system' in args and args['system']: + conf.system_include_paths.update( + conf.env['INCLUDES_' + nameify(args['uselib_store'])]) + def normpath(path): if sys.platform == 'win32': return os.path.normpath(path).replace('\\', '/') @@ -187,10 +227,6 @@ def normpath(path): return os.path.normpath(path) def configure(conf): - global g_step - if g_step > 1: - return - def append_cxx_flags(flags): conf.env.append_value('CFLAGS', flags) conf.env.append_value('CXXFLAGS', flags) @@ -238,16 +274,37 @@ def configure(conf): conf.env['CXXFLAGS'] = ['-O0', '-g'] else: if conf.env['MSVC_COMPILER']: - conf.env['CFLAGS'] = ['/MD', '/FS', '/DNDEBUG'] - conf.env['CXXFLAGS'] = ['/MD', '/FS', '/DNDEBUG'] + append_cxx_flags(['/MD', '/FS', '/DNDEBUG']) else: append_cxx_flags(['-DNDEBUG']) if conf.env.MSVC_COMPILER: Options.options.no_coverage = True - if Options.options.strict: - conf.env.append_value('CFLAGS', ['/Wall']) - conf.env.append_value('CXXFLAGS', ['/Wall']) + append_cxx_flags(['/nologo', + '/FS', + '/DNDEBUG', + '/D_CRT_SECURE_NO_WARNINGS', + '/experimental:external', + '/external:W0', + '/external:anglebrackets']) + conf.env.append_value('LINKFLAGS', '/nologo') + if Options.options.strict or Options.options.ultra_strict: + ms_strict_flags = ['/Wall', + '/wd4061', + '/wd4200', + '/wd4514', + '/wd4571', + '/wd4625', + '/wd4626', + '/wd4706', + '/wd4710', + '/wd4820', + '/wd5026', + '/wd5027', + '/wd5045'] + conf.env.append_value('CFLAGS', ms_strict_flags) + conf.env.append_value('CXXFLAGS', ms_strict_flags) + conf.env.append_value('CXXFLAGS', ['/EHsc']) else: if Options.options.ultra_strict: Options.options.strict = True @@ -310,16 +367,14 @@ def configure(conf): # Define version in configuration appname = getattr(Context.g_module, Context.APPNAME, 'noname') version = getattr(Context.g_module, Context.VERSION, '0.0.0') - define(conf, appname.upper() + '_VERSION', version) + defname = appname.upper().replace('-', '_').replace('.', '_') + define(conf, defname + '_VERSION', version) conf.env.prepend_value('CFLAGS', '-I' + os.path.abspath('.')) conf.env.prepend_value('CXXFLAGS', '-I' + os.path.abspath('.')) - g_step = 2 def display_summary(conf, msgs=None): - global g_is_child - Logs.pprint('', '') - if not g_is_child: + if len(conf.stack_path) == 1: display_msg(conf, "Install prefix", conf.env['PREFIX']) if 'COMPILER_CC' in conf.env: display_msg(conf, "C Flags", ' '.join(conf.env['CFLAGS'])) @@ -331,13 +386,11 @@ def display_summary(conf, msgs=None): if msgs is not None: display_msgs(conf, msgs) - Logs.pprint('', '') - def set_c_lang(conf, lang): "Set a specific C language standard, like 'c99' or 'c11'" if conf.env.MSVC_COMPILER: # MSVC has no hope or desire to compile C99, just compile as C++ - conf.env.append_unique('CFLAGS', ['-TP']) + conf.env.append_unique('CFLAGS', ['/TP']) else: flag = '-std=%s' % lang conf.check(cflags=['-Werror', flag], @@ -361,7 +414,7 @@ def set_modern_c_flags(conf): if 'COMPILER_CC' in conf.env: if conf.env.MSVC_COMPILER: # MSVC has no hope or desire to compile C99, just compile as C++ - conf.env.append_unique('CFLAGS', ['-TP']) + conf.env.append_unique('CFLAGS', ['/TP']) else: for flag in ['-std=c11', '-std=c99']: if conf.check(cflags=['-Werror', flag], mandatory=False, @@ -436,22 +489,16 @@ def set_lib_env(conf, name, version): major_ver = version.split('.')[0] pkg_var_name = 'PKG_' + name.replace('-', '_') + '_' + major_ver lib_name = '%s-%s' % (name, major_ver) + lib_path = [str(conf.path.get_bld())] if conf.env.PARDEBUG: lib_name += 'D' conf.env[pkg_var_name] = lib_name conf.env['INCLUDES_' + NAME] = ['${INCLUDEDIR}/%s-%s' % (name, major_ver)] - conf.env['LIBPATH_' + NAME] = [conf.env.LIBDIR] + conf.env['LIBPATH_' + NAME] = lib_path conf.env['LIB_' + NAME] = [lib_name] -def set_line_just(conf, width): - global line_just - line_just = max(line_just, width) - conf.line_just = line_just - -def display_header(title): - global g_is_child - if g_is_child: - Logs.pprint('BOLD', title) + conf.run_env.append_unique(lib_path_name, lib_path) + conf.define(NAME + '_VERSION', version) def display_msg(conf, msg, status=None, color=None): color = 'CYAN' @@ -495,7 +542,7 @@ def build_pc(bld, name, version, version_suffix, libs, subst_dict={}): """ pkg_prefix = bld.env['PREFIX'] - if pkg_prefix[-1] == '/': + if len(pkg_prefix) > 1 and pkg_prefix[-1] == '/': pkg_prefix = pkg_prefix[:-1] target = name.lower() @@ -584,7 +631,8 @@ def make_simple_dox(name): os.chdir(top) except Exception as e: Logs.error("Failed to fix up %s documentation: %s" % (name, e)) - + finally: + os.chdir(top) def build_dox(bld, name, version, srcdir, blddir, outdir='', versioned=True): """Build Doxygen API documentation""" @@ -715,211 +763,415 @@ def build_i18n(bld, srcdir, dir, name, sources, copyright_holder=None): build_i18n_po(bld, srcdir, dir, name, sources, copyright_holder) build_i18n_mo(bld, srcdir, dir, name, sources, copyright_holder) -def cd_to_build_dir(ctx, appname): - top_level = (len(ctx.stack_path) > 1) - if top_level: - os.chdir(os.path.join('build', appname)) - else: - os.chdir('build') - Logs.pprint('GREEN', ("Waf: Entering directory `%s'" % - os.path.abspath(os.getcwd()))) +class ExecutionEnvironment: + """Context that sets system environment variables for program execution""" + def __init__(self, changes): + self.original_environ = os.environ.copy() + + self.diff = {} + for path_name, paths in changes.items(): + value = os.pathsep.join(paths) + if path_name in os.environ: + value += os.pathsep + os.environ[path_name] + + self.diff[path_name] = value + + os.environ.update(self.diff) + + def __str__(self): + return '\n'.join({'%s="%s"' % (k, v) for k, v in self.diff.items()}) + + def __enter__(self): + return self -def cd_to_orig_dir(ctx, child): - if child: - os.chdir(os.path.join('..', '..')) + def __exit__(self, type, value, traceback): + os.environ = self.original_environ + +class RunContext(Build.BuildContext): + "runs an executable from the build directory" + cmd = 'run' + + def execute(self): + self.restore() + if not self.all_envs: + self.load_envs() + + with ExecutionEnvironment(self.env.AUTOWAF_RUN_ENV) as env: + if Options.options.verbose: + Logs.pprint('GREEN', str(env) + '\n') + + if Options.options.cmd: + Logs.pprint('GREEN', 'Running %s' % Options.options.cmd) + subprocess.call(Options.options.cmd, shell=True) + else: + Logs.error("error: Missing --cmd option for run command") + +def show_diff(from_lines, to_lines, from_filename, to_filename): + import difflib + import sys + + same = True + for line in difflib.unified_diff( + from_lines, to_lines, + fromfile=os.path.abspath(from_filename), + tofile=os.path.abspath(to_filename)): + sys.stderr.write(line) + same = False + + return same + +def test_file_equals(patha, pathb): + import filecmp + import io + + for path in (patha, pathb): + if not os.access(path, os.F_OK): + Logs.pprint('RED', 'error: missing file %s' % path) + return False + + if filecmp.cmp(patha, pathb, shallow=False): + return True + + with io.open(patha, 'rU', encoding='utf-8') as fa: + with io.open(pathb, 'rU', encoding='utf-8') as fb: + return show_diff(fa.readlines(), fb.readlines(), patha, pathb) + +def bench_time(): + if hasattr(time, 'perf_counter'): # Added in Python 3.3 + return time.perf_counter() else: - os.chdir('..') - -def pre_test(ctx, appname, dirs=['src']): - if not hasattr(ctx, 'autowaf_tests_total'): - ctx.autowaf_tests_total = 0 - ctx.autowaf_tests_failed = 0 - ctx.autowaf_local_tests_total = 0 - ctx.autowaf_local_tests_failed = 0 - ctx.autowaf_tests = {} - - ctx.autowaf_tests[appname] = {'total': 0, 'failed': 0} - - cd_to_build_dir(ctx, appname) - if not ctx.env.NO_COVERAGE: - diropts = '' - for i in dirs: - diropts += ' -d ' + i - clear_log = open('lcov-clear.log', 'w') - try: - try: - # Clear coverage data - subprocess.call(('lcov %s -z' % diropts).split(), - stdout=clear_log, stderr=clear_log) - except Exception: - Logs.warn('Failed to run lcov, no coverage report generated') - finally: - clear_log.close() - -def post_test(ctx, appname, dirs=['src'], remove=['*boost*', 'c++*']): - if not ctx.env.NO_COVERAGE: - diropts = '' - for i in dirs: - diropts += ' -d ' + i - coverage_log = open('lcov-coverage.log', 'w') - coverage_lcov = open('coverage.lcov', 'w') - coverage_stripped_lcov = open('coverage-stripped.lcov', 'w') + return time.time() + +class TestOutput: + """Test output that is truthy if result is as expected""" + def __init__(self, expected, result=None): + self.stdout = self.stderr = None + self.expected = expected + self.result = result + + def __bool__(self): + return self.expected is None or self.result == self.expected + + __nonzero__ = __bool__ + +def is_string(s): + if sys.version_info[0] < 3: + return isinstance(s, basestring) + return isinstance(s, str) + +class TestScope: + """Scope for running tests that maintains pass/fail statistics""" + def __init__(self, tst, name, defaults): + self.tst = tst + self.name = name + self.defaults = defaults + self.n_failed = 0 + self.n_total = 0 + + def run(self, test, **kwargs): + if type(test) == list and 'name' not in kwargs: + import pipes + kwargs['name'] = ' '.join(map(pipes.quote, test)) + + if Options.options.test_filter and 'name' in kwargs: + import re + found = False + for scope in self.tst.stack: + if re.search(Options.options.test_filter, scope.name): + found = True + break + + if (not found and + not re.search(Options.options.test_filter, self.name) and + not re.search(Options.options.test_filter, kwargs['name'])): + return True + + if callable(test): + output = self._run_callable(test, **kwargs) + elif type(test) == list: + + output = self._run_command(test, **kwargs) + else: + raise Exception("Unknown test type") + + if not output: + self.tst.log_bad('FAILED', kwargs['name']) + + return self.tst.test_result(output) + + def _run_callable(self, test, **kwargs): + expected = kwargs['expected'] if 'expected' in kwargs else True + return TestOutput(expected, test()) + + def _run_command(self, test, **kwargs): + if 'stderr' in kwargs and kwargs['stderr'] == NONEMPTY: + # Run with a temp file for stderr and check that it is non-empty + import tempfile + with tempfile.TemporaryFile() as stderr: + kwargs['stderr'] = stderr + output = self.run(test, **kwargs) + stderr.seek(0, 2) # Seek to end + return (output if not output else + self.run( + lambda: stderr.tell() > 0, + name=kwargs['name'] + ' error message')) + try: - try: - base = '.' - if g_is_child: - base = '..' - - # Generate coverage data - lcov_cmd = 'lcov -c %s -b %s' % (diropts, base) - if ctx.env.LLVM_COV: - lcov_cmd += ' --gcov-tool %s' % ctx.env.LLVM_COV[0] - subprocess.call(lcov_cmd.split(), - stdout=coverage_lcov, stderr=coverage_log) - - # Strip unwanted stuff - subprocess.call( - ['lcov', '--remove', 'coverage.lcov'] + remove, - stdout=coverage_stripped_lcov, stderr=coverage_log) - - # Generate HTML coverage output - if not os.path.isdir('coverage'): - os.makedirs('coverage') - subprocess.call( - 'genhtml -o coverage coverage-stripped.lcov'.split(), - stdout=coverage_log, stderr=coverage_log) - - except Exception: - Logs.warn('Failed to run lcov, no coverage report generated') + # Run with stdout and stderr set to the appropriate streams + out_stream = self._stream('stdout', kwargs) + err_stream = self._stream('stderr', kwargs) + return self._exec(test, **kwargs) finally: - coverage_stripped_lcov.close() - coverage_lcov.close() - coverage_log.close() - - if ctx.autowaf_tests[appname]['failed'] > 0: - Logs.pprint('RED', '\nSummary: %d / %d %s tests failed' % ( - ctx.autowaf_tests[appname]['failed'], - ctx.autowaf_tests[appname]['total'], - appname)) - else: - Logs.pprint('GREEN', '\nSummary: All %d %s tests passed' % ( - ctx.autowaf_tests[appname]['total'], appname)) - - if not ctx.env.NO_COVERAGE: - Logs.pprint('GREEN', 'Coverage: \n' - % os.path.abspath('coverage/index.html')) - - Logs.pprint('GREEN', ("Waf: Leaving directory `%s'" % - os.path.abspath(os.getcwd()))) - top_level = (len(ctx.stack_path) > 1) - if top_level: - cd_to_orig_dir(ctx, top_level) - -def run_test(ctx, - appname, - test, - desired_status=0, - dirs=['src'], - name='', - header=False, - quiet=False): - """Run an individual test. - - `test` is either a shell command string, or a list of [name, return status] - for displaying tests implemented in the calling Python code. - """ - - ctx.autowaf_tests_total += 1 - ctx.autowaf_local_tests_total += 1 - ctx.autowaf_tests[appname]['total'] += 1 + out_stream = out_stream.close() if out_stream else None + err_stream = err_stream.close() if err_stream else None + + def _stream(self, stream_name, kwargs): + s = kwargs[stream_name] if stream_name in kwargs else None + if is_string(s): + kwargs[stream_name] = open(s, 'wb') + return kwargs[stream_name] + return None + + def _exec(self, + test, + expected=0, + name='', + stdin=None, + stdout=None, + stderr=None, + verbosity=1): + def stream(s): + return open(s, 'wb') if type(s) == str else s + + if verbosity > 1: + self.tst.log_good('RUN ', name) - out = (None, None) - if type(test) == list: - name = test[0] - returncode = test[1] - elif callable(test): - returncode = test() - else: - s = test - if isinstance(test, type([])): - s = ' '.join(test) - if header and not quiet: - Logs.pprint('Green', '\n** Test %s' % s) - cmd = test if Options.options.test_wrapper: - cmd = Options.options.test_wrapper + ' ' + test - if name == '': - name = test - - proc = subprocess.Popen(cmd, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out = proc.communicate() - returncode = proc.returncode - - success = desired_status is None or returncode == desired_status - if success: - if not quiet: - Logs.pprint('GREEN', '** Pass %s' % name) - else: - Logs.pprint('RED', '** FAIL %s' % name) - ctx.autowaf_tests_failed += 1 - ctx.autowaf_tests[appname]['failed'] += 1 - if type(test) != list and not callable(test): - Logs.pprint('RED', test) + test = [Options.options.test_wrapper] + test - if Options.options.verbose_tests and type(test) != list and not callable(test): - sys.stdout.write(out[0]) - sys.stderr.write(out[1]) + output = TestOutput(expected) + with open(os.devnull, 'wb') as null: + out = null if verbosity < 3 and not stdout else stdout + err = null if verbosity < 2 and not stderr else stderr + proc = subprocess.Popen(test, stdin=stdin, stdout=out, stderr=err) + output.stdout, output.stderr = proc.communicate() + output.result = proc.returncode - return (success, out) + if output and verbosity > 0: + self.tst.log_good(' OK', name) -def tests_name(ctx, appname, name='*'): - if name == '*': - return appname - else: - return '%s.%s' % (appname, name) + return output -def begin_tests(ctx, appname, name='*'): - ctx.autowaf_local_tests_failed = 0 - ctx.autowaf_local_tests_total = 0 - Logs.pprint('GREEN', '\n** Begin %s tests' % ( - tests_name(ctx, appname, name))) +class TestContext(Build.BuildContext): + "runs test suite" + fun = cmd = 'test' - class Handle: - def __enter__(self): - pass + def __init__(self, **kwargs): + super(TestContext, self).__init__(**kwargs) + self.start_time = bench_time() + self.max_depth = 1 - def __exit__(self, type, value, traceback): - end_tests(ctx, appname, name) + defaults = {'verbosity': Options.options.verbose} + self.stack = [TestScope(self, Context.g_module.APPNAME, defaults)] - return Handle() + def defaults(self): + return self.stack[-1].defaults -def end_tests(ctx, appname, name='*'): - failures = ctx.autowaf_local_tests_failed - if failures == 0: - Logs.pprint('GREEN', '** Passed all %d %s tests' % ( - ctx.autowaf_local_tests_total, tests_name(ctx, appname, name))) - else: - Logs.pprint('RED', '** Failed %d / %d %s tests' % ( - failures, ctx.autowaf_local_tests_total, tests_name(ctx, appname, name))) - -def run_tests(ctx, - appname, - tests, - desired_status=0, - dirs=['src'], - name='*', - headers=False): - begin_tests(ctx, appname, name) - - diropts = '' - for i in dirs: - diropts += ' -d ' + i - - for i in tests: - run_test(ctx, appname, i, desired_status, dirs, i, headers) - - end_tests(ctx, appname, name) + def finalize(self): + if self.stack[-1].n_failed > 0: + sys.exit(1) + + super(TestContext, self).finalize() + + def __call__(self, test, **kwargs): + return self.stack[-1].run(test, **self.args(**kwargs)) + + def file_equals(self, from_path, to_path, **kwargs): + kwargs.update({'expected': True, + 'name': '%s == %s' % (from_path, to_path)}) + return self(lambda: test_file_equals(from_path, to_path), **kwargs) + + def log_good(self, title, fmt, *args): + Logs.pprint('GREEN', '[%s] %s' % (title.center(10), fmt % args)) + + def log_bad(self, title, fmt, *args): + Logs.pprint('RED', '[%s] %s' % (title.center(10), fmt % args)) + + def pre_recurse(self, node): + wscript_module = Context.load_module(node.abspath()) + group_name = wscript_module.APPNAME + self.stack.append(TestScope(self, group_name, self.defaults())) + self.max_depth = max(self.max_depth, len(self.stack) - 1) + + bld_dir = node.get_bld().parent + if bld_dir != self.path.get_bld(): + Logs.info('') + + self.original_dir = os.getcwd() + Logs.info("Waf: Entering directory `%s'\n", bld_dir) + os.chdir(str(bld_dir)) + + if not self.env.NO_COVERAGE and str(node.parent) == Context.top_dir: + self.clear_coverage() + + self.log_good('=' * 10, 'Running %s tests', group_name) + super(TestContext, self).pre_recurse(node) + + def test_result(self, success): + self.stack[-1].n_total += 1 + self.stack[-1].n_failed += 1 if not success else 0 + return success + + def pop(self): + scope = self.stack.pop() + self.stack[-1].n_total += scope.n_total + self.stack[-1].n_failed += scope.n_failed + return scope + + def post_recurse(self, node): + super(TestContext, self).post_recurse(node) + + scope = self.pop() + duration = (bench_time() - self.start_time) * 1000.0 + is_top = str(node.parent) == str(Context.top_dir) + + if is_top and self.max_depth > 1: + Logs.info('') + + self.log_good('=' * 10, '%d tests from %s ran (%d ms total)', + scope.n_total, scope.name, duration) + + if not self.env.NO_COVERAGE: + if is_top: + self.gen_coverage() + + if os.path.exists('coverage/index.html'): + self.log_good('REPORT', '', + os.path.abspath('coverage/index.html')) + + successes = scope.n_total - scope.n_failed + Logs.pprint('GREEN', '[ PASSED ] %d tests' % successes) + if scope.n_failed > 0: + Logs.pprint('RED', '[ FAILED ] %d tests' % scope.n_failed) + if is_top: + Logs.info("\nWaf: Leaving directory `%s'" % os.getcwd()) + + os.chdir(self.original_dir) + + def execute(self): + self.restore() + if not self.all_envs: + self.load_envs() + + if not self.env.BUILD_TESTS: + self.fatal('Configuration does not include tests') + + with ExecutionEnvironment(self.env.AUTOWAF_RUN_ENV) as env: + if self.defaults()['verbosity'] > 0: + Logs.pprint('GREEN', str(env) + '\n') + self.recurse([self.run_dir]) + + def src_path(self, path): + return os.path.relpath(os.path.join(str(self.path), path)) + + def args(self, **kwargs): + all_kwargs = self.defaults().copy() + all_kwargs.update(kwargs) + return all_kwargs + + def group(self, name, **kwargs): + return TestGroup( + self, self.stack[-1].name, name, **self.args(**kwargs)) + + def set_test_defaults(self, **kwargs): + """Set default arguments to be passed to all tests""" + self.stack[-1].defaults.update(kwargs) + + def clear_coverage(self): + """Zero old coverage data""" + try: + with open('cov-clear.log', 'w') as log: + subprocess.call(['lcov', '-z', '-d', str(self.path)], + stdout=log, stderr=log) + + except Exception: + Logs.warn('Failed to run lcov to clear old coverage data') + + def gen_coverage(self): + """Generate coverage data and report""" + try: + with open('cov.lcov', 'w') as out: + with open('cov.log', 'w') as err: + subprocess.call(['lcov', '-c', '--no-external', + '--rc', 'lcov_branch_coverage=1', + '-b', '.', + '-d', str(self.path)], + stdout=out, stderr=err) + + if not os.path.isdir('coverage'): + os.makedirs('coverage') + + with open('genhtml.log', 'w') as log: + subprocess.call(['genhtml', + '-o', 'coverage', + '--rc', 'genhtml_branch_coverage=1', + 'cov.lcov'], + stdout=log, stderr=log) + + summary = subprocess.check_output( + ['lcov', '--summary', + '--rc', 'lcov_branch_coverage=1', + 'cov.lcov'], + stderr=subprocess.STDOUT).decode('ascii') + + import re + lines = re.search('lines\.*: (.*)%.*', summary).group(1) + functions = re.search('functions\.*: (.*)%.*', summary).group(1) + branches = re.search('branches\.*: (.*)%.*', summary).group(1) + self.log_good('COVERAGE', '%s%% lines, %s%% functions, %s%% branches', + lines, functions, branches) + + except Exception: + Logs.warn('Failed to run lcov to generate coverage report') + +class TestGroup: + def __init__(self, tst, suitename, name, **kwargs): + self.tst = tst + self.suitename = suitename + self.name = name + self.kwargs = kwargs + self.start_time = bench_time() + tst.stack.append(TestScope(tst, name, tst.defaults())) + + def label(self): + return self.suitename + '.%s' % self.name if self.name else '' + + def args(self, **kwargs): + all_kwargs = self.tst.args(**self.kwargs) + all_kwargs.update(kwargs) + return all_kwargs + + def __enter__(self): + if 'verbosity' in self.kwargs and self.kwargs['verbosity'] > 0: + self.tst.log_good('-' * 10, self.label()) + return self + + def __call__(self, test, **kwargs): + return self.tst(test, **self.args(**kwargs)) + + def file_equals(self, from_path, to_path, **kwargs): + return self.tst.file_equals(from_path, to_path, **kwargs) + + def __exit__(self, type, value, traceback): + duration = (bench_time() - self.start_time) * 1000.0 + scope = self.tst.pop() + n_passed = scope.n_total - scope.n_failed + if scope.n_failed == 0: + self.tst.log_good('-' * 10, '%d tests from %s (%d ms total)', + scope.n_total, self.label(), duration) + else: + self.tst.log_bad('-' * 10, '%d/%d tests from %s (%d ms total)', + n_passed, scope.n_total, self.label(), duration) def run_ldconfig(ctx): should_run = (ctx.cmd == 'install' and diff --git a/extras/buildcopy.py b/extras/buildcopy.py index a6d9ac8..eaff7e6 100644 --- a/extras/buildcopy.py +++ b/extras/buildcopy.py @@ -22,7 +22,7 @@ Examples:: """ import os, shutil -from waflib import Errors, Task, TaskGen, Utils, Node +from waflib import Errors, Task, TaskGen, Utils, Node, Logs @TaskGen.before_method('process_source') @TaskGen.feature('buildcopy') @@ -58,10 +58,13 @@ def make_buildcopy(self): raise Errors.WafError('buildcopy: File not found in src: %s'%os.path.join(*lst)) nodes = [ to_src_nodes(n) for n in getattr(self, 'buildcopy_source', getattr(self, 'source', [])) ] + if not nodes: + Logs.warn('buildcopy: No source files provided to buildcopy in %s (set `buildcopy_source` or `source`)', + self) + return node_pairs = [(n, n.get_bld()) for n in nodes] self.create_task('buildcopy', [n[0] for n in node_pairs], [n[1] for n in node_pairs], node_pairs=node_pairs) - class buildcopy(Task.Task): """ Copy for each pair `n` in `node_pairs`: n[0] -> n[1]. diff --git a/extras/cpplint.py b/extras/cpplint.py index e3302e5..8cdd6dd 100644 --- a/extras/cpplint.py +++ b/extras/cpplint.py @@ -43,12 +43,12 @@ from waflib import Errors, Task, TaskGen, Logs, Options, Node, Utils critical_errors = 0 CPPLINT_FORMAT = '[CPPLINT] %(filename)s:\nline %(linenum)s, severity %(confidence)s, category: %(category)s\n%(message)s\n' -RE_EMACS = re.compile('(?P.*):(?P\d+): (?P.*) \[(?P.*)\] \[(?P\d+)\]') +RE_EMACS = re.compile(r'(?P.*):(?P\d+): (?P.*) \[(?P.*)\] \[(?P\d+)\]') CPPLINT_RE = { 'waf': RE_EMACS, 'emacs': RE_EMACS, - 'vs7': re.compile('(?P.*)\((?P\d+)\): (?P.*) \[(?P.*)\] \[(?P\d+)\]'), - 'eclipse': re.compile('(?P.*):(?P\d+): warning: (?P.*) \[(?P.*)\] \[(?P\d+)\]'), + 'vs7': re.compile(r'(?P.*)\((?P\d+)\): (?P.*) \[(?P.*)\] \[(?P\d+)\]'), + 'eclipse': re.compile(r'(?P.*):(?P\d+): warning: (?P.*) \[(?P.*)\] \[(?P\d+)\]'), } CPPLINT_STR = ('${CPPLINT} ' '--verbose=${CPPLINT_LEVEL} ' diff --git a/extras/cython.py b/extras/cython.py index 481d6f4..591c274 100644 --- a/extras/cython.py +++ b/extras/cython.py @@ -8,8 +8,9 @@ from waflib.TaskGen import extension cy_api_pat = re.compile(r'\s*?cdef\s*?(public|api)\w*') re_cyt = re.compile(r""" - (?:from\s+(\w+)\s+)? # optionally match "from foo" and capture foo - c?import\s(\w+|[*]) # require "import bar" and capture bar + ^\s* # must begin with some whitespace characters + (?:from\s+(\w+)(?:\.\w+)*\s+)? # optionally match "from foo(.baz)" and capture foo + c?import\s(\w+|[*]) # require "import bar" and capture bar """, re.M | re.VERBOSE) @extension('.pyx') @@ -85,12 +86,12 @@ class cython(Task.Task): node = self.inputs[0] txt = node.read() - mods = [] + mods = set() for m in re_cyt.finditer(txt): if m.group(1): # matches "from foo import bar" - mods.append(m.group(1)) + mods.add(m.group(1)) else: - mods.append(m.group(2)) + mods.add(m.group(2)) Logs.debug('cython: mods %r', mods) incs = getattr(self.generator, 'cython_includes', []) @@ -99,7 +100,7 @@ class cython(Task.Task): found = [] missing = [] - for x in mods: + for x in sorted(mods): for y in incs: k = y.find_resource(x + '.pxd') if k: diff --git a/extras/distnet.py b/extras/distnet.py index 09a31a6..ff3ed8e 100644 --- a/extras/distnet.py +++ b/extras/distnet.py @@ -44,7 +44,7 @@ TARFORMAT = 'w:bz2' TIMEOUT = 60 REQUIRES = 'requires.txt' -re_com = re.compile('\s*#.*', re.M) +re_com = re.compile(r'\s*#.*', re.M) def total_version_order(num): lst = num.split('.') diff --git a/extras/erlang.py b/extras/erlang.py index 49f6d5b..0b93d9a 100644 --- a/extras/erlang.py +++ b/extras/erlang.py @@ -51,7 +51,7 @@ class erl(Task.Task): if n.abspath() in scanned: continue - for i in re.findall('-include\("(.*)"\)\.', n.read()): + for i in re.findall(r'-include\("(.*)"\)\.', n.read()): for d in task.erlc_incnodes: r = d.find_node(i) if r: diff --git a/extras/fast_partial.py b/extras/fast_partial.py index b3af513..d5b6144 100644 --- a/extras/fast_partial.py +++ b/extras/fast_partial.py @@ -17,7 +17,7 @@ Usage:: def options(opt): opt.load('fast_partial') -Assuptions: +Assumptions: * Mostly for C/C++/Fortran targets with link tasks (object-only targets are not handled) * For full project builds: no --targets and no pruning from subfolders * The installation phase is ignored diff --git a/extras/fc_nfort.py b/extras/fc_nfort.py new file mode 100644 index 0000000..c25886b --- /dev/null +++ b/extras/fc_nfort.py @@ -0,0 +1,52 @@ +#! /usr/bin/env python +# encoding: utf-8 +# Detection of the NEC Fortran compiler for Aurora Tsubasa + +import re +from waflib.Tools import fc,fc_config,fc_scan +from waflib.Configure import conf +from waflib.Tools.compiler_fc import fc_compiler +fc_compiler['linux'].append('fc_nfort') + +@conf +def find_nfort(conf): + fc=conf.find_program(['nfort'],var='FC') + conf.get_nfort_version(fc) + conf.env.FC_NAME='NFORT' + conf.env.FC_MOD_CAPITALIZATION='lower' + +@conf +def nfort_flags(conf): + v=conf.env + v['_FCMODOUTFLAGS']=[] + v['FCFLAGS_DEBUG']=[] + v['FCFLAGS_fcshlib']=[] + v['LINKFLAGS_fcshlib']=[] + v['FCSTLIB_MARKER']='' + v['FCSHLIB_MARKER']='' + +@conf +def get_nfort_version(conf,fc): + version_re=re.compile(r"nfort\s*\(NFORT\)\s*(?P\d+)\.(?P\d+)\.",re.I).search + cmd=fc+['--version'] + out,err=fc_config.getoutput(conf,cmd,stdin=False) + if out: + match=version_re(out) + else: + match=version_re(err) + if not match: + return(False) + conf.fatal('Could not determine the NEC NFORT Fortran compiler version.') + else: + k=match.groupdict() + conf.env['FC_VERSION']=(k['major'],k['minor']) + +def configure(conf): + conf.find_nfort() + conf.find_program('nar',var='AR') + conf.add_os_flags('ARFLAGS') + if not conf.env.ARFLAGS: + conf.env.ARFLAGS=['rcs'] + conf.fc_flags() + conf.fc_add_flags() + conf.nfort_flags() diff --git a/extras/gccdeps.py b/extras/gccdeps.py index d9758ab..bfabe72 100644 --- a/extras/gccdeps.py +++ b/extras/gccdeps.py @@ -36,7 +36,7 @@ def scan(self): names = [] return (nodes, names) -re_o = re.compile("\.o$") +re_o = re.compile(r"\.o$") re_splitter = re.compile(r'(? '2': - m = re.search(r'^message\s+(\w*)\s*{*', line) - if m: - messages.append(m.groups()[0]) - - if javapkg: - nodename = javapkg - elif pkgname: - nodename = pkgname - else: - raise Errors.WafError('Cannot derive java name from protoc file') - - nodename = nodename.replace('.',os.sep) + os.sep - if javacn: - nodename += javacn + '.java' - else: - if self.env.PROTOC_MAJOR > '2' and node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title() in messages: - nodename += node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title().replace('_','') + 'OuterClass.java' - else: - nodename += node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title().replace('_','') + '.java' - - java_node = node.parent.find_or_declare(nodename) - out_nodes.append(java_node) - protoc_flags.append('--java_out=%s' % node.parent.get_bld().bldpath()) - # Make javac get also pick java code generated in build if not node.parent.get_bld() in self.javac_task.srcdir: self.javac_task.srcdir.append(node.parent.get_bld()) - if not out_nodes: - raise Errors.WafError('Feature %r not supported by protoc extra' % self.features) + protoc_flags.append('--java_out=%s' % node.parent.get_bld().bldpath()) tsk = self.create_task('protoc', node, out_nodes) tsk.env.append_value('PROTOC_FLAGS', protoc_flags) @@ -219,9 +186,22 @@ def process_protoc(self, node): # For C++ standard include files dirs are used, # but this doesn't apply to Python for example for incpath in getattr(self, 'protoc_includes', []): - incdirs.append(self.path.find_node(incpath).bldpath()) + incpath_node = self.path.find_node(incpath) + if incpath_node: + incdirs.append(incpath_node.bldpath()) + else: + # Check if relative to top-level for extra tg dependencies + incpath_node = self.bld.path.find_node(incpath) + if incpath_node: + incdirs.append(incpath_node.bldpath()) + else: + raise Errors.WafError('protoc: include path %r does not exist' % incpath) + tsk.env.PROTOC_INCPATHS = incdirs + # Include paths external to the waf project (ie. shared pb repositories) + tsk.env.PROTOC_EXTINCPATHS = getattr(self, 'protoc_extincludes', []) + # PR2115: protoc generates output of .proto files in nested # directories by canonicalizing paths. To avoid this we have to pass # as first include the full directory file of the .proto file diff --git a/extras/pyqt5.py b/extras/pyqt5.py index c21dfa7..80f43b8 100644 --- a/extras/pyqt5.py +++ b/extras/pyqt5.py @@ -111,9 +111,9 @@ def apply_pyqt5(self): """ The additional parameters are: - :param lang: list of translation files (\*.ts) to process + :param lang: list of translation files (\\*.ts) to process :type lang: list of :py:class:`waflib.Node.Node` or string without the .ts extension - :param langname: if given, transform the \*.ts files into a .qrc files to include in the binary file + :param langname: if given, transform the \\*.ts files into a .qrc files to include in the binary file :type langname: :py:class:`waflib.Node.Node` or string without the .qrc extension """ if getattr(self, 'lang', None): diff --git a/extras/qt4.py b/extras/qt4.py index 90cae7e..d19a4dd 100644 --- a/extras/qt4.py +++ b/extras/qt4.py @@ -290,11 +290,11 @@ def apply_qt4(self): The additional parameters are: - :param lang: list of translation files (\*.ts) to process + :param lang: list of translation files (\\*.ts) to process :type lang: list of :py:class:`waflib.Node.Node` or string without the .ts extension - :param update: whether to process the C++ files to update the \*.ts files (use **waf --translate**) + :param update: whether to process the C++ files to update the \\*.ts files (use **waf --translate**) :type update: bool - :param langname: if given, transform the \*.ts files into a .qrc files to include in the binary file + :param langname: if given, transform the \\*.ts files into a .qrc files to include in the binary file :type langname: :py:class:`waflib.Node.Node` or string without the .qrc extension """ if getattr(self, 'lang', None): diff --git a/extras/remote.py b/extras/remote.py index 3b038f7..f43b600 100644 --- a/extras/remote.py +++ b/extras/remote.py @@ -203,7 +203,7 @@ class remote(BuildContext): Options.commands.remove(k) def login_to_host(self, login): - return re.sub('(\w+@)', '', login) + return re.sub(r'(\w+@)', '', login) def variant_to_login(self, variant): """linux_32_debug -> search env.LINUX_32 and then env.LINUX""" diff --git a/extras/run_do_script.py b/extras/run_do_script.py index f3c5812..07e3aa2 100644 --- a/extras/run_do_script.py +++ b/extras/run_do_script.py @@ -101,7 +101,7 @@ class run_do_script(run_do_script_base): with open(**kwargs) as log: log_tail = log.readlines()[-10:] for line in log_tail: - error_found = re.match("r\(([0-9]+)\)", line) + error_found = re.match(r"r\(([0-9]+)\)", line) if error_found: return error_found.group(1), ''.join(log_tail) else: diff --git a/extras/swig.py b/extras/swig.py index fd3d6d2..740ab46 100644 --- a/extras/swig.py +++ b/extras/swig.py @@ -17,10 +17,10 @@ tasks have to be added dynamically: SWIG_EXTS = ['.swig', '.i'] -re_module = re.compile('%module(?:\s*\(.*\))?\s+(.+)', re.M) +re_module = re.compile(r'%module(?:\s*\(.*\))?\s+(.+)', re.M) re_1 = re.compile(r'^%module.*?\s+([\w]+)\s*?$', re.M) -re_2 = re.compile('[#%]include [<"](.*)[">]', re.M) +re_2 = re.compile(r'[#%](?:include|import(?:\(module=".*"\))+|python(?:begin|code)) [<"](.*)[">]', re.M) class swig(Task.Task): color = 'BLUE' -- cgit v1.2.1