fcc13a07eded9cb875547fe9442250de795fb163
[lldb.git] / lldb / test / API / lldbtest.py
1 from __future__ import absolute_import
2 import os
3 import tempfile
4 import subprocess
5 import sys
6 import platform
7
8 import lit.Test
9 import lit.TestRunner
10 import lit.util
11 from lit.formats.base import TestFormat
12
13 def getBuildDir(cmd):
14     found = False
15     for arg in cmd:
16         if found:
17             return arg
18         if arg == '--build-dir':
19             found = True
20     return None
21
22 def mkdir_p(path):
23     import errno
24     try:
25         os.makedirs(path)
26     except OSError as e:
27         if e.errno != errno.EEXIST:
28             raise
29     if not os.path.isdir(path):
30         raise OSError(errno.ENOTDIR, "%s is not a directory"%path)
31
32 class LLDBTest(TestFormat):
33     def __init__(self, dotest_cmd):
34         self.dotest_cmd = dotest_cmd
35
36     def getTestsInDirectory(self, testSuite, path_in_suite, litConfig,
37                             localConfig):
38         source_path = testSuite.getSourcePath(path_in_suite)
39         for filename in os.listdir(source_path):
40             # Ignore dot files and excluded tests.
41             if (filename.startswith('.') or filename in localConfig.excludes):
42                 continue
43
44             # Ignore files that don't start with 'Test'.
45             if not filename.startswith('Test'):
46                 continue
47
48             filepath = os.path.join(source_path, filename)
49             if not os.path.isdir(filepath):
50                 base, ext = os.path.splitext(filename)
51                 if ext in localConfig.suffixes:
52                     yield lit.Test.Test(testSuite, path_in_suite +
53                                         (filename, ), localConfig)
54
55     def execute(self, test, litConfig):
56         if litConfig.noExecute:
57             return lit.Test.PASS, ''
58
59         if not test.config.lldb_enable_python:
60             return (lit.Test.UNSUPPORTED, 'Python module disabled')
61
62         if test.config.unsupported:
63             return (lit.Test.UNSUPPORTED, 'Test is unsupported')
64
65         testPath, testFile = os.path.split(test.getSourcePath())
66
67         # The Python used to run lit can be different from the Python LLDB was
68         # build with.
69         executable = test.config.python_executable
70
71         # On Windows, the system does not always correctly interpret
72         # shebang lines.  To make sure we can execute the tests, add
73         # python exe as the first parameter of the command.
74         cmd = [executable] + self.dotest_cmd + [testPath, '-p', testFile]
75
76         builddir = getBuildDir(cmd)
77         mkdir_p(builddir)
78
79         # On macOS, we can't do the DYLD_INSERT_LIBRARIES trick with a shim
80         # python binary as the ASan interceptors get loaded too late. Also,
81         # when SIP is enabled, we can't inject libraries into system binaries
82         # at all, so we need a copy of the "real" python to work with.
83         #
84         # Find the "real" python binary, copy it, and invoke it.
85         if 'DYLD_INSERT_LIBRARIES' in test.config.environment and \
86                 platform.system() == 'Darwin':
87             copied_python = os.path.join(builddir, 'copied-system-python')
88             if not os.path.isfile(copied_python):
89                 import shutil, subprocess
90                 python = subprocess.check_output([
91                     executable,
92                     os.path.join(os.path.dirname(os.path.realpath(__file__)),
93                         'get_darwin_real_python.py')
94                 ]).decode('utf-8').strip()
95                 shutil.copy(python, copied_python)
96             cmd[0] = copied_python
97
98         if 'lldb-repro-capture' in test.config.available_features or \
99            'lldb-repro-replay' in test.config.available_features:
100             reproducer_root = os.path.join(builddir, 'reproducers')
101             mkdir_p(reproducer_root)
102             reproducer_path = os.path.join(reproducer_root, testFile)
103             if 'lldb-repro-capture' in test.config.available_features:
104                 cmd.extend(['--capture-path', reproducer_path])
105             else:
106                 cmd.extend(['--replay-path', reproducer_path])
107
108         timeoutInfo = None
109         try:
110             out, err, exitCode = lit.util.executeCommand(
111                 cmd,
112                 env=test.config.environment,
113                 timeout=litConfig.maxIndividualTestTime)
114         except lit.util.ExecuteCommandTimeoutException as e:
115             out = e.out
116             err = e.err
117             exitCode = e.exitCode
118             timeoutInfo = 'Reached timeout of {} seconds'.format(
119                 litConfig.maxIndividualTestTime)
120
121         if sys.version_info.major == 2:
122             # In Python 2, string objects can contain Unicode characters. Use
123             # the non-strict 'replace' decoding mode. We cannot use the strict
124             # mode right now because lldb's StringPrinter facility and the
125             # Python utf8 decoder have different interpretations of which
126             # characters are "printable". This leads to Python utf8 decoding
127             # exceptions even though lldb is behaving as expected.
128             out = out.decode('utf-8', 'replace')
129             err = err.decode('utf-8', 'replace')
130
131         output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (
132             ' '.join(cmd), exitCode)
133         if timeoutInfo is not None:
134             output += """Timeout: %s\n""" % (timeoutInfo,)
135         output += "\n"
136
137         if out:
138             output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
139         if err:
140             output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
141
142         if timeoutInfo:
143             return lit.Test.TIMEOUT, output
144
145         if exitCode:
146             if 'XPASS:' in out or 'XPASS:' in err:
147                 return lit.Test.XPASS, output
148
149             # Otherwise this is just a failure.
150             return lit.Test.FAIL, output
151
152         has_unsupported_tests = 'UNSUPPORTED:' in out or 'UNSUPPORTED:' in err
153         has_passing_tests = 'PASS:' in out or 'PASS:' in err
154         if has_unsupported_tests and not has_passing_tests:
155             return lit.Test.UNSUPPORTED, output
156
157         passing_test_line = 'RESULT: PASSED'
158         if passing_test_line not in out and passing_test_line not in err:
159             return lit.Test.UNRESOLVED, output
160
161         return lit.Test.PASS, output