1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
|
#! /usr/bin/env python
# encoding: utf-8
"""
Use extended attributes instead of database files
1. Input files will be made writable
2. This is only for systems providing extended filesystem attributes
3. By default, hashes are calculated only if timestamp/size change (HASH_CACHE below)
4. The module enables "deep_inputs" on all tasks by propagating task signatures
5. This module also skips task signature comparisons for task code changes due to point 4.
6. This module is for Python3/Linux only, but it could be extended to Python2/other systems
using the xattr library
7. For projects in which tasks always declare output files, it should be possible to
store the rest of build context attributes on output files (imp_sigs, raw_deps and node_deps)
but this is not done here
On a simple C++ project benchmark, the variations before and after adding waf_xattr.py were observed:
total build time: 20s -> 22s
no-op build time: 2.4s -> 1.8s
pickle file size: 2.9MB -> 2.6MB
"""
import os
from waflib import Logs, Node, Task, Utils, Errors
from waflib.Task import SKIP_ME, RUN_ME, CANCEL_ME, ASK_LATER, SKIPPED, MISSING
HASH_CACHE = True
SIG_VAR = 'user.waf.sig'
SEP = ','.encode()
TEMPLATE = '%b%d,%d'.encode()
try:
PermissionError
except NameError:
PermissionError = IOError
def getxattr(self):
return os.getxattr(self.abspath(), SIG_VAR)
def setxattr(self, val):
os.setxattr(self.abspath(), SIG_VAR, val)
def h_file(self):
try:
ret = getxattr(self)
except OSError:
if HASH_CACHE:
st = os.stat(self.abspath())
mtime = st.st_mtime
size = st.st_size
else:
if len(ret) == 16:
# for build directory files
return ret
if HASH_CACHE:
# check if timestamp and mtime match to avoid re-hashing
st = os.stat(self.abspath())
mtime, size = ret[16:].split(SEP)
if int(1000 * st.st_mtime) == int(mtime) and st.st_size == int(size):
return ret[:16]
ret = Utils.h_file(self.abspath())
if HASH_CACHE:
val = TEMPLATE % (ret, int(1000 * st.st_mtime), int(st.st_size))
try:
setxattr(self, val)
except PermissionError:
os.chmod(self.abspath(), st.st_mode | 128)
setxattr(self, val)
return ret
def runnable_status(self):
bld = self.generator.bld
if bld.is_install < 0:
return SKIP_ME
for t in self.run_after:
if not t.hasrun:
return ASK_LATER
elif t.hasrun < SKIPPED:
# a dependency has an error
return CANCEL_ME
# first compute the signature
try:
new_sig = self.signature()
except Errors.TaskNotReady:
return ASK_LATER
if not self.outputs:
# compare the signature to a signature computed previously
# this part is only for tasks with no output files
key = self.uid()
try:
prev_sig = bld.task_sigs[key]
except KeyError:
Logs.debug('task: task %r must run: it was never run before or the task code changed', self)
return RUN_ME
if new_sig != prev_sig:
Logs.debug('task: task %r must run: the task signature changed', self)
return RUN_ME
# compare the signatures of the outputs to make a decision
for node in self.outputs:
try:
sig = node.h_file()
except EnvironmentError:
Logs.debug('task: task %r must run: an output node does not exist', self)
return RUN_ME
if sig != new_sig:
Logs.debug('task: task %r must run: an output node is stale', self)
return RUN_ME
return (self.always_run and RUN_ME) or SKIP_ME
def post_run(self):
bld = self.generator.bld
sig = self.signature()
for node in self.outputs:
if not node.exists():
self.hasrun = MISSING
self.err_msg = '-> missing file: %r' % node.abspath()
raise Errors.WafError(self.err_msg)
os.setxattr(node.abspath(), 'user.waf.sig', sig)
if not self.outputs:
# only for task with no outputs
bld.task_sigs[self.uid()] = sig
if not self.keep_last_cmd:
try:
del self.last_cmd
except AttributeError:
pass
try:
os.getxattr
except AttributeError:
pass
else:
h_file.__doc__ = Node.Node.h_file.__doc__
# keep file hashes as file attributes
Node.Node.h_file = h_file
# enable "deep_inputs" on all tasks
Task.Task.runnable_status = runnable_status
Task.Task.post_run = post_run
Task.Task.sig_deep_inputs = Utils.nada
|
