1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
"""Phase 4: adjacent-version diffs over normalised decompilation.
Adjacent (not endpoint) comparison, so a change introduced and then reverted
inside the range cannot hide. Classes per pair:
CHANGED present in both, normalised text differs -> the interesting set
ADDED only in the newer build
REMOVED only in the older build
IDENTICAL present in both, text equal
Functions absent from a build (UNMATCHED in Phase 2) are reported as coverage,
never silently dropped.
"""
import json, glob, os, re, sys, collections
TOKRE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\S")
def tokens(t): return TOKRE.findall(t)
def semantically_equal(x, y):
"""Compare token streams treating SYM (an UNMATCHED callee, i.e. *unknown*)
as a wildcard. Porting coverage differs between builds, so the same callee can
be SYM in one and named in the other; that is a coverage artifact, not a
semantic change. Any other token mismatch -- including every numeric literal --
is a real difference."""
a, b = tokens(x), tokens(y)
if len(a) != len(b): return False
for p, q in zip(a, b):
if p == q: continue
if p == 'SYM' or q == 'SYM': continue # unknown vs known
return False
return True
def vkey(t):
m=re.findall(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', t)
return [int(x) for x in m[0]] if m else [0,0,0,0]
def load(fam):
out={}
for p in sorted(glob.glob('phase3/%s_*.json'%fam)):
out[os.path.basename(p)[:-5]]=json.load(open(p))
return dict(sorted(out.items(), key=lambda kv: vkey(kv[0])))
LIT = re.compile(r"\b(?:0[xX][0-9A-Fa-f]+|\d+)\b")
def skeleton(t):
"""Structure + literals, with ALL identifiers erased. Two functions whose
skeletons match differ only in which names were assigned -- which, for
duplicate stubs, is an arbitrary porting choice, not a semantic change.
Literals are retained, so a changed bound/mask/size still shows up."""
return TOKRE.sub(lambda m: (m.group(0) if (LIT.fullmatch(m.group(0)) or not m.group(0)[0].isalpha()
and m.group(0)[0] != '_') else 'ID'), t)
def diff_pair(a, b):
ka,kb=set(a),set(b)
raw=[k for k in ka&kb if not semantically_equal(a[k], b[k])]
# split: SEMANTIC (structure or literals differ) vs NAME-ONLY (identifier churn)
semantic=sorted(k for k in raw if skeleton(a[k]) != skeleton(b[k]))
nameonly=sorted(set(raw)-set(semantic))
return dict(changed=semantic, name_only=nameonly, added=sorted(kb-ka),
removed=sorted(ka-kb), identical=len(ka&kb)-len(raw), comparable=len(ka&kb))
if __name__=='__main__':
fams=sys.argv[1:] or ['sos','sys_drv','hdcp','asd']
allres={}
for fam in fams:
builds=load(fam)
names=list(builds)
print("\n### %s : %d builds"%(fam.upper(), len(names)))
for i in range(len(names)-1):
o,n=names[i],names[i+1]
d=diff_pair(builds[o], builds[n])
allres["%s -> %s"%(o,n)]=d
print(" %-34s -> %-34s cmp=%-5d SEMANTIC=%-4d name-only=%-4d ADDED=%-4d REM=%d"%(
o.replace(fam+'_',''), n.replace(fam+'_',''), d['comparable'],
len(d['changed']), len(d['name_only']), len(d['added']), len(d['removed'])))
json.dump(allres, open('phase4_diffs.json','w'), indent=1)
print("\nwrote phase4_diffs.json")
|
