"""Phase 4: adjacent-version diffs over normalised decompilation.

Adjacent (not endpoint) comparison, so a change introduced and then reverted
inside the range cannot hide. Classes per pair:
  CHANGED   present in both, normalised text differs  -> the interesting set
  ADDED     only in the newer build
  REMOVED   only in the older build
  IDENTICAL present in both, text equal
Functions absent from a build (UNMATCHED in Phase 2) are reported as coverage,
never silently dropped.
"""
import json, glob, os, re, sys, collections

TOKRE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*|\S")

def tokens(t): return TOKRE.findall(t)

def semantically_equal(x, y):
    """Compare token streams treating SYM (an UNMATCHED callee, i.e. *unknown*)
    as a wildcard. Porting coverage differs between builds, so the same callee can
    be SYM in one and named in the other; that is a coverage artifact, not a
    semantic change. Any other token mismatch -- including every numeric literal --
    is a real difference."""
    a, b = tokens(x), tokens(y)
    if len(a) != len(b): return False
    for p, q in zip(a, b):
        if p == q: continue
        if p == 'SYM' or q == 'SYM': continue   # unknown vs known
        return False
    return True

def vkey(t):
    m=re.findall(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', t)
    return [int(x) for x in m[0]] if m else [0,0,0,0]

def load(fam):
    out={}
    for p in sorted(glob.glob('phase3/%s_*.json'%fam)):
        out[os.path.basename(p)[:-5]]=json.load(open(p))
    return dict(sorted(out.items(), key=lambda kv: vkey(kv[0])))

LIT = re.compile(r"\b(?:0[xX][0-9A-Fa-f]+|\d+)\b")
def skeleton(t):
    """Structure + literals, with ALL identifiers erased. Two functions whose
    skeletons match differ only in which names were assigned -- which, for
    duplicate stubs, is an arbitrary porting choice, not a semantic change.
    Literals are retained, so a changed bound/mask/size still shows up."""
    return TOKRE.sub(lambda m: (m.group(0) if (LIT.fullmatch(m.group(0)) or not m.group(0)[0].isalpha()
                                               and m.group(0)[0] != '_') else 'ID'), t)

def diff_pair(a, b):
    ka,kb=set(a),set(b)
    raw=[k for k in ka&kb if not semantically_equal(a[k], b[k])]
    # split: SEMANTIC (structure or literals differ) vs NAME-ONLY (identifier churn)
    semantic=sorted(k for k in raw if skeleton(a[k]) != skeleton(b[k]))
    nameonly=sorted(set(raw)-set(semantic))
    return dict(changed=semantic, name_only=nameonly, added=sorted(kb-ka),
                removed=sorted(ka-kb), identical=len(ka&kb)-len(raw), comparable=len(ka&kb))

if __name__=='__main__':
    fams=sys.argv[1:] or ['sos','sys_drv','hdcp','asd']
    allres={}
    for fam in fams:
        builds=load(fam)
        names=list(builds)
        print("\n### %s : %d builds"%(fam.upper(), len(names)))
        for i in range(len(names)-1):
            o,n=names[i],names[i+1]
            d=diff_pair(builds[o], builds[n])
            allres["%s -> %s"%(o,n)]=d
            print("  %-34s -> %-34s cmp=%-5d SEMANTIC=%-4d name-only=%-4d ADDED=%-4d REM=%d"%(
                o.replace(fam+'_',''), n.replace(fam+'_',''), d['comparable'],
                len(d['changed']), len(d['name_only']), len(d['added']), len(d['removed'])))
    json.dump(allres, open('phase4_diffs.json','w'), indent=1)
    print("\nwrote phase4_diffs.json")