[Scripts] provide fuzzilli toolchain

Author: bongjunj

scripts/cfg_preprocess.py

@@ -21,36 +21,86 @@
 
 cfg_dir = Path(args.cfg_dir)
 
-targets = []
+def parse_target_file(targets_file):
+    targets = []
+    with open(targets_file, 'r') as f:
+        for line in f:
+            target_file, target_line = line.strip().split(':')
+            target_line = int(target_line)
+            targets.append((target_file, target_line))
+
+    for target in targets:
+        print(f'* user target {target[0]}:{target[1]}', file=sys.stderr)
+
+    return targets
+
+def translate_target(target_file, target_line, target_blocks_map_file):
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2933
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2933
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2933
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2933
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2933
+    # target-blocks-map.txt:simplified-lowering.cc:2927 ../../src/compiler/simplified-lowering.cc:2934
+    targets = []
+    assert target_blocks_map_file.exists()
+    with open(target_blocks_map_file, 'r') as f:
+        for line in f:
+            bbid, target = line.strip().split(' ')
+            if f'{target_file}:{target_line}' in target:
+                filename, lineno = bbid.split(':')
+                lineno = int(lineno)
+                targets.append((filename, lineno))
+    return list(set(targets))
 
-for line in open(args.targets_file):
-    target_file, target_line = line.strip().split(':')
-    target_line = int(target_line)
-    targets.append((target_file, target_line))
+user_targets = parse_target_file(args.targets_file)
+assert len(user_targets) > 0, f'No targets found in {args.targets_file}'
 
-    print(target_file, target_line, file=sys.stderr)
+targets = []
+for filename, lineno in user_targets:
+    t = translate_target(filename, lineno, cfg_dir / 'target-blocks-map.txt')
+    targets.extend(t)
+
+for t in targets:
+    print(f'* target {t[0]}:{t[1]}', file=sys.stderr)
 
-for target in targets:
-    logging.info(f'Target: {target[0]}:{target[1]}')
+assert len(targets) > 0, f'No targets found in {cfg_dir / "target-blocks-map.txt"}'
 
 # each line looks like
 # simplified-lowering.cc:_ZN2v88internal8compiler22RepresentationSelector12EnqueueInputILNS1_5PhaseE0EEEvPNS1_4NodeEiNS1_7UseInfoE:0:0:384103392 _ZN2v88internal8compiler22RepresentationSelector7GetInfoEPNS1_4NodeE
 def parse_callgraph(callgraph_txt):
+    calledges = []
     with open(callgraph_txt, 'r') as f:
         for line in f:
             caller, callee = line.strip().split()
-            # print(f'{caller} -> {callee}')
             filename, funcname, lineno, order, bbid = caller.split(':')
-            yield (filename, funcname, lineno, order, hex(int(bbid))), callee
+            edge = (filename, funcname, lineno, order, hex(int(bbid))), callee
+            calledges.append(edge)
+    return calledges
+
+
+def merge_cfgs(cfgs, callgraph):
+    merged = nx.MultiDiGraph()
+    for cfg in cfgs:
+        merged = nx.compose(merged, cfg)
+
+    for caller, callee in callgraph:
+        _filename, _funcname, _lineno, _order, bbid = caller
+        node = 'Node' + bbid
+        assert node in merged, f'Node {node} not found in entire CFG'
+        if callee in func_to_node:
+            merged.add_edge(node, func_to_node[callee], weight=10)
+
+    # print('nodes in the merged graph', merged.nodes(data=True), file=sys.stderr)
+    # print('edges in the merged graph', merged.edges(keys=True, data=True), file=sys.stderr)
+
+    return merged
 
 target_nodes = []
 
 # need to extract (function name -> first block id)
-def parse_cfg(cfg_file):
+def parse_cfg(cfg_file, targets):
     graph: pydot.Graph = pydot.graph_from_dot_file(cfg_file)[0]
     graph: nx.MultiDiGraph = nx.drawing.nx_pydot.from_pydot(graph)
-    # print('graph name', graph.name)
-    # set edge weight as 1
     for u, v, k, d in graph.edges(keys=True, data=True):
         d['weight'] = 1
 
@@ -68,48 +118,38 @@ def parse_cfg(cfg_file):
                 logging.info(f'Found target node {n} in {cfg_file}')
                 break
 
-    # print('graph nodes', graph.nodes(data=True))
-    # print('graph edges', graph.edges(keys=True, data=True))
     return graph
 
-callgraph = list(parse_callgraph(cfg_dir / 'callgraph.txt'))
-cfgs = []
-func_to_node = {}
-for f in cfg_dir.glob('*.dot'):
-    g = parse_cfg(f)
-    cfgs.append(g)
+def parse_cfgs(cfg_dir, targets):
+    cfgs = []
+    for f in cfg_dir.glob('*.dot'):
+        g = parse_cfg(f, targets)
+        cfgs.append(g)
+    return cfgs
 
-    the_node = min(g.nodes(data=True), key=lambda n: n[1]['order'])
-    # print(the_node)
-    func_to_node[g.name] = the_node[0]
+def build_funcmap(cfgs):
+    func_to_node = {}
+    for g in cfgs:
+        the_node = min(g.nodes(data=True), key=lambda n: n[1]['order'])
+        func_to_node[g.name] = the_node[0]
+    return func_to_node
 
-# for func, first_node in func_to_node.items():
-#     print(f'{func} -> {first_node}')
 
-logging.info(f'{len(target_nodes)} target nodes found')
+callgraph = parse_callgraph(cfg_dir / 'callgraph.txt')
+cfgs = parse_cfgs(cfg_dir, targets)
+func_to_node = build_funcmap(cfgs)
 
-logging.info(f'{len(callgraph)} call edges found in callgraph.txt')
-logging.info(f'{len(cfgs)} CFG files found in {cfg_dir}')
+print(f'{len(cfgs)} CFG files found in {cfg_dir}', file=sys.stderr)
+print(f'{len(target_nodes)} target nodes found', file=sys.stderr)
+print(f'{len(callgraph)} call edges found', file=sys.stderr)
 
 node_cnt = sum([cfg.number_of_nodes() for cfg in cfgs])
 edge_cnt = sum([cfg.number_of_edges() for cfg in cfgs])
 
-logging.info(f'Total number of nodes: {node_cnt}')
-logging.info(f'Total number of edges: {edge_cnt}')
-
-# merge cfgs into one
-entire_cfg = nx.MultiDiGraph()
-for cfg in cfgs:
-    entire_cfg = nx.compose(entire_cfg, cfg)
-
-for caller, callee in callgraph:
-    filename, funcname, lineno, order, bbid = caller
-    node = 'Node' + bbid
-    assert node in entire_cfg, f'Node {node} not found in entire CFG'
-    entire_cfg.add_edge(node, func_to_node[callee], weight=10)
+print(f'Total number of nodes: {node_cnt}', file=sys.stderr)
+print(f'Total number of edges: {edge_cnt}', file=sys.stderr)
 
-logging.info(f'Number of nodes in entire CFG: {entire_cfg.number_of_nodes()}')
-logging.info(f'Number of edges in entire CFG: {entire_cfg.number_of_edges()}')
+entire_cfg = merge_cfgs(cfgs, callgraph)
 
 fulldistmap = defaultdict(list)
 for v in target_nodes:

scripts/install-swift.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+cd $HOME
+
+if [[ ! -f $HOME/swiftly ]]; then
+    curl -O https://download.swift.org/swiftly/linux/swiftly-1.0.0-$(uname -m).tar.gz
+    tar -zxf swiftly-1.0.0-$(uname -m).tar.gz
+fi
+
+sudo apt-get -y install gnupg2 libcurl4-openssl-dev libxml2-dev
+./swiftly init

scripts/repro-turbofan.sh

@@ -9,22 +9,34 @@ if [[ -z $NUMBER ]]; then
 fi
 
 if [[ $(basename "$ROOT") != "optimuzz" ]]; then
-    echo "Please run this script from the root of the optimuzz repository."
+    echo -e "\033[31mPlease run this script from the root of the optimuzz repository.\033[0m"
     exit 1
 fi
 
 echo $ROOT
+export SCRIPTS=$ROOT/scripts
 
 set -e
 
+scripts/install-swift.sh
+
+export SWIFTLY_HOME_DIR="/home/user/.local/share/swiftly"
+export SWIFTLY_BIN_DIR="/home/user/.local/share/swiftly/bin"
+if [[ ":$PATH:" != *":$SWIFTLY_BIN_DIR:"* ]]; then
+    export PATH="$SWIFTLY_BIN_DIR:$PATH"
+fi
+
+swiftly install --use 6.0.1
+swift --version
+
 # Our TurboFan benchmark requires Python 2
 if [[ ! -d $HOME/.pyenv ]]; then
     curl https://pyenv.run | bash
 fi
 export PYENV_ROOT="$HOME/.pyenv"
 [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
 eval "$(pyenv init - bash)"
-# pyenv install 2
+pyenv install 2 --skip-existing
 
 
 TURBOFAN_BUILDS=$HOME/turbofan-builds
@@ -35,15 +47,18 @@ if [[ ! -d tools/depot_tools ]]; then
     git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git tools/depot_tools
 fi
 export PATH=$PWD/tools/depot_tools:$PATH
-echo "Fetching v8..."
-# fetch v8
 
-echo "Checking out v8..."
-# gclient sync
+if [[ ! -d v8 ]]; then
+    echo -e "\033[34mFetching v8...\033[0m"
+    fetch v8
+fi
+
+echo -e "\033[34mChecking out v8...\033[0m"
+gclient sync
 
 
 COMMIT_ID=$(grep "^$NUMBER" $ROOT/turbotv-fuzzilli/d8-targets/commits.txt | awk '{print $2}')
-echo $COMMIT_ID
+echo -e "\033[34mFound the commit sha: $COMMIT_ID\033[0m"
 pyenv global 2 # Our targets need python 2.7
 pyenv exec gclient sync -D -R --revision=$COMMIT_ID
 
@@ -78,17 +93,25 @@ make -C $ROOT/instrumentation/ clean
 make -C $ROOT/instrumentation/ BASE=$HOME/clang+llvm-13.0.0-x86_64-linux-gnu-ubuntu-20.04
 
 echo -e "\033[34mEditing build rules...\033[0m"
-export SCRIPTS=$ROOT/scripts
+
 TARGET_FILE=$(cat $ROOT/turbotv-fuzzilli/d8-targets/$NUMBER.txt | cut -d':' -f1)
 $SCRIPTS/edit_rule.py out.gn/x64.debug/toolchain.ninja cxx "VERBOSE=1 TARGET_FILE=$TARGET_FILE OUT_DIR=\"$PWD/out.gn/x64.debug/cfg\"" "-fexperimental-new-pass-manager -Xclang -fpass-plugin=$ROOT/instrumentation/inst-pass.so"
 $SCRIPTS/edit_rule.py out.gn/x64.debug/toolchain.ninja link "" "$ROOT/instrumentation/coverage.o"
 $SCRIPTS/edit_rule.py out.gn/x64.debug/toolchain.ninja solink "" "$ROOT/instrumentation/coverage.o"
 
 echo -e "\033[34mBuilding d8...\033[0m"
-ninja -C out.gn/x64.debug d8 > out.gn/x64.debug/ninja.log
-$SCRIPTS/cfg_preprocess.py out.gn/x64.debug/cfg $HOME/optimuzz-experiment/cases/turbofan/targets/$NUMBER.txt > distmap.txt
+# ninja -C out.gn/x64.debug d8
+$SCRIPTS/cfg_preprocess.py out.gn/x64.debug/cfg $ROOT/turbotv-fuzzilli/d8-targets/$NUMBER.txt > distmap.txt
+cat distmap.txt | tail -n 5
+
 cd $ROOT
-mv $TURBOFAN_BUILDS/v8/out.gn/x64.debug/ d8s/$NUMBER/
-cp $TURBOFAN_BUILDS/v8/distmap.txt d8s/$NUMBER/distmap.txt
 
-tools/turbofan-reproduce.py $NUMBER
+rm -rf $HOME/d8s/$NUMBER/
+mkdir -p $HOME/d8s/$NUMBER
+mv $TURBOFAN_BUILDS/v8/out.gn/x64.debug/* $HOME/d8s/$NUMBER/
+cp $TURBOFAN_BUILDS/v8/distmap.txt        $HOME/d8s/$NUMBER/distmap.txt
+
+python3 -m venv .env
+source .env/bin/activate
+pip install -r $ROOT/turbotv-fuzzilli-requirements.txt
+tools/turbofan-reproduce.py $NUMBER $HOME/d8s/$NUMBER/