#!/usr/bin/python3

"""Run various fuzzing operations on random inputs, using wasm-opt.

See "testcase_handlers" below for the list of fuzzing operations.

Usage:

./scripts/fuzz_opt.py

That will run forever or until it finds a problem.

You can put files in the local directory 'fuzz' (under the top level of the

binaryen repo) and the fuzzer will treat them as important content to fuzz

with high frequency.

Setup: Some tools are optional, like emcc and wasm2c. The v8 shell (d8),

however, is used in various sub-fuzzers and so it is mandatory.

Note: For afl-fuzz integration, you probably don't want this, and can use

something like

BINARYEN_CORES=1 BINARYEN_PASS_DEBUG=1 afl-fuzz -i afl-testcases/ -o afl-findings/ -m 100 -d -- bin/wasm-opt -ttf --fuzz-exec --Os @@

(that is on a fixed set of arguments to wasm-opt, though - this

script covers different options being passed)

"""

# ruff: noqa: COM819, ARG002

import contextlib

import difflib

import json

import math

import os

import pathlib

import random

import re

import shutil

import subprocess

import sys

import tarfile

import time

import traceback

from datetime import datetime, timedelta, timezone

from os.path import abspath

from typing import override

from test import fuzzing, shared, support

assert sys.version_info >= (3, 10), 'requires Python 3.10'

# parameters

# feature options that are always passed to the tools.

# XXX fp16 is not yet stable, remove from here when it is

CONSTANT_FEATURE_OPTS = ['--all-features', '--disable-fp16']

INPUT_SIZE_MIN = 1024

INPUT_SIZE_MEAN = 40 * 1024

INPUT_SIZE_MAX = 5 * INPUT_SIZE_MEAN

PRINT_WATS = False

given_seed = None

CLOSED_WORLD_FLAG = '--closed-world'

# V8 does not support shared memories when running with shared-everything

# enabled, so do not fuzz shared-everything for now. The remaining features are

# not yet implemented in v8.

#

# Keep this in sync with bundle_clusterfuzz.py.

DISALLOWED_FEATURES_IN_V8 = [

'shared-everything',

'fp16',

'strings',

'stack-switching',

'multibyte',

]

# utilities

def in_binaryen(*args):

return os.path.join(shared.options.binaryen_root, *args)

def in_bin(tool):

return os.path.join(shared.options.binaryen_bin, tool)

def random_size():

if random.random() < 0.25:

# sometimes do an exponential distribution, which prefers smaller sizes but may

# also get very high

ret = int(random.expovariate(1.0 / INPUT_SIZE_MEAN))

# if the result is valid, use it, otherwise do the normal thing

# (don't clamp, which would give us a lot of values on the borders)

if ret >= INPUT_SIZE_MIN and ret <= INPUT_SIZE_MAX:

return ret

# most of the time do a simple linear range around the mean

return random.randint(INPUT_SIZE_MIN, 2 * INPUT_SIZE_MEAN - INPUT_SIZE_MIN)

def make_random_input(input_size, raw_input_data):

with open(raw_input_data, 'wb') as f:

f.write(bytes([random.randint(0, 255) for x in range(input_size)]))

def run(cmd, stderr=None, silent=False):

if not silent:

print(' '.join(cmd))

return subprocess.check_output(cmd, stderr=stderr, text=True)

def run_unchecked(cmd):

print(' '.join(cmd))

return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True).stdout

def randomize_pass_debug():

if random.random() < 0.1:

print('[pass-debug]')

os.environ['BINARYEN_PASS_DEBUG'] = '1'

else:

os.environ['BINARYEN_PASS_DEBUG'] = '0'

del os.environ['BINARYEN_PASS_DEBUG']

print('randomized pass debug:', os.environ.get('BINARYEN_PASS_DEBUG', ''))

@contextlib.contextmanager

def no_pass_debug():

old_env = os.environ.copy()

if os.environ.get('BINARYEN_PASS_DEBUG'):

del os.environ['BINARYEN_PASS_DEBUG']

try:

yield

finally:

os.environ.update(old_env)

def randomize_feature_opts():

global FEATURE_OPTS

FEATURE_OPTS = CONSTANT_FEATURE_OPTS[:]

if random.random() < 0.1:

# 10% of the time disable all features, i.e., fuzz the MVP featureset.

# Fuzzing that is less and less important as more features get enabled

# by default, but we don't want to lose all coverage for it entirely

# (and the odds of randomly not selecting any feature, below, is too

# small - at 17 features it is far less than 1%).

FEATURE_OPTS += FEATURE_DISABLE_FLAGS

elif random.random() < 0.333:

# 1/3 of the remaining 90% pick each feature randomly.

for possible in FEATURE_DISABLE_FLAGS:

if random.random() < 0.5:

FEATURE_OPTS.append(possible)

if possible in IMPLIED_FEATURE_OPTS:

FEATURE_OPTS.extend(IMPLIED_FEATURE_OPTS[possible])

# Otherwise, use all the features. This is useful to maximize

# coverage, as enabling more features enables more optimizations and

# code paths, and also allows all initial contents to run. However,

# half the time disable the specific ones that are a problem in V8.

elif random.random() < 0.5:

# Disable features not allowed in V8 to increase V8 fuzzing.

FEATURE_OPTS.extend(f'--disable-{feature}' for feature in DISALLOWED_FEATURES_IN_V8)

# Relaxed SIMD's nondeterminism disables much but not

# all of our V8 fuzzing, so avoid it.

FEATURE_OPTS.append('--disable-relaxed-simd')

print('randomized feature opts:', '\n ' + '\n '.join(FEATURE_OPTS))

# Pick closed or open with equal probability as both matter.

#

# Closed world is not a feature flag, technically, since it only makes sense

# to pass to wasm-opt (and not other tools). But decide on whether we'll

# be fuzzing in that mode now, as it determinies how we set other things up.

global CLOSED_WORLD

CLOSED_WORLD = random.random() < 0.5

ALL_FEATURE_OPTS = ['--all-features', '-all', '--mvp-features', '-mvp']

def update_feature_opts(wasm):

global FEATURE_OPTS

# we will re-compute the features; leave all other things as they are

EXTRA = [x for x in FEATURE_OPTS if not x.startswith('--enable') and

not x.startswith('--disable') and x not in ALL_FEATURE_OPTS]

FEATURE_OPTS = run([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + ['--print-features']).strip().split('\n')

# filter out '', which can happen if no features are enabled

FEATURE_OPTS = [x for x in FEATURE_OPTS if x]

print(FEATURE_OPTS, EXTRA)

FEATURE_OPTS += EXTRA

def randomize_fuzz_settings():

# a list of the arguments to pass to wasm-opt -ttf when generating the wasm

global GEN_ARGS

GEN_ARGS = []

# a list of the optimizations to run on the wasm

global FUZZ_OPTS

FUZZ_OPTS = []

# a boolean whether NaN values are allowed, or we de-NaN them

global NANS

# a boolean whether out of bounds operations are allowed, or we bounds-enforce them

global OOB

# a boolean whether we legalize the wasm for JS

global LEGALIZE

if random.random() < 0.5:

NANS = True

else:

NANS = False

GEN_ARGS += ['--denan']

if random.random() < 0.5:

OOB = True

else:

OOB = False

GEN_ARGS += ['--no-fuzz-oob']

if random.random() < 0.5:

LEGALIZE = True

GEN_ARGS += ['--legalize-and-prune-js-interface']

else:

LEGALIZE = False

if CLOSED_WORLD:

GEN_ARGS += [CLOSED_WORLD_FLAG]

# Enclose the world much of the time when fuzzing closed-world, so that

# many types are private and hence optimizable.

if random.random() < 0.5:

GEN_ARGS += ['--enclose-world']

# Test JSPI somewhat rarely, as it may be slower, and disables some other

# fuzzing.

global JSPI

JSPI = random.random() < 0.1

if JSPI:

# Fuzzing the start function with JSPI is tricky: if the start function

# calls an import that is marked as suspending, then it traps on

# "SuspendError: trying to suspend without WebAssembly.promising" - the

# start function in fact *cannot* be wrapped as promising, as it is not

# even an export. To make the binaryen interpreter behave the same way,

# we'd need to know JSPI is enabled and act like it, which would add a

# bunch of complexity. Instead, fuzz JSPI without start functions.

GEN_ARGS += ['--remove-start']

print('randomized settings (NaNs, OOB, legalize, JSPI):', NANS, OOB, LEGALIZE, JSPI)

def init_important_initial_contents():

# Fuzz dir contents are always important to us.

fuzz_dir = os.path.join(shared.options.binaryen_root, 'fuzz')

fuzz_cases = shared.get_tests(fuzz_dir, shared.test_suffixes, recursive=True)

FIXED_IMPORTANT_INITIAL_CONTENTS = fuzz_cases

# If auto_initial_contents is set we'll also grab all test files that are

# recent.

RECENT_DAYS = 30

# Returns the list of test wast/wat files added or modified within the

# RECENT_DAYS number of days counting from the commit time of HEAD

def auto_select_recent_initial_contents():

# Print 'git log' with changed file status and without commit messages,

# with commits within RECENT_DAYS number of days, counting from the

# commit time of HEAD. The reason we use the commit time of HEAD instead

# of the current system time is to make the results deterministic given

# the Binaryen HEAD commit.

head_ts_str = run(['git', 'log', '-1', '--format=%cd', '--date=raw'],

silent=True).split()[0]

head_dt = datetime.utcfromtimestamp(int(head_ts_str))

start_dt = head_dt - timedelta(days=RECENT_DAYS)

start_ts = start_dt.replace(tzinfo=timezone.utc).timestamp()

log = run(['git', 'log', '--name-status', '--format=', '--date=raw', '--no-renames', f'--since={start_ts}'], silent=True).splitlines()

# Pick up lines in the form of

# A test/../something.wast

# M test/../something.wast

# (wat extension is also included)

p = re.compile(r'^[AM]\stest' + os.sep + r'(.*\.(wat|wast))$')

matches = [p.match(e) for e in log]

auto_set = {match.group(1) for match in matches if match}

auto_set = auto_set.difference(set(FIXED_IMPORTANT_INITIAL_CONTENTS))

return sorted(auto_set)

def is_git_repo():

try:

ret = run(['git', 'rev-parse', '--is-inside-work-tree'],

silent=True, stderr=subprocess.DEVNULL)

return ret == 'true\n'

except subprocess.CalledProcessError:

return False

if not is_git_repo() and shared.options.auto_initial_contents:

print('Warning: The current directory is not a git repository, ' +

'so not automatically selecting initial contents.')

shared.options.auto_initial_contents = False

print('- Important provided initial contents:')

for test in FIXED_IMPORTANT_INITIAL_CONTENTS:

print(' ' + test)

print()

recent_contents = []

print('- Recently added or modified initial contents ', end='')

if shared.options.auto_initial_contents:

print(f'(automatically selected: within last {RECENT_DAYS} days):')

recent_contents += auto_select_recent_initial_contents()

for test in recent_contents:

print(' ' + test)

print()

initial_contents = FIXED_IMPORTANT_INITIAL_CONTENTS + recent_contents

global IMPORTANT_INITIAL_CONTENTS

IMPORTANT_INITIAL_CONTENTS = [os.path.join(shared.get_test_dir('.'), t) for t in initial_contents]

all_tests = shared.get_all_tests()

def pick_initial_contents():

# if we use an initial wasm file's contents as the basis for the

# fuzzing, then that filename, or None if we start entirely from scratch

global INITIAL_CONTENTS

INITIAL_CONTENTS = None

# half the time don't use any initial contents

if random.random() < 0.5:

return

# some of the time use initial contents that are known to be especially

# important

if IMPORTANT_INITIAL_CONTENTS and random.random() < 0.5:

test_name = random.choice(IMPORTANT_INITIAL_CONTENTS)

else:

test_name = random.choice(all_tests)

print('initial contents:', test_name)

if shared.options.auto_initial_contents:

# when using auto initial contents, we look through the git history to

# find test files. if a test file was renamed or removed then it may

# no longer exist, and we should just skip it.

if not os.path.exists(test_name):

return

if not fuzzing.is_fuzzable(test_name):

return

assert os.path.exists(test_name)

if test_name.endswith('.wast'):

# this can contain multiple modules, pick one

split_parts = support.split_wast(test_name)

if len(split_parts) > 1:

index = random.randint(0, len(split_parts) - 1)

chosen = split_parts[index]

module, _asserts = chosen

if not module:

# there is no module in this choice (just asserts), ignore it

print('initial contents has no module')

return

test_name = abspath('initial.wat')

with open(test_name, 'w') as f:

f.write(module)

print(f' picked submodule {index} from multi-module wast')

global FEATURE_OPTS

FEATURE_OPTS += [

# avoid multivalue for now due to bad interactions with gc non-nullable

# locals in stacky code. for example, this fails to roundtrip as the

# tuple code ends up creating stacky binary code that needs to spill

# non-nullable references to locals, which is not allowed:

#

# (module

# (type $other (struct))

# (func $foo (result (ref $other))

# (select

# (struct.new $other)

# (struct.new $other)

# (tuple.extract 2 1

# (tuple.make 2

# (i32.const 0)

# (i32.const 0)

# )

# )

# )

# )

# )

'--disable-multivalue',

]

# the given wasm may not work with the chosen feature opts. for example, if

# we pick atomics.wast but want to run with --disable-atomics, then we'd

# error, so we need to test the wasm. first, make sure it doesn't have a

# features section, as that would enable a feature that we might want to

# be disabled, and our test would not error as we want it to.

if test_name.endswith('.wasm'):

temp_test_name = 'initial.wasm'

try:

run([in_bin('wasm-opt'), test_name, '-all', '--strip-target-features',

'-o', temp_test_name])

except Exception:

# the input can be invalid if e.g. it is raw data that is used with

# -ttf as fuzzer input

print('(initial contents are not valid wasm, ignoring)')

return

test_name = temp_test_name

# Next, test the wasm. Note that we must check for closed world explicitly

# here, as a testcase may only work in an open world, which means we need to

# skip it.

args = FEATURE_OPTS

if CLOSED_WORLD:

args.append(CLOSED_WORLD_FLAG)

try:

run([in_bin('wasm-opt'), test_name] + args,

stderr=subprocess.PIPE,

silent=True)

except Exception:

print('(initial contents not valid for features, ignoring)')

return

INITIAL_CONTENTS = test_name

# Test outputs we want to ignore are marked this way.

IGNORE = '[binaryen-fuzzer-ignore]'

# Traps are reported as [trap REASON]

TRAP_PREFIX = '[trap '

# Host limits are reported as [host limit REASON]

HOST_LIMIT_PREFIX = '[host limit '

# --fuzz-exec reports calls as [fuzz-exec] export foo

FUZZ_EXEC_EXPORT_PREFIX = '[fuzz-exec] export'

# --fuzz-exec reports a stack limit using this notation

STACK_LIMIT = '[trap stack limit]'

# JS exceptions are logged as exception thrown: REASON

EXCEPTION_PREFIX = 'exception thrown: '

# given an export line that includes FUZZ_EXEC_EXPORT_PREFIX, return the export

# that is called

def get_export_from_export_line(export_line):

assert FUZZ_EXEC_EXPORT_PREFIX in export_line

return export_line.split(FUZZ_EXEC_EXPORT_PREFIX)[1].strip()

# compare two strings, strictly

def compare(x, y, context, verbose=True):

if x != y and IGNORE not in {x, y}:

message = ''.join([a + '\n' for a in difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile='expected', tofile='actual')])

if verbose:

raise Exception(f"{context} comparison error, expected to have '{x}' == '{y}', diff:\n\n{message}")

else:

raise Exception(f"{context}\nDiff:\n\n{message}")

# converts a possibly-signed integer to an unsigned integer

def unsign(x, bits):

return x & ((1 << bits) - 1)

# numbers are "close enough" if they just differ in printing, as different

# vms may print at different precision levels and verbosity

def numbers_are_close_enough(x, y):

# handle nan comparisons like -nan:0x7ffff0 vs NaN, ignoring the bits

if 'nan' in x.lower() and 'nan' in y.lower():

return True

# if one input is a pair, then it is in fact a 64-bit integer that is

# reported as two 32-bit chunks. convert such 'low high' pairs into a 64-bit

# integer for comparison to the other value

if ' ' in x or ' ' in y:

def to_64_bit(a):

if ' ' not in a:

return unsign(int(a), bits=64)

low, high = a.split(' ')

return unsign(int(low), 32) + (1 << 32) * unsign(int(high), 32)

return to_64_bit(x) == to_64_bit(y)

# float() on the strings will handle many minor differences, like

# float('1.0') == float('1') , float('inf') == float('Infinity'), etc.

try:

return float(x) == float(y)

except Exception:

pass

# otherwise, try a full eval which can handle i64s too

try:

ex = eval(x)

ey = eval(y)

return ex == ey or float(ex) == float(ey)

except Exception as e:

print('failed to check if numbers are close enough:', e)

return False

FUZZ_EXEC_NOTE_RESULT = '[fuzz-exec] note result'

# compare between vms, which may slightly change how numbers are printed

def compare_between_vms(x, y, context):

x_lines = x.splitlines()

y_lines = y.splitlines()

if len(x_lines) != len(y_lines):

return compare(x, y, context + ' (note: different number of lines between vms)')

num_lines = len(x_lines)

for i in range(num_lines):

x_line = x_lines[i]

y_line = y_lines[i]

if x_line != y_line:

# this is different, but maybe it's a vm difference we can ignore

LOGGING_PREFIX = '[LoggingExternalInterface logging'

if x_line.startswith(LOGGING_PREFIX) and y_line.startswith(LOGGING_PREFIX):

x_val = x_line[len(LOGGING_PREFIX) + 1:-1]

y_val = y_line[len(LOGGING_PREFIX) + 1:-1]

if numbers_are_close_enough(x_val, y_val):

continue

if x_line.startswith(FUZZ_EXEC_NOTE_RESULT) and y_line.startswith(FUZZ_EXEC_NOTE_RESULT):

x_val = x_line.split(' ')[-1]

y_val = y_line.split(' ')[-1]

if numbers_are_close_enough(x_val, y_val):

continue

# this failed to compare. print a custom diff of the relevant lines

MARGIN = 3

start = max(i - MARGIN, 0)

end = min(i + MARGIN, num_lines)

return compare('\n'.join(x_lines[start:end]), '\n'.join(y_lines[start:end]), context)

def fix_output(out):

# large doubles may print slightly different on different VMs

def fix_double(x):

x = x.group(1)

if 'nan' in x or 'NaN' in x:

x = 'nan'

else:

x = x.replace('Infinity', 'inf')

x = str(float(x))

return 'f64.const ' + x

out = re.sub(r'f64\.const (-?[nanN:abcdefxIity\d+-.]+)', fix_double, out)

# mark traps from wasm-opt as exceptions, even though they didn't run in a vm

out = out.replace(TRAP_PREFIX, EXCEPTION_PREFIX + TRAP_PREFIX)

# funcref(0) has the index of the function in it, and optimizations can

# change that index, so ignore it

out = re.sub(r'funcref\([\d\w$+-_:]+\)', 'funcref()', out)

# JS prints i31 as just a number, so change "i31ref(N)" (which C++ emits)

# to "N".

out = re.sub(r'i31ref\((-?\d+)\)', r'\1', out)

# Tag names may change due to opts, so canonicalize them.

out = re.sub(r' tag\$\d+', ' tag', out)

# If we failed to instantiate, remove everything after it. VMs may print

# additional error info that cannot be compared between VMs, like a JS

# stack trace for where we tried to instantiate.

if INSTANTIATE_ERROR in out:

after = out.find(INSTANTIATE_ERROR)

after = out.find('\n', after)

out = out[:after + 1]

lines = out.splitlines()

for i in range(len(lines)):

line = lines[i]

if 'Warning: unknown flag' in line or 'Try --help for options' in line:

# ignore some VM warnings that don't matter, like if a newer V8 has

# removed a flag that is no longer needed. but print the line so the

# developer can see it.

print(line)

lines[i] = None

elif 'V8 is running with experimental features enabled. Stability and security will suffer.' in line:

# Ignore some boilerplate from VMs

lines[i] = None

elif EXCEPTION_PREFIX in line:

# exceptions may differ when optimizing, but an exception should

# occur, so ignore their types (also js engines print them out

# slightly differently)

lines[i] = ' *exception*'

return '\n'.join([line for line in lines if line is not None])

def fix_spec_output(out):

out = fix_output(out)

# spec shows a pointer when it traps, remove that

out = '\n'.join(ln if 'runtime trap' not in ln else ln[ln.find('runtime trap'):] for ln in out.splitlines())

# https://github.com/WebAssembly/spec/issues/543 , float consts are messed up

out = '\n'.join(ln if 'f32' not in ln and 'f64' not in ln else '' for ln in out.splitlines())

return out

ignored_vm_runs = 0

ignored_vm_run_reasons = {}

# Notes a VM run that we ignore, and the reason for it (for metrics purposes).

# Extra text can also be printed that is not included in the metrics.

def note_ignored_vm_run(reason, extra_text='', amount=1):

global ignored_vm_runs

print(f'(ignore VM run: {reason}{extra_text})')

ignored_vm_runs += amount

ignored_vm_run_reasons.setdefault(reason, 0)

ignored_vm_run_reasons[reason] += amount

# Run a VM command, and filter out known issues.

def run_vm(cmd, checked=True):

def filter_known_issues(output):

known_issues = [

# can be caused by flatten, ssa, etc. passes

'local count too large',

# can be caused by (array.new $type -1) etc.

'requested new array is too large',

# https://github.com/WebAssembly/binaryen/issues/3767

# note that this text is a little too broad, but the problem is rare

# enough that it's unlikely to hide an unrelated issue

'found br_if of type',

# this text is emitted from V8 when it runs out of memory during a

# GC allocation.

'out of memory',

# if the call stack is exceeded we must ignore this, as

# optimizations can change whether this happens or not (e.g. by

# removing locals, which makes stack frames smaller), which is

# noticeable.

'Maximum call stack size exceeded',

# all host limitations are arbitrary and may differ between VMs and

# also be affected by optimizations, so ignore them.

# this is the prefix that the binaryen interpreter emits. For V8,

# there is no single host-limit signal, and we have the earlier

# strings in this list for known issues (to which more need to be

# added as necessary).

HOST_LIMIT_PREFIX,

# V8 does not accept nullable stringviews

# (https://github.com/WebAssembly/binaryen/pull/6574)

'expected (ref stringview_wtf16), got nullref',

'expected type (ref stringview_wtf16), found ref.null of type nullref',

# wasm64 memories have a V8 limit

'larger than implementation limit',

]

for issue in known_issues:

if issue in output:

note_ignored_vm_run(issue)

return IGNORE

return output

try:

# some known issues do not cause the entire process to fail

if checked:

ret = run(cmd)

else:

ret = run_unchecked(cmd)

return filter_known_issues(ret)

except subprocess.CalledProcessError:

# Other known issues do make it fail, so re-run without checking for

# success and see if we should ignore it

raw = run_unchecked(cmd)

filtered = filter_known_issues(raw)

if filtered == IGNORE:

return IGNORE

# Otherwise, raise an error.

raise

MAX_INTERPRETER_ENV_VAR = 'BINARYEN_MAX_INTERPRETER_DEPTH'

MAX_INTERPRETER_DEPTH = 1000

def run_bynterp(wasm, args):

# increase the interpreter stack depth, to test more things

os.environ[MAX_INTERPRETER_ENV_VAR] = str(MAX_INTERPRETER_DEPTH)

try:

return run_vm([in_bin('wasm-opt'), wasm] + FEATURE_OPTS + args)

finally:

del os.environ['BINARYEN_MAX_INTERPRETER_DEPTH']

# Enable even more things than V8_OPTS. V8_OPTS are the flags we want to use

# when testing, on our fixed test suite, but when fuzzing we may want more.

def get_v8_extra_flags():

# It is important to use the --fuzzing flag because it does things like

# enable mixed old and new EH (which is an issue since

# https://github.com/WebAssembly/exception-handling/issues/344 )

flags = ['--fuzzing']

# Sometimes add --future, which may enable new JITs and such, which is good

# to fuzz for V8's sake.

if random.random() < 0.5:

flags += ['--future']

# Sometimes add V8's type assertions.

if random.random() < 0.5:

flags += ['--wasm-assert-types']

# Some other options make sense to use sometimes.

if random.random() < 0.5:

flags += ['--no-wasm-generic-wrapper']

return flags

V8_LIFTOFF_ARGS = ['--liftoff']

V8_NO_LIFTOFF_ARGS = ['--no-liftoff']

# Default to running with liftoff enabled, because we need to pick either

# liftoff or turbo* for consistency (otherwise running the same command twice

# may have different results due to NaN nondeterminism), and liftoff is faster

# for small things.

def run_d8_js(js, args=[], liftoff=True):

cmd = [shared.V8] + shared.V8_OPTS

cmd += get_v8_extra_flags()

if liftoff:

cmd += V8_LIFTOFF_ARGS

cmd += [js]

if args:

cmd += ['--'] + args

return run_vm(cmd)

# For JSPI, we must customize fuzz_shell.js. We do so the first time we need

# it, and save the filename here.

JSPI_JS_FILE = None

def get_fuzz_shell_js():

js = in_binaryen('scripts', 'fuzz_shell.js')

if not JSPI:

# Just use the normal fuzz shell script.

return js

global JSPI_JS_FILE

if JSPI_JS_FILE:

# Use the customized file we've already created.

return JSPI_JS_FILE

JSPI_JS_FILE = os.path.abspath('jspi_fuzz_shell.js')

with open(JSPI_JS_FILE, 'w') as f:

# Enable JSPI.

f.write('var JSPI = 1;\n\n')

# Un-comment the async and await keywords.

with open(js) as g:

code = g.read()

code = code.replace('/* async */', 'async')

code = code.replace('/* await */', 'await')

f.write(code)

return JSPI_JS_FILE

def run_d8_wasm(wasm, liftoff=True, args=[]):

return run_d8_js(get_fuzz_shell_js(), [wasm] + args, liftoff=liftoff)

def all_disallowed(features):

return not any(('--enable-' + x) in FEATURE_OPTS for x in features)

class TestCaseHandler:

# how frequent this handler will be run. 1 means always run it, 0.5 means half the

# time

frequency = 1

def __init__(self):

self.num_runs = 0

# If the core handle_pair() method is not overridden, it calls handle() on

# each of the items. That is useful if you just want the two wasms and don't

# care about their relationship.

@override

def handle_pair(self, input, before_wasm, after_wasm, opts):

self.handle(before_wasm)

# Add some visual space between the independent parts.

print('\n')

self.handle(after_wasm)

@override

def can_run_on_wasm(self, wasm):

return True

def increment_runs(self):

self.num_runs += 1

def count_runs(self):

return self.num_runs

# Fuzz the interpreter with --fuzz-exec.

class FuzzExec(TestCaseHandler):

frequency = 1

@override

def handle_pair(self, input, before_wasm, after_wasm, opts):

run([in_bin('wasm-opt'), before_wasm] + opts + ['--fuzz-exec'])

# VMs

class BinaryenInterpreter:

name = 'binaryen interpreter'

@override

def run(self, wasm):

output = run_bynterp(wasm, ['--fuzz-exec-before'])

if output != IGNORE:

calls = output.count(FUZZ_EXEC_EXPORT_PREFIX)

errors = output.count(TRAP_PREFIX) + output.count(HOST_LIMIT_PREFIX)

if errors > calls / 2:

# A significant amount of execution on this testcase

# simply trapped, and was not very useful, so mark it

# as ignored. Ideally the fuzzer testcases would be

# improved to reduce this number.

#

# Note that we don't change output=IGNORE as there may

# still be useful testing here (up to 50%), so we only

# note that this is a mostly-ignored run, but we do not

# ignore the parts that are useful.

#

# Note that we set amount to 0.5 because we are run both

# on the before wasm and the after wasm. Those will be

# in sync (because the optimizer does not remove traps)

# and so by setting 0.5 we only increment by 1 for the

# entire iteration.

note_ignored_vm_run('too many errors vs calls',

extra_text=f' ({calls} calls, {errors} errors)',

amount=0.5)

return output

@override

def can_run(self, wasm):

return True

@override

def can_compare_to_self(self):

return True

@override

def can_compare_to_other(self, other):

return True

class D8:

name = 'd8'

extra_d8_flags = []

def run_js(self, js, wasm, checked=True):

return run_vm([shared.V8, js] + shared.V8_OPTS + get_v8_extra_flags() + self.extra_d8_flags + ['--', wasm], checked=checked)

@override

def run(self, wasm):

return self.run_js(js=get_fuzz_shell_js(), wasm=wasm)

@override

def can_run(self, wasm):

return all_disallowed(DISALLOWED_FEATURES_IN_V8)

@override

def can_compare_to_self(self):

# With nans, VM differences can confuse us, so only very simple VMs

# can compare to themselves after opts in that case.

return not NANS

@override

def can_compare_to_other(self, other):

# Relaxed SIMD allows different behavior between VMs, so only

# allow comparisons to other d8 variants if it is enabled.

if not all_disallowed(['relaxed-simd']) and not other.name.startswith('d8'):

return False

# If not legalized, the JS will fail immediately, so no point to

# compare to others.

return self.can_compare_to_self() and LEGALIZE

class D8Liftoff(D8):

name = 'd8_liftoff'

extra_d8_flags = V8_LIFTOFF_ARGS

class D8Turboshaft(D8):

name = 'd8_turboshaft'

extra_d8_flags = V8_NO_LIFTOFF_ARGS

class Wasm2C:

name = 'wasm2c'

def __init__(self):

# look for wabt in the path. if it's not here, don't run wasm2c

try:

wabt_bin = shared.which('wasm2c')

wabt_root = os.path.dirname(os.path.dirname(wabt_bin))

self.wasm2c_dir = os.path.join(wabt_root, 'wasm2c')

if not os.path.isdir(self.wasm2c_dir):

print('wabt found, but not wasm2c support dir')

self.wasm2c_dir = None

except Exception as e:

print('warning: no wabt found:', e)

self.wasm2c_dir = None

@override

def can_run(self, wasm):

if self.wasm2c_dir is None:

return False

# if we legalize for JS, the ABI is not what C wants

if LEGALIZE:

return False

# relatively slow, so run it less frequently

if random.random() < 0.5:

return False

# wasm2c doesn't support most features

return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'custom-descriptors', 'relaxed-atomics', 'wide-arithmetic'])

@override

def run(self, wasm):

run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)

run(['wasm2c', wasm, '-o', 'wasm.c'])

compile_cmd = ['clang', 'main.c', 'wasm.c', os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'), '-I' + self.wasm2c_dir, '-lm', '-Werror']

run(compile_cmd)

return run_vm(['./a.out'])

@override

def can_compare_to_self(self):

# The binaryen optimizer changes NaNs in the ways that wasm

# expects, but that's not quite what C has

return not NANS

@override

def can_compare_to_other(self, other):

# C won't trap on OOB, and NaNs can differ from wasm VMs

return not OOB and not NANS

class Wasm2C2Wasm(Wasm2C):

name = 'wasm2c2wasm'

def __init__(self):

super().__init__()

self.has_emcc = shared.which('emcc') is not None

@override

def run(self, wasm):

run([in_bin('wasm-opt'), wasm, '--emit-wasm2c-wrapper=main.c'] + FEATURE_OPTS)

run(['wasm2c', wasm, '-o', 'wasm.c'])

compile_cmd = ['emcc', 'main.c', 'wasm.c',

os.path.join(self.wasm2c_dir, 'wasm-rt-impl.c'),

'-I' + self.wasm2c_dir,

'-lm',

'-s', 'ENVIRONMENT=shell',

'-s', 'ALLOW_MEMORY_GROWTH']

# disable the signal handler: emcc looks like unix, but wasm has

# no signals

compile_cmd += ['-DWASM_RT_MEMCHECK_SIGNAL_HANDLER=0']

if random.random() < 0.5:

compile_cmd += ['-O' + str(random.randint(1, 3))]

elif random.random() < 0.5:

if random.random() < 0.5:

compile_cmd += ['-Os']

else:

compile_cmd += ['-Oz']

# avoid pass-debug on the emcc invocation itself (which runs

# binaryen to optimize the wasm), as the wasm here can be very

# large and it isn't what we are focused on testing here

with no_pass_debug():

run(compile_cmd)

return run_d8_js(abspath('a.out.js'))

@override

def can_run(self, wasm):

# quite slow (more steps), so run it less frequently

if random.random() < 0.8:

return False

# prefer not to run if the wasm is very large, as it can OOM

# the JS engine.

return super().can_run(wasm) and self.has_emcc and \

os.path.getsize(wasm) <= INPUT_SIZE_MEAN

@override

def can_compare_to_other(self, other):

# NaNs can differ from wasm VMs

return not NANS

class CompareVMs(TestCaseHandler):

frequency = 1

def __init__(self):

super().__init__()