"""
R2B2's command line interface offers significant out-of-the-box functionality with
respect to executing audits and generating audit data without requiring the user to
write a single line of Python.
Note:
Why does this file exist, and why not put this in __main__?
You might be tempted to import things from __main__ later, but that will cause
problems: the code will get executed twice:
- When you run `python -m r2b2` python will execute
``__main__.py`` as a script. That means there won't be any
``r2b2.__main__`` in ``sys.modules``.
- When you import __main__ it will get executed again (as a module) because
there's no ``r2b2.__main__`` in ``sys.modules``.
Also see (1) from http://click.pocoo.org/5/setuptools/#setuptools-integration
"""
import math
import click
import pkg_resources
from r2b2.athena import Athena
from r2b2.audit import Audit
from r2b2.brla import BayesianRLA as BRLA
from r2b2.contest import Contest
from r2b2.contest import ContestType
from r2b2.election import Election
from r2b2.minerva import Minerva
from r2b2.tests import util
# Class for parsing space separated list of integers from command line
[docs]class IntList(click.ParamType):
name = 'integer list'
[docs] def convert(self, value, param, ctx):
try:
str_list = value.split(' ')
return [int(i) for i in str_list]
except TypeError:
self.fail('Expected space separated list of integers')
INT_LIST = IntList()
# Audit type choices
# TODO: add new audit types when they become available
audit_types = click.Choice(['brla', 'minerva', 'athena'], case_sensitive=False)
# Contest type choice
contest_types = click.Choice(['PLURALITY', 'MAJORITY'])
@click.group()
def cli():
click.echo('\nWelcome to the R2B2 auditing tool!\n')
pass
@cli.command('interactive', short_help='Interactive audit execution.')
@click.option('-e',
'--election-mode',
default=False,
is_flag=True,
flag_value=True,
show_default=True,
help='Use election mode instead of single-contest mode.')
@click.option('--election-file', type=str, help='Pass election data as JSON file.') # TODO: provide format
@click.option('--contest-file', type=str, help='Pass contest data as JSON file.') # TODO: provide format
@click.option('-a', '--audit-type', type=audit_types, prompt='Select an audit type', help='Type of audit to execute on given contest.')
@click.option('-r',
'--risk-limit',
type=click.FloatRange(0.0, 1.0),
prompt='Enter desired risk limit (e.g. use 0.1 for 10%)',
help='Risk limit (alpha) of audit. Should be value between 0 and 1.')
@click.option('-m',
'--max-fraction-to-draw',
type=click.FloatRange(0.0, 1.0),
prompt='Enter maximum fraction of ballots to draw during audit',
help='Maximum fraction of total contest ballots that could be drawn during the audit.')
@click.option('-v',
'--verbose',
default=False,
is_flag=True,
flag_value=True,
show_default=False,
help='Provides risk and stopping probability schedule of previous rounds, minimum and maximum sample size.')
def interactive(election_mode, election_file, contest_file, audit_type, risk_limit, max_fraction_to_draw, verbose):
"""Executes an audit round by round.
Depending on what options are passed to the interactive command, users may be prompted for
contest results, audit type, risk limit, and/or maximum fraction of contest ballots to draw
when initializing the contest and audit to run.
During execution, users will enter each round
size and results of the round's sample and subsequently receive information about the current
state of the audit. The process continues until either the stopping conditions are met or the
audit reaches the maximum sample size.
\f
For information on each option run ::
$ r2b2 interactive --help
Example:
Contest results can be passed as a JSON file rather than entering the data through the
prompt::
$ r2b2 interactive --contest-file example_contest.json
Tip:
To generate a template contest JSON file run::
$ r2b2 template contest
Example:
Audit parameters can be passed in as options rather than entering through the prompt::
$ r2b2 interactive --audit-type brla --risk-limit 0.1 --max-fraction-to-draw 0.2
$ r2b2 interactive -a brla -r 0.1 -m 0.2 // Shortened equivalent
Example:
Election mode allows users to enter all the results from an election then select a contest
from the election to audit::
$ r2b2 interactive -e
$ r2b2 interactive -e --election-file // pass election results as JSON file.
Warning:
Election mode simply allows you to enter an entire election's data, then select one
one contest from that election to run. Auditing multiple contests from an election
concurrently is not implemented.
"""
if election_mode:
if election_file is not None:
election = util.parse_election(str(election_file))
else:
election = input_election()
contest_choices = click.Choice(election.contests.keys(), case_sensitive=True)
click.echo('\n')
click.echo(election)
contest_name = click.prompt('Select a contest from the above election', type=contest_choices)
contest = election.contests[contest_name]
# Check contest file if provided, otherwise request contest input
elif contest_file is not None:
contest = util.parse_contest(str(contest_file))
else:
contest = input_contest()
# Confirm contest, if incorrect get new input
click.echo('\n')
if election_mode:
click.echo(contest_name)
click.echo(contest)
while not click.confirm('\nUse the above contest data?'):
contest = input_contest()
click.echo(contest)
# Create audit from prompted input
audit = input_audit(contest, risk_limit, max_fraction_to_draw, audit_type)
click.echo(audit)
if audit_type == 'athena':
click.echo('Delta: ' + str(audit.delta))
# Confirm audit, if incorrect get new audit
while not click.confirm('\nAre the audit parameters correct?'):
audit = input_audit(contest)
click.echo(audit)
# Run audit
audit.run(verbose)
@cli.command('bulk', short_help='Generate audit data for given round sizes.')
@click.option('-v',
'--verbose',
default=False,
is_flag=True,
flag_value=True,
show_default=False,
help='Provides risk and stopping probability schedule of previous rounds, minimum and maximum sample size.')
@click.option('-o', '--output', type=click.File('w'), default=None, help='Write output into given file.')
@click.option('-r', '--round-list', type=INT_LIST, default=None, help='Provide a list of round sizes to generate stopping sizes for.')
@click.option('-f', '--full-audit-limit', type=int, default=None, help='Set maximum size for ballot by ballot output data.')
@click.option('-p', '--pair', type=str, default=None, help='Select candidate pair to generate data for.')
@click.argument('contest_file', type=str)
@click.argument('audit_type', type=audit_types)
@click.argument('risk_limit', type=click.FloatRange(0.0, 1.0))
@click.argument('max_fraction_to_draw', type=click.FloatRange(0.0, 1.0))
def bulk(audit_type, risk_limit, max_fraction_to_draw, contest_file, output, round_list, full_audit_limit, pair, verbose):
"""Bulk auditing mode generates stopping sizes for a given fixed round schedule.
Either provide a list of round sizes for which to generate stopping sizes or
generate a ballot by ballot list of stopping sizes from the minimum valid sample
size to the default maximum sample size or a specified maximum sample size.
\f
Args:
contest_file: Contest results as JSON file.
audit_type: Which audit type to use to generate stopping sizes.
risk_limit: Risk limit (alpha) of audit.
max_fraction_to_draw: Maximum fraction of contest ballots that could be drawm during
the audit. Sets the default maximum size of the ballot by ballot output.
Tip:
To generate a template contest JSON file, run::
$ r2b2 template contest
Returns:
Formatted list of rounds and their associated stopping sizes. Default execution is
ballot by ballot from minimum valid sample size to the maximum sample size of audit.
Example:
To generate stopping sizes for a specific set of round sizes, provide the round sizes
as a space separated list of integers enclosed by quotes using the round list option::
$ r2b2 bulk -l '100 200 300' contest.json brla 0.1 0.5
Example:
To generate a ballot by ballot result from the minimum valid sample size to a specific
maximum (i.e. not the maximum fraction to draw of the audit), run::
$ r2b2 bulk -f 221 contest.json brla 0.1 0.5
Example:
To write the results to a file instead of to stdout, run::
$ r2b2 bulk -o output.txt contest.json brla 0.1 0.5
Tip:
Generating large or compute heavy data sets can take some time. To estimate run times,
use the verbose flag to display a progress bar::
$ r2b2 bulk -v contest.json brla 0.1 0.5
"""
# Parse audit and contest from arguments
contest = util.parse_contest(contest_file)
if audit_type == 'brla':
audit = BRLA(risk_limit, max_fraction_to_draw, contest)
else:
raise click.BadArgumentUsage('No valid audit type found.')
# Get loser for pairwise audit
if pair is not None and pair not in audit.sub_audits.keys():
click.echo('pair [{}] is not valid.'.format(pair))
pair = None
if pair is None:
pairs = click.Choice(audit.sub_audits.keys())
pair = click.prompt('Select a pair to audit', type=pairs)
out = '\n{:^20}|{:^20}\n'.format('Round Sizes', 'Stopping Sizes')
out += '--------------------|--------------------\n'
if round_list is not None:
kmins = audit.compute_min_winner_ballots(audit.sub_audits[pair], round_list, progress=verbose)
for i in range(len(kmins)):
out += '{:^20}|{:^20}\n'.format(round_list[i], kmins[i])
elif full_audit_limit is not None:
kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], full_audit_limit, progress=verbose)
for r in range(audit.sub_audits[pair].min_sample_size, full_audit_limit + 1):
out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])
else:
kmins = audit.compute_all_min_winner_ballots(audit.sub_audits[pair], progress=verbose)
for r in range(
audit.sub_audits[pair].min_sample_size,
min(audit.sub_audits[pair].sub_contest.contest_ballots,
math.ceil(max_fraction_to_draw * contest.contest_ballots) + 1)):
out += '{:^20}|{:^20}\n'.format(r, kmins[r - audit.sub_audits[pair].min_sample_size])
# Write or print output
if output is not None:
output.write(out)
else:
click.echo(out)
@cli.command('template', short_help='Generate template JSON input files')
@click.argument('style', type=click.Choice(['contest', 'election']))
@click.option('-o', '--output', type=click.File('wb'), default=None, help='Write output into given file.')
def template(style, output):
"""Generate JSON templates for possible input formats.
\f
Example:
To create a contest results JSON file, first generate the template as a new JSON file::
$ r2b2 template -o my_contest.json contest
Now the file my_contest.json will be created and contain::
{
"contest_ballots" : 100,
"tally" : {
"CandidateA" : 50,
"CandidateB" : 50
},
"num_winners" : 1,
"reported_winners" : ["CandidateA"],
"contest_type" : "PLURALITY"
}
Simply repopulate the fields with your contest results.
"""
# TODO: docstring
if style == 'contest':
template = pkg_resources.resource_string(__name__, 'tests/data/single_contest_template.json')
elif style == 'election':
template = pkg_resources.resource_string(__name__, 'tests/data/election_template.json')
else:
raise click.BadArgumentUsage('No valid template style found')
if output is not None:
output.write(template)
click.echo('Template written to {}'.format(output.name))
else:
click.echo(template)
def input_audit(contest: Contest,
alpha: float = None,
max_fraction_to_draw: float = None,
audit_type: str = None,
delta: float = None) -> Audit:
# Create an audit from user-input.
click.echo('\nCreate a new Audit')
click.echo('==================\n')
if alpha is None:
alpha = click.prompt('Enter the desired risk limit', type=click.FloatRange(0.0, 1.0))
if max_fraction_to_draw is None:
max_fraction_to_draw = click.prompt('Enter the maximum fraction of contest ballots to draw', type=click.FloatRange(0.0, 1.0))
if audit_type is None:
audit_type = click.prompt('Select an audit type', type=audit_types)
if delta is None and audit_type == 'athena':
delta = click.prompt('Enter the Athena delta value', type=click.FloatRange(0.0))
if audit_type == 'brla':
return BRLA(alpha, max_fraction_to_draw, contest)
elif audit_type == 'minerva':
return Minerva(alpha, max_fraction_to_draw, contest)
elif audit_type == 'athena':
return Athena(alpha, delta, max_fraction_to_draw, contest)
# TODO: add creation for other types of audits.
return None
def input_contest() -> Contest:
# Creates a contest from user-input.
click.echo('\nCreate a new Contest')
click.echo('====================\n')
contest_ballots = click.prompt('Enter number of contest ballots', type=click.IntRange(min=1))
num_candidates = click.prompt('Enter number of candidates', type=click.IntRange(min=2))
tally = {}
while len(tally) == 0:
running_total = 0
for i in range(num_candidates):
candidate = click.prompt('Enter candidate name', type=str)
candidate_votes = click.prompt('Enter number of votes reported for {}'.format(candidate),
type=click.IntRange(0, contest_ballots))
running_total += candidate_votes
if running_total > contest_ballots:
input_warning('Exceeded total ballots cast in contest.')
click.echo('Restarting tally process...\n')
tally.clear()
break
tally[candidate] = candidate_votes
num_winners = click.prompt('Enter number of winners', type=click.IntRange(1, num_candidates - 1))
reported_winners = []
candidates = list(tally.keys())
for i in range(num_winners):
winner = click.prompt('Enter winner name', type=click.Choice(candidates, case_sensitive=False))
candidates.remove(winner)
reported_winners.append(winner)
contest_type_in = click.prompt('Select contest type', type=contest_types)
contest_type = ContestType[contest_type_in]
return Contest(contest_ballots, tally, num_winners, reported_winners, contest_type)
def input_election() -> Election:
# Creates an election from user input
click.echo('\nCreate a new Election')
click.echo('=====================\n')
name = click.prompt('Enter election name', type=str)
total_ballots = click.prompt('Enter total ballots cast in election', type=click.IntRange(min=1))
contests = {}
while True:
contest_name = click.prompt('\nEnter new contest name', type=str)
contest = input_contest()
contests[contest_name] = contest
if not click.confirm('\nWould you like to enter another contest? '):
break
return Election(name, total_ballots, contests)
def input_warning(msg):
click.echo('\nINVALID INPUT: ' + msg)