measure_vulnerability.py

#!/usr/bin/env python3


# 1st-party
from datetime import timedelta
import logging
import os
import sys

# 2nd-party
import move_new_projects_to_unsafe_set
import partition_packages_by_abandoned
import partition_packages_by_popularity
import partition_packages_by_time
import plot_vulnerability
import vulnerability_counter


safe_packages_length = None


def get_points(partition_function, variable, simple_log_filename):
  global safe_packages_length

  # For easier reading of log, print a new line for every new curve.
  logging.info('')

  safe_packages, unsafe_packages = partition_function(variable)

  move_new_projects_to_unsafe_set.move(safe_packages, unsafe_packages)

  # We are memorizing the absolute number of safe projects in order to include
  # it in the plot.
  safe_packages_length = len(safe_packages)

  return vulnerability_counter.traverse_event_log(simple_log_filename,
                                                  safe_packages,
                                                  unsafe_packages)


# vulnerability by when a project claimed itself when it last
# added/updated/removed a package
def plot_claim_by_abandonment(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                              simple_log_filename, LEGACY_SECURITY_COLORS,
                              MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                              NUMBER_OF_USERS):

  plot_vulnerability.plot(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                          'TLS / GPG')

  # 2/1 years, 6/3/1 months
  year_in_days = 365
  month_in_days = 30
  time_deltas = (timedelta(days=year_in_days*2),
                 timedelta(days=year_in_days*1),
                 timedelta(days=month_in_days*6),
                 timedelta(days=month_in_days*3),
                 timedelta(days=month_in_days*1),)
  labels = ('legacy (> 2yr)',
            'legacy (> 1yr)',
            'legacy (> 6mo)',
            'legacy (> 3mo)',
            'legacy (> 1mo)')

  for i, time_delta in enumerate(time_deltas):
    points = get_points(partition_packages_by_abandoned.partition, time_delta,
                        simple_log_filename)
    color = LEGACY_SECURITY_COLORS[i]
    label = labels[i]
    plot_vulnerability.plot(points, color, label)

  plot_vulnerability.plot(MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                          'maximum security')
  plot_vulnerability.save(NUMBER_OF_USERS,
                          '/var/experiments-output/diplomat-claim-by-abandonment.pdf')


# vulnerability by when a project claimed itself when it last
# added/updated/removed a package
def plot_claim_over_time(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                         simple_log_filename, LEGACY_SECURITY_COLORS,
                         MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                         NUMBER_OF_USERS):

  plot_vulnerability.plot(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                          'TLS / GPG')

  # 1 month, 6 month, 1 year, 2 years
  time_deltas = (timedelta(days=90), timedelta(days=180),
                 timedelta(days=365), timedelta(days=730))
  labels = ('legacy (last 3mo)',
            'legacy (last 6mo)',
            'legacy (last 1yr)',
            'legacy (last 2yr)')

  for i, time_delta in enumerate(time_deltas):
    points = get_points(partition_packages_by_time.partition, time_delta,
                        simple_log_filename)
    color = LEGACY_SECURITY_COLORS[i]
    label = labels[i]
    plot_vulnerability.plot(points, color, label)

  plot_vulnerability.plot(MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                          'maximum security')
  plot_vulnerability.save(NUMBER_OF_USERS,
                          '/var/experiments-output/diplomat-claim-over-time.pdf')


# vulnerability by popularity
def plot_claim_by_popularity(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                             simple_log_filename, LEGACY_SECURITY_COLORS,
                             MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                             NUMBER_OF_USERS):

    plot_vulnerability.plot(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                            'TLS / GPG')

    # top 0.1%, 1%, 10% packages by popularity
    secure_fractions = [p/100 for p in (0.1, 1, 10)]
    labels = ('legacy (top 0.1%)',
              'legacy (top 1%)',
              'legacy (top 10%)')

    for i, secure_fraction in enumerate(secure_fractions):
      points = get_points(partition_packages_by_popularity.partition,
                          secure_fraction, simple_log_filename)
      color = LEGACY_SECURITY_COLORS[i]
      label = labels[i]
      plot_vulnerability.plot(points, color, label)

    plot_vulnerability.plot(MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                            'maximum security')
    plot_vulnerability.save(NUMBER_OF_USERS,
                            '/var/experiments-output/diplomat-claim-by-popularity.pdf')


# Claim the top X% of projects.
# Additionally, claim projects over Y days.
# Plot the results.
def plot_claimed_by_mixed_strategy(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                                   simple_log_filename, LEGACY_SECURITY_COLORS,
                                   MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                                   NUMBER_OF_USERS):

  plot_vulnerability.plot(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                          'TLS / GPG')

  # Get top 1% safe projects, and 99% unsafe projects.
  secure_fraction = 1/100
  popular_safe_packages, unpopular_unsafe_packages = \
                    partition_packages_by_popularity.partition(secure_fraction)
  move_new_projects_to_unsafe_set.move(popular_safe_packages,
                                       unpopular_unsafe_packages)

  # Get projects abandoned some years before compromise.
  years = 2
  year_in_days = 365
  time_delta = timedelta(days=year_in_days*years)
  abandoned_safe_packages, updated_unsafe_packages = \
                          partition_packages_by_abandoned.partition(time_delta)
  move_new_projects_to_unsafe_set.move(abandoned_safe_packages,
                                       updated_unsafe_packages)

  # Plot popularity.
  points = vulnerability_counter.traverse_event_log(simple_log_filename,
                                                  popular_safe_packages,
                                                  unpopular_unsafe_packages)
  color = LEGACY_SECURITY_COLORS[0]
  label = 'legacy (top 1%)'
  plot_vulnerability.plot(points, color, label)

  # TODO: Double-check correctness.
  safe_packages = popular_safe_packages|abandoned_safe_packages
  unsafe_packages = \
              (unpopular_unsafe_packages|updated_unsafe_packages)-safe_packages

  assert len(safe_packages&unsafe_packages) == 0, \
         'New safe and unsafe sets must be disjoint!'

  assert (safe_packages|unsafe_packages) == \
         (popular_safe_packages|unpopular_unsafe_packages| \
          abandoned_safe_packages|updated_unsafe_packages), \
         'New sets must be the same as old sets!'

  # Plot popularity + abandoned.
  points = vulnerability_counter.traverse_event_log(simple_log_filename,
                                                    safe_packages,
                                                    unsafe_packages)
  color = LEGACY_SECURITY_COLORS[1]
  label = 'legacy (top 1%, > 2yr)'
  plot_vulnerability.plot(points, color, label)

  # 1 month, 6 month, 1 year
  time_deltas = (timedelta(days=90), timedelta(days=180), timedelta(days=365))
  labels = ('legacy (top 1%, > 2yr, last 3mo)',
            'legacy (top 1%, > 2yr, last 6mo)',
            'legacy (top 1%, > 2yr, last 1yr)')

  for i, time_delta in enumerate(time_deltas):
    new_safe_packages, old_unsafe_packages = \
                              partition_packages_by_time.partition(time_delta)
    move_new_projects_to_unsafe_set.move(new_safe_packages,
                                         old_unsafe_packages)

    # TODO: Double-check correctness.
    safe_packages |= new_safe_packages
    unsafe_packages |= old_unsafe_packages
    unsafe_packages -= safe_packages

    assert len(safe_packages&unsafe_packages) == 0, \
           'New safe and unsafe sets must be disjoint!'

    assert (safe_packages|unsafe_packages) == \
           (popular_safe_packages|unpopular_unsafe_packages| \
            new_safe_packages|old_unsafe_packages), \
           'New sets must be the same as old sets!'

    # Plot popularity + abandoned + claimed over time.
    points = vulnerability_counter.traverse_event_log(simple_log_filename,
                                                      safe_packages,
                                                      unsafe_packages)

    color = LEGACY_SECURITY_COLORS[i+2]
    label = labels[i]
    plot_vulnerability.plot(points, color, label)

  plot_vulnerability.plot(MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                          'maximum security')
  plot_vulnerability.save(NUMBER_OF_USERS,
                          '/var/experiments-output/diplomat-claim-by-mixed.pdf')


if __name__ == '__main__':
  # rw for owner and group but not others
  os.umask(0o07)

  # write log to file
  logging.basicConfig(filename='/var/experiments-output/'\
                               'measure_vulnerability.log',
                      level=logging.DEBUG, filemode='w',
                      format='[%(asctime)s UTC] [%(name)s] [%(levelname)s] '\
                             '[%(funcName)s:%(lineno)s@%(filename)s] '\
                             '%(message)s')

  try:
    # Data source 1: This is where we see users querying project simple indices
    # and/or the packages themselves.
    assert len(sys.argv) == 2
    simple_log_filename = sys.argv[1]
    assert os.path.isfile(simple_log_filename)

    # Just some colours for plots.
    PRE_DIPLOMAT_COLOR = 'b-o'
    MAX_SECURITY_COLOR = 'r->'
    LEGACY_SECURITY_COLORS = ('m-p', 'g-^', 'c-v', 'k-s', 'y-D')

    # 0% safe projects == 100% unsafe projects
    PRE_DIPLOMAT_POINTS = \
      get_points(partition_packages_by_popularity.partition, 0,
                 simple_log_filename)

    # The total number of users is given by the end of the PyPI line.
    NUMBER_OF_USERS = PRE_DIPLOMAT_POINTS[-1]

    # 100% safe projects == 0% unsafe projects
    MAX_SECURITY_POINTS = \
      get_points(partition_packages_by_popularity.partition, 1,
                 simple_log_filename)

    # 1. What does claiming abandoned projects look like?
    plot_claim_by_abandonment(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                              simple_log_filename, LEGACY_SECURITY_COLORS,
                              MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                              NUMBER_OF_USERS)

    # 2. What does claiming top X% projects look like?
    plot_claim_by_popularity(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                             simple_log_filename, LEGACY_SECURITY_COLORS,
                             MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                             NUMBER_OF_USERS)

    # 3. What does claiming projects over Y days look like?
    plot_claim_over_time(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                         simple_log_filename, LEGACY_SECURITY_COLORS,
                         MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                         NUMBER_OF_USERS)

    # 4. What does (1)+(2)+(3) look like?
    plot_claimed_by_mixed_strategy(PRE_DIPLOMAT_POINTS, PRE_DIPLOMAT_COLOR,
                                   simple_log_filename, LEGACY_SECURITY_COLORS,
                                   MAX_SECURITY_POINTS, MAX_SECURITY_COLOR,
                                   NUMBER_OF_USERS)

  except:
    logging.exception('BLARG!')
    raise

  else:
    logging.info('Done.')