<commit_before>try:
    import logging
    import os
    import numpy as np
    import math
    from pyproj import Proj, transform
    import laspy
    import sys
    import laspy.file
    from libs import AssignProjection
except ImportError as err:
    print('Error import module: {0}'.format(err))
    exit(128)

class LasPyConverter:
    def __init__(self, source_filename, source_projection, destination_filename,
                 destination_projection):
        self.__SourceFileName = source_filename
        self.__DestinationFileName = destination_filename
        self.__SourceProjection = source_projection
        self.__SourceProjectionString = AssignProjection.AssignProjectionString(self.__SourceProjection)
        self.__SourceProj = Proj(self.__SourceProjectionString)

        self.__SourceFallbackProjectionString = AssignProjection.AssignFallbackProjectionString(self.__SourceProjection)
        if self.__SourceFallbackProjectionString:
            self.__SourceFallbackProjection = AssignProjection.AssignProjectionName(self.__SourceProjection)
            self.__SourceFallbackProj = Proj(self.__SourceFallbackProjectionString)
        else:
            self.__SourceFallbackProjection = AssignProjection.AssignProjectionName(self.__SourceProjection)
            self.__SourceFallbackProj = ''

        self.__DestinationProjection = destination_projection
        self.__DestinationProjectionString = AssignProjection.AssignProjectionString(self.__DestinationProjection)
        self.__DestinationProj = Proj(self.__DestinationProjectionString)

        self.__DestinationFallbackProjectionString = AssignProjection.AssignFallbackProjectionString(
            self.__DestinationProjection)
        if self.__DestinationFallbackProjectionString:
            self.__DestinationFallbackProjection = AssignProjection.AssignProjectionName(self.__DestinationProjection)
            self.__DestinationFallbackProj = Proj(self.__DestinationFallbackProjectionString)
        else:
            self.__DestinationFallbackProjection = AssignProjection.AssignProjectionName(self.__DestinationProjection)
            self.__DestinationFallbackProj = ''

    def Open(self):
        try:
            if self.__SourceFileName != 'stdin':
                self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            else:
                self.__SourceOpenedFile = laspy.file.File(sys.stdin, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='w',
                                                           header=self.__SourceOpenedFile.header)
        except Exception as err:
            raise

    def OpenReanOnly(self):
        try:
            if self.__SourceFileName != 'stdin':
                self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            else:
                self.__SourceOpenedFile = laspy.file.File(sys.stdin, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='r')
        except Exception as err:
            raise

    def DumpHeaderFormat(self):
        for spec in self.__SourceOpenedFile.header.header_format:
            in_spec = self.__SourceOpenedFile.header.get_schema()
            logging.info('Source setting: {0}: {1}'.format(spec.name, in_spec))
        for spec in self.__DestinationOpenedFile.header.header_format:
            in_spec = self.__DestinationOpenedFile.header.get_schema()
            logging.info('Destination setting: {0}: {1}'.format(spec.name, in_spec))

    def DumpPointFormat(self):
        for spec in self.__SourceOpenedFile.point_format:
            in_spec = self.__SourceOpenedFile.reader.get_dimension(spec.name)
            logging.info('Source setting: {0}: {1}'.format(spec.name, in_spec))
        for spec in self.__DestinationOpenedFile.point_format:
            in_spec = self.__DestinationOpenedFile.reader.get_dimension(spec.name)
            logging.info('Destination setting: {0}: {1}'.format(spec.name, in_spec))

    def GetSourcePointFormat(self):
        for spec in self.__SourceOpenedFile.reader.point_format:
            in_spec = self.__SourceOpenedFile.reader.get_dimension(spec.name)
        return (spec.name, in_spec)

    def GetDestinationPointFormat(self):
        for spec in self.__DestinationOpenedFile.reader.point_format:
            in_spec = self.__DestinationOpenedFile.reader.get_dimension(spec.name)
        return (spec.name, in_spec)

    def GetSourceScale(self):
        # Use scale as is as
        try:
            self.__SourceScale = [self.__SourceOpenedFile.header.scale[0], self.__SourceOpenedFile.header.scale[1],
                                  self.__SourceOpenedFile.header.scale[2]]
            # Use offset as is as
            self.__SourceOffset = [self.__SourceOpenedFile.header.offset[0], self.__SourceOpenedFile.header.offset[1],
                                   self.__SourceOpenedFile.header.offset[2]]
        except Exception:
            raise

    def SetDestinationScale(self):
        # Use scale as is as
        try:
            self.__DestinationScale = [self.__SourceOpenedFile.header.scale[0],
                                       self.__SourceOpenedFile.header.scale[1],
                                       self.__SourceOpenedFile.header.scale[2]]
            # Use offset as is as
            self.__DestinationOffset = np.array([0, 0, 0])
            if self.__DestinationFallbackProjection:
                self.__DestinationOffset[0], self.__DestinationOffset[1], self.__DestinationOffset[2] = transform(
                    self.__SourceProj, self.__DestinationFallbackProj,
                    self.__SourceOffset[0], self.__SourceOffset[1],
                    self.__SourceOffset[2])
            else:
                self.__DestinationOffset[0], self.__DestinationOffset[1], self.__DestinationOffset[2] = transform(
                    self.__SourceProj, self.__DestinationProj,
                    self.__SourceOffset[0], self.__SourceOffset[1],
                    self.__SourceOffset[2])
            self.__DestinationOffset = [math.floor(self.__DestinationOffset[0]),
                                        math.floor(self.__DestinationOffset[1]),
                                        math.floor(self.__DestinationOffset[2])]
            self.__DestinationOpenedFile.header.set_scale((
                self.__DestinationScale[0], self.__DestinationScale[1], self.__DestinationScale[2]))
            self.__DestinationOpenedFile.header.set_offset((
                self.__DestinationOffset[0], self.__DestinationOffset[1], self.__DestinationOffset[2]))
        except Exception:
            raise
        else:
            return self.__SourceOffset, self.__DestinationOffset

    def TransformPointCloud(self):
        # Transforming PointCloud
        try:
            self.__DestinationOpenedFile.points = self.__SourceOpenedFile.points
            self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z = transform(
                self.__SourceProj, self.__DestinationProj,
                self.__SourceOpenedFile.x, self.__SourceOpenedFile.y,
                self.__SourceOpenedFile.z)
            self.UpdateDestinationMinMax()
        except Exception:
            raise

    def TransformPointCloudCoordsOnly(self):
        try:
            self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z = transform(
                self.__SourceProj,
                self.__DestinationProj,
                self.__SourceOpenedFile.x, self.__SourceOpenedFile.y,
                self.__SourceOpenedFile.z)
            self.UpdateDestinationMinMax()
        except Exception:
            raise

    def ReturnOriginalMin(self):
        return np.amin([self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z], axis=1)

    def ReturnOriginalMax(self):
        return np.amax([self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z], axis=1)

    def ReturnTransformedMin(self):
        return np.amin([self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z],
                       axis=1)

    def ReturnTransformedMax(self):
        return np.amax([self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z],
                       axis=1)

    def UpdateDestinationMinMax(self):
        try:
            self.__DestinationOpenedFile.header.update_min_max()
        except Exception:
            raise

    def CloseSourceFile(self):
        try:
            self.__SourceOpenedFile.close()
            del self.__SourceOpenedFile
        except Exception:
            raise

    def CloseDestinationFile(self, full_header_update=False):
        try:
            self.__DestinationOpenedFile.close(ignore_header_changes=not (full_header_update))
            del self.__DestinationOpenedFile
        except Exception:
            raise

    def Close(self, full_header_update=False):
        try:
            self.CloseSourceFile()
            self.CloseDestinationFile(full_header_update)
        except Exception:
            raise

class LasPyCompare:
    def __init__(self, source_filename, destination_filename):
        try:
            self.__SourceFileName = source_filename
            self.__DestinationFileName = destination_filename
        except Exception:
            raise

    def OpenReanOnly(self):
        try:
            self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='r')
        except Exception:
            raise

    def ComparePointCloud(self):
        try:
            diff = [self.__SourceOpenedFile.x - self.__DestinationOpenedFile.x,
                    self.__SourceOpenedFile.y - self.__DestinationOpenedFile.y,
                    self.__SourceOpenedFile.z - self.__DestinationOpenedFile.z]
            diff_min, diff_max, diff_avg, diff_std = [], [], [], []
            for i in range(0, 3):
                diff_min.append(np.min(diff[i]))
                diff_max.append(np.max(diff[i]))
                diff_avg.append(np.mean(diff[i]))
                diff_std.append(np.std(diff[i]))
            logging.info('{0} file differences:'.format(
                os.path.basename(self.__SourceFileName)))
            logging.info(
                'Xmin/max/avg/std: {0[0]:.4f}/{1[0]:.4f}/{2[0]:.4f}/{3[0]:.4f}, Ymin/max/avg/std: {0[1]:.4f}/{1[1]:.4f}/{2[1]:.4f}/{3[1]:.4f}, Zmin/max/avg/std: {0[2]:.4f}/{1[2]:.4f}/{2[2]:.4f}/{3[2]:.4f}'.format(
                    diff_min, diff_max, diff_avg, diff_std))
            return diff_min, diff_max, diff_avg, diff_std

        except Exception as err:
            print(err)
            raise

    def Close(self):
        try:
            self.__SourceOpenedFile.close()
            del self.__SourceOpenedFile
            self.__DestinationOpenedFile.close()
            del self.__DestinationOpenedFile
        except Exception:
            raise
<commit_msg>Optimize variable usage, and format code<commit_after>try:
    import logging
    import os
    import numpy as np
    import math
    from pyproj import Proj, transform
    import laspy
    import sys
    import laspy.file
    from libs import AssignProjection
except ImportError as err:
    print('Error import module: {0}'.format(err))
    exit(128)

class LasPyConverter:
    def __init__(self, source_filename, source_projection, destination_filename,
                 destination_projection):
        self.__SourceFileName = source_filename
        self.__DestinationFileName = destination_filename
        self.__SourceProjection = source_projection
        self.__SourceProjectionString = AssignProjection.AssignProjectionString(self.__SourceProjection)
        self.__SourceProj = Proj(self.__SourceProjectionString)

        self.__SourceFallbackProjectionString = AssignProjection.AssignFallbackProjectionString(self.__SourceProjection)
        if self.__SourceFallbackProjectionString:
            self.__SourceFallbackProjection = AssignProjection.AssignProjectionName(self.__SourceProjection)
            self.__SourceFallbackProj = Proj(self.__SourceFallbackProjectionString)
        else:
            self.__SourceFallbackProjection = AssignProjection.AssignProjectionName(self.__SourceProjection)
            self.__SourceFallbackProj = ''

        self.__DestinationProjection = destination_projection
        self.__DestinationProjectionString = AssignProjection.AssignProjectionString(self.__DestinationProjection)
        self.__DestinationProj = Proj(self.__DestinationProjectionString)

        self.__DestinationFallbackProjectionString = AssignProjection.AssignFallbackProjectionString(
            self.__DestinationProjection)
        if self.__DestinationFallbackProjectionString:
            self.__DestinationFallbackProjection = AssignProjection.AssignProjectionName(self.__DestinationProjection)
            self.__DestinationFallbackProj = Proj(self.__DestinationFallbackProjectionString)
        else:
            self.__DestinationFallbackProjection = AssignProjection.AssignProjectionName(self.__DestinationProjection)
            self.__DestinationFallbackProj = ''

    def Open(self):
        try:
            if self.__SourceFileName != 'stdin':
                self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            else:
                self.__SourceOpenedFile = laspy.file.File(sys.stdin, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='w',
                                                           header=self.__SourceOpenedFile.header)
        except Exception as err:
            raise

    def OpenReanOnly(self):
        try:
            if self.__SourceFileName != 'stdin':
                self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            else:
                self.__SourceOpenedFile = laspy.file.File(sys.stdin, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='r')
        except Exception as err:
            raise

    def DumpHeaderFormat(self):
        for spec in self.__SourceOpenedFile.header.header_format:
            in_spec = self.__SourceOpenedFile.header.get_schema()
            logging.info('Source setting: {0}: {1}'.format(spec.name, in_spec))
        for spec in self.__DestinationOpenedFile.header.header_format:
            in_spec = self.__DestinationOpenedFile.header.get_schema()
            logging.info('Destination setting: {0}: {1}'.format(spec.name, in_spec))

    def DumpPointFormat(self):
        for spec in self.__SourceOpenedFile.point_format:
            in_spec = self.__SourceOpenedFile.reader.get_dimension(spec.name)
            logging.info('Source setting: {0}: {1}'.format(spec.name, in_spec))
        for spec in self.__DestinationOpenedFile.point_format:
            in_spec = self.__DestinationOpenedFile.reader.get_dimension(spec.name)
            logging.info('Destination setting: {0}: {1}'.format(spec.name, in_spec))

    def GetSourcePointFormat(self):
        for spec in self.__SourceOpenedFile.reader.point_format:
            in_spec = self.__SourceOpenedFile.reader.get_dimension(spec.name)
        return (spec.name, in_spec)

    def GetDestinationPointFormat(self):
        for spec in self.__DestinationOpenedFile.reader.point_format:
            in_spec = self.__DestinationOpenedFile.reader.get_dimension(spec.name)
        return (spec.name, in_spec)

    def GetSourceScale(self):
        # Use scale as is as
        try:
            self.__SourceScale = self.__SourceOpenedFile.header.scale
            # Use offset as is as
            self.__SourceOffset = self.__SourceOpenedFile.header.offset
        except Exception:
            raise

    def SetDestinationScale(self):
        # Use scale as is as
        try:
            self.__DestinationScale = self.__SourceScale
            # Use offset as is as
            self.__DestinationOffset = np.array([0, 0, 0])
            if self.__DestinationFallbackProjection:
                self.__DestinationOffset[0], self.__DestinationOffset[1], self.__DestinationOffset[2] = transform(
                    self.__SourceProj, self.__DestinationFallbackProj,
                    self.__SourceOffset[0], self.__SourceOffset[1], self.__SourceOffset[2])
            else:
                self.__DestinationOffset[0], self.__DestinationOffset[1], self.__DestinationOffset[2] = transform(
                    self.__SourceProj, self.__DestinationProj,
                    self.__SourceOffset[0], self.__SourceOffset[1], self.__SourceOffset[2])
            self.__DestinationOffset = [math.floor(self.__DestinationOffset[0]),
                                        math.floor(self.__DestinationOffset[1]),
                                        math.floor(self.__DestinationOffset[2])]
            self.__DestinationOpenedFile.header.set_scale(self.__DestinationScale)
            self.__DestinationOpenedFile.header.set_offset(self.__DestinationOffset)
        except Exception:
            raise
        else:
            return self.__SourceOffset, self.__DestinationOffset

    def TransformPointCloud(self):
        # Transforming PointCloud
        try:
            self.__DestinationOpenedFile.points = self.__SourceOpenedFile.points
            self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z = transform(
                self.__SourceProj, self.__DestinationProj,
                self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z)
            self.UpdateDestinationMinMax()
        except Exception:
            raise

    def TransformPointCloudCoordsOnly(self):
        try:
            self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z = transform(
                self.__SourceProj, self.__DestinationProj,
                self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z)
            self.UpdateDestinationMinMax()
        except Exception:
            raise

    def ReturnOriginalMin(self):
        return np.amin([self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z], axis=1)

    def ReturnOriginalMax(self):
        return np.amax([self.__SourceOpenedFile.x, self.__SourceOpenedFile.y, self.__SourceOpenedFile.z], axis=1)

    def ReturnTransformedMin(self):
        return np.amin([self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z],
                       axis=1)

    def ReturnTransformedMax(self):
        return np.amax([self.__DestinationOpenedFile.x, self.__DestinationOpenedFile.y, self.__DestinationOpenedFile.z],
                       axis=1)

    def UpdateDestinationMinMax(self):
        try:
            self.__DestinationOpenedFile.header.update_min_max()
        except Exception:
            raise

    def CloseSourceFile(self):
        try:
            self.__SourceOpenedFile.close()
            del self.__SourceOpenedFile
        except Exception:
            raise

    def CloseDestinationFile(self, full_header_update=False):
        try:
            self.__DestinationOpenedFile.close(ignore_header_changes=not (full_header_update))
            del self.__DestinationOpenedFile
        except Exception:
            raise

    def Close(self, full_header_update=False):
        try:
            self.CloseSourceFile()
            self.CloseDestinationFile(full_header_update)
        except Exception:
            raise

class LasPyCompare:
    def __init__(self, source_filename, destination_filename):
        try:
            self.__SourceFileName = source_filename
            self.__DestinationFileName = destination_filename
        except Exception:
            raise

    def OpenReanOnly(self):
        try:
            self.__SourceOpenedFile = laspy.file.File(self.__SourceFileName, mode='r')
            self.__DestinationOpenedFile = laspy.file.File(self.__DestinationFileName, mode='r')
        except Exception:
            raise

    def ComparePointCloud(self):
        try:
            diff = [self.__SourceOpenedFile.x - self.__DestinationOpenedFile.x,
                    self.__SourceOpenedFile.y - self.__DestinationOpenedFile.y,
                    self.__SourceOpenedFile.z - self.__DestinationOpenedFile.z]
            diff_min, diff_max, diff_avg, diff_std = [], [], [], []
            for i in range(0, 3):
                diff_min.append(np.min(diff[i]))
                diff_max.append(np.max(diff[i]))
                diff_avg.append(np.mean(diff[i]))
                diff_std.append(np.std(diff[i]))
            logging.info('{0} file differences:'.format(
                os.path.basename(self.__SourceFileName)))
            logging.info(
                'Xmin/max/avg/std: {0[0]:.4f}/{1[0]:.4f}/{2[0]:.4f}/{3[0]:.4f}, Ymin/max/avg/std: {0[1]:.4f}/{1[1]:.4f}/{2[1]:.4f}/{3[1]:.4f}, Zmin/max/avg/std: {0[2]:.4f}/{1[2]:.4f}/{2[2]:.4f}/{3[2]:.4f}'.format(
                    diff_min, diff_max, diff_avg, diff_std))
            return diff_min, diff_max, diff_avg, diff_std

        except Exception as err:
            raise

    def is_equal(self):
        return np.array_equal(self.__SourceOpenedFile.points, self.__DestinationOpenedFile.points)

    def Close(self):
        try:
            self.__SourceOpenedFile.close()
            del self.__SourceOpenedFile
            self.__DestinationOpenedFile.close()
            del self.__DestinationOpenedFile
        except Exception:
            raise
<|endoftext|>


<commit_before>"""Functions for processing CUES data in AppEngine JSON format into CSV format

Has one public function, ProcessResults, used like this from interpreter:
    ProcessResults('2015-01-06cuesDogfoodData.json', '2015-01-06-')
"""

import csv
import dateutil.parser
from datetime import datetime
import json
import re

DOGFOOD_START_DATE = datetime(2014, 12, 01, 0, 0, 0, 0)
CONDITIONS = [
    'ssl-overridable-proceed', 'ssl-overridable-noproceed',
    'ssl-nonoverridable', 'malware-proceed', 'malware-noproceed',
    'phishing-proceed', 'phishing-noproceed', 'extension-proceed',
    'extension-noproceed']
ATTRIBUTE_QUESTION_PREFIX = 'To what degree do each of the following'

def ProcessResults(json_in_file, csv_prefix):
  """Take results from AppEngine JSON file, process, and write to CSV file.

  Results from the input JSON file will be filtered into the 9 experimental
  conditions and written to CSV output files named in the format
  <csv_prefix><condition>.csv. CSV column headers will be the meta data fields
  date_received, date_taken, participant_id, and survey_type, plus the
  questions asked of the participant (with [URL] replacing actual URLs in the
  question text). Responses where questions were replaced with PLACEHOLDER
  will be ignored and not copied to the CSV file.

  Args:
    json_in_file: File with raw AppEngine results in JSON format
    csv_prefix: Prefix to output CSV files. These files will be named
        in the format <csv_prefix><condition>.csv. For example, if
        csv_prefix is '2015-01-15-' then one output file would be
        named 2015-01-15-ssl-overridable-proceed.csv

  Returns:
    None
  """
  parsed_events = _ParseSurveyResults(json_in_file)
  parsed_events = _DiscardResultsBeforeDate(parsed_events, DOGFOOD_START_DATE)

  for c in CONDITIONS:
    try:
      filtered_results = _FilterByCondition(c, parsed_events)
      canonical_index = _CanonicalizeQuestions(filtered_results)
      _WriteToCsv(filtered_results, canonical_index, csv_prefix + c + '.csv')
    # Print UnexpectedFormatException and continue, since they are usually
    # due to lack of data for a condition.
    except UnexpectedFormatException as e:
      print 'Exception in %s: %s' % (c, e.value)

def _ParseSurveyResults(in_file):
  with open(in_file, 'r') as json_file:
    s = json_file.read()
    parsed = json.loads(s)
  events = filter(lambda x: x['survey_type'] != 'setup.js', parsed)
  return events

def _DiscardResultsBeforeDate(results, date):
  """Return a list of results that occur after the given date.

  Args:
    results: Results parsed from a raw JSON file into list of dicts
    date: A date of type datetime.datetime

  Returns:
    List of results whose date_taken value comes after the given date.
  """
  return [
    r for r in results
    if dateutil.parser.parse(r['date_taken']) >= date]

def _FilterByCondition(cond, results):
  """Return a list of results for the given condition only.

  Args:
    cond: Condition to filter on; e.g., 'ssl-overridable-proceed',
      'malware-noproceed', etc.
    results: Results parsed from raw JSON file into list of dicts

  Returns:
    List of results for the given condition.

  Raises:
    ValueError: A unrecognized condition was passed in.
  """
  if cond not in CONDITIONS:
    raise ValueError(cond + ' is not a valid condition')

  return [pr for pr in results if pr['survey_type'] == cond + '.js']

class QuestionError(BaseException):
  def __init__(self, value):
    BaseException.__init__(self, value)
    self.value = value
  def __str__(self):
    return(repr(self.value))

class UnexpectedFormatException(BaseException):
  def __init__(self, value):
    BaseException.__init__(self, value)
    self.value = value
  def __str__(self):
    return(repr(self.value))

def _CanonicalizeQuestions(results):
  """Apply various fixes to questions in results

  To write a set of survey responses to a CSV file, we need the questions
  to look the same for each response, so we can match answers to each other.
  But questions can differ from survey to survey, even within a condition,
  because:
    -- some questions were replaced with PLACEHOLDER in early dogfood versions
    -- attribute questions are presented in randomized order
    -- sometimes domains in the first question were supposed to be replaced
           by a placeholder but weren't
    -- some question lists are longer than others because the participant
           gave consent to collect URL and the URL was sent back as an extra
           question
  This function modifies results to fix these issues as follows:
    -- eliminates responses where questions that were replaced with PLACEHOLDER
    -- within each response, sorts attribute questions alphabetically
    -- replaces domains in first questions with [URL]
    -- question lists stay their original length, but a list of max length
           is selected as a canonical list for use as CSV headers
  The function does some light error checking to flag unexpected differences
  in questions.

  Args:
    results: List of results, where each result is a dict
      containing metadata and response data from a survey response.
      The results list should be filtered for one condition.

  Returns:
    Integer index into the results list indicating which list
    element's questions can be considered canonical and complete.
    Also modifies the input results to canonicalize all questions.

  Raises:
    QuestionError: A question in some response didn't match the expected
        canonical version of the question
    UnexpectedFormatError: Either results has no responses or all
        responses use PLACEHOLDER as the text for all questions
  """
  # Find responses that didn't use 'PLACEHOLDER' as the text for every question
  results[:] = [r for r in results
                      if r['responses'][0]['question'] != 'PLACEHOLDER']
  if not results:
    raise UnexpectedFormatException('No results with questions found')

  _ReorderAttributeQuestions(results)

  _ReplaceUrlWithPlaceholder(results)

  # Any response with the max number of questions should now be fine as
  # the canonical list of questions; find one such response.
  max_list_len = len(results[0]['responses'])
  canonical_index = 0
  for i, r in enumerate(results):
    if len(r['responses']) > max_list_len:
      max_list_len = len(r['responses'])
      canonical_index = i

  # Do some light error checking; all questions lists should now be the same,
  # except some question lists may have extra questions. So, check each list
  # of questions against the canonical list of questions.
  for i, qa_pair in enumerate(
      results[canonical_index]['responses']):
    for j, r in enumerate(results):
      if i < len(r['responses']):
        if qa_pair['question'] != r['responses'][i]['question']:
          raise QuestionError(
              'Question text differs: Result %d question %d is {%s}. Result '
              '%d question %d is {%s}.'
              % (canonical_index, i, qa_pair['question'],
                 j, i, r['responses'][i]['question']))

  return canonical_index

def _WriteToCsv(results, canonical_index, out_file):
  """Write results for a given condition to a CSV file

  Given a list of results in the expected format, writes
  date_received, date_taken,  participant_id, survey_type, and
  answers to each survey question to a CSV file. Uses
  canonical_questions as column headers for corresponding answers.

  Args:
    results: List of results, where each result is a dict
        containing metadata and response data from a survey response.
        The results list should be filtered for one condition, and
        questions in the response data should be canonicalized.
    canonical_index: Index of element in results whose questions can
        be considered canonical for use as CSV column headers.
    out_file: CSV file to write output to.

  Returns:
    None.
  """
  canonical_questions = [
      qa_pair['question']
      for qa_pair in results[canonical_index]['responses']]

  # Use the Python csv library to write to csv format. The writer in this
  # library requires data to be a list of rows in flat field_name:data form.
  # So, we put it in the required form, open the CSV file, and write the data.
  results_for_csv_writer = []
  for r in results:
    dict_for_csv_writer = {}
    for qa_pair in r['responses']:
      dict_for_csv_writer[qa_pair['question']] = qa_pair['answer']
    dict_for_csv_writer.update(r)
    del dict_for_csv_writer['responses']
    results_for_csv_writer.append(dict_for_csv_writer)

  with open(out_file, 'w') as csv_file:
    field_names = ['date_received', 'date_taken', 'participant_id',
        'survey_type']
    field_names.extend(canonical_questions)
    csv_writer = csv.DictWriter(csv_file, field_names)
    csv_writer.writeheader()
    csv_writer.writerows(results_for_csv_writer)

def _ReorderAttributeQuestions(results):
  """Alphabetize attribute questions

  The attribute questions are presented in random order, so we
  need to reorder them into a canonical order for the output CSV
  file. This uses alphabetical order as canonical order.

  Args:
    results: A list of dicts containing parsed and filtered results.
        It is assumed that results has been filtered for a given survey
        condition, such that attributes questions should all appear in the
        same place.

  Returns:
    The altered results. Modifies the input results list as well.

  Raises:
    UnexpectedFormatException: Something unexpected found in input.
  """
  # Gather a list of lists of the indices of the attribute questions in
  # each result
  attribute_question_indices = []
  for r in results:
    index_list = []
    for i, qa_pair in enumerate(r['responses']):
      if ATTRIBUTE_QUESTION_PREFIX in qa_pair['question']:
        index_list.append(i)
      attribute_question_indices.append(index_list)

  # Do some error checking; attribute questions should have the same indices
  # for all results and should appear consecutively. Since they should be the
  # same for all results, we grab the min and max index from the first result.
  if not attribute_question_indices:
    raise UnexpectedFormatException ('Attributes questions not found.')
  min_index = attribute_question_indices[0][0]
  max_index = attribute_question_indices[0][-1]
  for i, index_list in enumerate(attribute_question_indices):
    if min(index_list) != min_index or max(index_list) != max_index:
      raise UnexpectedFormatException(
          'min and max indices in list %d not equal to min and max index '
          'in list 0' % (i))
    if index_list != range(min_index, max_index+1): # Check for consecutive
      raise UnexpectedFormatException(
          'indices in list %d not consecutive' % (i))

  for r in results:
    r['responses'][min_index:max_index+1] = sorted(
        r['responses'][min_index:max_index+1],
        key=lambda x: x['question'])

  return results

def _ReplaceUrlWithPlaceholder(results):
  """Fix a bug by replacing domain names with [URL]

  There seems to be a bug that URLs were included in questions where they
  were supposed to have a placeholder. Specifically, text like
  "Proceed to www.example.com" should be replaced with "Proceed to [URL]".
  These questions were the first question asked, so this function will only
  do the replacement in the first question in each result.

  Args:
    results:  A list of dicts containing parsed and filtered results.
        Is it assumed that results has been filtered for a given survey
        condition, such that attributes questions should all appear in the
        same place.

  Returns:
    The fixed results. Changes the input results list as well.
  """
  for r in results:
    q = r['responses'][0]['question'] # Do replacement in first question only
    m = re.search('\"Proceed to.*?\"', q)
    if m:
      q = q.replace(m.group(0), '\"Proceed to [URL]\"')
      r['responses'][0]['question'] = q

  return results
<commit_msg>A few more changes based on code review<commit_after>"""Functions for processing CUES data in AppEngine JSON format into CSV format

Has one public function, ProcessResults, used like this from interpreter:
    ProcessResults('2015-01-06cuesDogfoodData.json', '2015-01-06-')
"""

import csv
import dateutil.parser
from datetime import datetime
import json
import re

DOGFOOD_START_DATE = datetime(2014, 12, 01, 0, 0, 0, 0)
CONDITIONS = [
    'ssl-overridable-proceed', 'ssl-overridable-noproceed',
    'ssl-nonoverridable', 'malware-proceed', 'malware-noproceed',
    'phishing-proceed', 'phishing-noproceed', 'extension-proceed',
    'extension-noproceed']
ATTRIBUTE_QUESTION_PREFIX = 'To what degree do each of the following'

def ProcessResults(json_in_file, csv_prefix):
  """Take results from AppEngine JSON file, process, and write to CSV file.

  Results from the input JSON file will be filtered into the 9 experimental
  conditions and written to CSV output files named in the format
  <csv_prefix><condition>.csv. CSV column headers will be the meta data fields
  date_received, date_taken, participant_id, and survey_type, plus the
  questions asked of the participant (with [URL] replacing actual URLs in the
  question text). Responses where questions were replaced with PLACEHOLDER
  will be ignored and not copied to the CSV file.

  Args:
    json_in_file: File with raw AppEngine results in JSON format
    csv_prefix: Prefix to output CSV files. These files will be named
        in the format <csv_prefix><condition>.csv. For example, if
        csv_prefix is '2015-01-15-' then one output file would be
        named 2015-01-15-ssl-overridable-proceed.csv

  Returns:
    None
  """
  parsed_events = _ParseSurveyResults(json_in_file)
  parsed_events = _DiscardResultsBeforeDate(parsed_events, DOGFOOD_START_DATE)

  for c in CONDITIONS:
    try:
      filtered_results = _FilterByCondition(c, parsed_events)
      canonical_index = _CanonicalizeQuestions(filtered_results)
      _WriteToCsv(filtered_results, canonical_index, csv_prefix + c + '.csv')
    except UnexpectedFormatException as e:
      # Print UnexpectedFormatException and continue, since they are usually
      # due to lack of data for a condition.
      print 'Exception in %s: %s' % (c, e.value)

def _ParseSurveyResults(in_file):
  with open(in_file, 'r') as json_file:
    parsed = json.loads(json_file)
  events = filter(lambda x: x['survey_type'] != 'setup.js', parsed)
  return events

def _DiscardResultsBeforeDate(results, date):
  """Return a list of results that occur after the given date.

  Args:
    results: Results parsed from a raw JSON file into list of dicts
    date: A date of type datetime.datetime

  Returns:
    List of results whose date_taken value comes after the given date.
  """
  return [
      r for r in results
      if dateutil.parser.parse(r['date_taken']) >= date]

def _FilterByCondition(cond, results):
  """Return a list of results for the given condition only.

  Args:
    cond: Condition to filter on; e.g., 'ssl-overridable-proceed',
      'malware-noproceed', etc.
    results: Results parsed from raw JSON file into list of dicts

  Returns:
    List of results for the given condition.

  Raises:
    ValueError: A unrecognized condition was passed in.
  """
  if cond not in CONDITIONS:
    raise ValueError(cond + ' is not a valid condition')

  return [pr for pr in results if pr['survey_type'] == cond + '.js']

class QuestionError(BaseException):
  def __init__(self, value):
    BaseException.__init__(self, value)
    self.value = value
  def __str__(self):
    return(repr(self.value))

class UnexpectedFormatException(BaseException):
  def __init__(self, value):
    BaseException.__init__(self, value)
    self.value = value
  def __str__(self):
    return(repr(self.value))

def _CanonicalizeQuestions(results):
  """Apply various fixes to questions in results

  To write a set of survey responses to a CSV file, we need the questions
  to look the same for each response, so we can match answers to each other.
  But questions can differ from survey to survey, even within a condition,
  because:
    -- some questions were replaced with PLACEHOLDER in early dogfood versions
    -- attribute questions are presented in randomized order
    -- sometimes domains in the first question were supposed to be replaced
           by a placeholder but weren't
    -- some question lists are longer than others because the participant
           gave consent to collect URL and the URL was sent back as an extra
           question
  This function modifies results to fix these issues as follows:
    -- eliminates responses where questions that were replaced with PLACEHOLDER
    -- within each response, sorts attribute questions alphabetically
    -- replaces domains in first questions with [URL]
    -- question lists stay their original length, but a list of max length
           is selected as a canonical list for use as CSV headers
  The function does some light error checking to flag unexpected differences
  in questions.

  Args:
    results: List of results, where each result is a dict
      containing metadata and response data from a survey response.
      The results list should be filtered for one condition.

  Returns:
    Integer index into the results list indicating which list
    element's questions can be considered canonical and complete.
    Also modifies the input results to canonicalize all questions.

  Raises:
    QuestionError: A question in some response didn't match the expected
        canonical version of the question
    UnexpectedFormatError: Either results has no responses or all
        responses use PLACEHOLDER as the text for all questions
  """
  # Find responses that didn't use 'PLACEHOLDER' as the text for every question
  results[:] = [r for r in results
                      if r['responses'][0]['question'] != 'PLACEHOLDER']
  if not results:
    raise UnexpectedFormatException('No results with questions found')

  _ReorderAttributeQuestions(results)

  _ReplaceUrlWithPlaceholder(results)

  # Any response with the max number of questions should now be fine as
  # the canonical list of questions; find one such response.
  max_list_len = len(results[0]['responses'])
  canonical_index = 0
  for i, r in enumerate(results):
    if len(r['responses']) > max_list_len:
      max_list_len = len(r['responses'])
      canonical_index = i

  # Do some light error checking; all questions lists should now be the same,
  # except some question lists may have extra questions. So, check each list
  # of questions against the canonical list of questions.
  for i, qa_pair in enumerate(
      results[canonical_index]['responses']):
    for j, r in enumerate(results):
      if i < len(r['responses']):
        if qa_pair['question'] != r['responses'][i]['question']:
          raise QuestionError(
              'Question text differs: Result %d question %d is {%s}. Result '
              '%d question %d is {%s}.'
              % (canonical_index, i, qa_pair['question'],
                 j, i, r['responses'][i]['question']))

  return canonical_index

def _WriteToCsv(results, canonical_index, out_file):
  """Write results for a given condition to a CSV file

  Given a list of results in the expected format, writes
  date_received, date_taken,  participant_id, survey_type, and
  answers to each survey question to a CSV file. Uses
  canonical_questions as column headers for corresponding answers.

  Args:
    results: List of results, where each result is a dict
        containing metadata and response data from a survey response.
        The results list should be filtered for one condition, and
        questions in the response data should be canonicalized.
    canonical_index: Index of element in results whose questions can
        be considered canonical for use as CSV column headers.
    out_file: CSV file to write output to.

  Returns:
    None.
  """
  canonical_questions = [
      qa_pair['question']
      for qa_pair in results[canonical_index]['responses']]

  # Use the Python csv library to write to csv format. The writer in this
  # library requires data to be a list of rows in flat field_name:data form.
  # So, we put it in the required form, open the CSV file, and write the data.
  results_for_csv_writer = []
  for r in results:
    dict_for_csv_writer = {}
    for qa_pair in r['responses']:
      dict_for_csv_writer[qa_pair['question']] = qa_pair['answer']
    dict_for_csv_writer.update(r)
    del dict_for_csv_writer['responses']
    results_for_csv_writer.append(dict_for_csv_writer)

  with open(out_file, 'w') as csv_file:
    field_names = ['date_received', 'date_taken', 'participant_id',
        'survey_type']
    field_names.extend(canonical_questions)
    csv_writer = csv.DictWriter(csv_file, field_names)
    csv_writer.writeheader()
    csv_writer.writerows(results_for_csv_writer)

def _ReorderAttributeQuestions(results):
  """Alphabetize attribute questions

  The attribute questions are presented in random order, so we
  need to reorder them into a canonical order for the output CSV
  file. This uses alphabetical order as canonical order.

  Args:
    results: A list of dicts containing parsed and filtered results.
        It is assumed that results has been filtered for a given survey
        condition, such that attributes questions should all appear in the
        same place.

  Returns:
    The altered results. Modifies the input results list as well.

  Raises:
    UnexpectedFormatException: Something unexpected found in input.
  """
  # Gather a list of lists of the indices of the attribute questions in
  # each result
  attribute_question_indices = []
  for r in results:
    index_list = []
    for i, qa_pair in enumerate(r['responses']):
      if ATTRIBUTE_QUESTION_PREFIX in qa_pair['question']:
        index_list.append(i)
      attribute_question_indices.append(index_list)

  # Do some error checking; attribute questions should have the same indices
  # for all results and should appear consecutively. Since they should be the
  # same for all results, we grab the min and max index from the first result.
  if not attribute_question_indices:
    raise UnexpectedFormatException ('Attributes questions not found.')
  min_index = attribute_question_indices[0][0]
  max_index = attribute_question_indices[0][-1]
  for i, index_list in enumerate(attribute_question_indices):
    if min(index_list) != min_index or max(index_list) != max_index:
      raise UnexpectedFormatException(
          'min and max indices in list %d not equal to min and max index '
          'in list 0' % (i))
    if index_list != range(min_index, max_index+1): # Check for consecutive
      raise UnexpectedFormatException(
          'indices in list %d not consecutive' % (i))

  for r in results:
    r['responses'][min_index:max_index+1] = sorted(
        r['responses'][min_index:max_index+1],
        key=lambda x: x['question'])

  return results

def _ReplaceUrlWithPlaceholder(results):
  """Fix a bug by replacing domain names with [URL]

  There seems to be a bug that URLs were included in questions where they
  were supposed to have a placeholder. Specifically, text like
  "Proceed to www.example.com" should be replaced with "Proceed to [URL]".
  These questions were the first question asked, so this function will only
  do the replacement in the first question in each result.

  Args:
    results:  A list of dicts containing parsed and filtered results.
        Is it assumed that results has been filtered for a given survey
        condition, such that attributes questions should all appear in the
        same place.

  Returns:
    The fixed results. Changes the input results list as well.
  """
  for r in results:
    q = r['responses'][0]['question'] # Do replacement in first question only
    m = re.search('\"Proceed to.*?\"', q)
    if m:
      q = q.replace(m.group(0), '\"Proceed to [URL]\"')
      r['responses'][0]['question'] = q

  return results
<|endoftext|>


<commit_before>#!/usr/bin/python
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Library General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# TestServer.py
# Copyright (C) 2012 Ravindra Nath Kakarla

from wsgiref.simple_server import make_server
import json
import urlparse
import TestRunner
import sys
import textwrap
from ola import PidStore
from ola.ClientWrapper import ClientWrapper
from ola.UID import UID
from optparse import OptionParser, OptionGroup, OptionValueError

__author__ = 'ravindhranath@gmail.com (Ravindra Nath Kakarla)'

settings = {
  'PORT': 9999,
  'headers': [('Content-type', 'application/json')],
}

status = {
  '200': '200 OK',
  '404': '404 Not Found',
  '500': '500 Internal Server Error',
}

paths = {
  '/RunTests': 'run_tests',
  '/GetDevices': 'get_devices',
}

"""
  An instance of this class is created to serve every request.
"""
class TestServerApplication(object):
  def __init__(self, environ, start_response):
    self.environ = environ
    self.start = start_response
    self.wrapper = ClientWrapper()
    self.get_params = {}
    self.response = {}
    self.__request_handler()

  def __set_response_status(self, bool_value):
    if type(bool_value) == bool:
      self.response.update({'status': bool_value})

  def __set_response_message(self, message):
    if type(message) == str:
      self.response.update({'message': message})

  def __request_handler(self):
    self.request = self.environ['PATH_INFO']

    if self.request not in paths.keys():
      self.status = status['404']
    else:
      self.status = status['200']
      self.get_params = urlparse.parse_qs(self.environ['QUERY_STRING'])

    return self.__response_handler()

  def __response_handler(self):
    if self.status == status['404']:
      self.__set_response_status(False)
      self.__set_response_message('Invalid request!')

    elif self.status == status['200']:
      try:
        self.__getattribute__(paths[self.request])(self.get_params)
      except AttributeError:
        self.status = status['500']
        self.__response_handler()

    elif self.status == status['500']:
      self.__set_response_status(False)
      self.__set_response_message('Error 500: Internal failure')

  def run_tests(self, params):
    pass

  def get_devices(self, params):
    def format_uids(state, uids):
      if state.Succeeded():
        self.__set_response_status(True)
        self.response.update({'uids': [str(uid) for uid in uids]})
      else:
        self.__set_response_status(False)
        self.__set_response_message('Invalid Universe id!')

      self.wrapper.Stop()

    universe = int(params['u'][0])
    self.wrapper.Client().FetchUIDList(universe, format_uids)
    self.wrapper.Run()
    self.wrapper.Reset()

  def __iter__(self):
    self.start(self.status, settings['headers'])
    yield(json.dumps(self.response))

def parse_options():
  """
    Parse Command Line options
  """
  usage = 'Usage: %prog [options]'
  description = textwrap.dedent("""\
    Starts the TestServer (A simple Web Server) which run a series of tests on a RDM responder and returns the results to Web UI.
    This requires the OLA server to be running, and the RDM device to have been
    detected. You can confirm this by running ola_rdm_discover -u
    UNIVERSE. This will send SET commands to the broadcast UIDs which means
    the start address, device label etc. will be changed for all devices
    connected to the responder. Think twice about running this on your
    production lighting rig.
  """)
  parser = OptionParser(usage, description=description)
  parser.add_option('-p', '--pid_file', metavar='FILE',
                    help='The file to load the PID definitions from.')

  options, args = parser.parse_args()

  return options

def main():
  options = parse_options()
  settings.update(options.__dict__)
  httpd = make_server('', settings['PORT'], TestServerApplication)
  httpd.serve_forever()

if __name__ == '__main__':
  main()
<commit_msg>A new route handler to serve universe info<commit_after>#!/usr/bin/python
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Library General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# TestServer.py
# Copyright (C) 2012 Ravindra Nath Kakarla

from wsgiref.simple_server import make_server
import json
import urlparse
import TestRunner
import sys
import textwrap
from ola import PidStore
from ola.ClientWrapper import ClientWrapper
from ola.UID import UID
from optparse import OptionParser, OptionGroup, OptionValueError

__author__ = 'ravindhranath@gmail.com (Ravindra Nath Kakarla)'

settings = {
  'PORT': 9999,
  'headers': [('Content-type', 'application/json')],
}

status = {
  '200': '200 OK',
  '404': '404 Not Found',
  '500': '500 Internal Server Error',
}

paths = {
  '/RunTests': 'run_tests',
  '/GetDevices': 'get_devices',
  '/GetUnivInfo': 'get_univ_info',
}

"""
  An instance of this class is created to serve every request.
"""
class TestServerApplication(object):
  def __init__(self, environ, start_response):
    self.environ = environ
    self.start = start_response
    self.wrapper = ClientWrapper()
    self.get_params = {}
    self.response = {}
    self.__request_handler()

  def __set_response_status(self, bool_value):
    if type(bool_value) == bool:
      self.response.update({'status': bool_value})

  def __set_response_message(self, message):
    if type(message) == str:
      self.response.update({'message': message})

  def __request_handler(self):
    self.request = self.environ['PATH_INFO']

    if self.request not in paths.keys():
      self.status = status['404']
    else:
      self.status = status['200']
      self.get_params = urlparse.parse_qs(self.environ['QUERY_STRING'])

    return self.__response_handler()

  def __response_handler(self):
    if self.status == status['404']:
      self.__set_response_status(False)
      self.__set_response_message('Invalid request!')

    elif self.status == status['200']:
      try:
        self.__getattribute__(paths[self.request])(self.get_params)
      except AttributeError:
        self.status = status['500']
        self.__response_handler()

    elif self.status == status['500']:
      self.__set_response_status(False)
      self.__set_response_message('Error 500: Internal failure')

  def get_univ_info(self, params):
    def format_univ_info(state, universes):
      if state.Succeeded():
        self.__set_response_status(True)
        self.response.update({'universes': [univ.__dict__ for univ in universes]})
      else:
        self.__set_response_status(False)
        self.__set_response_message('Something\'s wrong with olad, Is it running?')

      self.wrapper.Stop()

    self.wrapper.Client().FetchUniverses(format_univ_info)
    self.wrapper.Run()
    self.wrapper.Reset()

  def run_tests(self, params):
    pass

  def get_devices(self, params):
    def format_uids(state, uids):
      if state.Succeeded():
        self.__set_response_status(True)
        self.response.update({'uids': [str(uid) for uid in uids]})
      else:
        self.__set_response_status(False)
        self.__set_response_message('Invalid Universe id!')

      self.wrapper.Stop()

    universe = int(params['u'][0])
    self.wrapper.Client().FetchUIDList(universe, format_uids)
    self.wrapper.Run()
    self.wrapper.Reset()

  def __iter__(self):
    self.start(self.status, settings['headers'])
    yield(json.dumps(self.response))

def parse_options():
  """
    Parse Command Line options
  """
  usage = 'Usage: %prog [options]'
  description = textwrap.dedent("""\
    Starts the TestServer (A simple Web Server) which run a series of tests on a RDM responder and returns the results to Web UI.
    This requires the OLA server to be running, and the RDM device to have been
    detected. You can confirm this by running ola_rdm_discover -u
    UNIVERSE. This will send SET commands to the broadcast UIDs which means
    the start address, device label etc. will be changed for all devices
    connected to the responder. Think twice about running this on your
    production lighting rig.
  """)
  parser = OptionParser(usage, description=description)
  parser.add_option('-p', '--pid_file', metavar='FILE',
                    help='The file to load the PID definitions from.')

  options, args = parser.parse_args()

  return options

def main():
  options = parse_options()
  settings.update(options.__dict__)
  httpd = make_server('', settings['PORT'], TestServerApplication)
  httpd.serve_forever()

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import threading, time, Queue, os, sys, shutil, random
from util import user_dir, appdata_dir, print_error, print_msg
from bitcoin import *
import interface
from blockchain import Blockchain

DEFAULT_PORTS = {'t':'50001', 's':'50002', 'h':'8081', 'g':'8082'}

DEFAULT_SERVERS = {
    #'electrum.coinwallet.me': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.hachre.de': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.novit.ro': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.stepkrav.pw': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    #'ecdsa.org': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.no-ip.org': {'h': '80', 's': '50002', 't': '50001', 'g': '443'},
    'electrum.drollette.com': {'h': '5000', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.random.re': {'h': '80', 's': '110', 't': '50001', 'g': '443'},
    'btc.medoix.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.stupidfoot.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    #'electrum.pdmc.net': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.be': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}
}

def filter_protocol(servers, p):
    l = []
    for k, protocols in servers.items():
        if p in protocols:
            l.append( ':'.join([k, protocols[p], p]) )
    return l

def pick_random_server(p='s'):
    return random.choice( filter_protocol(DEFAULT_SERVERS,p) )

from simple_config import SimpleConfig

class Network(threading.Thread):

    def __init__(self, config = {}):
        threading.Thread.__init__(self)
        self.daemon = True
        self.config = SimpleConfig(config) if type(config) == type({}) else config
        self.lock = threading.Lock()
        self.num_server = 8 if not self.config.get('oneserver') else 0
        self.blockchain = Blockchain(self.config, self)
        self.interfaces = {}
        self.queue = Queue.Queue()
        self.callbacks = {}
        self.protocol = self.config.get('protocol','s')
        self.running = False

        # Server for addresses and transactions
        self.default_server = self.config.get('server')
        if not self.default_server:
            self.default_server = pick_random_server(self.protocol)

        self.irc_servers = [] # returned by interface (list from irc)
        self.pending_servers = set([])
        self.disconnected_servers = set([])
        self.recent_servers = self.config.get('recent_servers',[]) # successful connections

        self.banner = ''
        self.interface = None
        self.proxy = self.config.get('proxy')
        self.heights = {}
        self.server_lag = 0

        dir_path = os.path.join( self.config.path, 'certs')
        if not os.path.exists(dir_path):
            os.mkdir(dir_path)

        # default subscriptions
        self.subscriptions = {}
        self.subscriptions[self.on_banner] = [('server.banner',[])]
        self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]

    def is_connected(self):
        return self.interface and self.interface.is_connected

    def is_up_to_date(self):
        return self.interface.is_up_to_date()

    def main_server(self):
        return self.interface.server

    def send_subscriptions(self):
        for cb, sub in self.subscriptions.items():
            self.interface.send(sub, cb)

    def subscribe(self, messages, callback):
        with self.lock:
            if self.subscriptions.get(callback) is None:
                self.subscriptions[callback] = []
            for message in messages:
                if message not in self.subscriptions[callback]:
                    self.subscriptions[callback].append(message)

        if self.is_connected():
            self.interface.send( messages, callback )

    def send(self, messages, callback):
        if self.is_connected():
            self.interface.send( messages, callback )
            return True
        else:
            return False

    def register_callback(self, event, callback):
        with self.lock:
            if not self.callbacks.get(event):
                self.callbacks[event] = []
            self.callbacks[event].append(callback)

    def trigger_callback(self, event):
        with self.lock:
            callbacks = self.callbacks.get(event,[])[:]
        if callbacks:
            [callback() for callback in callbacks]

    def random_server(self):
        choice_list = []
        l = filter_protocol(self.get_servers(), self.protocol)
        for s in l:
            if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
                continue
            else:
                choice_list.append(s)

        if not choice_list:
            if not self.interfaces:
                # we are probably offline, retry later
                self.disconnected_servers = set([])
            return

        server = random.choice( choice_list )
        return server

    def get_servers(self):
        out = self.irc_servers if self.irc_servers else DEFAULT_SERVERS
        for s in self.recent_servers:
            host, port, protocol = s.split(':')
            if host not in out:
                out[host] = { protocol:port }
        return out

    def start_interface(self, server):
        if server in self.interfaces.keys():
            return
        i = interface.Interface(server, self.config)
        self.pending_servers.add(server)
        i.start(self.queue)
        return i

    def start_random_interface(self):
        server = self.random_server()
        if server:
            self.start_interface(server)

    def start_interfaces(self):
        self.interface = self.start_interface(self.default_server)

        for i in range(self.num_server):
            self.start_random_interface()

    def start(self, wait=False):
        self.start_interfaces()
        threading.Thread.start(self)
        if wait:
            return self.wait_until_connected()

    def wait_until_connected(self):
        "wait until connection status is known"
        if self.config.get('auto_cycle'):
            # self.random_server() returns None if all servers have been tried
            while not self.is_connected() and self.random_server():
                time.sleep(0.1)
        else:
            self.interface.connect_event.wait()

        return self.interface.is_connected

    def set_parameters(self, host, port, protocol, proxy, auto_connect):

        self.config.set_key('auto_cycle', auto_connect, True)
        self.config.set_key("proxy", proxy, True)
        self.config.set_key("protocol", protocol, True)
        server = ':'.join([ host, port, protocol ])
        self.config.set_key("server", server, True)

        if self.proxy != proxy or self.protocol != protocol:
            self.proxy = proxy
            self.protocol = protocol
            for i in self.interfaces.values(): i.stop()
            if auto_connect:
                #self.interface = None
                return

        if auto_connect:
            if not self.interface.is_connected:
                self.switch_to_random_interface()
            else:
                if self.server_lag > 0:
                    self.stop_interface()
        else:
            self.set_server(server)

    def switch_to_random_interface(self):
        if self.interfaces:
            self.switch_to_interface(random.choice(self.interfaces.values()))

    def switch_to_interface(self, interface):
        assert not self.interface.is_connected
        server = interface.server
        print_error("switching to", server)
        self.interface = interface
        h =  self.heights.get(server)
        if h:
            self.server_lag = self.blockchain.height() - h
        self.config.set_key('server', server, False)
        self.default_server = server
        self.send_subscriptions()
        self.trigger_callback('connected')

    def stop_interface(self):
        self.interface.stop()

    def set_server(self, server):
        if self.default_server == server and self.interface.is_connected:
            return

        if self.protocol != server.split(':')[2]:
            return

        # stop the interface in order to terminate subscriptions
        if self.interface.is_connected:
            self.stop_interface()

        # notify gui
        self.trigger_callback('disconnecting')
        # start interface
        self.default_server = server
        self.config.set_key("server", server, True)

        if server in self.interfaces.keys():
            self.switch_to_interface( self.interfaces[server] )
        else:
            self.interface = self.start_interface(server)

    def add_recent_server(self, i):
        # list is ordered
        s = i.server
        if s in self.recent_servers:
            self.recent_servers.remove(s)
        self.recent_servers.insert(0,s)
        self.recent_servers = self.recent_servers[0:20]
        self.config.set_key('recent_servers', self.recent_servers)

    def new_blockchain_height(self, blockchain_height, i):
        if self.is_connected():
            h = self.heights.get(self.interface.server)
            if h:
                self.server_lag = blockchain_height - h
                if self.server_lag > 1:
                    print_error( "Server is lagging", blockchain_height, h)
                    if self.config.get('auto_cycle'):
                        self.set_server(i.server)
            else:
                print_error('no height for main interface')

        self.trigger_callback('updated')

    def run(self):
        self.blockchain.start()

        with self.lock:
            self.running = True

        while self.is_running():
            try:
                i = self.queue.get(timeout = 30 if self.interfaces else 3)
            except Queue.Empty:
                if len(self.interfaces) < self.num_server:
                    self.start_random_interface()
                continue

            if i.server in self.pending_servers:
                self.pending_servers.remove(i.server)

            if i.is_connected:
                #if i.server in self.interfaces: raise
                self.interfaces[i.server] = i
                self.add_recent_server(i)
                i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
                if i == self.interface:
                    print_error('sending subscriptions to', self.interface.server)
                    self.send_subscriptions()
                    self.trigger_callback('connected')
            else:
                self.disconnected_servers.add(i.server)
                if i.server in self.interfaces:
                    self.interfaces.pop(i.server)
                if i.server in self.heights:
                    self.heights.pop(i.server)
                if i == self.interface:
                    #self.interface = None
                    self.trigger_callback('disconnected')

            if not self.interface.is_connected and self.config.get('auto_cycle'):
                self.switch_to_random_interface()

    def on_header(self, i, r):
        result = r.get('result')
        if not result: return
        height = result.get('block_height')
        self.heights[i.server] = height
        # notify blockchain about the new height
        self.blockchain.queue.put((i,result))

        if i == self.interface:
            self.server_lag = self.blockchain.height() - height
            if self.server_lag > 1 and self.config.get('auto_cycle'):
                print_error( "Server lagging, stopping interface")
                self.stop_interface()

            self.trigger_callback('updated')

    def on_peers(self, i, r):
        if not r: return
        self.irc_servers = self.parse_servers(r.get('result'))
        self.trigger_callback('peers')

    def on_banner(self, i, r):
        self.banner = r.get('result')
        self.trigger_callback('banner')

    def stop(self):
        with self.lock: self.running = False

    def is_running(self):
        with self.lock: return self.running

    def synchronous_get(self, requests, timeout=100000000):
        queue = Queue.Queue()
        ids = self.interface.send(requests, lambda i,r: queue.put(r))
        id2 = ids[:]
        res = {}
        while ids:
            r = queue.get(True, timeout)
            _id = r.get('id')
            if _id in ids:
                ids.remove(_id)
                res[_id] = r.get('result')
        out = []
        for _id in id2:
            out.append(res[_id])
        return out

    #def retrieve_transaction(self, tx_hash, tx_height=0):
    #    import transaction
    #    r = self.synchronous_get([ ('blockchain.transaction.get',[tx_hash, tx_height]) ])[0]
    #    if r:
    #        return transaction.Transaction(r)

    def parse_servers(self, result):
        """ parse servers list into dict format"""
        from version import PROTOCOL_VERSION
        servers = {}
        for item in result:
            host = item[1]
            out = {}
            version = None
            pruning_level = '-'
            if len(item) > 2:
                for v in item[2]:
                    if re.match("[stgh]\d*", v):
                        protocol, port = v[0], v[1:]
                        if port == '': port = DEFAULT_PORTS[protocol]
                        out[protocol] = port
                    elif re.match("v(.?)+", v):
                        version = v[1:]
                    elif re.match("p\d*", v):
                        pruning_level = v[1:]
                    if pruning_level == '': pruning_level = '0'
            try:
                is_recent = float(version)>=float(PROTOCOL_VERSION)
            except Exception:
                is_recent = False

            if out and is_recent:
                out['pruning'] = pruning_level
                servers[host] = out

        return servers

class NetworkProxy:
    # interface to the network object.
    # handle subscriptions and callbacks
    # the network object can be jsonrpc server
    def __init__(self, network):
        self.network = network

if __name__ == "__main__":
    import simple_config
    config = simple_config.SimpleConfig({'verbose':True, 'server':'ecdsa.org:50002:s'})
    network = Network(config)
    network.start()

    while 1:
        time.sleep(1)

<commit_msg>store merkle roots and utxo roots<commit_after>import threading, time, Queue, os, sys, shutil, random
from util import user_dir, appdata_dir, print_error, print_msg
from bitcoin import *
import interface
from blockchain import Blockchain

DEFAULT_PORTS = {'t':'50001', 's':'50002', 'h':'8081', 'g':'8082'}

DEFAULT_SERVERS = {
    #'electrum.coinwallet.me': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.hachre.de': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.novit.ro': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.stepkrav.pw': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    #'ecdsa.org': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.no-ip.org': {'h': '80', 's': '50002', 't': '50001', 'g': '443'},
    'electrum.drollette.com': {'h': '5000', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.random.re': {'h': '80', 's': '110', 't': '50001', 'g': '443'},
    'btc.medoix.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.stupidfoot.com': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    #'electrum.pdmc.net': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'},
    'electrum.be': {'h': '8081', 's': '50002', 't': '50001', 'g': '8082'}
}

def filter_protocol(servers, p):
    l = []
    for k, protocols in servers.items():
        if p in protocols:
            l.append( ':'.join([k, protocols[p], p]) )
    return l

def pick_random_server(p='s'):
    return random.choice( filter_protocol(DEFAULT_SERVERS,p) )

from simple_config import SimpleConfig

class Network(threading.Thread):

    def __init__(self, config = {}):
        threading.Thread.__init__(self)
        self.daemon = True
        self.config = SimpleConfig(config) if type(config) == type({}) else config
        self.lock = threading.Lock()
        self.num_server = 8 if not self.config.get('oneserver') else 0
        self.blockchain = Blockchain(self.config, self)
        self.interfaces = {}
        self.queue = Queue.Queue()
        self.callbacks = {}
        self.protocol = self.config.get('protocol','s')
        self.running = False

        # Server for addresses and transactions
        self.default_server = self.config.get('server')
        if not self.default_server:
            self.default_server = pick_random_server(self.protocol)

        self.irc_servers = [] # returned by interface (list from irc)
        self.pending_servers = set([])
        self.disconnected_servers = set([])
        self.recent_servers = self.config.get('recent_servers',[]) # successful connections

        self.banner = ''
        self.interface = None
        self.proxy = self.config.get('proxy')
        self.heights = {}
        self.merkle_roots = {}
        self.utxo_roots = {}
        self.server_lag = 0

        dir_path = os.path.join( self.config.path, 'certs')
        if not os.path.exists(dir_path):
            os.mkdir(dir_path)

        # default subscriptions
        self.subscriptions = {}
        self.subscriptions[self.on_banner] = [('server.banner',[])]
        self.subscriptions[self.on_peers] = [('server.peers.subscribe',[])]

    def is_connected(self):
        return self.interface and self.interface.is_connected

    def is_up_to_date(self):
        return self.interface.is_up_to_date()

    def main_server(self):
        return self.interface.server

    def send_subscriptions(self):
        for cb, sub in self.subscriptions.items():
            self.interface.send(sub, cb)

    def subscribe(self, messages, callback):
        with self.lock:
            if self.subscriptions.get(callback) is None:
                self.subscriptions[callback] = []
            for message in messages:
                if message not in self.subscriptions[callback]:
                    self.subscriptions[callback].append(message)

        if self.is_connected():
            self.interface.send( messages, callback )

    def send(self, messages, callback):
        if self.is_connected():
            self.interface.send( messages, callback )
            return True
        else:
            return False

    def register_callback(self, event, callback):
        with self.lock:
            if not self.callbacks.get(event):
                self.callbacks[event] = []
            self.callbacks[event].append(callback)

    def trigger_callback(self, event):
        with self.lock:
            callbacks = self.callbacks.get(event,[])[:]
        if callbacks:
            [callback() for callback in callbacks]

    def random_server(self):
        choice_list = []
        l = filter_protocol(self.get_servers(), self.protocol)
        for s in l:
            if s in self.pending_servers or s in self.disconnected_servers or s in self.interfaces.keys():
                continue
            else:
                choice_list.append(s)

        if not choice_list:
            if not self.interfaces:
                # we are probably offline, retry later
                self.disconnected_servers = set([])
            return

        server = random.choice( choice_list )
        return server

    def get_servers(self):
        out = self.irc_servers if self.irc_servers else DEFAULT_SERVERS
        for s in self.recent_servers:
            host, port, protocol = s.split(':')
            if host not in out:
                out[host] = { protocol:port }
        return out

    def start_interface(self, server):
        if server in self.interfaces.keys():
            return
        i = interface.Interface(server, self.config)
        self.pending_servers.add(server)
        i.start(self.queue)
        return i

    def start_random_interface(self):
        server = self.random_server()
        if server:
            self.start_interface(server)

    def start_interfaces(self):
        self.interface = self.start_interface(self.default_server)

        for i in range(self.num_server):
            self.start_random_interface()

    def start(self, wait=False):
        self.start_interfaces()
        threading.Thread.start(self)
        if wait:
            return self.wait_until_connected()

    def wait_until_connected(self):
        "wait until connection status is known"
        if self.config.get('auto_cycle'):
            # self.random_server() returns None if all servers have been tried
            while not self.is_connected() and self.random_server():
                time.sleep(0.1)
        else:
            self.interface.connect_event.wait()

        return self.interface.is_connected

    def set_parameters(self, host, port, protocol, proxy, auto_connect):

        self.config.set_key('auto_cycle', auto_connect, True)
        self.config.set_key("proxy", proxy, True)
        self.config.set_key("protocol", protocol, True)
        server = ':'.join([ host, port, protocol ])
        self.config.set_key("server", server, True)

        if self.proxy != proxy or self.protocol != protocol:
            self.proxy = proxy
            self.protocol = protocol
            for i in self.interfaces.values(): i.stop()
            if auto_connect:
                #self.interface = None
                return

        if auto_connect:
            if not self.interface.is_connected:
                self.switch_to_random_interface()
            else:
                if self.server_lag > 0:
                    self.stop_interface()
        else:
            self.set_server(server)

    def switch_to_random_interface(self):
        if self.interfaces:
            self.switch_to_interface(random.choice(self.interfaces.values()))

    def switch_to_interface(self, interface):
        assert not self.interface.is_connected
        server = interface.server
        print_error("switching to", server)
        self.interface = interface
        h =  self.heights.get(server)
        if h:
            self.server_lag = self.blockchain.height() - h
        self.config.set_key('server', server, False)
        self.default_server = server
        self.send_subscriptions()
        self.trigger_callback('connected')

    def stop_interface(self):
        self.interface.stop()

    def set_server(self, server):
        if self.default_server == server and self.interface.is_connected:
            return

        if self.protocol != server.split(':')[2]:
            return

        # stop the interface in order to terminate subscriptions
        if self.interface.is_connected:
            self.stop_interface()

        # notify gui
        self.trigger_callback('disconnecting')
        # start interface
        self.default_server = server
        self.config.set_key("server", server, True)

        if server in self.interfaces.keys():
            self.switch_to_interface( self.interfaces[server] )
        else:
            self.interface = self.start_interface(server)

    def add_recent_server(self, i):
        # list is ordered
        s = i.server
        if s in self.recent_servers:
            self.recent_servers.remove(s)
        self.recent_servers.insert(0,s)
        self.recent_servers = self.recent_servers[0:20]
        self.config.set_key('recent_servers', self.recent_servers)

    def new_blockchain_height(self, blockchain_height, i):
        if self.is_connected():
            h = self.heights.get(self.interface.server)
            if h:
                self.server_lag = blockchain_height - h
                if self.server_lag > 1:
                    print_error( "Server is lagging", blockchain_height, h)
                    if self.config.get('auto_cycle'):
                        self.set_server(i.server)
            else:
                print_error('no height for main interface')

        self.trigger_callback('updated')

    def run(self):
        self.blockchain.start()

        with self.lock:
            self.running = True

        while self.is_running():
            try:
                i = self.queue.get(timeout = 30 if self.interfaces else 3)
            except Queue.Empty:
                if len(self.interfaces) < self.num_server:
                    self.start_random_interface()
                continue

            if i.server in self.pending_servers:
                self.pending_servers.remove(i.server)

            if i.is_connected:
                #if i.server in self.interfaces: raise
                self.interfaces[i.server] = i
                self.add_recent_server(i)
                i.send([ ('blockchain.headers.subscribe',[])], self.on_header)
                if i == self.interface:
                    print_error('sending subscriptions to', self.interface.server)
                    self.send_subscriptions()
                    self.trigger_callback('connected')
            else:
                self.disconnected_servers.add(i.server)
                if i.server in self.interfaces:
                    self.interfaces.pop(i.server)
                if i.server in self.heights:
                    self.heights.pop(i.server)
                if i == self.interface:
                    #self.interface = None
                    self.trigger_callback('disconnected')

            if not self.interface.is_connected and self.config.get('auto_cycle'):
                self.switch_to_random_interface()

    def on_header(self, i, r):
        result = r.get('result')
        if not result: return
        height = result.get('block_height')
        self.heights[i.server] = height
        self.merkle_roots[i.server] = result.get('merkle_root')
        self.utxo_roots[i.server] = result.get('utxo_root')
        # notify blockchain about the new height
        self.blockchain.queue.put((i,result))

        if i == self.interface:
            self.server_lag = self.blockchain.height() - height
            if self.server_lag > 1 and self.config.get('auto_cycle'):
                print_error( "Server lagging, stopping interface")
                self.stop_interface()

            self.trigger_callback('updated')

    def on_peers(self, i, r):
        if not r: return
        self.irc_servers = self.parse_servers(r.get('result'))
        self.trigger_callback('peers')

    def on_banner(self, i, r):
        self.banner = r.get('result')
        self.trigger_callback('banner')

    def stop(self):
        with self.lock: self.running = False

    def is_running(self):
        with self.lock: return self.running

    def synchronous_get(self, requests, timeout=100000000):
        queue = Queue.Queue()
        ids = self.interface.send(requests, lambda i,r: queue.put(r))
        id2 = ids[:]
        res = {}
        while ids:
            r = queue.get(True, timeout)
            _id = r.get('id')
            if _id in ids:
                ids.remove(_id)
                res[_id] = r.get('result')
        out = []
        for _id in id2:
            out.append(res[_id])
        return out

    #def retrieve_transaction(self, tx_hash, tx_height=0):
    #    import transaction
    #    r = self.synchronous_get([ ('blockchain.transaction.get',[tx_hash, tx_height]) ])[0]
    #    if r:
    #        return transaction.Transaction(r)

    def parse_servers(self, result):
        """ parse servers list into dict format"""
        from version import PROTOCOL_VERSION
        servers = {}
        for item in result:
            host = item[1]
            out = {}
            version = None
            pruning_level = '-'
            if len(item) > 2:
                for v in item[2]:
                    if re.match("[stgh]\d*", v):
                        protocol, port = v[0], v[1:]
                        if port == '': port = DEFAULT_PORTS[protocol]
                        out[protocol] = port
                    elif re.match("v(.?)+", v):
                        version = v[1:]
                    elif re.match("p\d*", v):
                        pruning_level = v[1:]
                    if pruning_level == '': pruning_level = '0'
            try:
                is_recent = float(version)>=float(PROTOCOL_VERSION)
            except Exception:
                is_recent = False

            if out and is_recent:
                out['pruning'] = pruning_level
                servers[host] = out

        return servers

class NetworkProxy:
    # interface to the network object.
    # handle subscriptions and callbacks
    # the network object can be jsonrpc server
    def __init__(self, network):
        self.network = network

if __name__ == "__main__":
    import simple_config
    config = simple_config.SimpleConfig({'verbose':True, 'server':'ecdsa.org:50002:s'})
    network = Network(config)
    network.start()

    while 1:
        time.sleep(1)

<|endoftext|>


<commit_before>"""
Utilities for making merged interaction tables of choosers and
the alternatives from which they are choosing.
Used for location choice models.

"""
import random
import sys
import time

import numpy as np
import pandas as pd

import mnl
import nl
import pmat

GPU = 0

def enable_gpu():
    global GPU
    GPU = 1
    pmat.initialize_gpu()

# if nested logit, add the field names for the additional nesting params

def add_fnames(fnames, est_params):
    if est_params[0] == 'nl':
        fnames = ['mu%d' % (i + 1)
                  for i in range(est_params[2].numnests())] + fnames
    return fnames

# TODO: split this out into separate functions for estimation
# and simulation.
def mnl_interaction_dataset(choosers, alternatives, SAMPLE_SIZE,
                            chosenalts=None):
    # filter choosers and their current choices if they point to
    # something that isn't in the alternatives table
    if chosenalts is not None:
        isin = chosenalts.isin(alternatives.index)
        removing = isin.value_counts()[False]
        if removing:
            print (
                "Removing {} choice situations because chosen "
                "alt doesn't exist"
            ).format(removing)
            choosers = choosers[isin]
            chosenalts = chosenalts[isin]

    numchoosers = choosers.shape[0]
    numalts = alternatives.shape[0]

    # TODO: this is currently broken in a situation where
    # SAMPLE_SIZE >= numalts. That may not happen often in
    # practical situations but it should be supported
    # because a) why not? and b) testing.
    if SAMPLE_SIZE < numalts:
        sample = np.random.choice(
            alternatives.index.values, SAMPLE_SIZE * numchoosers)
        if chosenalts is not None:
            # replace the first row for each chooser with
            # the currently chosen alternative.
            sample[::SAMPLE_SIZE] = chosenalts
    else:
        assert chosenalts is None  # if not sampling, must be simulating
        # we're about to do a huge join - do this with a discretized population
        assert numchoosers < 10
        sample = np.tile(alternatives.index.values, numchoosers)

    if not choosers.index.is_unique:
        raise Exception(
            "ERROR: choosers index is not unique, "
            "sample will not work correctly")
    if not alternatives.index.is_unique:
        raise Exception(
            "ERROR: alternatives index is not unique, "
            "sample will not work correctly")

    alts_sample = alternatives.loc[sample]
    assert len(alts_sample.index) == SAMPLE_SIZE * len(choosers.index)
    try:
        alts_sample['join_index'] = np.repeat(choosers.index, SAMPLE_SIZE)
    except:
        # TODO: log the error here and re-raise the original exception
        raise Exception(
            "ERROR: An exception here means agents and "
            "alternatives aren't merging correctly")

    alts_sample = pd.merge(
        alts_sample, choosers, left_on='join_index', right_index=True,
        suffixes=('', '_r'))

    chosen = np.zeros((numchoosers, SAMPLE_SIZE))
    chosen[:, 0] = 1

    return sample, alts_sample, chosen

def nl_interaction_dataset(choosers, alternatives, SAMPLE_SIZE, nestcol,
                           chosenalts=None, left_on='nodeid', presample=None,
                           nestcounts=None):
    nests = alternatives[nestcol].value_counts()
    print "Alternatives in each nest\n", nests
    sample_size_per_nest = SAMPLE_SIZE / len(nests)
    print "Sample size per nest", sample_size_per_nest
    assert SAMPLE_SIZE % len(nests) == 0  # divides evenly

    if presample is None:
        sample = None
        for m in np.sort(nests.keys().values):
            # full sampled set
            nsample = np.random.choice(
                alternatives[alternatives[nestcol] == m].index.values,
                sample_size_per_nest * choosers.shape[0])
            nsample = np.reshape(
                nsample, (nsample.size / sample_size_per_nest,
                          sample_size_per_nest))
            if sample is None:
                sample = nsample
            else:
                sample = np.concatenate((sample, nsample), axis=1)
    else:
        sample = presample

    # means we're estimating, not simulating
    if chosenalts is not None:
        chosen = np.zeros((choosers.shape[0], SAMPLE_SIZE))
        if isinstance(left_on, str):
            assert left_on in choosers.columns
        chosennest = pd.merge(
            choosers, alternatives, left_on=left_on, right_index=True,
            how="left"
        ).set_index(choosers.index)[nestcol]  # need to maintain same index
        print "Chosen alternatives by nest\n", chosennest.value_counts()
        assert sample.shape == chosen.shape
        # this restriction should be removed in the future - for now nestids
        # have to count from 0 to numnests-1
        assert min(nests.keys()) == 0 and max(nests.keys()) == len(nests) - 1
        # replace with chosen alternative
        sample[range(sample.shape[0]), chosennest *
               sample_size_per_nest] = chosenalts
        chosen[range(chosen.shape[0]), chosennest * sample_size_per_nest] = 1

    sample = sample.flatten().astype('object')
    alts_sample = alternatives.ix[sample]
    alts_sample['join_index'] = np.repeat(choosers.index, SAMPLE_SIZE)

    print "Merging sample (takes a while)"
    t1 = time.time()
    alts_sample = pd.merge(
        alts_sample, choosers, left_on='join_index', right_index=True)
    print "Done merging sample in %f" % (time.time() - t1)

    nestinfo = nl.NestInfo(nests, sample_size_per_nest, chosennest, nestcounts)

    return sample, alts_sample, ('nl', chosen, nestinfo)
<commit_msg>small fix in print statement in interaction<commit_after>"""
Utilities for making merged interaction tables of choosers and
the alternatives from which they are choosing.
Used for location choice models.

"""
import random
import sys
import time

import numpy as np
import pandas as pd

import mnl
import nl
import pmat

GPU = 0

def enable_gpu():
    global GPU
    GPU = 1
    pmat.initialize_gpu()

# if nested logit, add the field names for the additional nesting params

def add_fnames(fnames, est_params):
    if est_params[0] == 'nl':
        fnames = ['mu%d' % (i + 1)
                  for i in range(est_params[2].numnests())] + fnames
    return fnames

# TODO: split this out into separate functions for estimation
# and simulation.
def mnl_interaction_dataset(choosers, alternatives, SAMPLE_SIZE,
                            chosenalts=None):
    # filter choosers and their current choices if they point to
    # something that isn't in the alternatives table
    if chosenalts is not None:
        isin = chosenalts.isin(alternatives.index)
        try:
            removing = isin.value_counts().loc[False]
        except:
            removing = None
        if removing:
            print (
                "Removing {} choice situations because chosen "
                "alt doesn't exist"
            ).format(removing)
            choosers = choosers[isin]
            chosenalts = chosenalts[isin]

    numchoosers = choosers.shape[0]
    numalts = alternatives.shape[0]

    # TODO: this is currently broken in a situation where
    # SAMPLE_SIZE >= numalts. That may not happen often in
    # practical situations but it should be supported
    # because a) why not? and b) testing.
    if SAMPLE_SIZE < numalts:
        sample = np.random.choice(
            alternatives.index.values, SAMPLE_SIZE * numchoosers)
        if chosenalts is not None:
            # replace the first row for each chooser with
            # the currently chosen alternative.
            sample[::SAMPLE_SIZE] = chosenalts
    else:
        assert chosenalts is None  # if not sampling, must be simulating
        # we're about to do a huge join - do this with a discretized population
        assert numchoosers < 10
        sample = np.tile(alternatives.index.values, numchoosers)

    if not choosers.index.is_unique:
        raise Exception(
            "ERROR: choosers index is not unique, "
            "sample will not work correctly")
    if not alternatives.index.is_unique:
        raise Exception(
            "ERROR: alternatives index is not unique, "
            "sample will not work correctly")

    alts_sample = alternatives.loc[sample]
    assert len(alts_sample.index) == SAMPLE_SIZE * len(choosers.index)
    try:
        alts_sample['join_index'] = np.repeat(choosers.index, SAMPLE_SIZE)
    except:
        # TODO: log the error here and re-raise the original exception
        raise Exception(
            "ERROR: An exception here means agents and "
            "alternatives aren't merging correctly")

    alts_sample = pd.merge(
        alts_sample, choosers, left_on='join_index', right_index=True,
        suffixes=('', '_r'))

    chosen = np.zeros((numchoosers, SAMPLE_SIZE))
    chosen[:, 0] = 1

    return sample, alts_sample, chosen

def nl_interaction_dataset(choosers, alternatives, SAMPLE_SIZE, nestcol,
                           chosenalts=None, left_on='nodeid', presample=None,
                           nestcounts=None):
    nests = alternatives[nestcol].value_counts()
    print "Alternatives in each nest\n", nests
    sample_size_per_nest = SAMPLE_SIZE / len(nests)
    print "Sample size per nest", sample_size_per_nest
    assert SAMPLE_SIZE % len(nests) == 0  # divides evenly

    if presample is None:
        sample = None
        for m in np.sort(nests.keys().values):
            # full sampled set
            nsample = np.random.choice(
                alternatives[alternatives[nestcol] == m].index.values,
                sample_size_per_nest * choosers.shape[0])
            nsample = np.reshape(
                nsample, (nsample.size / sample_size_per_nest,
                          sample_size_per_nest))
            if sample is None:
                sample = nsample
            else:
                sample = np.concatenate((sample, nsample), axis=1)
    else:
        sample = presample

    # means we're estimating, not simulating
    if chosenalts is not None:
        chosen = np.zeros((choosers.shape[0], SAMPLE_SIZE))
        if isinstance(left_on, str):
            assert left_on in choosers.columns
        chosennest = pd.merge(
            choosers, alternatives, left_on=left_on, right_index=True,
            how="left"
        ).set_index(choosers.index)[nestcol]  # need to maintain same index
        print "Chosen alternatives by nest\n", chosennest.value_counts()
        assert sample.shape == chosen.shape
        # this restriction should be removed in the future - for now nestids
        # have to count from 0 to numnests-1
        assert min(nests.keys()) == 0 and max(nests.keys()) == len(nests) - 1
        # replace with chosen alternative
        sample[range(sample.shape[0]), chosennest *
               sample_size_per_nest] = chosenalts
        chosen[range(chosen.shape[0]), chosennest * sample_size_per_nest] = 1

    sample = sample.flatten().astype('object')
    alts_sample = alternatives.ix[sample]
    alts_sample['join_index'] = np.repeat(choosers.index, SAMPLE_SIZE)

    print "Merging sample (takes a while)"
    t1 = time.time()
    alts_sample = pd.merge(
        alts_sample, choosers, left_on='join_index', right_index=True)
    print "Done merging sample in %f" % (time.time() - t1)

    nestinfo = nl.NestInfo(nests, sample_size_per_nest, chosennest, nestcounts)

    return sample, alts_sample, ('nl', chosen, nestinfo)
<|endoftext|>