<commit_before>"""
Analysis tools, specific for the *Puppy* experiment.
"""

import pylab
import h5py
import numpy as np
import itertools

sensor_names = ['trg0','trg1','trg2','trg3','accelerometer_x','accelerometer_y','accelerometer_z','compass_x','compass_y','compass_z','gyro_x','gyro_y','gyro_z','hip0','hip1','hip2','hip3','knee0','knee1','knee2','knee3','puppyGPS_x','puppyGPS_y','puppyGPS_z','touch0','touch0','touch1','touch2','touch3']

def puppy_plot_trajectory(analysis, axis, episode, step_width=1, offset=0, legend=True, **kwargs):
    """Plot the trajectory of an episode
    """
    gps_x = analysis[episode]['puppyGPS_x'][offset+step_width-1::step_width]
    gps_y = analysis[episode]['puppyGPS_y'][offset+step_width-1::step_width]
    if step_width > 1:
        gps_x = np.concatenate(([analysis[episode]['puppyGPS_x'][offset]], gps_x))
        gps_y = np.concatenate(([analysis[episode]['puppyGPS_y'][offset]], gps_y))

    col = kwargs.pop('color', 'k')
    label = kwargs.pop('label', 'Trajectory')
    axis.plot(gps_x, gps_y, color=col, **kwargs)
    axis.axis('equal')
    if legend:
        axis.plot(gps_x[0], gps_y[0], 'ks', label='Start')
        axis.plot(gps_x[-1], gps_y[-1], 'kv', label='End')

    return axis

def puppy_plot_all_trajectories(analysis, axis, step_width=1, **kwargs):
    """Plot all trajectories in ``analysis`` into ``axis``.
    """
    gps_x = analysis.get_data('puppyGPS_x')
    gps_y = analysis.get_data('puppyGPS_y')

    N = len(gps_x)-1
    kwargs.pop('color', None) # remove color argument
    for idx, (x, y) in enumerate(zip(gps_x, gps_y)):
        col = 0.75 - (0.75 * (idx - 1))/N

        x_plot = np.concatenate(([x[0]], x[step_width-1::step_width]))
        y_plot = np.concatenate(([y[0]], y[step_width-1::step_width]))

        axis.plot(x_plot, y_plot, color=str(col), **kwargs)

    return axis

def puppy_plot_linetarget(axis, origin=(2.0, 0.0), direction=(1.0, 1.0), range_=(-5.0, 5.0)):
    """Plot a line given by ``origin`` and ``direction``. The ``range_``
    may be supplid, which corresponds to the length of the line (from
    the origin).
    """
    origin = np.array(origin)
    dir_ = np.array(direction)
    dir_ /= np.linalg.norm(dir_)
    line = [origin + t * dir_ for t in range_]
    line_x, line_y = zip(*line)
    axis.plot(line_x, line_y, 'k', label='Target')
    return axis

def puppy_plot_locationtarget(axis, target=(4.0, 4.0), distance=0.5, **kwargs):
    """Plot the ``target`` location with a sphere of radius ``distance``
    into ``axis`` to mark the target location. ``kwargs`` will be passed
    to all :py:mod:`pylab` calls."""
    linewidth = kwargs.pop('linewidth', 2)
    color = kwargs.pop('facecolor', 'k')
    fill = kwargs.pop('fill', False)
    lbl = kwargs.pop('label', 'Target')
    axis.plot([target[0]], [target[1]], 'kD', **kwargs)
    if distance > 0.0:
        trg_field = pylab.Circle(target, distance, fill=fill, facecolor=color, linewidth=linewidth, label=lbl, **kwargs)
        axis.add_artist(trg_field)

    return axis

def puppy_plot_landmarks(axis, landmarks, **kwargs):
    """Plot markers at ``landmark`` locations in ``axis``."""
    color = kwargs.pop('color', 'k')
    lbl = kwargs.pop('label', '')
    marker = kwargs.pop('marker','^')
    for x,y in landmarks:
        axis.plot([x],[y], marker=marker, color=color)
    return axis

def puppy_offline_playback(pth_data, critic, samples_per_action, ms_per_step, episode_start=None, episode_end=None, min_episode_len=0):
    """Simulate an experiment run for the critic by using offline data.
    The data has to be collected in webots, using the respective
    robot and supervisor. Note that the behaviour of the simulation
    should match what's expected by the critic. The critic is fed the
    sensor data, in order. Of course, it can't react to it since
    the next action is predefined.

    Additional to the sensor fields, the 'tumbling' dataset is expected
    which indicates, if and when the robot has tumbled. It is used such
    that the respective signals can be sent to the critic.

    The critic won't store any sensory data again.

    ``pth_data``
        Path to the datafile with the sensory information (HDF5).

    ``critic``
        PuppyHDP instance.

    ``samples_per_action``
        Number of samples per control step. Must correspond to the data.

    ``ms_per_step``
        Sensor sampling period.

    ``episode_start``
        Defines a lower limit on the episode number. Passed as int,
        is with respect to the episode index, not its identifier.

    ``episode_stop``
        Defines an upper limit on the episode number. Passed as int,
        is with respect to the episode index, not its identifier.

    ``min_episode_len``
        Only pick episodes longer than this threshold.

    """
    # Open data file, get valid experiments
    f = h5py.File(pth_data,'r')
    storages = map(str, sorted(map(int, f.keys())))
    storages = filter(lambda s: len(f[s]) > 0, storages)
    if min_episode_len > 0:
        storages = filter(lambda s: f[s]['a_curr'].shape[0] > min_episode_len, storages)

    if episode_end is not None:
        storages = storages[:episode_end]

    if episode_start is not None:
        storages = storages[episode_start:]

    assert len(storages) > 0

    # Prepare critic; redirect hooks to avoid storing epoch data twice
    # and feed the actions
    next_action = None
    episode = None
    critic._pre_increment_hook_orig = critic._pre_increment_hook
    critic._next_action_hook_orig = critic._next_action_hook

    def pre_increment_hook(epoch, **kwargs):
        kwargs['offline_episode'] = np.array([episode])
        critic._pre_increment_hook_orig(dict(), **kwargs)
    def next_action_hook(a_next):
        #print "(next)", a_next.T, next_action.T
        return next_action

    critic._next_action_hook = next_action_hook
    critic._pre_increment_hook = pre_increment_hook

    # Main loop, feed data to the critic
    time_step_ms = ms_per_step * samples_per_action
    time_start_ms = 0
    for episode_idx, episode in enumerate(storages):

        data_grp = f[episode]
        N = data_grp['trg0'].shape[0]
        assert N % samples_per_action == 0

        # get tumbled infos
        if 'tumbled' in data_grp:
            time_tumbled = data_grp['tumbled'][0] * samples_per_action
        else:
            time_tumbled = -1

        # initial, empty call
        if 'init_step' in data_grp:
            print "Simulation was started/reverted"
            time_start_ms = 0
            critic(dict(), time_start_ms, time_start_ms + samples_per_action, ms_per_step)
            time_tumbled -= samples_per_action

        # initial action
        critic.a_curr = np.atleast_2d(data_grp['a_curr'][0]).T

        # loop through data, incrementally feed the critic
        for num_iter in np.arange(0, N, samples_per_action):
            # next action
            next_action = np.atleast_2d(data_grp['a_next'][num_iter/samples_per_action]).T

            # get data
            time_start_ms += time_step_ms
            time_end_ms = time_start_ms + time_step_ms
            chunk = dict([(k, data_grp[k][num_iter:(num_iter+samples_per_action)]) for k in sensor_names])

            # send tumbled message
            if num_iter == time_tumbled:
                critic.event_handler(None, dict(), time_tumbled, 'tumbled_grace_start')

            # update critic
            critic(chunk, time_start_ms, time_end_ms, time_step_ms)

        # send reset after episode has finished
        if episode_idx < len(storages) - 1:
            critic.event_handler(None, dict(), ms_per_step * N, 'reset')

    # cleanup
    critic._pre_increment_hook = critic._pre_increment_hook_orig
    critic._next_action_hook = critic._next_action_hook_orig
    del critic._pre_increment_hook_orig
    del critic._next_action_hook_orig

def puppy_plot_action(analysis, episode, critic, reservoir, inspect_epochs, actions_range_x, actions_range_y, step_width, obs_offset, actions=None):
    """

    .. todo::
        offset in case of offline data?

    .. todo::
        implementation unfinished, untested, undocumented

    """
    grp = analysis[episode]
    for trg_epoch in inspect_epochs:
        reservoir.reset()
        reservoir.states = np.atleast_2d(grp['x_curr'][trg_epoch-1,:])

        # evaluate actions
        # Note: epoch is one step ahead (of a_curr, same time as a_next)!
        # Note: sensor values are shifted w.r.t a_curr by obs_offset
        s_curr = dict([(sensor,grp[sensor][obs_offset+step_width*(trg_epoch-1):obs_offset+trg_epoch*step_width]) for sensor in sensor_names])
        a_ret = np.zeros((len(actions_range_x), len(actions_range_y)))

        actions_iter = itertools.product(range(len(actions_range_x)), range(len(actions_range_y)))
        for idx_x, idx_y in actions_iter:
            action_candidate = np.atleast_2d((actions_range_x[idx_x], actions_range_y[idx_y])).T
            j_curr = critic(s_curr, action_candidate, simulate=True)
            a_ret[idx_x, idx_y] = j_curr[0, 0]
            #print actions_range_x[idx_x], actions_range_y[idx_y], j_curr[0, 0]

        # plot results
        fig = pylab.figure()
        pylab.gray()
        # In the image, the y-axis is the rows, the x-axis the columns of the matrix
        # Having index (0,0) in the left/bottom corner: origin='lower'
        pylab.plot((0, len(actions_range_x)-1), (0, len(actions_range_y)-1), 'b')
        pylab.imshow(a_ret, origin='lower')
        pylab.colorbar()
        pylab.xticks(range(len(actions_range_y)), actions_range_y)
        pylab.yticks(range(len(actions_range_x)), actions_range_x)
        pylab.title('Expected Return per action at epoch ' + str(trg_epoch))
        pylab.xlabel('Amplitude right legs') # cols are idx_y, right legs
        pylab.ylabel('Amplitude left legs') # rows are idx_x, left legs

    if actions is not None:
        a_left, a_right = zip(*actions)
        pylab.plot(a_left, a_right, 'r')
        pylab.plot([a_left[0]], [a_right[0]], 'rs')

    return fig
<commit_msg>Pre-vid commit<commit_after>"""
Analysis tools, specific for the *Puppy* experiment.
"""

import pylab
import h5py
import numpy as np
import itertools

sensor_names = ['trg0','trg1','trg2','trg3','accelerometer_x','accelerometer_y','accelerometer_z','compass_x','compass_y','compass_z','gyro_x','gyro_y','gyro_z','hip0','hip1','hip2','hip3','knee0','knee1','knee2','knee3','puppyGPS_x','puppyGPS_y','puppyGPS_z','touch0','touch0','touch1','touch2','touch3']

def puppy_plot_trajectory(analysis, axis, episode, step_width=1, offset=0, legend=True, **kwargs):
    """Plot the trajectory of an episode
    """
    gps_x = analysis[episode]['puppyGPS_x'][offset+step_width-1::step_width]
    gps_y = analysis[episode]['puppyGPS_y'][offset+step_width-1::step_width]
    if step_width > 1:
        gps_x = np.concatenate(([analysis[episode]['puppyGPS_x'][offset]], gps_x))
        gps_y = np.concatenate(([analysis[episode]['puppyGPS_y'][offset]], gps_y))

    col = kwargs.pop('color', 'k')
    label = kwargs.pop('label', 'Trajectory')
    axis.plot(gps_x, gps_y, color=col, **kwargs)
    axis.axis('equal')
    if legend:
        axis.plot(gps_x[0], gps_y[0], 'ks', label='Start')
        axis.plot(gps_x[-1], gps_y[-1], 'kv', label='End')

    return axis

def puppy_plot_all_trajectories(analysis, axis, step_width=1, **kwargs):
    """Plot all trajectories in ``analysis`` into ``axis``.
    """
    gps_x = analysis.get_data('puppyGPS_x')
    gps_y = analysis.get_data('puppyGPS_y')

    N = len(gps_x)-1
    kwargs.pop('color', None) # remove color argument
    for idx, (x, y) in enumerate(zip(gps_x, gps_y)):
        col = 0.75 - (0.75 * (idx - 1))/N

        x_plot = np.concatenate(([x[0]], x[step_width-1::step_width]))
        y_plot = np.concatenate(([y[0]], y[step_width-1::step_width]))

        axis.plot(x_plot, y_plot, color=str(col), **kwargs)

    return axis

def puppy_plot_linetarget(axis, origin=(2.0, 0.0), direction=(1.0, 1.0), range_=(-5.0, 5.0)):
    """Plot a line given by ``origin`` and ``direction``. The ``range_``
    may be supplid, which corresponds to the length of the line (from
    the origin).
    """
    origin = np.array(origin)
    dir_ = np.array(direction)
    dir_ /= np.linalg.norm(dir_)
    line = [origin + t * dir_ for t in range_]
    line_x, line_y = zip(*line)
    axis.plot(line_x, line_y, 'k', label='Target')
    return axis

def puppy_plot_locationtarget(axis, target=(4.0, 4.0), distance=0.5, **kwargs):
    """Plot the ``target`` location with a sphere of radius ``distance``
    into ``axis`` to mark the target location. ``kwargs`` will be passed
    to all :py:mod:`pylab` calls."""
    linewidth = kwargs.pop('linewidth', 2)
    color = kwargs.pop('facecolor', 'k')
    fill = kwargs.pop('fill', False)
    lbl = kwargs.pop('label', 'Target')
    axis.plot([target[0]], [target[1]], 'kD', **kwargs)
    if distance > 0.0:
        trg_field = pylab.Circle(target, distance, fill=fill, facecolor=color, linewidth=linewidth, label=lbl, **kwargs)
        axis.add_artist(trg_field)

    return axis

def puppy_plot_landmarks(axis, landmarks, **kwargs):
    """Plot markers at ``landmark`` locations in ``axis``."""
    color = kwargs.pop('color', 'k')
    lbl = kwargs.pop('label', '')
    marker = kwargs.pop('marker','^')
    for x,y in landmarks:
        axis.plot([x],[y], marker=marker, color=color)
    return axis

def puppy_offline_playback(pth_data, critic, samples_per_action, ms_per_step, episode_start=None, episode_end=None, min_episode_len=0):
    """Simulate an experiment run for the critic by using offline data.
    The data has to be collected in webots, using the respective
    robot and supervisor. Note that the behaviour of the simulation
    should match what's expected by the critic. The critic is fed the
    sensor data, in order. Of course, it can't react to it since
    the next action is predefined.

    Additional to the sensor fields, the 'tumbling' dataset is expected
    which indicates, if and when the robot has tumbled. It is used such
    that the respective signals can be sent to the critic.

    The critic won't store any sensory data again.

    ``pth_data``
        Path to the datafile with the sensory information (HDF5).

    ``critic``
        PuppyHDP instance.

    ``samples_per_action``
        Number of samples per control step. Must correspond to the data.

    ``ms_per_step``
        Sensor sampling period.

    ``episode_start``
        Defines a lower limit on the episode number. Passed as int,
        is with respect to the episode index, not its identifier.

    ``episode_stop``
        Defines an upper limit on the episode number. Passed as int,
        is with respect to the episode index, not its identifier.

    ``min_episode_len``
        Only pick episodes longer than this threshold.

    """
    # Open data file, get valid experiments
    f = h5py.File(pth_data,'r')
    storages = map(str, sorted(map(int, f.keys())))
    storages = filter(lambda s: len(f[s]) > 0, storages)
    if min_episode_len > 0:
        storages = filter(lambda s: f[s]['a_curr'].shape[0] > min_episode_len, storages)

    if episode_end is not None:
        storages = storages[:episode_end]

    if episode_start is not None:
        storages = storages[episode_start:]

    assert len(storages) > 0

    # Prepare critic; redirect hooks to avoid storing epoch data twice
    # and feed the actions
    next_action = None
    episode = None
    critic._pre_increment_hook_orig = critic._pre_increment_hook
    critic._next_action_hook_orig = critic._next_action_hook

    def pre_increment_hook(epoch, **kwargs):
        kwargs['offline_episode'] = np.array([episode])
        critic._pre_increment_hook_orig(dict(), **kwargs)
    def next_action_hook(a_next):
        #print "(next)", a_next.T, next_action.T
        return next_action

    critic._next_action_hook = next_action_hook
    critic._pre_increment_hook = pre_increment_hook

    # Main loop, feed data to the critic
    time_step_ms = ms_per_step * samples_per_action
    time_start_ms = 0
    for episode_idx, episode in enumerate(storages):

        data_grp = f[episode]
        N = data_grp['trg0'].shape[0]
        assert N % samples_per_action == 0

        # get tumbled infos
        if 'tumbled' in data_grp:
            time_tumbled = data_grp['tumbled'][0] * samples_per_action
        else:
            time_tumbled = -1

        # initial, empty call
        if 'init_step' in data_grp:
            print "Simulation was started/reverted"
            time_start_ms = 0
            critic(dict(), time_start_ms, time_start_ms + samples_per_action, ms_per_step)
            time_tumbled -= samples_per_action

        # initial action
        critic.a_curr = np.atleast_2d(data_grp['a_curr'][0]).T

        # loop through data, incrementally feed the critic
        for num_iter in np.arange(0, N, samples_per_action):
            # next action
            next_action = np.atleast_2d(data_grp['a_next'][num_iter/samples_per_action]).T

            # get data
            time_start_ms += time_step_ms
            time_end_ms = time_start_ms + time_step_ms
            chunk = dict([(k, data_grp[k][num_iter:(num_iter+samples_per_action)]) for k in sensor_names])

            # send tumbled message
            if num_iter == time_tumbled:
                critic.event_handler(None, dict(), time_tumbled, 'tumbled_grace_start')

            # update critic
            critic(chunk, time_start_ms, time_end_ms, time_step_ms)

        # send reset after episode has finished
        if episode_idx < len(storages) - 1:
            critic.event_handler(None, dict(), ms_per_step * N, 'reset')

    # cleanup
    critic._pre_increment_hook = critic._pre_increment_hook_orig
    critic._next_action_hook = critic._next_action_hook_orig
    del critic._pre_increment_hook_orig
    del critic._next_action_hook_orig

def puppy_plot_action(analysis, episode, critic, reservoir, inspect_epochs, actions_range_x, actions_range_y, step_width, obs_offset, epoch_actions=None):
    """

    .. todo::
        documentation, video mode

    """
    grp = analysis[episode]
    if epoch_actions is None:
        epoch_actions = [None] * len(inspect_epochs)

    for trg_epoch, actions in zip(inspect_epochs, epoch_actions):
        reservoir.reset()
        reservoir.states = np.atleast_2d(grp['x_curr'][trg_epoch-1,:])

        # evaluate actions
        # Note: epoch is one step ahead (of a_curr, same time as a_next)!
        # Note: sensor values are shifted w.r.t a_curr by obs_offset
        s_curr = dict([(sensor,grp[sensor][obs_offset+step_width*(trg_epoch-1):obs_offset+trg_epoch*step_width]) for sensor in sensor_names])
        a_ret = np.zeros((len(actions_range_x), len(actions_range_y)))

        actions_iter = itertools.product(range(len(actions_range_x)), range(len(actions_range_y)))
        for idx_x, idx_y in actions_iter:
            action_candidate = np.atleast_2d((actions_range_x[idx_x], actions_range_y[idx_y])).T
            j_curr = critic(s_curr, action_candidate, simulate=True)
            a_ret[idx_x, idx_y] = j_curr[0, 0]
            #print actions_range_x[idx_x], actions_range_y[idx_y], j_curr[0, 0]

        # plot results
        fig = pylab.figure()
        pylab.gray()
        # In the image, the y-axis is the rows, the x-axis the columns of the matrix
        # Having index (0,0) in the left/bottom corner: origin='lower'
        pylab.plot((0, len(actions_range_x)-1), (0, len(actions_range_y)-1), 'b')
        pylab.imshow(a_ret, origin='lower')
        pylab.colorbar()
        pylab.xticks(range(len(actions_range_y)), actions_range_y)
        pylab.yticks(range(len(actions_range_x)), actions_range_x)
        pylab.title('Expected Return per action at epoch ' + str(trg_epoch))
        pylab.xlabel('Amplitude right legs') # cols are idx_y, right legs
        pylab.ylabel('Amplitude left legs') # rows are idx_x, left legs

        if actions is not None:
            a_left, a_right = zip(*actions)
            pylab.plot(a_left, a_right, 'r')
            pylab.plot([a_left[0]], [a_right[0]], 'rs')

    return fig

def puppy_plot_inspected_trajectory(analysis, episode_idx, step_width, axis, inspect_epochs, obs_offset):
    puppy_plot_trajectory(analysis, axis, episode_idx, step_width, color='b')
    trg_x = [analysis[episode_idx]['puppyGPS_x'][obs_offset + step_width*trg_epoch+step_width-1] for trg_epoch in inspect_epochs]
    trg_y = [analysis[episode_idx]['puppyGPS_y'][obs_offset + step_width*trg_epoch+step_width-1] for trg_epoch in inspect_epochs]
    axis.plot(trg_x, trg_y, 'k*', label='Inspected states')
    return axis
<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Turbinia Evidence objects."""

import json

# TODO(aarontp): Add serialization so we can pass these in messages
class Evidence(object):

  def __init__(self, name=None, description=None, local_path=None):
    self.name = name
    self.local_path = local_path
    self.description = description

    # List of jobs that have processed this evidence
    self.processed_by = []
    self.type = self.__class__.__name__

  def __str__(self):
    return u'{0:s} {1:s}'.format(self.name, self.type)

class RawDisk(Evidence):
  pass

class EncryptedDisk(Evidence):
  """Encrypted disk file evidence."""

  def __init__(self, encryption_type=None, encryption_key=None,
               unencrypted_path=None, *args, **kwargs):
    self.encryption_type = encryption_type
    self.encryption_key = encryption_key
    # self.local_path will be the encrypted path
    self.unencrypted_path = unencrypted_path
    super(EncryptedDisk, self).__init__(*args, **kwargs)

class GoogleCloudDisk(Evidence):

  def __init__(self, project=None, zone=None, device_name=None,
               cloud_path=None, *args, **kwargs):
    self.project = project
    self.zone = zone
    self.device_name = device_name
    self.cloud_path = cloud_path
    super(GoogleCloudDisk, self).__init__(*args, **kwargs)

class PlasoFile(Evidence):
  """Plaso output file evidence."""

  def __init__(self, plaso_version=None, *args, **kwargs):
    self.plaso_version = plaso_version
    super(PlasoFile, self).__init__(*args, **kwargs)
<commit_msg>Improve some of the evidence objects<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Turbinia Evidence objects."""

import json

class Evidence(object):
  """Evidence object for processing.

  In most cases, these objects will just contain metadata about the actual
  evidence.
  """

  def __init__(self, name=None, description=None, source=None, local_path=None,
               tags=None):
    """Initialization for Evidence.

    Args:
      name: Name of evidence.
      description: Description of evidence.
      source: String indicating where evidence came from (including tool version
              that created it, if appropriate).
      local_path: A string of the local_path to the evidence.
      tags: dict of extra tags assocated with this evidence.
    """
    self.name = name
    self.local_path = local_path
    self.source = source
    self.description = description
    self.tags = tags if tags else {}

    # List of jobs that have processed this evidence
    self.processed_by = []
    self.type = self.__class__.__name__

  def __str__(self):
    return u'{0:s} {1:s}'.format(self.type, self.name)

  def to_json(self):
    """Convert object to JSON."""
    return json.dumps(self.__dict__)

class RawDisk(Evidence):
  """Evidence object for Disk based evidence."""

  def __init__(self, mount_path=None, size=None, *args, **kwargs):
    """Initialization for raw disk evidence object.

    Args:
      mount_path: The mount path for this disk (if any).
      size:  The size of the disk in bytes.
    """
    self.mount_path = mount_path
    self.size = size
    super(RawDisk, self).__init__(*args, **kwargs)

class EncryptedDisk(RawDisk):
  """Encrypted disk file evidence."""

  def __init__(self, encryption_type=None, encryption_key=None,
               unencrypted_path=None, *args, **kwargs):
    """Initialization for Encrypted disk evidence objects.

    Args:
      encryption_type: The type of encryption used, e.g. FileVault or Bitlocker.
      encryption_key: A string of the encryption key used for this disk.
    """
    # TODO(aarontp): Make this an enum, or limited list
    self.encryption_type = encryption_type
    self.encryption_key = encryption_key
    # self.local_path will be the encrypted path
    self.unencrypted_path = unencrypted_path
    super(EncryptedDisk, self).__init__(*args, **kwargs)

class GoogleCloudDisk(RawDisk):
  """Evidence object for Google Cloud Disks."""

  def __init__(self, project=None, zone=None, disk_name=None, type_=None,
               *args, **kwargs):
    """Initialization for Google Cloud Disk.

    Args:
      project: The cloud project name this disk is associated with.
      zone: The geographic zone.
      disk_name: The cloud disk name.
      type_: The type of cloud disk.
    """
    self.project = project
    self.zone = zone
    self.disk_name = disk_name
    self.type = type_
    super(GoogleCloudDisk, self).__init__(*args, **kwargs)

class PlasoFile(Evidence):
  """Plaso output file evidence."""

  def __init__(self, plaso_version=None, *args, **kwargs):
    self.plaso_version = plaso_version
    super(PlasoFile, self).__init__(*args, **kwargs)
<|endoftext|>


<commit_before>"""
This tutorial illustrates a client request round trip with a simple consensus
pool.
"""
from tempfile import TemporaryDirectory

from ioflo.base.consoling import getConsole

from zeno.client.client import Client
from zeno.common.looper import Looper
from zeno.server.node import Node
from zeno.test.malicious_behaviors_node import changesRequest, makeNodeFaulty

console = getConsole()
console.reinit(verbosity=console.Wordage.terse)

"""
Nodes persist keys when bootstrapping to other nodes and reconnecting using an
ephemeral temporary directory when proving a concept is a nice way to keep
things tidy.
"""
with TemporaryDirectory() as tmpdir:

    """
    Looper runs an asynchronous message loop that services the nodes and client.
    It's also a context manager, so it cleans up after itself.
    """
    with Looper(debug=False) as looper:

        """
        A node registry is a dictionary of Node names and their IP addresses
        and port numbers.
        """
        nodeReg = {
            'Alpha': ('127.0.0.1', 7560),
            'Beta': ('127.0.0.1', 7562),
            'Gamma': ('127.0.0.1', 7564),
            'Delta': ('127.0.0.1', 7566)}

        """
        Create a node called Alpha
        """
        alpha = Node('Alpha', nodeReg, basedirpath=tmpdir)

        """
        Add the Alpha node to the looper, so it can be serviced.
        """
        looper.add(alpha)

        """
        Start key sharing among nodes. Key sharing is a way to bootstrap a
        consensus pool when you don't want to manually construct keys
        beforehand. See the github wiki for more details on key sharing.
        """
        alpha.startKeySharing()

        """
        Do the same process for the other nodes. Ordinarily, we would never have
        more than one node on a machine, but this is for demonstration purposes.
        """
        beta = Node('Beta', nodeReg, basedirpath=tmpdir)
        looper.add(beta)
        beta.startKeySharing()

        gamma = Node('Gamma', nodeReg, basedirpath=tmpdir)
        looper.add(gamma)
        gamma.startKeySharing()

        delta = Node('Delta', nodeReg, basedirpath=tmpdir)
        looper.add(delta)
        delta.startKeySharing()

        """
        Give the nodes time to come up, find each other, share long-term keys,
        and establish connections.
        """
        looper.runFor(5)

        """
        The client has a slightly different node registry than the nodes. The
        Nodes have two network interfaces, one for other nodes, and one for
        client. This registry points to the nodes' client-facing interfaces.
        """
        cliNodeReg = {
            'AlphaC': ('127.0.0.1', 7561),
            'BetaC': ('127.0.0.1', 7563),
            'GammaC': ('127.0.0.1', 7565),
            'DeltaC': ('127.0.0.1', 7567)}

        """
        A bi-directional connection is made from the client. This is the ip
        address and port for the client's interfact to the nodes.
        """
        client_addr = ("127.0.0.1", 8000)

        """
        Create a client.
        """
        clientId = "my_client_id"
        client = Client(clientId=clientId,
                        ha=client_addr,
                        nodeReg=cliNodeReg,
                        basedirpath=tmpdir)
        looper.add(client)

        """
        A client signs its requests. By default, a simple yet secure signing
        mechanism is created for a client.
        """
        idAndKey = client.signer.identifier, client.signer.verkey

        """
        A client's signature verification key must be bootstrapped out of band
        into the consensus pool. For demonstration, we'll add it directly to
        each node.
        """
        for node in alpha, beta, gamma, delta:
            node.clientAuthNr.addClient(*idAndKey)

        """
        We give the client a little time to connect
        """
        looper.runFor(3)

        """
        Create a simple message.
        """
        msg = {'life_answer': 42}

        """
        And submit it to the pool.
        """
        request, = client.submit(msg)

        """
        Allow some time for the request to be executed.
        """
        looper.runFor(3)

        """
        Let's get the reply.
        """
        reply, status = client.getReply(request.reqId)

        """
        Check the reply and see if consensus has been reached.
        """
        print("Reply: {}\n".format(reply))
        print("Status: {}\n".format(status))

        """
        See the reply details of a request.
        """
        client.showReplyDetails(request.reqId)

        """
        As we are using 4 nodes, we have an f-value of 1, which means that
        consensus can be still achieved with one faulty node. In this example,
        we're going to cause Beta to be malicious, altering a client's request
        before propagating to other nodes.
        """
        makeNodeFaulty(beta, changesRequest)

        """
        Create a new message.
        """
        msg = {"type": "sell", "amount": 101}

        """
        And submit it.
        """
        request2, = client.submit(msg)

        """
        Allow time for the message to be executed.
        """
        looper.runFor(3)

        """
        Observe that consensus is still reached
        """
        reply, consensusReached = client.getReply(request2.reqId)
        print("Reply for the request: {}\n".format(reply))
        print("Consensus Reached?: {}\n".format(consensusReached))

        client.showReplyDetails(request2.reqId)
<commit_msg>Tweaking the tutorial to make a node to send a wrong response<commit_after>"""
This tutorial illustrates a client request round trip with a simple consensus
pool.
"""
from tempfile import TemporaryDirectory

from ioflo.base.consoling import getConsole

from zeno.client.client import Client
from zeno.common.looper import Looper
from zeno.server.node import Node
from zeno.test.malicious_behaviors_node import faultyReply, makeNodeFaulty

console = getConsole()
console.reinit(verbosity=console.Wordage.terse)

"""
Nodes persist keys when bootstrapping to other nodes and reconnecting using an
ephemeral temporary directory when proving a concept is a nice way to keep
things tidy.
"""
with TemporaryDirectory() as tmpdir:

    """
    Looper runs an asynchronous message loop that services the nodes and client.
    It's also a context manager, so it cleans up after itself.
    """
    with Looper(debug=False) as looper:

        """
        A node registry is a dictionary of Node names and their IP addresses
        and port numbers.
        """
        nodeReg = {
            'Alpha': ('127.0.0.1', 7560),
            'Beta': ('127.0.0.1', 7562),
            'Gamma': ('127.0.0.1', 7564),
            'Delta': ('127.0.0.1', 7566)}

        """
        Create a node called Alpha
        """
        alpha = Node('Alpha', nodeReg, basedirpath=tmpdir)

        """
        Add the Alpha node to the looper, so it can be serviced.
        """
        looper.add(alpha)

        """
        Start key sharing among nodes. Key sharing is a way to bootstrap a
        consensus pool when you don't want to manually construct keys
        beforehand. See the github wiki for more details on key sharing.
        """
        alpha.startKeySharing()

        """
        Do the same process for the other nodes. Ordinarily, we would never have
        more than one node on a machine, but this is for demonstration purposes.
        """
        beta = Node('Beta', nodeReg, basedirpath=tmpdir)
        looper.add(beta)
        beta.startKeySharing()

        gamma = Node('Gamma', nodeReg, basedirpath=tmpdir)
        looper.add(gamma)
        gamma.startKeySharing()

        delta = Node('Delta', nodeReg, basedirpath=tmpdir)
        looper.add(delta)
        delta.startKeySharing()

        """
        Give the nodes time to come up, find each other, share long-term keys,
        and establish connections.
        """
        looper.runFor(5)

        """
        The client has a slightly different node registry than the nodes. The
        Nodes have two network interfaces, one for other nodes, and one for
        client. This registry points to the nodes' client-facing interfaces.
        """
        cliNodeReg = {
            'AlphaC': ('127.0.0.1', 7561),
            'BetaC': ('127.0.0.1', 7563),
            'GammaC': ('127.0.0.1', 7565),
            'DeltaC': ('127.0.0.1', 7567)}

        """
        A bi-directional connection is made from the client. This is the ip
        address and port for the client's interfact to the nodes.
        """
        client_addr = ("127.0.0.1", 8000)

        """
        Create a client.
        """
        clientId = "my_client_id"
        client = Client(clientId=clientId,
                        ha=client_addr,
                        nodeReg=cliNodeReg,
                        basedirpath=tmpdir)
        looper.add(client)

        """
        A client signs its requests. By default, a simple yet secure signing
        mechanism is created for a client.
        """
        idAndKey = client.signer.identifier, client.signer.verkey

        """
        A client's signature verification key must be bootstrapped out of band
        into the consensus pool. For demonstration, we'll add it directly to
        each node.
        """
        for node in alpha, beta, gamma, delta:
            node.clientAuthNr.addClient(*idAndKey)

        """
        We give the client a little time to connect
        """
        looper.runFor(3)

        """
        Create a simple message.
        """
        msg = {'life_answer': 42}

        """
        And submit it to the pool.
        """
        request, = client.submit(msg)

        """
        Allow some time for the request to be executed.
        """
        looper.runFor(3)

        """
        Let's get the reply.
        """
        reply, status = client.getReply(request.reqId)

        """
        Check the reply and see if consensus has been reached.
        """
        print("Reply: {}\n".format(reply))
        print("Status: {}\n".format(status))

        """
        See the reply details of a request.
        """
        client.showReplyDetails(request.reqId)

        """
        As we are using 4 nodes, we have an f-value of 1, which means that
        consensus can be still achieved with one faulty node. In this example,
        we're going to cause Beta to be malicious, altering a client's request
        before propagating to other nodes.
        """
        makeNodeFaulty(beta, faultyReply)

        """
        Create a new message.
        """
        msg = {"type": "sell", "amount": 101}

        """
        And submit it to the pool.
        """
        request2, = client.submit(msg)

        """
        Allow time for the message to be executed.
        """
        looper.runFor(10)

        """
        Observe that consensus is still reached with one node replying with a different response.
        """
        reply, consensusReached = client.getReply(request2.reqId)
        print("Reply for the request: {}\n".format(reply))
        print("Consensus Reached?: {}\n".format(consensusReached))

        client.showReplyDetails(request2.reqId)
<|endoftext|>


<commit_before>import w3lib.url
import w3lib.html

from lxml import etree
import re, sys
from hashlib import md5

from feedgenerator import Rss201rev2Feed, Enclosure
import datetime

import MySQLdb
from contextlib import closing
from settings import DATABASES, DOWNLOADER_USER_AGENT
from twisted.logger import Logger

from .db import get_conn

log = Logger()

class Feed(object):

    url_hash_regexp = re.compile('(#.*)?$')

    POST_TIME_DISTANCE = 15 # minutes, RSS Feed Reader skip same titles created in 10 min interval

    FIELD_IDS = {'title': 1, 'description': 2, 'link': 3}

    def __init__(self, db_creds):
        self.db_creds = db_creds

    def save_post(self, conn, created, feed_id, post_fields):
        with conn as cur:
            cur.execute("""insert into frontend_post (md5sum, created, feed_id)
                            values (unhex(%s), %s, %s)""", (post_fields['md5'], created, feed_id))
            post_id = conn.insert_id()
            log.info('Post saved id:{id!r}', id=post_id)
            return post_id

    def fill_time(self, feed_id, items):
        if not items:
            return 0

        new_post_cnt = 0
        for item in items:
            #create md5
            h = md5('')
            for key in ['title', 'description', 'link']:
                if key in item:
                    h.update(item[key].encode('utf-8'))
            item['md5'] = h.hexdigest()

        #fetch dates from db
        fetched_dates = {}
        with closing(get_conn(self.db_creds)) as conn:
            with conn as cur:
                quoted_hashes = ','.join(["unhex('%s')" % (i['md5']) for i in items])

                cur.execute("""select lower(hex(p.md5sum)), p.created, p.id
                               from frontend_post p
                               where p.md5sum in (%s)
                               and p.feed_id=%s""" % (quoted_hashes, feed_id,))
                rows = cur.fetchall()
                log.debug('Selected {count!r} posts', count=len(rows))
                for row in rows:
                    md5hash = row[0]
                    created = row[1]
                    post_id = row[2]
                    fetched_dates[md5hash] = created

            cur_time = datetime.datetime.utcnow()
            for item in items:
                if item['md5'] in fetched_dates:
                    item['time'] = fetched_dates[item['md5']]
                else:
                    item['time'] = cur_time
                    self.save_post(conn, cur_time, feed_id, item)
                    new_post_cnt += 1
                    cur_time -= datetime.timedelta(minutes=self.POST_TIME_DISTANCE)
        return new_post_cnt

    def _build_link(self, html, doc_url, url):
        base_url = w3lib.html.get_base_url(html, doc_url)
        return w3lib.url.urljoin_rfc(base_url, url).decode('utf-8')

    def buildFeed(self, selector, page_unicode, feed_config):
        selector.remove_namespaces()

        selector = selector
        tree = selector.root.getroottree()
        # get data from html
        items = []
        for node in selector.xpath(feed_config['xpath']):
            item = {}
            required_count = 0
            required_found = 0
            for field_name in ['title', 'description', 'link']:
                if field_name in feed_config['fields']:
                    if feed_config['required'][field_name]:
                        required_count += 1

                    extracted = node.xpath(feed_config['fields'][field_name]).extract()
                    if extracted:
                        item[field_name] = u''.join(extracted)
                        if feed_config['required'][field_name]:
                            required_found += 1
                        if field_name == 'link':
                            item['link'] = self._build_link(page_unicode, feed_config['uri'], item[field_name])

            if required_count == required_found:
                items.append(item)

        title = selector.xpath('//title/text()').extract_first()

        if not feed_config['name'] and title:
            self.update_feed_name(feed_config['id'], title.encode('utf-8'))

        #build feed
        feed = Rss201rev2Feed(
            title = title if title else 'PolitePol: ' + feed_config['uri'],
            link=feed_config['uri'],
            description="Generated by PolitePol.\n"+\
                "Source page: " + feed_config['uri'],
            language="en",
        )
        new_post_cnt = self.fill_time(feed_config['id'], items)

        for item in items:
            title = item['title'] if 'title' in item else ''
            desc = item['description'] if 'description' in item else ''
            time = item['time'] if 'time' in item else datetime.datetime.utcnow()
            if 'link' in item:
                link = item['link']
            else:
                link = self.url_hash_regexp.sub('#' + md5((title+desc).encode('utf-8')).hexdigest(), feed_config['uri'])
            feed.add_item(
                title = title,
                link = link,
                unique_id = link,
                description = desc,
                #enclosure=Enclosure(fields[4], "32000", "image/jpeg") if  4 in fields else None, #"Image"
                pubdate = time
            )
        return [feed.writeString('utf-8'), len(items), new_post_cnt]

    def update_feed_name(self, feed_id, name):
        with closing(get_conn(self.db_creds)) as conn, conn as cur:
            cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id))

    def getFeedData(self, feed_id):
        # get url, xpathes
        feed = {}

        with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur:
            cur.execute("""select f.name as feed_name, f.uri, f.js, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
                           from frontend_feed f
                           right join frontend_feedfield ff on ff.feed_id=f.id
                           left join frontend_field fi on fi.id=ff.field_id
                           where f.id=%s""", (feed_id,))
            rows = cur.fetchall()

            for row in rows:
                if not feed:
                    feed['id'] = feed_id
                    feed['name'] = row['feed_name']
                    feed['uri'] = row['uri']
                    feed['js'] = row['js']
                    feed['xpath'] = row['feed_xpath']
                    feed['fields'] = {}
                    feed['required'] = {}
                feed['fields'][row['name']] = row['xpath']
                feed['required'][row['name']] = row['required']

        if feed:
            return [feed['uri'], feed]
        else:
            return 'Feed generator error: config of feed is empty'
<commit_msg>no js<commit_after>import w3lib.url
import w3lib.html

from lxml import etree
import re, sys
from hashlib import md5

from feedgenerator import Rss201rev2Feed, Enclosure
import datetime

import MySQLdb
from contextlib import closing
from settings import DATABASES, DOWNLOADER_USER_AGENT
from twisted.logger import Logger

from .db import get_conn

log = Logger()

class Feed(object):

    url_hash_regexp = re.compile('(#.*)?$')

    POST_TIME_DISTANCE = 15 # minutes, RSS Feed Reader skip same titles created in 10 min interval

    FIELD_IDS = {'title': 1, 'description': 2, 'link': 3}

    def __init__(self, db_creds):
        self.db_creds = db_creds

    def save_post(self, conn, created, feed_id, post_fields):
        with conn as cur:
            cur.execute("""insert into frontend_post (md5sum, created, feed_id)
                            values (unhex(%s), %s, %s)""", (post_fields['md5'], created, feed_id))
            post_id = conn.insert_id()
            log.info('Post saved id:{id!r}', id=post_id)
            return post_id

    def fill_time(self, feed_id, items):
        if not items:
            return 0

        new_post_cnt = 0
        for item in items:
            #create md5
            h = md5('')
            for key in ['title', 'description', 'link']:
                if key in item:
                    h.update(item[key].encode('utf-8'))
            item['md5'] = h.hexdigest()

        #fetch dates from db
        fetched_dates = {}
        with closing(get_conn(self.db_creds)) as conn:
            with conn as cur:
                quoted_hashes = ','.join(["unhex('%s')" % (i['md5']) for i in items])

                cur.execute("""select lower(hex(p.md5sum)), p.created, p.id
                               from frontend_post p
                               where p.md5sum in (%s)
                               and p.feed_id=%s""" % (quoted_hashes, feed_id,))
                rows = cur.fetchall()
                log.debug('Selected {count!r} posts', count=len(rows))
                for row in rows:
                    md5hash = row[0]
                    created = row[1]
                    post_id = row[2]
                    fetched_dates[md5hash] = created

            cur_time = datetime.datetime.utcnow()
            for item in items:
                if item['md5'] in fetched_dates:
                    item['time'] = fetched_dates[item['md5']]
                else:
                    item['time'] = cur_time
                    self.save_post(conn, cur_time, feed_id, item)
                    new_post_cnt += 1
                    cur_time -= datetime.timedelta(minutes=self.POST_TIME_DISTANCE)
        return new_post_cnt

    def _build_link(self, html, doc_url, url):
        base_url = w3lib.html.get_base_url(html, doc_url)
        return w3lib.url.urljoin_rfc(base_url, url).decode('utf-8')

    def buildFeed(self, selector, page_unicode, feed_config):
        selector.remove_namespaces()

        selector = selector
        tree = selector.root.getroottree()
        # get data from html
        items = []
        for node in selector.xpath(feed_config['xpath']):
            item = {}
            required_count = 0
            required_found = 0
            for field_name in ['title', 'description', 'link']:
                if field_name in feed_config['fields']:
                    if feed_config['required'][field_name]:
                        required_count += 1

                    extracted = node.xpath(feed_config['fields'][field_name]).extract()
                    if extracted:
                        item[field_name] = u''.join(extracted)
                        if feed_config['required'][field_name]:
                            required_found += 1
                        if field_name == 'link':
                            item['link'] = self._build_link(page_unicode, feed_config['uri'], item[field_name])

            if required_count == required_found:
                items.append(item)

        title = selector.xpath('//title/text()').extract_first()

        if not feed_config['name'] and title:
            self.update_feed_name(feed_config['id'], title.encode('utf-8'))

        #build feed
        feed = Rss201rev2Feed(
            title = title if title else 'PolitePol: ' + feed_config['uri'],
            link=feed_config['uri'],
            description="Generated by PolitePol.\n"+\
                "Source page: " + feed_config['uri'],
            language="en",
        )
        new_post_cnt = self.fill_time(feed_config['id'], items)

        for item in items:
            title = item['title'] if 'title' in item else ''
            desc = item['description'] if 'description' in item else ''
            time = item['time'] if 'time' in item else datetime.datetime.utcnow()
            if 'link' in item:
                link = item['link']
            else:
                link = self.url_hash_regexp.sub('#' + md5((title+desc).encode('utf-8')).hexdigest(), feed_config['uri'])
            feed.add_item(
                title = title,
                link = link,
                unique_id = link,
                description = desc,
                #enclosure=Enclosure(fields[4], "32000", "image/jpeg") if  4 in fields else None, #"Image"
                pubdate = time
            )
        return [feed.writeString('utf-8'), len(items), new_post_cnt]

    def update_feed_name(self, feed_id, name):
        with closing(get_conn(self.db_creds)) as conn, conn as cur:
            cur.execute("""update frontend_feed set name=%s where id=%s""", (name[:255], feed_id))

    def getFeedData(self, feed_id):
        # get url, xpathes
        feed = {}

        with closing(get_conn(self.db_creds, dict_result=True)) as conn, conn as cur:
            cur.execute("""select f.name as feed_name, f.uri, f.xpath as feed_xpath, fi.name, ff.xpath, fi.required
                           from frontend_feed f
                           right join frontend_feedfield ff on ff.feed_id=f.id
                           left join frontend_field fi on fi.id=ff.field_id
                           where f.id=%s""", (feed_id,))
            rows = cur.fetchall()

            for row in rows:
                if not feed:
                    feed['id'] = feed_id
                    feed['name'] = row['feed_name']
                    feed['uri'] = row['uri']
                    feed['xpath'] = row['feed_xpath']
                    feed['fields'] = {}
                    feed['required'] = {}
                feed['fields'][row['name']] = row['xpath']
                feed['required'][row['name']] = row['required']

        if feed:
            return [feed['uri'], feed]
        else:
            return 'Feed generator error: config of feed is empty'
<|endoftext|>


<commit_before>import socket
import requests
import sys
from globalvars import GlobalVars

def watchCi():
    HOST = ''
    PORT = 49494

    s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    print "CI Socket Created"

    try:
        s.bind((HOST, PORT))
    except socket.error as msg:
        print 'Bind Failed. Error code: ' + str(msg[0])
        return

    s.listen(10)
    print 'listening for ci changes'

    while 1:
        conn, addr = s.accept()
        circleci_addr = socket.getaddrinfo("circleci.com", 80, 0, 0, socket.IPPROTO_TCP)[0][4][0]
        is_circleci = addr == circleci_addr
        print 'Received request from ' + addr[0] + " ; " + "verified as CircleCI" if is_circleci else "NOT verified as CircleCI!"
        if not is_circleci:
            GlobalVars.charcoal_hq.send_message("WARNING: got socket that doesn't come from CircleCI")
            s.close()
            continue
        r=requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/git/refs/heads/master')
        latest_sha = r.json()["object"]["sha"]
        r = requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/commits/' + latest_sha + '/statuses')
        states = []
        for status in r.json():
            state = status["state"]
            states.append(state)
        if "success" in states:
            GlobalVars.charcoal_hq.send_message("CI build passed. Ready to pull!")
        elif "error" in states or "failure" in states:
            GlobalVars.charcoal_hq.send_message("CI build failed, *someone* (prolly Undo) borked something!")
    s.close()
<commit_msg>Don't close the socket too soon<commit_after>import socket
import requests
import sys
from globalvars import GlobalVars

def watchCi():
    HOST = ''
    PORT = 49494

    s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    print "CI Socket Created"

    try:
        s.bind((HOST, PORT))
    except socket.error as msg:
        print 'Bind Failed. Error code: ' + str(msg[0])
        return

    s.listen(10)
    print 'listening for ci changes'

    while 1:
        conn, addr = s.accept()
        circleci_addr = socket.getaddrinfo("circleci.com", 80, 0, 0, socket.IPPROTO_TCP)[0][4][0]
        is_circleci = addr == circleci_addr
        print 'Received request from ' + addr[0] + " ; " + "verified as CircleCI" if is_circleci else "NOT verified as CircleCI!"
        if not is_circleci:
            GlobalVars.charcoal_hq.send_message("WARNING: got socket that doesn't come from CircleCI")
            continue
        r=requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/git/refs/heads/master')
        latest_sha = r.json()["object"]["sha"]
        r = requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/commits/' + latest_sha + '/statuses')
        states = []
        for status in r.json():
            state = status["state"]
            states.append(state)
        if "success" in states:
            GlobalVars.charcoal_hq.send_message("CI build passed. Ready to pull!")
        elif "error" in states or "failure" in states:
            GlobalVars.charcoal_hq.send_message("CI build failed, *someone* (prolly Undo) borked something!")
    s.close()
<|endoftext|>


<commit_before>
import os
import pytest

from shapefile import ShapefileException

from landlab.io.shapefile import read_shapefile

_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")

def test_read_methow():
    file = os.path.join(_TEST_DATA_DIR, "Methow_Network.shp")
    grid = read_shapefile(file)
    assert grid.number_of_nodes == 721
    assert grid.number_of_links == 720

    assert grid.x_of_node[0] == -1672349.0889982011
    assert grid.y_of_node[0] == 1160800.240247
    assert "x_of_polyline" in grid.at_link
    assert "y_of_polyline" in grid.at_link

def test_bad_file():
    file = os.path.join(_TEST_DATA_DIR, "bad_file.shp")
    with pytest.raises(ShapefileException):
        read_shapefile(file)

def test_points():
    file = os.path.join(_TEST_DATA_DIR, "points.shp")
    with pytest.raises(ValueError):
        read_shapefile(file)

def test_multipart():
    file = os.path.join(_TEST_DATA_DIR, "multipartpolyline.shp")
    with pytest.raises(ValueError):
        read_shapefile(file)
<commit_msg>adding test for fields that are in the shapefile. <commit_after>
import os
import pytest

from shapefile import ShapefileException

from landlab.io.shapefile import read_shapefile

_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")

def test_read_methow():
    file = os.path.join(_TEST_DATA_DIR, "Methow_Network.shp")
    grid = read_shapefile(file)
    assert grid.number_of_nodes == 721
    assert grid.number_of_links == 720

    assert grid.x_of_node[0] == -1672349.0889982011
    assert grid.y_of_node[0] == 1160800.240247
    assert "x_of_polyline" in grid.at_link
    assert "y_of_polyline" in grid.at_link

    fields = ["Length_m", "ToLink", "usarea_km2", "uselev_m", "dselev_m", "Slope"]
    for field in fields:
        assert field in grid.at_link

def test_bad_file():
    file = os.path.join(_TEST_DATA_DIR, "bad_file.shp")
    with pytest.raises(ShapefileException):
        read_shapefile(file)

def test_points():
    file = os.path.join(_TEST_DATA_DIR, "points.shp")
    with pytest.raises(ValueError):
        read_shapefile(file)

def test_multipart():
    file = os.path.join(_TEST_DATA_DIR, "multipartpolyline.shp")
    with pytest.raises(ValueError):
        read_shapefile(file)
<|endoftext|>


<commit_before># Tweepy
# Copyright 2009-2022 Joshua Roesslein
# See LICENSE for details.

from math import inf

class Paginator:
    """:class:`Paginator` can be used to paginate for any :class:`Client`
    methods that support pagination

    .. versionadded:: 4.0

    Parameters
    ----------
    method
        :class:`Client` method to paginate for
    args
        Positional arguments to pass to ``method``
    kwargs
        Keyword arguments to pass to ``method``
    """

    def __init__(self, method, *args, **kwargs):
        self.method = method
        self.args = args
        self.kwargs = kwargs

    def __iter__(self):
        return PaginationIterator(self.method, *self.args, **self.kwargs)

    def __reversed__(self):
        return PaginationIterator(self.method, *self.args, reverse=True,
                                  **self.kwargs)

    def flatten(self, limit=inf):
        """Flatten paginated data

        Parameters
        ----------
        limit
            Maximum number of results to yield
        """
        if limit <= 0:
            return

        count = 0
        for response in PaginationIterator(self.method, *self.args,
                                           **self.kwargs):
            if response.data is not None:
                for data in response.data:
                    yield data
                    count += 1
                    if count == limit:
                        return

class PaginationIterator:

    def __init__(self, method, *args, limit=inf, pagination_token=None,
                 reverse=False, **kwargs):
        self.method = method
        self.args = args
        self.limit = limit
        self.kwargs = kwargs
        self.reverse = reverse

        if reverse:
            self.previous_token = pagination_token
            self.next_token = None
        else:
            self.previous_token = None
            self.next_token = pagination_token

        self.count = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self.reverse:
            pagination_token = self.previous_token
        else:
            pagination_token = self.next_token

        if self.count >= self.limit or self.count and pagination_token is None:
            raise StopIteration

        # https://twittercommunity.com/t/why-does-timeline-use-pagination-token-while-search-uses-next-token/150963
        if self.method.__name__ in ("search_all_tweets",
                                    "search_recent_tweets",
                                    "get_all_tweets_count"):
            self.kwargs["next_token"] = pagination_token
        else:
            self.kwargs["pagination_token"] = pagination_token

        response = self.method(*self.args, **self.kwargs)

        self.previous_token = response.meta.get("previous_token")
        self.next_token = response.meta.get("next_token")
        self.count += 1

        return response
<commit_msg>Improve PaginationIterator.__next__ formatting<commit_after># Tweepy
# Copyright 2009-2022 Joshua Roesslein
# See LICENSE for details.

from math import inf

class Paginator:
    """:class:`Paginator` can be used to paginate for any :class:`Client`
    methods that support pagination

    .. versionadded:: 4.0

    Parameters
    ----------
    method
        :class:`Client` method to paginate for
    args
        Positional arguments to pass to ``method``
    kwargs
        Keyword arguments to pass to ``method``
    """

    def __init__(self, method, *args, **kwargs):
        self.method = method
        self.args = args
        self.kwargs = kwargs

    def __iter__(self):
        return PaginationIterator(self.method, *self.args, **self.kwargs)

    def __reversed__(self):
        return PaginationIterator(self.method, *self.args, reverse=True,
                                  **self.kwargs)

    def flatten(self, limit=inf):
        """Flatten paginated data

        Parameters
        ----------
        limit
            Maximum number of results to yield
        """
        if limit <= 0:
            return

        count = 0
        for response in PaginationIterator(self.method, *self.args,
                                           **self.kwargs):
            if response.data is not None:
                for data in response.data:
                    yield data
                    count += 1
                    if count == limit:
                        return

class PaginationIterator:

    def __init__(self, method, *args, limit=inf, pagination_token=None,
                 reverse=False, **kwargs):
        self.method = method
        self.args = args
        self.limit = limit
        self.kwargs = kwargs
        self.reverse = reverse

        if reverse:
            self.previous_token = pagination_token
            self.next_token = None
        else:
            self.previous_token = None
            self.next_token = pagination_token

        self.count = 0

    def __iter__(self):
        return self

    def __next__(self):
        if self.reverse:
            pagination_token = self.previous_token
        else:
            pagination_token = self.next_token

        if self.count >= self.limit or self.count and pagination_token is None:
            raise StopIteration

        # https://twittercommunity.com/t/why-does-timeline-use-pagination-token-while-search-uses-next-token/150963
        if self.method.__name__ in (
            "search_all_tweets", "search_recent_tweets",
            "get_all_tweets_count"
        ):
            self.kwargs["next_token"] = pagination_token
        else:
            self.kwargs["pagination_token"] = pagination_token

        response = self.method(*self.args, **self.kwargs)

        self.previous_token = response.meta.get("previous_token")
        self.next_token = response.meta.get("next_token")
        self.count += 1

        return response
<|endoftext|>


<commit_before>"""
The owsproxy is based on `papyrus_ogcproxy <https://github.com/elemoine/papyrus_ogcproxy>`_
"""

import urllib
from httplib2 import Http

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPBadRequest, HTTPBadGateway, HTTPNotAcceptable
from pyramid.response import Response

from twitcher.registry import registry_factory

import logging
logger = logging.getLogger(__name__)

allowed_content_types = (
    "application/xml", "text/xml",
    "application/vnd.ogc.se_xml",           # OGC Service Exception
    "application/vnd.ogc.se+xml",           # OGC Service Exception
    #"application/vnd.ogc.success+xml",      # OGC Success (SLD Put)
    #"application/vnd.ogc.wms_xml",          # WMS Capabilities
    #"application/vnd.ogc.gml",              # GML
    #"application/vnd.ogc.sld+xml",          # SLD
    #"application/vnd.google-earth.kml+xml", # KML
    )

class OWSProxy(object):
    def __init__(self, request):
        self.request = request

    def send_request(self, service):
        # TODO: fix way to build url
        logger.debug('params = %s', self.request.params)
        url = service['url'] + '?' + urllib.urlencode(self.request.params)

        # forward request to target (without Host Header)
        http = Http(disable_ssl_certificate_validation=True)
        h = dict(self.request.headers)
        h.pop("Host", h)
        try:
            resp, content = http.request(url, method=self.request.method, body=self.request.body, headers=h)
        except:
            return HTTPBadGateway()

        # check for allowed content types
        if resp.has_key("content-type"):
            ct = resp["content-type"]
            if not ct.split(";")[0] in allowed_content_types:
                return HTTPForbidden()
        else:
            return HTTPNotAcceptable()

        # replace urls in xml content
        if 'xml' in ct:
            content = content.replace(service['url'], service['proxy_url'])

        return Response(content, status=resp.status, headers={"Content-Type": ct})

    @view_config(route_name='owsproxy')
    @view_config(route_name='owsproxy_secured')
    def owsproxy(self):
        service_name = self.request.matchdict.get('service_name')
        if service_name is None:
            return HTTPBadRequest('Param service_name is required')

        service = None
        try:
            registry = registry_factory(request)
            service = reqistry.get_service(service_name)
        except Exception as err:
            return HTTPBadRequest("Could not find service: %s" % (err.message))

        return self.send_request(service)

def includeme(config):
    # include mongodb
    config.include('twitcher.db')

    config.add_route('owsproxy', '/ows/proxy/{service_name}')
    config.add_route('owsproxy_secured', '/ows/proxy/{service_name}/{access_token}')
<commit_msg>fixed owsproxy ... request, registry not found<commit_after>"""
The owsproxy is based on `papyrus_ogcproxy <https://github.com/elemoine/papyrus_ogcproxy>`_
"""

import urllib
from httplib2 import Http

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPBadRequest, HTTPBadGateway, HTTPNotAcceptable
from pyramid.response import Response

from twitcher.registry import registry_factory

import logging
logger = logging.getLogger(__name__)

allowed_content_types = (
    "application/xml", "text/xml",
    "application/vnd.ogc.se_xml",           # OGC Service Exception
    "application/vnd.ogc.se+xml",           # OGC Service Exception
    #"application/vnd.ogc.success+xml",      # OGC Success (SLD Put)
    #"application/vnd.ogc.wms_xml",          # WMS Capabilities
    #"application/vnd.ogc.gml",              # GML
    #"application/vnd.ogc.sld+xml",          # SLD
    #"application/vnd.google-earth.kml+xml", # KML
    )

class OWSProxy(object):
    def __init__(self, request):
        self.request = request

    def send_request(self, service):
        # TODO: fix way to build url
        logger.debug('params = %s', self.request.params)
        url = service['url'] + '?' + urllib.urlencode(self.request.params)

        # forward request to target (without Host Header)
        http = Http(disable_ssl_certificate_validation=True)
        h = dict(self.request.headers)
        h.pop("Host", h)
        try:
            resp, content = http.request(url, method=self.request.method, body=self.request.body, headers=h)
        except:
            return HTTPBadGateway()

        # check for allowed content types
        if resp.has_key("content-type"):
            ct = resp["content-type"]
            if not ct.split(";")[0] in allowed_content_types:
                return HTTPForbidden()
        else:
            return HTTPNotAcceptable()

        # replace urls in xml content
        if 'xml' in ct:
            content = content.replace(service['url'], service['proxy_url'])

        return Response(content, status=resp.status, headers={"Content-Type": ct})

    @view_config(route_name='owsproxy')
    @view_config(route_name='owsproxy_secured')
    def owsproxy(self):
        service_name = self.request.matchdict.get('service_name')
        if service_name is None:
            return HTTPBadRequest('Param service_name is required')

        service = None
        try:
            registry = registry_factory(self.request)
            service = registry.get_service(service_name)
        except Exception as err:
            return HTTPBadRequest("Could not find service: %s" % (err.message))

        return self.send_request(service)

def includeme(config):
    # include mongodb
    config.include('twitcher.db')

    config.add_route('owsproxy', '/ows/proxy/{service_name}')
    config.add_route('owsproxy_secured', '/ows/proxy/{service_name}/{access_token}')
<|endoftext|>


<commit_before># Copyright 2004-2010 OpenERP SA
# Copyright 2014-2018 Tecnativa - Pedro M. Baeza
# Copyright 2015 Domatix
# Copyright 2016-2018 Tecnativa - Carlos Dauden
# Copyright 2017 Tecnativa - Vicent Cubells
# Copyright 2016-2017 LasLabs Inc.
# Copyright 2018-2019 ACSONE SA/NV
# Copyright 2020-2021 Tecnativa - Pedro M. Baeza
# Copyright 2020 Tecnativa - Víctor Martínez
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Recurring - Contracts Management",
    "version": "14.0.2.3.3",
    "category": "Contract Management",
    "license": "AGPL-3",
    "author": "Tecnativa, ACSONE SA/NV, Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/contract",
    "depends": ["base", "account", "product", "portal"],
    "data": [
        "security/groups.xml",
        "security/contract_tag.xml",
        "security/ir.model.access.csv",
        "security/contract_security.xml",
        "security/contract_terminate_reason.xml",
        "report/report_contract.xml",
        "report/contract_views.xml",
        "data/contract_cron.xml",
        "data/contract_renew_cron.xml",
        "data/mail_template.xml",
        "data/template_mail_notification.xml",
        "data/mail_message_subtype.xml",
        "data/ir_ui_menu.xml",
        "wizards/contract_line_wizard.xml",
        "wizards/contract_manually_create_invoice.xml",
        "wizards/contract_contract_terminate.xml",
        "views/contract_tag.xml",
        "views/assets.xml",
        "views/abstract_contract_line.xml",
        "views/contract.xml",
        "views/contract_line.xml",
        "views/contract_template.xml",
        "views/contract_template_line.xml",
        "views/res_partner_view.xml",
        "views/res_config_settings.xml",
        "views/contract_terminate_reason.xml",
        "views/contract_portal_templates.xml",
    ],
    "installable": True,
}
<commit_msg>contract 14.0.2.3.4<commit_after># Copyright 2004-2010 OpenERP SA
# Copyright 2014-2018 Tecnativa - Pedro M. Baeza
# Copyright 2015 Domatix
# Copyright 2016-2018 Tecnativa - Carlos Dauden
# Copyright 2017 Tecnativa - Vicent Cubells
# Copyright 2016-2017 LasLabs Inc.
# Copyright 2018-2019 ACSONE SA/NV
# Copyright 2020-2021 Tecnativa - Pedro M. Baeza
# Copyright 2020 Tecnativa - Víctor Martínez
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

{
    "name": "Recurring - Contracts Management",
    "version": "14.0.2.3.4",
    "category": "Contract Management",
    "license": "AGPL-3",
    "author": "Tecnativa, ACSONE SA/NV, Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/contract",
    "depends": ["base", "account", "product", "portal"],
    "data": [
        "security/groups.xml",
        "security/contract_tag.xml",
        "security/ir.model.access.csv",
        "security/contract_security.xml",
        "security/contract_terminate_reason.xml",
        "report/report_contract.xml",
        "report/contract_views.xml",
        "data/contract_cron.xml",
        "data/contract_renew_cron.xml",
        "data/mail_template.xml",
        "data/template_mail_notification.xml",
        "data/mail_message_subtype.xml",
        "data/ir_ui_menu.xml",
        "wizards/contract_line_wizard.xml",
        "wizards/contract_manually_create_invoice.xml",
        "wizards/contract_contract_terminate.xml",
        "views/contract_tag.xml",
        "views/assets.xml",
        "views/abstract_contract_line.xml",
        "views/contract.xml",
        "views/contract_line.xml",
        "views/contract_template.xml",
        "views/contract_template_line.xml",
        "views/res_partner_view.xml",
        "views/res_config_settings.xml",
        "views/contract_terminate_reason.xml",
        "views/contract_portal_templates.xml",
    ],
    "installable": True,
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, osv
import re
from openerp.report.render.rml2pdf import customfonts

class base_config_settings(osv.osv_memory):
    _name = 'base.config.settings'
    _inherit = 'res.config.settings'

    _columns = {
        'module_multi_company': fields.boolean('Manage multiple companies',
            help='Work in multi-company environments, with appropriate security access between companies.\n'
                 '-This installs the module multi_company.'),
        'module_share': fields.boolean('Allow documents sharing',
            help="""Share or embbed any screen of openerp."""),
        'module_portal': fields.boolean('Activate the customer portal',
            help="""Give your customers access to their documents."""),
        'module_portal_anonymous': fields.boolean('Activate the public portal',
            help="""Enable the public part of openerp, openerp becomes a public website."""),
        'module_auth_oauth': fields.boolean('Use external authentication providers, sign in with google, facebook, ...'),
        'module_base_import': fields.boolean("Allow users to import data from CSV files"),
        'module_google_drive': fields.boolean('Attach Google documents to any record',
                                              help="""This installs the module google_docs."""),
        'font': fields.many2one('res.font', string="Report Font", help="Set the font into the report header, it will be used as default font in the RML reports of the user company"),
    }

    _defaults= {
        'font': lambda self,cr,uid,c: self.pool.get('res.users').browse(cr, uid, uid, c).company_id.font.id,
    }

    def open_company(self, cr, uid, ids, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context)
        return {
            'type': 'ir.actions.act_window',
            'name': 'Your Company',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'res.company',
            'res_id': user.company_id.id,
            'target': 'current',
        }

    def _change_header(self, header,font):
        """ Replace default fontname use in header and setfont tag """

        default_para = re.sub('fontName.?=.?".*"', 'fontName="%s"'% font,header)
        return re.sub('(<setFont.?name.?=.?)(".*?")(.)', '\g<1>"%s"\g<3>'% font,default_para)

    def set_base_defaults(self, cr, uid, ids, context=None):
        ir_model_data = self.pool.get('ir.model.data')
        wizard = self.browse(cr, uid, ids)[0]
        if wizard.font:
            user = self.pool.get('res.users').browse(cr, uid, uid, context)
            font_name = wizard.font.name
            user.company_id.write({'font': wizard.font.id,'rml_header': self._change_header(user.company_id.rml_header,font_name), 'rml_header2': self._change_header(user.company_id.rml_header2, font_name), 'rml_header3': self._change_header(user.company_id.rml_header3, font_name)})
        return {}

    def act_discover_fonts(self, cr, uid, ids, context=None):
        return self.pool.get("res.font").init_scan(cr, uid, context)

# Preferences wizard for Sales & CRM.
# It is defined here because it is inherited independently in modules sale, crm,
# plugin_outlook and plugin_thunderbird.
class sale_config_settings(osv.osv_memory):
    _name = 'sale.config.settings'
    _inherit = 'res.config.settings'
    _columns = {
        'module_web_linkedin': fields.boolean('Get contacts automatically from linkedIn',
            help="""When you create a new contact (person or company), you will be able to load all the data from LinkedIn (photos, address, etc)."""),
        'module_crm': fields.boolean('CRM'),
        'module_sale' : fields.boolean('SALE'),
        'module_plugin_thunderbird': fields.boolean('Enable Thunderbird plug-in',
            help='The plugin allows you archive email and its attachments to the selected '
                 'OpenERP objects. You can select a partner, or a lead and '
                 'attach the selected mail as a .eml file in '
                 'the attachment of a selected record. You can create documents for CRM Lead, '
                 'Partner from the selected emails.\n'
                 '-This installs the module plugin_thunderbird.'),
        'module_plugin_outlook': fields.boolean('Enable Outlook plug-in',
            help='The Outlook plugin allows you to select an object that you would like to add '
                 'to your email and its attachments from MS Outlook. You can select a partner, '
                 'or a lead object and archive a selected email into an OpenERP mail message with attachments.\n'
                 '-This installs the module plugin_outlook.'),
        'module_mass_mailing': fields.boolean(
            'Manage mass mailing campaigns',
            help='Get access to statistics with your mass mailing, manage campaigns.'),
    }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<commit_msg>[FIX] base_setup: adapt api<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Business Applications
#    Copyright (C) 2004-2012 OpenERP S.A. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, osv
import re
from openerp.report.render.rml2pdf import customfonts

class base_config_settings(osv.osv_memory):
    _name = 'base.config.settings'
    _inherit = 'res.config.settings'

    _columns = {
        'module_multi_company': fields.boolean('Manage multiple companies',
            help='Work in multi-company environments, with appropriate security access between companies.\n'
                 '-This installs the module multi_company.'),
        'module_share': fields.boolean('Allow documents sharing',
            help="""Share or embbed any screen of openerp."""),
        'module_portal': fields.boolean('Activate the customer portal',
            help="""Give your customers access to their documents."""),
        'module_portal_anonymous': fields.boolean('Activate the public portal',
            help="""Enable the public part of openerp, openerp becomes a public website."""),
        'module_auth_oauth': fields.boolean('Use external authentication providers, sign in with google, facebook, ...'),
        'module_base_import': fields.boolean("Allow users to import data from CSV files"),
        'module_google_drive': fields.boolean('Attach Google documents to any record',
                                              help="""This installs the module google_docs."""),
        'font': fields.many2one('res.font', string="Report Font", help="Set the font into the report header, it will be used as default font in the RML reports of the user company"),
    }

    _defaults= {
        'font': lambda self,cr,uid,c: self.pool.get('res.users').browse(cr, uid, uid, c).company_id.font.id,
    }

    def open_company(self, cr, uid, ids, context=None):
        user = self.pool.get('res.users').browse(cr, uid, uid, context)
        return {
            'type': 'ir.actions.act_window',
            'name': 'Your Company',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'res.company',
            'res_id': user.company_id.id,
            'target': 'current',
        }

    def _change_header(self, header,font):
        """ Replace default fontname use in header and setfont tag """

        default_para = re.sub('fontName.?=.?".*"', 'fontName="%s"'% font,header)
        return re.sub('(<setFont.?name.?=.?)(".*?")(.)', '\g<1>"%s"\g<3>'% font,default_para)

    def set_base_defaults(self, cr, uid, ids, context=None):
        ir_model_data = self.pool.get('ir.model.data')
        wizard = self.browse(cr, uid, ids)[0]
        if wizard.font:
            user = self.pool.get('res.users').browse(cr, uid, uid, context)
            font_name = wizard.font.name
            user.company_id.write({'font': wizard.font.id,'rml_header': self._change_header(user.company_id.rml_header,font_name), 'rml_header2': self._change_header(user.company_id.rml_header2, font_name), 'rml_header3': self._change_header(user.company_id.rml_header3, font_name)})
        return {}

    def act_discover_fonts(self, cr, uid, ids, context=None):
        return self.pool.get("res.font").font_scan(cr, uid, context)

# Preferences wizard for Sales & CRM.
# It is defined here because it is inherited independently in modules sale, crm,
# plugin_outlook and plugin_thunderbird.
class sale_config_settings(osv.osv_memory):
    _name = 'sale.config.settings'
    _inherit = 'res.config.settings'
    _columns = {
        'module_web_linkedin': fields.boolean('Get contacts automatically from linkedIn',
            help="""When you create a new contact (person or company), you will be able to load all the data from LinkedIn (photos, address, etc)."""),
        'module_crm': fields.boolean('CRM'),
        'module_sale' : fields.boolean('SALE'),
        'module_plugin_thunderbird': fields.boolean('Enable Thunderbird plug-in',
            help='The plugin allows you archive email and its attachments to the selected '
                 'OpenERP objects. You can select a partner, or a lead and '
                 'attach the selected mail as a .eml file in '
                 'the attachment of a selected record. You can create documents for CRM Lead, '
                 'Partner from the selected emails.\n'
                 '-This installs the module plugin_thunderbird.'),
        'module_plugin_outlook': fields.boolean('Enable Outlook plug-in',
            help='The Outlook plugin allows you to select an object that you would like to add '
                 'to your email and its attachments from MS Outlook. You can select a partner, '
                 'or a lead object and archive a selected email into an OpenERP mail message with attachments.\n'
                 '-This installs the module plugin_outlook.'),
        'module_mass_mailing': fields.boolean(
            'Manage mass mailing campaigns',
            help='Get access to statistics with your mass mailing, manage campaigns.'),
    }

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
from tools.translate import _

class product(osv.osv):
    _inherit = 'product.product'
    _columns = {
        'event_ok': fields.boolean('Event Subscription', help='Determine if a product needs to create automatically an event registration at the confirmation of a sale order line.'),
        'event_type_id': fields.many2one('event.type', 'Type of Event', help='Filter the list of event on this category only, in the sale order lines'),
    }

    def onchange_event_ok(self, cr, uid, ids, event_ok, context=None):
        return {'value': {'type': event_ok and 'service' or False}}

product()

class sale_order_line(osv.osv):
    _inherit = 'sale.order.line'
    _columns = {
        'event_id': fields.many2one('event.event', 'Event', help="Choose an event and it will authomaticaly create a registration for this event"),
        #those 2 fields are used for dynamic domains and filled by onchange
        'event_type_id': fields.related('event_type_id', type='many2one', relation="event.type", string="Event Type"),
        'event_ok': fields.related('event_ok', string='event_ok', type='boolean'),
    }

    def product_id_change(self, cr, uid, ids,
                          pricelist,
                          product, qty=0,
                          uom=False,
                          qty_uos=0,
                          uos=False,
                          name='',
                          partner_id=False,
                          lang=False,
                          update_tax=True,
                          date_order=False,
                          packaging=False,
                          fiscal_position=False,
                          flag=False, context=None):
        """
        check product if event type
        """
        res = super(sale_order_line,self).product_id_change(cr, uid, ids, pricelist, product, qty=qty, uom=uom, qty_uos=qty_uos, uos=uos, name=name, partner_id=partner_id, lang=lang, update_tax=update_tax, date_order=date_order, packaging=packaging, fiscal_position=fiscal_position, flag=flag, context=context)
        if product:
            product_res = self.pool.get('product.product').browse(cr, uid, product, context=context)
            if product_res.event_ok:
                res['value'].update({'event_type_id': product_res.event_type_id.id, 'event_ok':product_res.event_ok})
        return res

    def button_confirm(self, cr, uid, ids, context=None):
        '''
        create registration with sale order
        '''
        registration_obj = self.pool.get('event.registration')
        sale_obj = self.pool.get('sale.order')
        for order_line in self.browse(cr, uid, ids, context=context):
            if order_line.event_id.id:
                dic = {
                    'name': order_line.order_id.partner_invoice_id.name,
                    'partner_id': order_line.order_id.partner_id.id,
                    'contact_id': order_line.order_id.partner_invoice_id.id,
                    'nb_register': int(order_line.product_uom_qty),
                    'email': order_line.order_id.partner_id.email,
                    'phone': order_line.order_id.partner_id.phone,
                    'street': order_line.order_id.partner_invoice_id.street,
                    'city': order_line.order_id.partner_invoice_id.city,
                    'origin': order_line.order_id.name,
                    'event_id': order_line.event_id.id,
                }
                registration_id = registration_obj.create(cr, uid, dic, context=context)
                message = _("The registration %s has been created from the Sale Order %s.") % (registration_id, order_line.order_id.name)
                registration_obj.log(cr, uid, registration_id, message)
        return super(sale_order_line, self).button_confirm(cr, uid, ids, context=context)
<commit_msg>[IMP]event_sale: remove contact_id<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
from tools.translate import _

class product(osv.osv):
    _inherit = 'product.product'
    _columns = {
        'event_ok': fields.boolean('Event Subscription', help='Determine if a product needs to create automatically an event registration at the confirmation of a sale order line.'),
        'event_type_id': fields.many2one('event.type', 'Type of Event', help='Filter the list of event on this category only, in the sale order lines'),
    }

    def onchange_event_ok(self, cr, uid, ids, event_ok, context=None):
        return {'value': {'type': event_ok and 'service' or False}}

product()

class sale_order_line(osv.osv):
    _inherit = 'sale.order.line'
    _columns = {
        'event_id': fields.many2one('event.event', 'Event', help="Choose an event and it will authomaticaly create a registration for this event"),
        #those 2 fields are used for dynamic domains and filled by onchange
        'event_type_id': fields.related('event_type_id', type='many2one', relation="event.type", string="Event Type"),
        'event_ok': fields.related('event_ok', string='event_ok', type='boolean'),
    }

    def product_id_change(self, cr, uid, ids,
                          pricelist,
                          product, qty=0,
                          uom=False,
                          qty_uos=0,
                          uos=False,
                          name='',
                          partner_id=False,
                          lang=False,
                          update_tax=True,
                          date_order=False,
                          packaging=False,
                          fiscal_position=False,
                          flag=False, context=None):
        """
        check product if event type
        """
        res = super(sale_order_line,self).product_id_change(cr, uid, ids, pricelist, product, qty=qty, uom=uom, qty_uos=qty_uos, uos=uos, name=name, partner_id=partner_id, lang=lang, update_tax=update_tax, date_order=date_order, packaging=packaging, fiscal_position=fiscal_position, flag=flag, context=context)
        if product:
            product_res = self.pool.get('product.product').browse(cr, uid, product, context=context)
            if product_res.event_ok:
                res['value'].update({'event_type_id': product_res.event_type_id.id, 'event_ok':product_res.event_ok})
        return res

    def button_confirm(self, cr, uid, ids, context=None):
        '''
        create registration with sale order
        '''
        registration_obj = self.pool.get('event.registration')
        sale_obj = self.pool.get('sale.order')
        for order_line in self.browse(cr, uid, ids, context=context):
            if order_line.event_id.id:
                dic = {
                    'name': order_line.order_id.partner_invoice_id.name,
                    'partner_id': order_line.order_id.partner_id.id,
                    'nb_register': int(order_line.product_uom_qty),
                    'email': order_line.order_id.partner_id.email,
                    'phone': order_line.order_id.partner_id.phone,
                    'street': order_line.order_id.partner_invoice_id.street,
                    'city': order_line.order_id.partner_invoice_id.city,
                    'origin': order_line.order_id.name,
                    'event_id': order_line.event_id.id,
                }
                registration_id = registration_obj.create(cr, uid, dic, context=context)
                message = _("The registration %s has been created from the Sale Order %s.") % (registration_id, order_line.order_id.name)
                registration_obj.message_append_note(cr, uid, [registration_id], body=message, context=context)
        return super(sale_order_line, self).button_confirm(cr, uid, ids, context=context)
<|endoftext|>