<commit_before>from django.conf import settings
from django.db import models
from django.utils import timezone
from phonenumber_field.modelfields import PhoneNumberField
from solo.models import SingletonModel
from .utils import class_name_pk

# Create your models here.

class Client(models.Model):
    """
    """
    active = models.BooleanField(default=False)
    name = models.CharField(max_length=300, blank=True, null=True)
    address = models.TextField(blank=True, null=True)

    def __unicode__(self):
        return self.name

class Company(SingletonModel):
    """
    """
    name = models.CharField(max_length=255)
    address = models.TextField()

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name = u'Company'

class Contact(models.Model):
    """
    Client, First Name, Last Name, Title, Email, Office Phone, Mobile Phone,
    Fax
    """
    active = models.BooleanField(default=False)
    client = models.ForeignKey(Client, blank=True, null=True)
    first_name = models.CharField(max_length=300, blank=True, null=True)
    last_name = models.CharField(max_length=300, blank=True, null=True)
    title = models.CharField(max_length=300, blank=True, null=True)
    email = models.EmailField(blank=True, null=True)
    mobile_phone = PhoneNumberField(blank=True, null=True)
    office_phone = PhoneNumberField(blank=True, null=True)
    fax = PhoneNumberField(blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Contract(models.Model):
    """
    """

    def __unicode__(self):
        return class_name_pk(self)

class Estimate(models.Model):
    """
    Issue Date, Estimate ID, Client, Subject, Estimate Amount, Subtotal,
    Discount, Tax, Tax2, Currency, Accepted Date, Declined Date
    """
    issue_date = models.DateField(blank=True, null=True)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    subject = models.CharField(max_length=300, blank=True, null=True)
    estimate_id = models.IntegerField(blank=True, null=True)
    estimate_amount = models.DecimalField(blank=True,
                                          null=True,
                                          max_digits=12,
                                          decimal_places=2)
    subject = models.CharField(max_length=300, blank=True, null=True)
    discount = models.IntegerField(blank=True, null=True)
    tax = models.IntegerField(blank=True, null=True)
    tax2 = models.IntegerField(blank=True, null=True)
    currency = models.CharField(max_length=300,
                                blank=True,
                                default='United States Dollar - USD',
                                null=True, )
    accepted_date = models.DateField(blank=True, null=True)
    declined_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Invoice(models.Model):
    """
    Issue Date, Last Payment Date, Invoice ID, PO Number, Client, Subject,
    Invoice Amount, Paid Amount, Balance, Subtotal, Discount, Tax, Tax2,
    Currency, Currency Symbol, Document Type
    """
    issue_date = models.DateField(blank=True, null=True)
    last_payment_date = models.DateField(blank=True, null=True)
    invoice_id = models.IntegerField(blank=True, null=True)
    po_number = models.CharField(max_length=300, blank=True, null=True)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    subject = models.CharField(max_length=300, blank=True, null=True)
    invoice_amount = models.DecimalField(blank=True,
                                         null=True,
                                         max_digits=12,
                                         decimal_places=2)
    paid_amount = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    balance = models.DecimalField(blank=True,
                                  null=True,
                                  max_digits=12,
                                  decimal_places=2)
    subtotal = models.DecimalField(blank=True,
                                   null=True,
                                   max_digits=12,
                                   decimal_places=2)
    discount = models.IntegerField(blank=True, null=True)
    tax = models.IntegerField(blank=True, null=True)
    tax2 = models.IntegerField(blank=True, null=True)
    project = models.ForeignKey('Project', blank=True, null=True)
    currency = models.CharField(max_length=300, blank=True, null=True)
    currency_symbol = models.CharField(max_length=300, blank=True, null=True)
    document_type = models.CharField(max_length=300, blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Project(models.Model):
    """
    Client, Project, Project Code, Start Date, End Date, Project Notes,
    Total Hours, Billable Hours, Billable Amount, Budget, Budget Spent,
    Budget Remaining, Total Costs, Team Costs, Expenses
    """
    active = models.BooleanField(default=False)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    name = models.CharField(max_length=300, blank=True, null=True, unique=True)
    code = models.IntegerField(blank=True, null=True)
    start_date = models.DateField(blank=True, null=True)
    end_date = models.DateField(blank=True, null=True)
    notes = models.TextField(blank=True, null=True)
    total_hours = models.FloatField(blank=True, null=True)
    billable_hours = models.FloatField(blank=True, null=True)
    billable_amount = models.DecimalField(blank=True,
                                          null=True,
                                          max_digits=12,
                                          decimal_places=2)
    budget = models.DecimalField(blank=True,
                                 null=True,
                                 max_digits=12,
                                 decimal_places=2)
    budget_spent = models.DecimalField(blank=True,
                                       null=True,
                                       max_digits=12,
                                       decimal_places=2)
    budget_remaining = models.DecimalField(blank=True,
                                           null=True,
                                           max_digits=12,
                                           decimal_places=2)
    total_costs = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    team_costs = models.DecimalField(blank=True,
                                     null=True,
                                     max_digits=12,
                                     decimal_places=2)
    expenses = models.DecimalField(blank=True,
                                   null=True,
                                   max_digits=12,
                                   decimal_places=2)

    def __unicode__(self):
        return self.name

class Task(models.Model):
    """
    """
    active = models.BooleanField(default=False)
    billable = models.BooleanField(default=True)
    name = models.CharField(max_length=300, blank=True, null=True)
    rate = models.DecimalField(blank=True,
                               null=True,
                               max_digits=12,
                               decimal_places=2)
    unit = models.DurationField('Unit', default='01:00', blank=True, null=True)

    def __unicode__(self):
        return self.name

class Time(models.Model):
    """
    Date, Client, Project, Project Code, Task, Notes, Hours, Billable?,
    Invoiced?, First Name, Last Name, Department, Employee?, Billable
    Rate, Billable Amount, Cost Rate, Cost Amount, Currency,
    External Reference URL
    """
    date = models.DateField(default=timezone.now)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    project = models.ForeignKey(Project,
                                blank=True,
                                null=True,
                                limit_choices_to={'active': True}, )
    project_code = models.IntegerField(blank=True, null=True)
    task = models.ForeignKey(Task,
                             blank=True,
                             null=True,
                             limit_choices_to={'active': True}, )
    notes = models.TextField(blank=True, null=True)
    hours = models.DurationField('Hours',
                                 default='01:00',
                                 blank=True,
                                 null=True)
    billable = models.BooleanField(default=True)
    invoiced = models.BooleanField(default=True)
    department = models.CharField(max_length=300, blank=True, null=True)
    employee = models.BooleanField(default=True)
    cost_rate = models.DecimalField(blank=True,
                                    null=True,
                                    max_digits=12,
                                    decimal_places=2)
    cost_amount = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    currency = models.CharField(max_length=300, blank=True, null=True)
    external_reference_url = models.URLField(blank=True, null=True)
    user = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)
<commit_msg>Update<commit_after>from django.conf import settings
from django.db import models
from django.utils import timezone
from phonenumber_field.modelfields import PhoneNumberField
from solo.models import SingletonModel
from .utils import class_name_pk

# Create your models here.

class Client(models.Model):
    """
    """
    active = models.BooleanField(default=False)
    name = models.CharField(max_length=300, blank=True, null=True)
    address = models.TextField(blank=True, null=True)

    def __unicode__(self):
        return self.name

class Company(SingletonModel):
    """
    """
    name = models.CharField(max_length=255)
    address = models.TextField()

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name = u'Company'

class Contact(models.Model):
    """
    Client, First Name, Last Name, Title, Email, Office Phone, Mobile Phone,
    Fax
    """
    active = models.BooleanField(default=False)
    client = models.ForeignKey(Client, blank=True, null=True)
    first_name = models.CharField(max_length=300, blank=True, null=True)
    last_name = models.CharField(max_length=300, blank=True, null=True)
    title = models.CharField(max_length=300, blank=True, null=True)
    email = models.EmailField(blank=True, null=True)
    mobile_phone = PhoneNumberField(blank=True, null=True)
    office_phone = PhoneNumberField(blank=True, null=True)
    fax = PhoneNumberField(blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Contract(models.Model):
    """
    """

    def __unicode__(self):
        return class_name_pk(self)

class Estimate(models.Model):
    """
    Issue Date, Estimate ID, Client, Subject, Estimate Amount, Subtotal,
    Discount, Tax, Tax2, Currency, Accepted Date, Declined Date
    """
    issue_date = models.DateField(blank=True, null=True)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    subject = models.CharField(max_length=300, blank=True, null=True)
    estimate_amount = models.DecimalField('Estimate Amount',
                                          blank=True,
                                          null=True,
                                          max_digits=12,
                                          decimal_places=2)
    estimate_id = models.IntegerField('Estimate Id', blank=True, null=True)
    discount = models.IntegerField(blank=True, null=True)
    tax = models.IntegerField(blank=True, null=True)
    tax2 = models.IntegerField(blank=True, null=True)
    currency = models.CharField(max_length=300,
                                blank=True,
                                default='United States Dollar - USD',
                                null=True, )
    accepted_date = models.DateField(blank=True, null=True)
    declined_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Invoice(models.Model):
    """
    Issue Date, Last Payment Date, Invoice ID, PO Number, Client, Subject,
    Invoice Amount, Paid Amount, Balance, Subtotal, Discount, Tax, Tax2,
    Currency, Currency Symbol, Document Type
    """
    issue_date = models.DateField(blank=True, null=True)
    last_payment_date = models.DateField(blank=True, null=True)
    invoice_id = models.IntegerField(blank=True, null=True)
    po_number = models.CharField(max_length=300, blank=True, null=True)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    subject = models.CharField(max_length=300, blank=True, null=True)
    invoice_amount = models.DecimalField(blank=True,
                                         null=True,
                                         max_digits=12,
                                         decimal_places=2)
    paid_amount = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    balance = models.DecimalField(blank=True,
                                  null=True,
                                  max_digits=12,
                                  decimal_places=2)
    subtotal = models.DecimalField(blank=True,
                                   null=True,
                                   max_digits=12,
                                   decimal_places=2)
    discount = models.IntegerField(blank=True, null=True)
    tax = models.IntegerField(blank=True, null=True)
    tax2 = models.IntegerField(blank=True, null=True)
    project = models.ForeignKey('Project', blank=True, null=True)
    currency = models.CharField(max_length=300, blank=True, null=True)
    currency_symbol = models.CharField(max_length=300, blank=True, null=True)
    document_type = models.CharField(max_length=300, blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)

class Project(models.Model):
    """
    Client, Project, Project Code, Start Date, End Date, Project Notes,
    Total Hours, Billable Hours, Billable Amount, Budget, Budget Spent,
    Budget Remaining, Total Costs, Team Costs, Expenses
    """
    active = models.BooleanField(default=False)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    name = models.CharField(max_length=300, blank=True, null=True, unique=True)
    code = models.IntegerField(blank=True, null=True)
    start_date = models.DateField(blank=True, null=True)
    end_date = models.DateField(blank=True, null=True)
    notes = models.TextField(blank=True, null=True)
    total_hours = models.FloatField(blank=True, null=True)
    billable_hours = models.FloatField(blank=True, null=True)
    billable_amount = models.DecimalField(blank=True,
                                          null=True,
                                          max_digits=12,
                                          decimal_places=2)
    budget = models.DecimalField(blank=True,
                                 null=True,
                                 max_digits=12,
                                 decimal_places=2)
    budget_spent = models.DecimalField(blank=True,
                                       null=True,
                                       max_digits=12,
                                       decimal_places=2)
    budget_remaining = models.DecimalField(blank=True,
                                           null=True,
                                           max_digits=12,
                                           decimal_places=2)
    total_costs = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    team_costs = models.DecimalField(blank=True,
                                     null=True,
                                     max_digits=12,
                                     decimal_places=2)
    expenses = models.DecimalField(blank=True,
                                   null=True,
                                   max_digits=12,
                                   decimal_places=2)

    def __unicode__(self):
        return self.name

class Task(models.Model):
    """
    """
    active = models.BooleanField(default=False)
    billable = models.BooleanField(default=True)
    name = models.CharField(max_length=300, blank=True, null=True)
    rate = models.DecimalField(blank=True,
                               null=True,
                               max_digits=12,
                               decimal_places=2)
    unit = models.DurationField('Unit', default='01:00', blank=True, null=True)

    def __unicode__(self):
        return self.name

class Time(models.Model):
    """
    Date, Client, Project, Project Code, Task, Notes, Hours, Billable?,
    Invoiced?, First Name, Last Name, Department, Employee?, Billable
    Rate, Billable Amount, Cost Rate, Cost Amount, Currency,
    External Reference URL
    """
    date = models.DateField(default=timezone.now)
    client = models.ForeignKey(Client,
                               blank=True,
                               null=True,
                               limit_choices_to={'active': True}, )
    project = models.ForeignKey(Project,
                                blank=True,
                                null=True,
                                limit_choices_to={'active': True}, )
    project_code = models.IntegerField(blank=True, null=True)
    task = models.ForeignKey(Task,
                             blank=True,
                             null=True,
                             limit_choices_to={'active': True}, )
    notes = models.TextField(blank=True, null=True)
    hours = models.DurationField('Hours',
                                 default='01:00',
                                 blank=True,
                                 null=True)
    billable = models.BooleanField(default=True)
    invoiced = models.BooleanField(default=True)
    department = models.CharField(max_length=300, blank=True, null=True)
    employee = models.BooleanField(default=True)
    cost_rate = models.DecimalField(blank=True,
                                    null=True,
                                    max_digits=12,
                                    decimal_places=2)
    cost_amount = models.DecimalField(blank=True,
                                      null=True,
                                      max_digits=12,
                                      decimal_places=2)
    currency = models.CharField(max_length=300, blank=True, null=True)
    external_reference_url = models.URLField(blank=True, null=True)
    user = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    def __unicode__(self):
        return class_name_pk(self)
<|endoftext|>


<commit_before>"""NoOp Starts wrapper to allow stochastic initial state for deterministic Python environments."""

from typing import Optional

from acme.wrappers import base
import dm_env
import numpy as np

class NoopStartsWrapper(base.EnvironmentWrapper):
  """Implements random noop starts to episodes."""

  def __init__(self,
               environment: dm_env.Environment,
               noop_action: int = 0,
               noop_max: int = 30,
               seed: Optional[int] = None):
    """Initializes a `_NoopReset` wrapper.

    Args:
      environment: An Atari environment.
      noop_action: The noop action used to step the environment for random initialisation.
      noop_max: The maximal number of noop actions at the start of an episode.
      seed: The random seed used to sample the number of noops.
    """
    if noop_max < 0:
      raise ValueError(
        "Maximal number of no-ops after reset cannot be negative. "
        f"Received noop_max={noop_max}"
      )

    super().__init__(environment)
    self.np_random = np.random.RandomState(seed)
    self._noop_max = noop_max
    self._noop_action = noop_action

  def reset(self) -> dm_env.TimeStep:
    """Resets environment and provides the first timestep."""
    noops = self.np_random.randint(self._noop_max + 1)
    timestep = self.environment.reset()
    for _ in range(noops):
      timestep = self.environment.step(self._noop_action)
      if timestep.last():
        timestep = self.environment.reset()

    return timestep._replace(step_type=dm_env.StepType.FIRST)
<commit_msg>docs: correct documentation now wrapper is not Atari specific<commit_after>"""NoOp Starts wrapper to allow stochastic initial state for deterministic Python environments."""

from typing import Optional

from acme.wrappers import base
import dm_env
import numpy as np

class NoopStartsWrapper(base.EnvironmentWrapper):
  """Implements random noop starts to episodes.

  This introduces randomness into an otherwise deterministic environment.

  Note that the base environment must support a no-op action and the value
  of this action must be known and provided to this wrapper.
  """

  def __init__(self,
               environment: dm_env.Environment,
               noop_action: int = 0,
               noop_max: int = 30,
               seed: Optional[int] = None):
    """Initializes a `NoopStartsWrapper` wrapper.

    Args:
      environment: An environment conforming to the dm_env.Environment interface.
      noop_action: The noop action used to step the environment for random initialisation.
      noop_max: The maximal number of noop actions at the start of an episode.
      seed: The random seed used to sample the number of noops.
    """
    if noop_max < 0:
      raise ValueError(
        "Maximal number of no-ops after reset cannot be negative. "
        f"Received noop_max={noop_max}"
      )

    super().__init__(environment)
    self.np_random = np.random.RandomState(seed)
    self._noop_max = noop_max
    self._noop_action = noop_action

  def reset(self) -> dm_env.TimeStep:
    """Resets environment and provides the first timestep."""
    noops = self.np_random.randint(self._noop_max + 1)
    timestep = self.environment.reset()
    for _ in range(noops):
      timestep = self.environment.step(self._noop_action)
      if timestep.last():
        timestep = self.environment.reset()

    return timestep._replace(step_type=dm_env.StepType.FIRST)
<|endoftext|>


<commit_before># Copyright (C) 2010-2012 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import logging

from lib.cuckoo.common.exceptions import CuckooMachineError
from lib.cuckoo.common.exceptions import CuckooOperationalError
from lib.cuckoo.common.exceptions import CuckooReportError
from lib.cuckoo.common.objects import Dictionary
from lib.cuckoo.common.utils import create_folder
from lib.cuckoo.core.database import Database

try:
    import libvirt
except ImportError:
    raise CuckooDependencyError("Unable to import libvirt")

log = logging.getLogger(__name__)

class MachineManager(object):
    """Base abstract class for analysis machine manager."""

    def __init__(self):
        self.module_name = ""
        self.options = None
        # Database pointer.
        self.db = Database()
        # Machine table is cleaned to be filled from configuration file at each start.
        self.db.clean_machines()

    def set_options(self, options):
        """Set machine manager options.
        @param options: machine manager options dict.
        """
        self.options = options

    def initialize(self, module_name):
        """Read and load machines configuration, try to check the configuration.
        @param module_name: module name.
        """
        # Load.
        self._initialize(module_name)

        # Run initialization checks.
        self._initialize_check()

    def _initialize(self, module_name):
        """Read configuration.
        @param module_name: module name.
        """
        self.module_name = module_name
        mmanager_opts = self.options.get(module_name)

        for machine_id in mmanager_opts["machines"].strip().split(","):
            # Parse from configuration file.
            try:
                machine_opts = self.options.get(machine_id.strip())
                machine = Dictionary()
                machine.id = machine_id.strip()
                machine.label = machine_opts["label"].strip()
                machine.platform = machine_opts["platform"].strip()
                machine.ip = machine_opts["ip"].strip()
                # Insert in db.
                self.db.add_machine(name=machine.id,
                                    label=machine.label,
                                    ip=machine.ip,
                                    platform=machine.platform)
            except (AttributeError, CuckooOperationalError):
                log.warning("Configuration details about machine %s are "
                            "missing. Continue" % machine_id)
                continue

    def _initialize_check(self):
        """Runs checks against virtualization software when a machine manager
        is initialized.
        @note: in machine manager modules you may override or superclass
               his method.
        @raise CuckooMachineError: if a misconfiguration or a unkown vm state
                                   is found.
        """
        try:
            configured_vm = self._list()
            for machine in self.machines():
                if machine.label not in configured_vm:
                    raise CuckooMachineError("Configured machine %s was not "
                        "detected or it's not in proper state" % machine.label)
        except NotImplementedError:
            pass

    def machines(self):
        """List virtual machines.
        @return: virtual machines list
        """
        return self.db.list_machines()

    def availables(self):
        """How many machines are free.
        @return: free machines count.
        """
        return self.db.count_machines_available()

    def acquire(self, machine_id=None, platform=None):
        """Acquire a machine to start analysis.
        @param machine_id: machine ID.
        @param platform: machine platform.
        @return: machine or None.
        """
        if machine_id:
            return self.db.lock_machine_by_name(machine_id)
        elif platform:
            return self.db.lock_machine_by_platform(platform)
        else:
            return self.db.lock_machine()

    def release(self, label=None):
        """Release a machine.
        @param label: machine name.
        """
        self.db.unlock_machine(label)

    def running(self):
        """Returns running virtual machines.
        @return: running virtual machines list.
        """
        return self.db.list_machines_locked()

    def shutdown(self):
        """Shutdown the machine manager. Kills all alive machines.
        @raise CuckooMachineError: if unable to stop machine.
        """
        if self.running().count() > 0:
            log.info("Still %s guests alive. Shutting down" % self.running().count())
            for machine in self.running():
                try:
                    self.stop(machine.label)
                except CuckooMachineError as e:
                    log.error("Unable to shutdown machine %s, please check "
                              "manually. Error: %s" % (machine.label, e))

    def set_status(self, label, status):
        """Set status for a virtual machine.
        @param label: virtual machine label
        @param status: new virtual machine status
        """
        self.db.set_machine_status(label, status)

    def start(self, label=None):
        """Start a machine.
        @param label: machine name.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

    def stop(self, label=None):
        """Stop a machine.
        @param label: machine name.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

    def _list(self):
        """Lists virtual machines configured.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class LibVirtMachineManager(MachineManager):
    """Libvirt based machine manager.

    If you want to write a custom module for a virtualization software supported
    by libvirt you have just to inherit this machine manager and change the
    connection string.
    """

    def initialize(self, module):
        """Initialize machine manager module. Ovverride defualt to set proper
        connection string.
        @param module:  machine manager module
        """
        self.set_dsn()
        super(LibVirtMachineManager, self).initialize(module)

    def _initialize_check(self):
        """Runs all checks when a machine manager is initialized.
        @raise CuckooMachineError: if libvirt version is not supported.
        """
        # Version checks.
        if not self._version_check():
            raise CuckooMachineError("Libvirt version is not supported, please get an updated version")
        # Preload VMs
        self.vms = self._fetch_machines()
        # Base checks.
        super(LibVirtMachineManager, self)._initialize_check()

    def start(self, label):
        """Starts a virtual machine.
        @param label: virtual machine name.
        @raise CuckooMachineError: if unable to start virtual machine.
        """
        log.debug("Staring vm %s" % label)
        # Get current snapshot.
        conn = self._connect()
        print conn
        print self.vms
        try:
            snap = self.vms[label].hasCurrentSnapshot(flags=0)
        except libvirt.libvirtError:
            self._disconnect(conn)
            raise CuckooMachineError("Unable to get current snapshot for virtual machine %s" % label)
        print snap
        print 'ehia'
        # Revert to latest snapshot.
        if snap:
            try:
                current = self.vms[label].snapshotCurrent(flags=0)
                print current
                self.vms[label].revertToSnapshot(current, flags=0)
                print 'reverted'
            except libvirt.libvirtError:
                raise CuckooMachineError("Unable to restore snapshot on virtual machine %s" % label)
            finally:
                self._disconnect(conn)
            print 'bbbb'
        else:
            self._disconnect(conn)
            raise CuckooMachineError("No snapshot found for virtual machine %s" % label)
        print 'aaa'

    def stop(self, label):
        """Stops a virtual machine. Kill them all.
        @param label: virtual machine name.
        @raise CuckooMachineError: if unable to stop virtual machine.
        """
        log.debug("Stopping vm %s" % label)
        # Force virtual machine shutdown.
        conn = self._connect()
        try:
            if not self.vms[label].isActive():
                log.debug("Trying to stop an already stopped vm %s. Skip" % label)
            else:
                self.vms[label].destroy() # Machete's way!
        except libvirt.libvirtError as e:
            raise CuckooMachineError("Error stopping virtual machine %s: %s" % (label, e))
        finally:
            self._disconnect(conn)

    def shutdown(self):
        """Override shutdown to free libvirt handlers, anyway they print errors."""
        super(LibVirtMachineManager, self).shutdown()
        # Free handlers.
        self.vms = None

    def _connect(self):
        """Connects to libvirt subsystem.
        @raise CuckooMachineError: if cannot connect to libvirt.
        """
        if not self.dsn:
            raise CuckooMachineError("You must provide a proper connection string")
        print self.dsn
        try:
            return libvirt.open(self.dsn)
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot connect to libvirt")

    def _disconnect(self, conn):
        """Disconnects to libvirt subsystem.
        @raise CuckooMachineError: if cannot disconnect from libvirt.
        """
        try:
            conn.close()
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot disconnect from libvirt")

    def _fetch_machines(self):
        """Fetch machines handlers.
        @return: dict with machine label as key and handle as value.
        """
        vms = {}
        for vm in self.machines():
            vms[vm.label] = self._lookup(vm.label)
        return vms

    def _lookup(self, label):
        """Search for a virtual machine.
        @param conn: libvirt connection handle.
        @param label: virtual machine name.
        @raise CuckooMachineError: if virtual machine is not found.
        """
        conn = self._connect()
        try:
            vm = conn.lookupByName(label)
        except libvirt.libvirtError:
                raise CuckooMachineError("Cannot found machine %s" % label)
        finally:
            self._disconnect(conn)
        return vm

    def _list(self):
        """List available virtual machines.
        @raise CuckooMachineError: if unable to list virtual machines.
        """
        conn = self._connect()
        try:
            names = conn.listDefinedDomains()
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot list domains")
        finally:
            self._disconnect(conn)
        return names

    def _version_check(self):
        """Check if libvirt release supports snapshots.
        @return: True or false.
        """
        if libvirt.getVersion() >= 8000:
            return True
        else:
            return False

    def set_dsn(self):
        """Set libvirt connection string.
        @raise NotImplementedError: abstract interface
        """
        raise NotImplementedError

class Processing(object):
    """Base abstract class for processing module."""
    order = 1
    enabled = True

    def __init__(self):
        self.analysis_path = ""
        self.logs_path = ""

    def set_path(self, analysis_path):
        """Set paths.
        @param analysis_path: analysis folder path.
        """
        self.analysis_path = analysis_path
        self.log_path = os.path.join(self.analysis_path, "analysis.log")
        self.conf_path = os.path.join(self.analysis_path, "analysis.conf")
        self.file_path = os.path.realpath(os.path.join(self.analysis_path,
                                                       "binary"))
        self.dropped_path = os.path.join(self.analysis_path, "files")
        self.logs_path = os.path.join(self.analysis_path, "logs")
        self.shots_path = os.path.join(self.analysis_path, "shots")
        self.pcap_path = os.path.join(self.analysis_path, "dump.pcap")

    def run(self):
        """Start processing.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class Signature(object):
    """Base abstract class for signature."""

    name = ""
    description = ""
    severity = 1
    categories = []
    authors = []
    references = []
    alert = False
    enabled = True
    minimum = None
    maximum = None
    families = []

    def __init__(self):
        self.data = []

    def run(self, results=None):
        """Start signature processing.
        @param results: analysis results.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class Report(object):
    """Base abstract class for reporting module."""
    order = 1

    def __init__(self):
        self.analysis_path = ""
        self.reports_path = ""
        self.options = None

    def set_path(self, analysis_path):
        """Set analysis folder path.
        @param analysis_path: analysis folder path.
        """
        self.analysis_path = analysis_path
        self.conf_path = os.path.join(self.analysis_path, "analysis.conf")
        self.reports_path = os.path.join(self.analysis_path, "reports")

        try:
            create_folder(folder=self.reports_path)
        except CuckooOperationalError as e:
            CuckooReportError(e)

    def set_options(self, options):
        """Set report options.
        @param options: report options dict.
        """
        self.options = options

    def run(self):
        """Start report processing.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError
<commit_msg>Missed debug statements<commit_after># Copyright (C) 2010-2012 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import logging

from lib.cuckoo.common.exceptions import CuckooMachineError
from lib.cuckoo.common.exceptions import CuckooOperationalError
from lib.cuckoo.common.exceptions import CuckooReportError
from lib.cuckoo.common.objects import Dictionary
from lib.cuckoo.common.utils import create_folder
from lib.cuckoo.core.database import Database

try:
    import libvirt
except ImportError:
    raise CuckooDependencyError("Unable to import libvirt")

log = logging.getLogger(__name__)

class MachineManager(object):
    """Base abstract class for analysis machine manager."""

    def __init__(self):
        self.module_name = ""
        self.options = None
        # Database pointer.
        self.db = Database()
        # Machine table is cleaned to be filled from configuration file at each start.
        self.db.clean_machines()

    def set_options(self, options):
        """Set machine manager options.
        @param options: machine manager options dict.
        """
        self.options = options

    def initialize(self, module_name):
        """Read and load machines configuration, try to check the configuration.
        @param module_name: module name.
        """
        # Load.
        self._initialize(module_name)

        # Run initialization checks.
        self._initialize_check()

    def _initialize(self, module_name):
        """Read configuration.
        @param module_name: module name.
        """
        self.module_name = module_name
        mmanager_opts = self.options.get(module_name)

        for machine_id in mmanager_opts["machines"].strip().split(","):
            # Parse from configuration file.
            try:
                machine_opts = self.options.get(machine_id.strip())
                machine = Dictionary()
                machine.id = machine_id.strip()
                machine.label = machine_opts["label"].strip()
                machine.platform = machine_opts["platform"].strip()
                machine.ip = machine_opts["ip"].strip()
                # Insert in db.
                self.db.add_machine(name=machine.id,
                                    label=machine.label,
                                    ip=machine.ip,
                                    platform=machine.platform)
            except (AttributeError, CuckooOperationalError):
                log.warning("Configuration details about machine %s are "
                            "missing. Continue" % machine_id)
                continue

    def _initialize_check(self):
        """Runs checks against virtualization software when a machine manager
        is initialized.
        @note: in machine manager modules you may override or superclass
               his method.
        @raise CuckooMachineError: if a misconfiguration or a unkown vm state
                                   is found.
        """
        try:
            configured_vm = self._list()
            for machine in self.machines():
                if machine.label not in configured_vm:
                    raise CuckooMachineError("Configured machine %s was not "
                        "detected or it's not in proper state" % machine.label)
        except NotImplementedError:
            pass

    def machines(self):
        """List virtual machines.
        @return: virtual machines list
        """
        return self.db.list_machines()

    def availables(self):
        """How many machines are free.
        @return: free machines count.
        """
        return self.db.count_machines_available()

    def acquire(self, machine_id=None, platform=None):
        """Acquire a machine to start analysis.
        @param machine_id: machine ID.
        @param platform: machine platform.
        @return: machine or None.
        """
        if machine_id:
            return self.db.lock_machine_by_name(machine_id)
        elif platform:
            return self.db.lock_machine_by_platform(platform)
        else:
            return self.db.lock_machine()

    def release(self, label=None):
        """Release a machine.
        @param label: machine name.
        """
        self.db.unlock_machine(label)

    def running(self):
        """Returns running virtual machines.
        @return: running virtual machines list.
        """
        return self.db.list_machines_locked()

    def shutdown(self):
        """Shutdown the machine manager. Kills all alive machines.
        @raise CuckooMachineError: if unable to stop machine.
        """
        if self.running().count() > 0:
            log.info("Still %s guests alive. Shutting down" % self.running().count())
            for machine in self.running():
                try:
                    self.stop(machine.label)
                except CuckooMachineError as e:
                    log.error("Unable to shutdown machine %s, please check "
                              "manually. Error: %s" % (machine.label, e))

    def set_status(self, label, status):
        """Set status for a virtual machine.
        @param label: virtual machine label
        @param status: new virtual machine status
        """
        self.db.set_machine_status(label, status)

    def start(self, label=None):
        """Start a machine.
        @param label: machine name.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

    def stop(self, label=None):
        """Stop a machine.
        @param label: machine name.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

    def _list(self):
        """Lists virtual machines configured.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class LibVirtMachineManager(MachineManager):
    """Libvirt based machine manager.

    If you want to write a custom module for a virtualization software supported
    by libvirt you have just to inherit this machine manager and change the
    connection string.
    """

    def initialize(self, module):
        """Initialize machine manager module. Ovverride defualt to set proper
        connection string.
        @param module:  machine manager module
        """
        self.set_dsn()
        super(LibVirtMachineManager, self).initialize(module)

    def _initialize_check(self):
        """Runs all checks when a machine manager is initialized.
        @raise CuckooMachineError: if libvirt version is not supported.
        """
        # Version checks.
        if not self._version_check():
            raise CuckooMachineError("Libvirt version is not supported, please get an updated version")
        # Preload VMs
        self.vms = self._fetch_machines()
        # Base checks.
        super(LibVirtMachineManager, self)._initialize_check()

    def start(self, label):
        """Starts a virtual machine.
        @param label: virtual machine name.
        @raise CuckooMachineError: if unable to start virtual machine.
        """
        log.debug("Staring vm %s" % label)
        # Get current snapshot.
        conn = self._connect()

        try:
            snap = self.vms[label].hasCurrentSnapshot(flags=0)
        except libvirt.libvirtError:
            self._disconnect(conn)
            raise CuckooMachineError("Unable to get current snapshot for virtual machine %s" % label)

        # Revert to latest snapshot.
        if snap:
            try:
                current = self.vms[label].snapshotCurrent(flags=0)
                self.vms[label].revertToSnapshot(current, flags=0)
            except libvirt.libvirtError:
                raise CuckooMachineError("Unable to restore snapshot on virtual machine %s" % label)
            finally:
                self._disconnect(conn)
        else:
            self._disconnect(conn)
            raise CuckooMachineError("No snapshot found for virtual machine %s" % label)

    def stop(self, label):
        """Stops a virtual machine. Kill them all.
        @param label: virtual machine name.
        @raise CuckooMachineError: if unable to stop virtual machine.
        """
        log.debug("Stopping vm %s" % label)
        # Force virtual machine shutdown.
        conn = self._connect()
        try:
            if not self.vms[label].isActive():
                log.debug("Trying to stop an already stopped vm %s. Skip" % label)
            else:
                self.vms[label].destroy() # Machete's way!
        except libvirt.libvirtError as e:
            raise CuckooMachineError("Error stopping virtual machine %s: %s" % (label, e))
        finally:
            self._disconnect(conn)

    def shutdown(self):
        """Override shutdown to free libvirt handlers, anyway they print errors."""
        super(LibVirtMachineManager, self).shutdown()
        # Free handlers.
        self.vms = None

    def _connect(self):
        """Connects to libvirt subsystem.
        @raise CuckooMachineError: if cannot connect to libvirt.
        """
        if not self.dsn:
            raise CuckooMachineError("You must provide a proper connection string")
        print self.dsn
        try:
            return libvirt.open(self.dsn)
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot connect to libvirt")

    def _disconnect(self, conn):
        """Disconnects to libvirt subsystem.
        @raise CuckooMachineError: if cannot disconnect from libvirt.
        """
        try:
            conn.close()
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot disconnect from libvirt")

    def _fetch_machines(self):
        """Fetch machines handlers.
        @return: dict with machine label as key and handle as value.
        """
        vms = {}
        for vm in self.machines():
            vms[vm.label] = self._lookup(vm.label)
        return vms

    def _lookup(self, label):
        """Search for a virtual machine.
        @param conn: libvirt connection handle.
        @param label: virtual machine name.
        @raise CuckooMachineError: if virtual machine is not found.
        """
        conn = self._connect()
        try:
            vm = conn.lookupByName(label)
        except libvirt.libvirtError:
                raise CuckooMachineError("Cannot found machine %s" % label)
        finally:
            self._disconnect(conn)
        return vm

    def _list(self):
        """List available virtual machines.
        @raise CuckooMachineError: if unable to list virtual machines.
        """
        conn = self._connect()
        try:
            names = conn.listDefinedDomains()
        except libvirt.libvirtError:
            raise CuckooMachineError("Cannot list domains")
        finally:
            self._disconnect(conn)
        return names

    def _version_check(self):
        """Check if libvirt release supports snapshots.
        @return: True or false.
        """
        if libvirt.getVersion() >= 8000:
            return True
        else:
            return False

    def set_dsn(self):
        """Set libvirt connection string.
        @raise NotImplementedError: abstract interface
        """
        raise NotImplementedError

class Processing(object):
    """Base abstract class for processing module."""
    order = 1
    enabled = True

    def __init__(self):
        self.analysis_path = ""
        self.logs_path = ""

    def set_path(self, analysis_path):
        """Set paths.
        @param analysis_path: analysis folder path.
        """
        self.analysis_path = analysis_path
        self.log_path = os.path.join(self.analysis_path, "analysis.log")
        self.conf_path = os.path.join(self.analysis_path, "analysis.conf")
        self.file_path = os.path.realpath(os.path.join(self.analysis_path,
                                                       "binary"))
        self.dropped_path = os.path.join(self.analysis_path, "files")
        self.logs_path = os.path.join(self.analysis_path, "logs")
        self.shots_path = os.path.join(self.analysis_path, "shots")
        self.pcap_path = os.path.join(self.analysis_path, "dump.pcap")

    def run(self):
        """Start processing.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class Signature(object):
    """Base abstract class for signature."""

    name = ""
    description = ""
    severity = 1
    categories = []
    authors = []
    references = []
    alert = False
    enabled = True
    minimum = None
    maximum = None
    families = []

    def __init__(self):
        self.data = []

    def run(self, results=None):
        """Start signature processing.
        @param results: analysis results.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError

class Report(object):
    """Base abstract class for reporting module."""
    order = 1

    def __init__(self):
        self.analysis_path = ""
        self.reports_path = ""
        self.options = None

    def set_path(self, analysis_path):
        """Set analysis folder path.
        @param analysis_path: analysis folder path.
        """
        self.analysis_path = analysis_path
        self.conf_path = os.path.join(self.analysis_path, "analysis.conf")
        self.reports_path = os.path.join(self.analysis_path, "reports")

        try:
            create_folder(folder=self.reports_path)
        except CuckooOperationalError as e:
            CuckooReportError(e)

    def set_options(self, options):
        """Set report options.
        @param options: report options dict.
        """
        self.options = options

    def run(self):
        """Start report processing.
        @raise NotImplementedError: this method is abstract.
        """
        raise NotImplementedError
<|endoftext|>


<commit_before>#   -*- coding: utf-8 -*-
#
#   This file is part of PyBuilder
#
#   Copyright 2011-2015 PyBuilder Team
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Plugin for Sphinx.
"""
import os
from pybuilder.core import after
from pybuilder.core import depends
from pybuilder.core import init
from pybuilder.core import task
from pybuilder.core import use_plugin
from pybuilder.errors import BuildFailedException
from pybuilder.utils import assert_can_execute
from pybuilder.utils import execute_command
from pybuilder import scaffolding as SCAFFODING

__author__ = 'Thomas Prebble', 'Marcel Wolf'

use_plugin("core")

DEFAULT_SPHINX_BUILDER = "html"
DEFAULT_SPHINX_OUTPUT_DIR = SCAFFODING.DEFAULT_DOCS_DIRECTORY + "/_build/"
PROJECT_NAME = os.path.basename(os.getcwd())
__version__ = '${version}'
AUTHORS = '${doc_author}'

@init
def initialize_sphinx_plugin(project):
    project.build_depends_on("sphinx")
    project.set_property_if_unset("sphinx_builder", DEFAULT_SPHINX_BUILDER)
    project.set_property_if_unset(
        "sphinx_source_dir", SCAFFODING.DEFAULT_DOCS_DIRECTORY)
    project.set_property_if_unset(
        "sphinx_output_dir", DEFAULT_SPHINX_OUTPUT_DIR)
    project.set_property_if_unset(
        "sphinx_config_path", SCAFFODING.DEFAULT_DOCS_DIRECTORY)

@after("prepare")
def assert_sphinx_is_available(logger):
    """Asserts that the sphinx-build script is available.
    """
    logger.debug("Checking if sphinx-build is available.")

    assert_can_execute(
        ["sphinx-build", "--version"], "sphinx", "plugin python.sphinx")

@after("prepare")
def assert_sphinx_quickstart_is_available(logger):
    """Asserts that the sphinx-quickstart script is available.
    """
    logger.debug("Checking if sphinx-quickstart is available.")

    assert_can_execute(
        ["sphinx-quickstart", "--version"], "sphinx", "plugin python.sphinx")

@task("sphinx_quickstart", "starts a new phinx project")
@depends("prepare")
def sphinx_quickstart_generate(project, logger):
    """Runs sphinx-build against rst sources for the given project.
    """
    logger.info("Running sphinx-quickstart")

    log_file = project.expand_path(
        "$dir_target/reports/{0}".format("sphinx-quickstart"))
    build_command = get_sphinx_quickstart_command(project)
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
    if exit_code != 0:
        raise BuildFailedException(
            "Sphinx build command failed. See %s for details.", log_file)

def get_sphinx_quickstart_command(project):
    """Builds the sphinx-quickstart command using project properties.
    """
    options = ["-q",
               "-p %s" % PROJECT_NAME,
               "-a %s" % project.doc_author,
               "-v %s" % project.version,
               "%s" % project.expand_path
               (project.get_property("sphinx_source_dir"))]
    return "sphinx-quickstart %s" % " ".join(options)

@task("sphinx_generate_documentation", "Generates documentation with sphinx")
@depends("prepare")
def sphinx_generate(project, logger):
    """Runs sphinx-build against rst sources for the given project.
    """
    logger.info("Running sphinx-build")

    log_file = project.expand_path(
        "$dir_target/reports/{0}".format("sphinx-build"))
    build_command = get_sphinx_build_command(project)
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
        logger.info("documentation was build as %s"
                    % DEFAULT_SPHINX_BUILDER + ", check %s"
                    % DEFAULT_SPHINX_OUTPUT_DIR)
    if exit_code != 0:
        raise BuildFailedException(
            "Sphinx build command failed. See %s for details.", log_file)

def get_sphinx_build_command(project):
    """Builds the sphinx-build command using project properties.
    """
    options = ["-b %s" % project.get_property("sphinx_builder"),
               project.expand_path(project.get_property("sphinx_config_path")),
               project.expand_path(project.get_property("sphinx_output_dir"))]
    return "sphinx-build %s" % " ".join(options)
<commit_msg>project.name doc.author doc.builder can now be configured via build.py<commit_after>#   -*- coding: utf-8 -*-
#
#   This file is part of PyBuilder
#
#   Copyright 2011-2015 PyBuilder Team
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Sphinx-plugin for PyBuilder to make a sphinx-quickstart and sphinx-generate_documentation.
"""

from pybuilder.core import after
from pybuilder.core import depends
from pybuilder.core import init
from pybuilder.core import task
from pybuilder.core import use_plugin
from pybuilder.errors import BuildFailedException
from pybuilder.utils import assert_can_execute
from pybuilder.utils import execute_command
from pybuilder import scaffolding as SCAFFODING

__author__ = 'Thomas Prebble', 'Marcel Wolf'

use_plugin("core")

DEFAULT_SPHINX_OUTPUT_DIR = SCAFFODING.DEFAULT_DOCS_DIRECTORY + "/_build/"

"""
                           build.py:
    :param ${name}:        name = "project name"
    :param ${version}:     version = "0.01"
    :param ${doc_author}   project.doc_author = "project team"
    :param ${doc_builder}  project.doc_builder = "html"

"""

AUTHORS = '${doc_author}'
DEFAULT_SPHINX_BUILDER = '${doc_builder}'
PROJECT_NAME = '${name}'

__version__ = '${version}'

@init
def initialize_sphinx_plugin(project):
    project.build_depends_on("sphinx")
    project.set_property_if_unset(
        "sphinx_source_dir", SCAFFODING.DEFAULT_DOCS_DIRECTORY)
    project.set_property_if_unset(
        "sphinx_output_dir", DEFAULT_SPHINX_OUTPUT_DIR)
    project.set_property_if_unset(
        "sphinx_config_path", SCAFFODING.DEFAULT_DOCS_DIRECTORY)

@after("prepare")
def assert_sphinx_is_available(logger):
    """Asserts that the sphinx-build script is available.
    """
    logger.debug("Checking if sphinx-build is available.")

    assert_can_execute(
        ["sphinx-build", "--version"], "sphinx", "plugin python.sphinx")

@after("prepare")
def assert_sphinx_quickstart_is_available(logger):
    """Asserts that the sphinx-quickstart script is available.
    """
    logger.debug("Checking if sphinx-quickstart is available.")

    assert_can_execute(
        ["sphinx-quickstart", "--version"], "sphinx", "plugin python.sphinx")

@task("sphinx_quickstart", "starts a new sphinx project")
@depends("prepare")
def sphinx_quickstart_generate(project, logger):
    """Runs sphinx-build against rst sources for the given project.
    """
    logger.info("Running sphinx-quickstart")

    log_file = project.expand_path(
        "$dir_target/reports/{0}".format("sphinx-quickstart"))
    build_command = get_sphinx_quickstart_command(project)
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
    if exit_code != 0:
        raise BuildFailedException(
            "Sphinx build command failed. See %s for details.", log_file)

def get_sphinx_quickstart_command(project):
    """Builds the sphinx-quickstart command using project properties.
        :param -q: Quiet mode that will skips interactive wizard to specify options.
        :param -p: Project name will be set.
        :param -a: Author names.
        :param -v: Version of project.
    """
    options = ["-q",
               "-p %s" % project.name,
               "-a %s" % project.doc_author,
               "-v %s" % project.version,
               "%s" % project.expand_path
               (project.get_property("sphinx_source_dir"))]
    return "sphinx-quickstart %s" % " ".join(options)

@task("sphinx_generate_documentation", "Generates documentation with sphinx")
@depends("prepare")
def sphinx_generate(project, logger):
    """Runs sphinx-build against rst sources for the given project.
    """
    logger.info("Running sphinx-build")
    log_file = project.expand_path(
        "$dir_target/reports/{0}".format("sphinx-build"))
    build_command = get_sphinx_build_command(project)
    if project.get_property("verbose"):
        logger.info(build_command)
        exit_code = execute_command(build_command, log_file, shell=True)
        logger.info("documentation was build as %s"
                    % project.doc_builder + ", check %s"
                    % DEFAULT_SPHINX_OUTPUT_DIR)
    if exit_code != 0:
        raise BuildFailedException(
            "Sphinx build command failed. See %s for details.", log_file)

def get_sphinx_build_command(project):
    """Builds the sphinx-build command using properties.
    """
    options = ["-b %s" % project.doc_builder,
               project.expand_path(project.get_property("sphinx_config_path")),
               project.expand_path(project.get_property("sphinx_output_dir"))]
    return "sphinx-build %s" % " ".join(options)
<|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set fileencoding=utf8

# Crossword Puzzle Loader Demo
# Copyright (C) 2011 Camille Dalmeras
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import pyjd

from pyjamas.ui.HTML import HTML
from pyjamas.ui.Grid import Grid
from pyjamas.ui.FocusPanel import FocusPanel
from pyjamas.ui import HasAlignment
from pyjamas.ui import KeyboardListener
from pyjamas.Timer import Timer
from pyjamas import DOM
from pyjamas import Factory

import re

class InputBox(FocusPanel):

    _props = [ ("maxLength", "Max Length", "MaxLength", int),
               ("text", "Text", "Text", None),
               ("matchPattern", "Match Pattern", "MatchPattern", None),
             ]

    @classmethod
    def _getProps(self):
        return FocusPanel._getProps() + self._props

    def __init__(self, **kwargs):
        """ setMatchPattern - defaults to '' to match everything
            match pattern examples: '^[0-9]*$' is for digits only
                                    '^[0-9,A-Z]*$' is for digits and uppercase
            setMaxLength
            setText
        """

        kwargs['MatchPattern'] = kwargs.pop('MatchPattern', '')
        gs = kwargs.pop('StyleName', 'inputbox')
        self.tp = Grid(StyleName=gs)
        self.cf = self.tp.getCellFormatter()
        FocusPanel.__init__(self, Widget=self.tp, **kwargs)

        self.addTableListener(self)
        self.addKeyboardListener(self)
        self.addFocusListener(self)

        self.word_selected_pos = 0
        self.ctimer = Timer(notify=self.cursorFlash)
        self.focusset = False
        self.cstate = False
        self._keypressListeners = []

    def addKeypressListener(self, listener):
        self._keypressListeners.append(listener)

    def removeKeypressListener(self, listener):
        self._keypressListeners.remove(listener)

    def getMatchPattern(self):
        return self.mp

    def setMatchPattern(self, mp):
        self.mp = mp
        self.rexp = re.compile(self.mp)

    def addDblTableListener(self, listener):
        self.tp.addDblTableListener(listener)

    def addTableListener(self, listener):
        self.tp.addTableListener(listener)

    def _highlight_cursor(self, row, col, highlight):
        """ highlights (or dehighlights) the currently selected cell
        """
        print "_highlight", row, col, highlight
        self.cf.setStyleName(row, col, "inputbox-square-word-cursor", highlight)

    def resize(self, width, height):
        self.tp.resize(width, height)

    def set_grid_value(self, v, y, x):

        v = v or ' '
        self.tp.setWidget(y, x, HTML(v, StyleName="inputbox-square"))
        self.cf.setAlignment(y, x,  HasAlignment.ALIGN_CENTER,
                                    HasAlignment.ALIGN_MIDDLE)

    def onKeyDown(self, sender, keycode, modifiers):

        evt = DOM.eventGetCurrentEvent()
        DOM.eventPreventDefault(evt)

        if self.word_selected_pos is None:
            return

        val = chr(keycode)
        done = False

        if keycode == KeyboardListener.KEY_DELETE:
            self.shift_letters_back()
            done = True
        elif keycode == KeyboardListener.KEY_BACKSPACE:
            if not self.nasty_hack():
                if self.moveCursor(-1):
                    self.shift_letters_back()
            done = True
        elif keycode == KeyboardListener.KEY_LEFT:
            self.moveCursor(-1)
            done = True
        elif keycode == KeyboardListener.KEY_RIGHT:
            self.moveCursor(1)
            done = True

        print "onKeyDown", keycode, val, self.rexp.match(val)

        if not done:
            if self.rexp.match(val):
                self.press_letter(val)

        for listener in self._keypressListeners:
            listener.onKeyPressed(sender, keycode, modifiers)

    def press_letter(self, val):

        print "press letter", val
        # check the key is a letter, and there's a grid position highlighted
        if self.word_selected_pos is None:
            return

        row = 0
        col = self.word_selected_pos

        self.set_grid_value(val, row, col)
        self.moveCursor(1)

    def nasty_hack(self):
        """ breaking of backspace/delete rules for the final character
        """

        row = 0
        col = self.word_selected_pos

        txt = self.get_char(col)
        if txt is None or txt == " ":
            return False

        self.set_grid_value(" ", row, col)
        return True

    def shift_letters_back(self):
        """ this function is used by del and backspace, to move the letters
            backwards from after the cursor.  the only difference between
            backspace and delete is that backspace moves the cursor and
            _then_ does letter-moving.
        """

        row = 0
        col = self.word_selected_pos
        x2 = self.tp.getColumnCount()-1

        while (x2 != col):
            txt = self.get_char(col+1)
            self.set_grid_value(txt, row, col)
            col += 1
        self.set_grid_value(" ", row, col)

    def setCursorPos(self, col):

        x2 = self.tp.getColumnCount()

        col = min(x2, col)
        col = max(col, 0)

        if self.get_char(0) is None or self.get_char(0) == ' ':
            col = 0

        while (self.get_char(col-1) is None or \
              self.get_char(col-1) == ' ') and col > 1:
            col -= 1

        self.highlight_cursor(False)
        self.word_selected_pos = col
        self.highlight_cursor(self.focusset)

        return True

    def getCursorPos(self):
        return self.word_selected_pos

    def moveCursor(self, dirn):

        x2 = self.tp.getColumnCount()

        row = 0
        col = self.word_selected_pos

        if dirn == 1 and x2 == col+1:
            return False

        if dirn == -1 and 0 == col+1:
            return False

        return self.setCursorPos(col+dirn)

    def onKeyUp(self, sender, keycode, modifiers):
        evt = DOM.eventGetCurrentEvent()
        DOM.eventCancelBubble(evt, True)
        DOM.eventPreventDefault(evt)

    def onKeyPress(self, sender, keycode, modifiers):
        evt = DOM.eventGetCurrentEvent()
        DOM.eventPreventDefault(evt)

    def highlight_cursor(self, highlight):
        """ highlights (or dehighlights) the currently selected cell
        """
        if self.word_selected_pos is None:
            return
        row = 0
        col = self.word_selected_pos
        self._highlight_cursor(row, col, highlight)

    def getMaxLength(self):
        return self.tp.getColumnCount()

    def setMaxLength(self, x):
        l = self.tp.getColumnCount()
        self.tp.resize(1, x)
        self.clear(l)

    def clear(self, fromrange=0):
        for i in range(fromrange, self.tp.getColumnCount()):
            self.set_grid_value(" ", 0, i)

    def onCellClicked(self, listener, row, col, direction=None):
        self.setCursorPos(col)

    def cursorFlash(self, timr):
        if not self.focusset:
            return
        self.highlight_cursor(self.cstate)
        self.cstate = not self.cstate

    def onFocus(self, sender):
        print "onFocus", sender
        self.focusset = True
        self.ctimer.scheduleRepeating(800)

    def onLostFocus(self, sender):
        print "onLostFocus", sender
        self.focusset = False
        self.ctimer.cancel()
        self.highlight_cursor(False)

    def get_char(self, x):
        w = self.tp.getWidget(0, x)
        return w and w.getHTML()

    def getText(self):
        txt = ''
        for i in range(self.tp.getColumnCount()):
            c = self.get_char(i)
            if c is None or c == ' ':
                break
            txt += c
        return txt

    def setText(self, txt):
        self.clear()
        if len(txt) > self.getMaxLength(): # hm... cheat...
            self.setMaxLength(len(txt))
        for (i, c) in enumerate(txt):
            self.set_grid_value(c, 0, i)
        self.setCursorPos(min(self.getMaxLength()-1, len(txt)))

Factory.registerClass('pyjamas.ui.InputBox', 'InputBox', InputBox)

<commit_msg>restrict text length to max size<commit_after>#!/usr/bin/python
# vim: set fileencoding=utf8

# Crossword Puzzle Loader Demo
# Copyright (C) 2011 Camille Dalmeras
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU Affero General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Affero General Public License for more details.
#
#   You should have received a copy of the GNU Affero General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.

import pyjd

from pyjamas.ui.HTML import HTML
from pyjamas.ui.Grid import Grid
from pyjamas.ui.FocusPanel import FocusPanel
from pyjamas.ui import HasAlignment
from pyjamas.ui import KeyboardListener
from pyjamas.Timer import Timer
from pyjamas import DOM
from pyjamas import Factory

import re

class InputBox(FocusPanel):

    _props = [ ("maxLength", "Max Length", "MaxLength", int),
               ("text", "Text", "Text", None),
               ("matchPattern", "Match Pattern", "MatchPattern", None),
             ]

    @classmethod
    def _getProps(self):
        return FocusPanel._getProps() + self._props

    def __init__(self, **kwargs):
        """ setMatchPattern - defaults to '' to match everything
            match pattern examples: '^[0-9]*$' is for digits only
                                    '^[0-9,A-Z]*$' is for digits and uppercase
            setMaxLength
            setText
        """

        kwargs['MatchPattern'] = kwargs.pop('MatchPattern', '')
        gs = kwargs.pop('StyleName', 'inputbox')
        self.tp = Grid(StyleName=gs)
        self.cf = self.tp.getCellFormatter()
        FocusPanel.__init__(self, Widget=self.tp, **kwargs)

        self.addTableListener(self)
        self.addKeyboardListener(self)
        self.addFocusListener(self)

        self.word_selected_pos = 0
        self.ctimer = Timer(notify=self.cursorFlash)
        self.focusset = False
        self.cstate = False
        self._keypressListeners = []

    def addKeypressListener(self, listener):
        self._keypressListeners.append(listener)

    def removeKeypressListener(self, listener):
        self._keypressListeners.remove(listener)

    def getMatchPattern(self):
        return self.mp

    def setMatchPattern(self, mp):
        self.mp = mp
        self.rexp = re.compile(self.mp)

    def addDblTableListener(self, listener):
        self.tp.addDblTableListener(listener)

    def addTableListener(self, listener):
        self.tp.addTableListener(listener)

    def _highlight_cursor(self, row, col, highlight):
        """ highlights (or dehighlights) the currently selected cell
        """
        print "_highlight", row, col, highlight
        self.cf.setStyleName(row, col, "inputbox-square-word-cursor", highlight)

    def resize(self, width, height):
        self.tp.resize(width, height)

    def set_grid_value(self, v, y, x):

        v = v or ' '
        self.tp.setWidget(y, x, HTML(v, StyleName="inputbox-square"))
        self.cf.setAlignment(y, x,  HasAlignment.ALIGN_CENTER,
                                    HasAlignment.ALIGN_MIDDLE)

    def onKeyDown(self, sender, keycode, modifiers):

        evt = DOM.eventGetCurrentEvent()
        DOM.eventPreventDefault(evt)

        if self.word_selected_pos is None:
            return

        val = chr(keycode)
        done = False

        if keycode == KeyboardListener.KEY_DELETE:
            self.shift_letters_back()
            done = True
        elif keycode == KeyboardListener.KEY_BACKSPACE:
            if not self.nasty_hack():
                if self.moveCursor(-1):
                    self.shift_letters_back()
            done = True
        elif keycode == KeyboardListener.KEY_LEFT:
            self.moveCursor(-1)
            done = True
        elif keycode == KeyboardListener.KEY_RIGHT:
            self.moveCursor(1)
            done = True

        print "onKeyDown", keycode, val, self.rexp.match(val)

        if not done:
            if self.rexp.match(val):
                self.press_letter(val)

        for listener in self._keypressListeners:
            listener.onKeyPressed(sender, keycode, modifiers)

    def press_letter(self, val):

        print "press letter", val
        # check the key is a letter, and there's a grid position highlighted
        if self.word_selected_pos is None:
            return

        row = 0
        col = self.word_selected_pos

        self.set_grid_value(val, row, col)
        self.moveCursor(1)

    def nasty_hack(self):
        """ breaking of backspace/delete rules for the final character
        """

        row = 0
        col = self.word_selected_pos

        txt = self.get_char(col)
        if txt is None or txt == " ":
            return False

        self.set_grid_value(" ", row, col)
        return True

    def shift_letters_back(self):
        """ this function is used by del and backspace, to move the letters
            backwards from after the cursor.  the only difference between
            backspace and delete is that backspace moves the cursor and
            _then_ does letter-moving.
        """

        row = 0
        col = self.word_selected_pos
        x2 = self.tp.getColumnCount()-1

        while (x2 != col):
            txt = self.get_char(col+1)
            self.set_grid_value(txt, row, col)
            col += 1
        self.set_grid_value(" ", row, col)

    def setCursorPos(self, col):

        x2 = self.tp.getColumnCount()

        col = min(x2, col)
        col = max(col, 0)

        if self.get_char(0) is None or self.get_char(0) == ' ':
            col = 0

        while (self.get_char(col-1) is None or \
              self.get_char(col-1) == ' ') and col > 1:
            col -= 1

        self.highlight_cursor(False)
        self.word_selected_pos = col
        self.highlight_cursor(self.focusset)

        return True

    def getCursorPos(self):
        return self.word_selected_pos

    def moveCursor(self, dirn):

        x2 = self.tp.getColumnCount()

        row = 0
        col = self.word_selected_pos

        if dirn == 1 and x2 == col+1:
            return False

        if dirn == -1 and 0 == col+1:
            return False

        return self.setCursorPos(col+dirn)

    def onKeyUp(self, sender, keycode, modifiers):
        evt = DOM.eventGetCurrentEvent()
        DOM.eventCancelBubble(evt, True)
        DOM.eventPreventDefault(evt)

    def onKeyPress(self, sender, keycode, modifiers):
        evt = DOM.eventGetCurrentEvent()
        DOM.eventPreventDefault(evt)

    def highlight_cursor(self, highlight):
        """ highlights (or dehighlights) the currently selected cell
        """
        if self.word_selected_pos is None:
            return
        row = 0
        col = self.word_selected_pos
        self._highlight_cursor(row, col, highlight)

    def getMaxLength(self):
        return self.tp.getColumnCount()

    def setMaxLength(self, x):
        l = self.tp.getColumnCount()
        self.tp.resize(1, x)
        self.clear(l)

    def clear(self, fromrange=0):
        for i in range(fromrange, self.tp.getColumnCount()):
            self.set_grid_value(" ", 0, i)

    def onCellClicked(self, listener, row, col, direction=None):
        self.setCursorPos(col)

    def cursorFlash(self, timr):
        if not self.focusset:
            return
        self.highlight_cursor(self.cstate)
        self.cstate = not self.cstate

    def onFocus(self, sender):
        print "onFocus", sender
        self.focusset = True
        self.ctimer.scheduleRepeating(800)

    def onLostFocus(self, sender):
        print "onLostFocus", sender
        self.focusset = False
        self.ctimer.cancel()
        self.highlight_cursor(False)

    def get_char(self, x):
        w = self.tp.getWidget(0, x)
        return w and w.getHTML()

    def getText(self):
        txt = ''
        for i in range(self.tp.getColumnCount()):
            c = self.get_char(i)
            if c is None or c == ' ':
                break
            txt += c
        return txt

    def setText(self, txt):
        self.clear()
        txt = txt[:self.getMaxLength()]
        for (i, c) in enumerate(txt):
            self.set_grid_value(c, 0, i)
        self.setCursorPos(min(self.getMaxLength()-1, len(txt)))

Factory.registerClass('pyjamas.ui.InputBox', 'InputBox', InputBox)

<|endoftext|>


<commit_before># Copyright 2006 James Tauber and contributors
# Copyright 2009 Luke Kenneth Casson Leighton
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

class HasHorizontalAlignment:
    ALIGN_LEFT = "left"
    ALIGN_CENTER = "center"
    ALIGN_RIGHT = "right"

class HasVerticalAlignment:
    ALIGN_TOP = "top"
    ALIGN_MIDDLE = "middle"
    ALIGN_BOTTOM = "bottom"

class HasAlignment:
    ALIGN_BOTTOM = "bottom"
    ALIGN_MIDDLE = "middle"
    ALIGN_TOP = "top"
    ALIGN_CENTER = "center"
    ALIGN_LEFT = "left"
    ALIGN_RIGHT = "right"

PROP_NAME = 0
PROP_DESC = 1
PROP_FNAM = 2
PROP_TYPE = 3

ELPROP_NAME = 0
ELPROP_DESC = 1
ELPROP_FNAM = 2
ELPROP_TYPE = 3
ELPROP_DFLT = 4

def get_list_columns(props, cols):
    res = []
    for p in props:
        r = ()
        for idx in cols:
            r.append(p[idx])
        res.append(r)
    return res

def get_prop_widget_function_names(props):
    return get_list_columns(props, (PROP_FNAM,))

class Applier(object):

    _props = []
    _elem_props = []

    def __init__(self, **kwargs):
        """ use this to apply properties as a dictionary, e.g.
                x = klass(..., StyleName='class-name')
            will do:
                x = klass(...)
                x.setStyleName('class-name')

            and:
                x = klass(..., Size=("100%", "20px"), Visible=False)
            will do:
                x = klass(...)
                x.setSize("100%", "20px")
                x.setVisible(False)
        """

        self.applyValues(**kwargs)

    def applyValues(self, **kwargs):

        for (prop, args) in kwargs.items():
            fn = getattr(self, "set%s" % prop, None)
            if not fn:
                raise Exception("setter function for %s does not exist" % prop)
            args = kwargs[prop]
            if isinstance(args, tuple):
                fn(*args)
            else:
                fn(args)

    def retrieveValues(self, *args):
        """ use this function to obtain a dictionary of properties, as
            stored in getXXX functions.
        """

        res = {}
        for prop in args:
            fn = getattr(self, "get%s" % prop, None)
            if not fn:
                raise Exception("getter function for %s does not exist" % prop)
            res[prop] = fn()

        return res

    @classmethod
    def _getProps(self):
        return self._props

    @classmethod
    def _getElementProps(self):
        return self._elem_props

    def setDefaults(self, defaults):
        divs = self.retrieveValues(wnames)
        for p in get_prop_widget_function_names(self._getProps()):
            defaults[p[PROP_NAME]] = divs[p[PROP_FNAM]]

    def updateInstance(self, app_context):
        args = {}
        for p in self._getProps():
            val = app_context.getAppProperty(p[0])
            convert_to_type = p[PROP_TYPE]
            if convert_to_type:
                 val = convert_to_type(val) if val else None
            args[p[PROP_FNAM]] = val

        self.applyValues(**args)

    def setElementProperties(self, context, elemProps):
        args = {}
        for p in self._getElementProps():
            if elemProps.has_key(p):
                val = elemProps[p]
                convert_to_type = p[ELPROP_TYPE]
                if convert_to_type:
                     val = convert_to_type(val) if val else None
            else:
                val = p[ELPROP_DFLT]
                if val is None:
                    continue
            args[p[ELPROP_FNAM]] = (context, val,)

        self.applyValues(**args)

<commit_msg>UIApplier.setElementProperties never worked! have to look for property by name<commit_after># Copyright 2006 James Tauber and contributors
# Copyright 2009 Luke Kenneth Casson Leighton
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

class HasHorizontalAlignment:
    ALIGN_LEFT = "left"
    ALIGN_CENTER = "center"
    ALIGN_RIGHT = "right"

class HasVerticalAlignment:
    ALIGN_TOP = "top"
    ALIGN_MIDDLE = "middle"
    ALIGN_BOTTOM = "bottom"

class HasAlignment:
    ALIGN_BOTTOM = "bottom"
    ALIGN_MIDDLE = "middle"
    ALIGN_TOP = "top"
    ALIGN_CENTER = "center"
    ALIGN_LEFT = "left"
    ALIGN_RIGHT = "right"

PROP_NAME = 0
PROP_DESC = 1
PROP_FNAM = 2
PROP_TYPE = 3

ELPROP_NAME = 0
ELPROP_DESC = 1
ELPROP_FNAM = 2
ELPROP_TYPE = 3
ELPROP_DFLT = 4

def get_list_columns(props, cols):
    res = []
    for p in props:
        r = ()
        for idx in cols:
            r.append(p[idx])
        res.append(r)
    return res

def get_prop_widget_function_names(props):
    return get_list_columns(props, (PROP_FNAM,))

class Applier(object):

    _props = []
    _elem_props = []

    def __init__(self, **kwargs):
        """ use this to apply properties as a dictionary, e.g.
                x = klass(..., StyleName='class-name')
            will do:
                x = klass(...)
                x.setStyleName('class-name')

            and:
                x = klass(..., Size=("100%", "20px"), Visible=False)
            will do:
                x = klass(...)
                x.setSize("100%", "20px")
                x.setVisible(False)
        """

        self.applyValues(**kwargs)

    def applyValues(self, **kwargs):

        for (prop, args) in kwargs.items():
            fn = getattr(self, "set%s" % prop, None)
            if not fn:
                raise Exception("setter function for %s does not exist" % prop)
            args = kwargs[prop]
            if isinstance(args, tuple):
                fn(*args)
            else:
                fn(args)

    def retrieveValues(self, *args):
        """ use this function to obtain a dictionary of properties, as
            stored in getXXX functions.
        """

        res = {}
        for prop in args:
            fn = getattr(self, "get%s" % prop, None)
            if not fn:
                raise Exception("getter function for %s does not exist" % prop)
            res[prop] = fn()

        return res

    @classmethod
    def _getProps(self):
        return self._props

    @classmethod
    def _getElementProps(self):
        return self._elem_props

    def setDefaults(self, defaults):
        divs = self.retrieveValues(wnames)
        for p in get_prop_widget_function_names(self._getProps()):
            defaults[p[PROP_NAME]] = divs[p[PROP_FNAM]]

    def updateInstance(self, app_context):
        args = {}
        for p in self._getProps():
            val = app_context.getAppProperty(p[0])
            convert_to_type = p[PROP_TYPE]
            if convert_to_type:
                 val = convert_to_type(val) if val else None
            args[p[PROP_FNAM]] = val

        self.applyValues(**args)

    def setElementProperties(self, context, elemProps):
        args = {}
        for p in self._getElementProps():
            if elemProps.has_key(p[ELPROP_NAME]):
                val = elemProps[p[ELPROP_NAME]]
                convert_to_type = p[ELPROP_TYPE]
                if convert_to_type:
                     val = convert_to_type(val) if val else None
            else:
                val = p[ELPROP_DFLT]
                if val is None:
                    continue
            args[p[ELPROP_FNAM]] = (context, val,)

        self.applyValues(**args)

<|endoftext|>


<commit_before>from django.db import models
from django.conf import settings
from django.core import serializers
import tempfile
import time
import os

verbose = False
RASTDIR = os.path.join(os.path.dirname(__file__), 'test_data')
RASTER_TYPES = (
                ("continuous", "continuous"),
                ("categorical", "catgorical"),
               )
try:
    STARSPAN_BIN = settings.STARSPAN_BIN
except:
    STARSPAN_BIN = 'starspan'

class RasterDataset(models.Model):
    name = models.CharField(max_length=30, unique=True)
    filepath = models.FilePathField(path=RASTDIR, recursive=True)
    type = models.CharField(max_length=30, choices=RASTER_TYPES)

    def __unicode__(self):
        return unicode(self.name + " raster at " + self.filepath)

class ZonalStatsCache(models.Model):
    geom_hash = models.CharField(max_length=255)
    raster = models.ForeignKey('RasterDataset')
    avg = models.FloatField(null=True, blank=True)
    min = models.FloatField(null=True, blank=True)
    max = models.FloatField(null=True, blank=True)
    mode = models.FloatField(null=True, blank=True)
    median = models.FloatField(null=True, blank=True)
    stdev = models.FloatField(null=True, blank=True)
    nulls = models.FloatField(null=True, blank=True)
    pixels = models.FloatField(null=True, blank=True)
    date_modified = models.DateTimeField(auto_now=True)

    @property
    def json(self):
        return serializers.serialize("json", self)

    def __unicode__(self):
        return unicode("Zonal Stats - avg:%s , min:%s, max:%s" % (self.avg, self.min, self.max))

    class Meta:
        unique_together = ('geom_hash', 'raster')

def geom_to_file(geom, filepath):
    json = """{
"type": "FeatureCollection",
"features": [
{ "type": "Feature", "properties": { "id": 1 }, "geometry": %s }
]
}""" % geom.json
    fh = open(filepath,'w')
    fh.write(json)
    fh.close()
    assert os.path.exists(filepath)

def run_starspan_zonal(geom, rasterds, write_cache=False):
    """
    Consider this a 'private' method .. dont call directly, use zonal_stats() instead
    Runs starspan and returns a ZonalStatsCache object
    If not write_cache, just return an unsaved object
    """
    # Create tempdir and cd in
    tmpdir = tempfile.gettempdir()
    os.chdir(tmpdir)

    # Output geom to temp dataset
    timestamp = str(time.time())
    out_json = os.path.join(tmpdir, 'geom_%s.json' % timestamp)
    geom_to_file(geom, out_json)

    # Run starspan
    out_csv = os.path.join(tmpdir, 'output_%s_stats.csv' % timestamp)
    if os.path.exists(out_csv):
        os.remove(out_csv)
    cmd = '%s --vector %s --where "id=1" --out-prefix %s/output_%s --out-type table --summary-suffix _stats.csv --raster %s --stats avg mode median min max sum stdev nulls ' % (STARSPAN_BIN,out_json,tmpdir, timestamp, rasterds.filepath)
    if verbose: print cmd
    starspan_out = os.popen(cmd).read()
    if verbose: print starspan_out

    res = open(out_csv,'r').readlines()
    if verbose: print res

    # Create zonal model
    hash = geom.wkt.__hash__()
    zonal = ZonalStatsCache(raster=rasterds, geom_hash=hash)

    # Make sure we have valid results output by starspan
    if len(res) == 2 and "Intersecting features: 0" not in starspan_out:
        headers = [x.strip() for x in res[0].split(',')]
        vals = [x.strip() for x in res[1].split(',')]
        assert len(headers) == len(vals)

        # loop and populate model
        for i in range(len(headers)):
            if "_Band1" in headers[i]:
                stat_type = headers[i].replace("_Band1",'')
                zonal.__dict__[stat_type] = float(vals[i])
            elif headers[i] == 'numPixels':
                zonal.pixels = float(vals[i])

    # return zonal object (caching it if needed)
    if write_cache:
        try:
            zonal.save()
        except:
            # Most likely another zonal stats cache for this geom/raster
            # was saved to the cache before this one completed.
            pass
    return zonal

def clear_cache():
    objs = ZonalStatsCache.objects.all()
    if verbose: print "Clearing %s objects from cache" % len(objs)
    objs.delete()

def zonal_stats(geom, rasterds, write_cache=True, read_cache=True):
    """
    Given a GEOSGeometry and a RasterDataset,
    compute the zonal stats and return json like
     { 'raster': 'elevation', 'stats': {'sum': 10234.2, 'mean': 12.4}}
    result can be stored in cache (write_cache)
     and cache value is returned if read_cache
    """
    if not geom.valid:
        return None

    hash = geom.wkt.__hash__()

    cached = None
    if read_cache:
        try:
            cached = ZonalStatsCache.objects.get(geom_hash=hash, raster=rasterds)
        except ZonalStatsCache.DoesNotExist:
            cached = None
    else:
        write_cache = False #If we're not reading the cache, we're not going to write to it either

    if cached:
        result = cached
        result.from_cache = True
    else:
        result = run_starspan_zonal(geom, rasterds, write_cache=write_cache)
        result.from_cache = False

    return result
<commit_msg>Exception handler to pass in race condition where the geom/raster has already been cached (again)<commit_after>from django.db import models
from django.conf import settings
from django.core import serializers
from django.db.utils import DatabaseError
import tempfile
import time
import os

verbose = False
RASTDIR = os.path.join(os.path.dirname(__file__), 'test_data')
RASTER_TYPES = (
                ("continuous", "continuous"),
                ("categorical", "catgorical"),
               )
try:
    STARSPAN_BIN = settings.STARSPAN_BIN
except:
    STARSPAN_BIN = 'starspan'

class RasterDataset(models.Model):
    name = models.CharField(max_length=30, unique=True)
    filepath = models.FilePathField(path=RASTDIR, recursive=True)
    type = models.CharField(max_length=30, choices=RASTER_TYPES)

    def __unicode__(self):
        return unicode(self.name + " raster at " + self.filepath)

class ZonalStatsCache(models.Model):
    geom_hash = models.CharField(max_length=255)
    raster = models.ForeignKey('RasterDataset')
    avg = models.FloatField(null=True, blank=True)
    min = models.FloatField(null=True, blank=True)
    max = models.FloatField(null=True, blank=True)
    mode = models.FloatField(null=True, blank=True)
    median = models.FloatField(null=True, blank=True)
    stdev = models.FloatField(null=True, blank=True)
    nulls = models.FloatField(null=True, blank=True)
    pixels = models.FloatField(null=True, blank=True)
    date_modified = models.DateTimeField(auto_now=True)

    @property
    def json(self):
        return serializers.serialize("json", self)

    def __unicode__(self):
        return unicode("Zonal Stats - avg:%s , min:%s, max:%s" % (self.avg, self.min, self.max))

    class Meta:
        unique_together = ('geom_hash', 'raster')

def geom_to_file(geom, filepath):
    json = """{
"type": "FeatureCollection",
"features": [
{ "type": "Feature", "properties": { "id": 1 }, "geometry": %s }
]
}""" % geom.json
    fh = open(filepath,'w')
    fh.write(json)
    fh.close()
    assert os.path.exists(filepath)

def run_starspan_zonal(geom, rasterds, write_cache=False):
    """
    Consider this a 'private' method .. dont call directly, use zonal_stats() instead
    Runs starspan and returns a ZonalStatsCache object
    If not write_cache, just return an unsaved object
    """
    # Create tempdir and cd in
    tmpdir = tempfile.gettempdir()
    os.chdir(tmpdir)

    # Output geom to temp dataset
    timestamp = str(time.time())
    out_json = os.path.join(tmpdir, 'geom_%s.json' % timestamp)
    geom_to_file(geom, out_json)

    # Run starspan
    out_csv = os.path.join(tmpdir, 'output_%s_stats.csv' % timestamp)
    if os.path.exists(out_csv):
        os.remove(out_csv)
    cmd = '%s --vector %s --where "id=1" --out-prefix %s/output_%s --out-type table --summary-suffix _stats.csv --raster %s --stats avg mode median min max sum stdev nulls ' % (STARSPAN_BIN,out_json,tmpdir, timestamp, rasterds.filepath)
    if verbose: print cmd
    starspan_out = os.popen(cmd).read()
    if verbose: print starspan_out

    res = open(out_csv,'r').readlines()
    if verbose: print res

    # Create zonal model
    hash = geom.wkt.__hash__()
    zonal = ZonalStatsCache(raster=rasterds, geom_hash=hash)

    # Make sure we have valid results output by starspan
    if len(res) == 2 and "Intersecting features: 0" not in starspan_out:
        headers = [x.strip() for x in res[0].split(',')]
        vals = [x.strip() for x in res[1].split(',')]
        assert len(headers) == len(vals)

        # loop and populate model
        for i in range(len(headers)):
            if "_Band1" in headers[i]:
                stat_type = headers[i].replace("_Band1",'')
                zonal.__dict__[stat_type] = float(vals[i])
            elif headers[i] == 'numPixels':
                zonal.pixels = float(vals[i])

    # return zonal object (caching it if needed)
    if write_cache:
        try:
            zonal.save()
        except:
            # Most likely another zonal stats cache for this geom/raster
            # was saved to the cache before this one completed.
            pass
    return zonal

def clear_cache():
    objs = ZonalStatsCache.objects.all()
    if verbose: print "Clearing %s objects from cache" % len(objs)
    objs.delete()

def zonal_stats(geom, rasterds, write_cache=True, read_cache=True):
    """
    Given a GEOSGeometry and a RasterDataset,
    compute the zonal stats and return json like
     { 'raster': 'elevation', 'stats': {'sum': 10234.2, 'mean': 12.4}}
    result can be stored in cache (write_cache)
     and cache value is returned if read_cache
    """
    if not geom.valid:
        return None

    hash = geom.wkt.__hash__()

    cached = None
    if read_cache:
        try:
            cached = ZonalStatsCache.objects.get(geom_hash=hash, raster=rasterds)
        except (ZonalStatsCache.DoesNotExist, DatabaseError):
            cached = None

    else:
        write_cache = False #If we're not reading the cache, we're not going to write to it either

    if cached:
        result = cached
        result.from_cache = True
    else:
        result = run_starspan_zonal(geom, rasterds, write_cache=write_cache)
        result.from_cache = False

    return result
<|endoftext|>