<commit_before># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re

import os

__author__ = "Rubens Ulysse <urubens@uliege.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@uliege.be>", "Mormont Romain <r.mormont@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

from cytomine.cytomine import Cytomine
from cytomine.models.collection import Collection
from cytomine.models.model import Model

class Software(Model):
    def __init__(self, name=None, result_name=None, **attributes):
        super(Software, self).__init__()
        self.name = name
        self.resultName = result_name
        self.softwareUserRepository = None
        self.defaultProcessingServer = None
        self.executeCommand = None
        self.pullingCommand = None
        self.deprecated = None
        self.softwareVersion = None
        self.executable = None
        self.parameters = None
        self.numberOfJob = None
        self.numberOfNotLaunch = None
        self.numberOfInQueue = None
        self.numberOfRunning = None
        self.numberOfSuccess = None
        self.numberOfFailed = None
        self.numberOfIndeterminate = None
        self.numberOfWait = None
        self.populate(attributes)

class SoftwareCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareCollection, self).__init__(Software, filters, max, offset)
        self._allowed_filters = [None, "project"]
        self.set_parameters(parameters)

class SoftwareProject(Model):
    def __init__(self, id_software=None, id_project=None, **attributes):
        super(SoftwareProject, self).__init__()
        self.software = id_software
        self.project = id_project
        self.name = None
        self.softwareVersion = None
        self.populate(attributes)

    def update(self, *args, **kwargs):
        raise NotImplementedError("Cannot update a software-project by client.")

class SoftwareProjectCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareProjectCollection, self).__init__(SoftwareProject, filters, max, offset)
        self._allowed_filters = [None, "project"]
        self.set_parameters(parameters)

class SoftwareParameter(Model):
    def __init__(self, name=None, type=None, id_software=None, default_value=None,
                 required=None, index=None, set_by_server=None, uri=None, uri_sort_attribut=None,
                 uri_print_attribut=None, server_parameter=None, human_name=None, value_key=None,
                 command_line_flag=None, **attributes):
        super(SoftwareParameter, self).__init__()
        self.name = name
        self.type = type
        self.software = id_software
        self.defaultValue = default_value
        self.required = required
        self.index = index
        self.uri_ = uri
        self.uriSortAttribut = uri_sort_attribut
        self.uriPrintAttribut = uri_print_attribut
        self.setByServer = set_by_server
        self.serverParameter = server_parameter
        self.humanName = human_name
        self.valueKey = value_key
        self.commandLineFlag = command_line_flag
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_parameter"

class SoftwareParameterCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareParameterCollection, self).__init__(SoftwareParameter, filters, max, offset)
        self._allowed_filters = ["software"]
        self.set_parameters(parameters)

class SoftwareParameterConstraint(Model):
    def __init__(self, parameter_constraint_id=None, software_parameter_id=None, value=None, **attributes):
        super(SoftwareParameterConstraint, self).__init__()
        self.parameterConstraint = parameter_constraint_id
        self.softwareParameter = software_parameter_id
        self.value = value
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_parameter_constraint"

class SoftwareParameterConstraintCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareParameterConstraintCollection, self).__init__(SoftwareParameterConstraint, filters, max, offset)
        self._allowed_filters = ["softwareparameter"]
        self.set_parameters(parameters)

_HUMAN_READABLE_JOB_STATUS = {
    0: "NOT_LAUNCH",
    1: "INQUEUE",
    2: "RUNNING",
    3: "SUCCESS/TERMINATED",
    4: "FAILED",
    5: "INDETERMINATE",
    6: "WAIT",
    7: "PREVIEW_DONE"
}

class Job(Model):
    NOT_LAUNCH = 0
    INQUEUE = 1
    RUNNING = 2
    SUCCESS = TERMINATED = 3
    FAILED = 4
    INDETERMINATE = 5
    WAIT = 6
    PREVIEW_DONE = 7

    def __init__(self, project_id=None, software_id=None, **attributes):
        super(Job, self).__init__()
        self.algoType = None
        self.progress = None
        self.status = None
        self.number = None
        self.statusComment = None
        self.project = project_id
        self.software = software_id
        self.softwareNone = None
        self.rate = None
        self.dataDeleted = None
        self.username = None
        self.userJob = None
        self.jobParameters = None
        self.populate(attributes)

    def execute(self):
        if self.is_new():
            raise ValueError("Cannot execute job if no ID was provided.")
        response = Cytomine.get_instance().post(uri="{}/{}/execute.json"
                                        .format(self.callback_identifier, self.id),
                                        data=self.to_json(),
                                        query_parameters={id: self.id})
        self.populate(response)
        return self

    def set_running(self):
        self.status = Job.RUNNING
        self.update()

    def set_terminated(self):
        self.status = Job.TERMINATED
        self.update()

    def update(self, id=None, **attributes):
        Cytomine.get_instance().log(
            "Job (id:{job_id}) status update: \"{statusComment}\" (status: {status}, progress: {progress}%)".format(
                job_id=self.id,
                statusComment=attributes.get("statusComment", self.statusComment),
                status=_HUMAN_READABLE_JOB_STATUS[attributes.get("status", self.status)],
                progress=attributes.get("progess", self.progress)
            )
        )
        return super(Job, self).update(id=id, **attributes)

class JobCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobCollection, self).__init__(Job, filters, max, offset)
        self._allowed_filters = [None]

        self.project = None
        self.software = None
        self.light = None

        self.set_parameters(parameters)

    def save(self, *args, **kwargs):
        raise NotImplementedError("Cannot save a job collection by client.")

class JobParameter(Model):
    def __init__(self, id_job=None, id_software_parameter=None, value=None, **attributes):
        super(JobParameter, self).__init__()
        self.job = id_job
        self.softwareParameter = id_software_parameter
        self.value = value
        self.populate(attributes)

class JobParameterCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobParameterCollection, self).__init__(JobParameter, filters, max, offset)
        self._allowed_filters = ["job"]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        # HACK to save on collection
        if len(self._data) > 0:
            return "jobparameter"
        return "parameter"

class JobTemplate(Model):
    def __init__(self, name=None, id_software=None, id_project=None, **attributes):
        super(JobTemplate, self).__init__()
        self.name = name
        self.software = id_software
        self.project = id_project
        self.populate(attributes)

class JobTemplateCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobTemplateCollection, self).__init__(JobTemplate, filters, max, offset)
        self._allowed_filters = ["project"]
        self.set_parameters(parameters)

class JobData(Model):
    def __init__(self, id_job=None, key=None, filename=None, **attributes):
        super(JobData, self).__init__()
        self.job = id_job
        self.key = key
        self.filename = filename
        self.size = None
        self.dir = None
        self.value = None
        self.populate(attributes)

    def upload(self, filename):
        if self.is_new():
            raise ValueError("Cannot upload file if not existing ID.")
        return Cytomine.get_instance().upload_file(self, filename,
                                                   uri="{}/{}/upload".format(self.callback_identifier, self.id))

    def download(self, destination="{filename}", override=False):
        if self.is_new():
            raise ValueError("Cannot download file if not existing ID.")

        self.filename = os.path.basename(self.filename)
        pattern = re.compile("{(.*?)}")
        destination = re.sub(pattern, lambda m: str(getattr(self, str(m.group(0))[1:-1], "_")), destination)

        return Cytomine.get_instance().download_file("{}/{}/download".format(self.callback_identifier, self.id),
                                                     destination, override)

class JobDataCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobDataCollection, self).__init__(JobData, filters, max, offset)
        self._allowed_filters = ["job"]
        self.set_parameters(parameters)

class SoftwareUserRepository(Model):
    def __init__(self, provider=None, username=None, docker_username=None, prefix=None, **attributes):
        super(SoftwareUserRepository, self).__init__()
        self.provider = provider
        self.username = username
        self.dockerUsername = docker_username
        self.prefix = prefix
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_user_repository"

class SoftwareUserRepositoryCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareUserRepositoryCollection, self).__init__(SoftwareUserRepository, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class ProcessingServer(Model):
    def __init__(self, name=None, host=None, username=None, port=None, type=None, processing_method_name=None,
                 persistent_directory=None, working_directory=None, index=None, **attributes):
        super(ProcessingServer, self).__init__()
        self.name = name
        self.host = host
        self.username = username
        self.port = port
        self.type = type
        self.processingMethodName = processing_method_name
        self.persistentDirectory = persistent_directory
        self.workingDirectory = working_directory
        self.index = index
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "processing_server"

class ProcessingServerCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(ProcessingServerCollection, self).__init__(ProcessingServer, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)
<commit_msg>override url for collection too<commit_after># -*- coding: utf-8 -*-

# * Copyright (c) 2009-2018. Authors: see NOTICE file.
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re

import os

__author__ = "Rubens Ulysse <urubens@uliege.be>"
__contributors__ = ["Marée Raphaël <raphael.maree@uliege.be>", "Mormont Romain <r.mormont@uliege.be>"]
__copyright__ = "Copyright 2010-2018 University of Liège, Belgium, http://www.cytomine.be/"

from cytomine.cytomine import Cytomine
from cytomine.models.collection import Collection
from cytomine.models.model import Model

class Software(Model):
    def __init__(self, name=None, result_name=None, **attributes):
        super(Software, self).__init__()
        self.name = name
        self.resultName = result_name
        self.softwareUserRepository = None
        self.defaultProcessingServer = None
        self.executeCommand = None
        self.pullingCommand = None
        self.deprecated = None
        self.softwareVersion = None
        self.executable = None
        self.parameters = None
        self.numberOfJob = None
        self.numberOfNotLaunch = None
        self.numberOfInQueue = None
        self.numberOfRunning = None
        self.numberOfSuccess = None
        self.numberOfFailed = None
        self.numberOfIndeterminate = None
        self.numberOfWait = None
        self.populate(attributes)

class SoftwareCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareCollection, self).__init__(Software, filters, max, offset)
        self._allowed_filters = [None, "project"]
        self.set_parameters(parameters)

class SoftwareProject(Model):
    def __init__(self, id_software=None, id_project=None, **attributes):
        super(SoftwareProject, self).__init__()
        self.software = id_software
        self.project = id_project
        self.name = None
        self.softwareVersion = None
        self.populate(attributes)

    def update(self, *args, **kwargs):
        raise NotImplementedError("Cannot update a software-project by client.")

class SoftwareProjectCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareProjectCollection, self).__init__(SoftwareProject, filters, max, offset)
        self._allowed_filters = [None, "project"]
        self.set_parameters(parameters)

class SoftwareParameter(Model):
    def __init__(self, name=None, type=None, id_software=None, default_value=None,
                 required=None, index=None, set_by_server=None, uri=None, uri_sort_attribut=None,
                 uri_print_attribut=None, server_parameter=None, human_name=None, value_key=None,
                 command_line_flag=None, **attributes):
        super(SoftwareParameter, self).__init__()
        self.name = name
        self.type = type
        self.software = id_software
        self.defaultValue = default_value
        self.required = required
        self.index = index
        self.uri_ = uri
        self.uriSortAttribut = uri_sort_attribut
        self.uriPrintAttribut = uri_print_attribut
        self.setByServer = set_by_server
        self.serverParameter = server_parameter
        self.humanName = human_name
        self.valueKey = value_key
        self.commandLineFlag = command_line_flag
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_parameter"

class SoftwareParameterCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareParameterCollection, self).__init__(SoftwareParameter, filters, max, offset)
        self._allowed_filters = ["software"]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        return "software_parameter"

class SoftwareParameterConstraint(Model):
    def __init__(self, parameter_constraint_id=None, software_parameter_id=None, value=None, **attributes):
        super(SoftwareParameterConstraint, self).__init__()
        self.parameterConstraint = parameter_constraint_id
        self.softwareParameter = software_parameter_id
        self.value = value
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_parameter_constraint"

class SoftwareParameterConstraintCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareParameterConstraintCollection, self).__init__(SoftwareParameterConstraint, filters, max, offset)
        self._allowed_filters = ["softwareparameter"]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        return "software_parameter_constraint"

_HUMAN_READABLE_JOB_STATUS = {
    0: "NOT_LAUNCH",
    1: "INQUEUE",
    2: "RUNNING",
    3: "SUCCESS/TERMINATED",
    4: "FAILED",
    5: "INDETERMINATE",
    6: "WAIT",
    7: "PREVIEW_DONE"
}

class Job(Model):
    NOT_LAUNCH = 0
    INQUEUE = 1
    RUNNING = 2
    SUCCESS = TERMINATED = 3
    FAILED = 4
    INDETERMINATE = 5
    WAIT = 6
    PREVIEW_DONE = 7

    def __init__(self, project_id=None, software_id=None, **attributes):
        super(Job, self).__init__()
        self.algoType = None
        self.progress = None
        self.status = None
        self.number = None
        self.statusComment = None
        self.project = project_id
        self.software = software_id
        self.softwareNone = None
        self.rate = None
        self.dataDeleted = None
        self.username = None
        self.userJob = None
        self.jobParameters = None
        self.populate(attributes)

    def execute(self):
        if self.is_new():
            raise ValueError("Cannot execute job if no ID was provided.")
        response = Cytomine.get_instance().post(uri="{}/{}/execute.json"
                                        .format(self.callback_identifier, self.id),
                                        data=self.to_json(),
                                        query_parameters={id: self.id})
        self.populate(response)
        return self

    def set_running(self):
        self.status = Job.RUNNING
        self.update()

    def set_terminated(self):
        self.status = Job.TERMINATED
        self.update()

    def update(self, id=None, **attributes):
        Cytomine.get_instance().log(
            "Job (id:{job_id}) status update: \"{statusComment}\" (status: {status}, progress: {progress}%)".format(
                job_id=self.id,
                statusComment=attributes.get("statusComment", self.statusComment),
                status=_HUMAN_READABLE_JOB_STATUS[attributes.get("status", self.status)],
                progress=attributes.get("progess", self.progress)
            )
        )
        return super(Job, self).update(id=id, **attributes)

class JobCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobCollection, self).__init__(Job, filters, max, offset)
        self._allowed_filters = [None]

        self.project = None
        self.software = None
        self.light = None

        self.set_parameters(parameters)

    def save(self, *args, **kwargs):
        raise NotImplementedError("Cannot save a job collection by client.")

class JobParameter(Model):
    def __init__(self, id_job=None, id_software_parameter=None, value=None, **attributes):
        super(JobParameter, self).__init__()
        self.job = id_job
        self.softwareParameter = id_software_parameter
        self.value = value
        self.populate(attributes)

class JobParameterCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobParameterCollection, self).__init__(JobParameter, filters, max, offset)
        self._allowed_filters = ["job"]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        # HACK to save on collection
        if len(self._data) > 0:
            return "jobparameter"
        return "parameter"

class JobTemplate(Model):
    def __init__(self, name=None, id_software=None, id_project=None, **attributes):
        super(JobTemplate, self).__init__()
        self.name = name
        self.software = id_software
        self.project = id_project
        self.populate(attributes)

class JobTemplateCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobTemplateCollection, self).__init__(JobTemplate, filters, max, offset)
        self._allowed_filters = ["project"]
        self.set_parameters(parameters)

class JobData(Model):
    def __init__(self, id_job=None, key=None, filename=None, **attributes):
        super(JobData, self).__init__()
        self.job = id_job
        self.key = key
        self.filename = filename
        self.size = None
        self.dir = None
        self.value = None
        self.populate(attributes)

    def upload(self, filename):
        if self.is_new():
            raise ValueError("Cannot upload file if not existing ID.")
        return Cytomine.get_instance().upload_file(self, filename,
                                                   uri="{}/{}/upload".format(self.callback_identifier, self.id))

    def download(self, destination="{filename}", override=False):
        if self.is_new():
            raise ValueError("Cannot download file if not existing ID.")

        self.filename = os.path.basename(self.filename)
        pattern = re.compile("{(.*?)}")
        destination = re.sub(pattern, lambda m: str(getattr(self, str(m.group(0))[1:-1], "_")), destination)

        return Cytomine.get_instance().download_file("{}/{}/download".format(self.callback_identifier, self.id),
                                                     destination, override)

class JobDataCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(JobDataCollection, self).__init__(JobData, filters, max, offset)
        self._allowed_filters = ["job"]
        self.set_parameters(parameters)

class SoftwareUserRepository(Model):
    def __init__(self, provider=None, username=None, docker_username=None, prefix=None, **attributes):
        super(SoftwareUserRepository, self).__init__()
        self.provider = provider
        self.username = username
        self.dockerUsername = docker_username
        self.prefix = prefix
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "software_user_repository"

class SoftwareUserRepositoryCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(SoftwareUserRepositoryCollection, self).__init__(SoftwareUserRepository, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

class ProcessingServer(Model):
    def __init__(self, name=None, host=None, username=None, port=None, type=None, processing_method_name=None,
                 persistent_directory=None, working_directory=None, index=None, **attributes):
        super(ProcessingServer, self).__init__()
        self.name = name
        self.host = host
        self.username = username
        self.port = port
        self.type = type
        self.processingMethodName = processing_method_name
        self.persistentDirectory = persistent_directory
        self.workingDirectory = working_directory
        self.index = index
        self.populate(attributes)

    @property
    def callback_identifier(self):
        return "processing_server"

class ProcessingServerCollection(Collection):
    def __init__(self, filters=None, max=0, offset=0, **parameters):
        super(ProcessingServerCollection, self).__init__(ProcessingServer, filters, max, offset)
        self._allowed_filters = [None]
        self.set_parameters(parameters)

    @property
    def callback_identifier(self):
        return "processing_server"

<|endoftext|>


<commit_before># collectd-cloudstack - cloudstack.py
#
# Author : Antoine Coetsier @ exoscale
# Description : This is a collectd python module to gather stats from cloudstack
# inspired by collectd-haproxy from Michael Leinartas - https://github.com/mleinart/collectd-haproxy

import collectd
import urllib2
import urllib
import json
import hmac
import base64
import hashlib
import re
import CloudStack

NAME = 'cloudstack'

DEFAULT_API = 'http://localh:8096/client/api'
DEFAULT_AUTH = False
DEFAULT_APIKEY = ''
DEFAULT_SECRET = ''
VERBOSE_LOGGING = False

METRIC_TYPES = {
  'memoryused': ('h_memory_used', 'memory'),
  'memorytotal': ('h_memory_total', 'memory'),
  'memoryallocated': ('h_memory_allocated', 'memory'),
  'activeviewersessions': ('console_active_sessions', 'current')
}

METRIC_DELIM = '.'

hypervisors = []

def get_stats():
  stats = dict()

  logger('verb', "get_stats calls API %s KEY %s SECRET %s" % (API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS))
  cloudstack = CloudStack.Client(API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS)
  try:
 	hypervisors = cloudstack.listHosts({
                        'type': 'Routing'
                })
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)
  for  h in hypervisors:
	metricnameMemUsed = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'], 'memoryused' ])
	metricnameMemTotal = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'], 'memorytotal' ])
	metricnameMemAlloc = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'], 'memoryallocated' ])
	try:
        	stats[metricnameMemUsed] = h['memoryused']
        	stats[metricnameMemTotal] = h['memorytotal']
        	stats[metricnameMemAlloc] = h['memoryallocated']
  		logger('verb', "readings :  %s memory used %s " % (h['name'], h['memoryused']))
	except (TypeError, ValueError), e:
        	pass
  try
	systemvms = cloudstack.listSystemVms({
		'systemvmtype': 'consoleproxy'
		})
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)

  for systemvm in systemvms:
	metricnameSessions = METRIC_DELIM.join([ systemvm['name'].lower(), h['podid'].lower(), h['zonename'], 'activeviewersessions' ])
	if 'activeviewersessions' in systemvm:
		stats[metricnameSessions] = systemvm['activeviewersessions']

  return stats

# callback configuration for module
def configure_callback(conf):
  global API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS, AUTH_MONITORS, VERBOSE_LOGGING
  API_MONITORS = ''
  APIKEY_MONITORS = ''
  SECRET_MONITORS = ''
  AUTH_MONITORS = DEFAULT_AUTH
  VERBOSE_LOGGING = False

  for node in conf.children:
    if node.key == "Api":
      API_MONITORS = node.values[0]
    elif node.key == "ApiKey":
      APIKEY_MONITORS = node.values[0]
    elif node.key == "Secret":
      SECRET_MONITORS = node.values[0]
    elif node.key == "Auth":
      AUTH_MONITORS = node.values[0]
    elif node.key == "Verbose":
      VERBOSE_LOGGING = bool(node.values[0])
    else:
      logger('warn', 'Unknown config key: %s' % node.key)

  if not API_MONITORS:
    API_MONITORS += DEFAULT_API

def read_callback():
  logger('verb', "beginning read_callback")
  info = get_stats()

  if not info:
    logger('warn', "%s: No data received" % NAME)
    return

  for key,value in info.items():
    key_prefix = ''
    key_root = key
    logger('verb', "read_callback key %s" % (key))
    logger('verb', "read_callback value %s" % (value))
    if not value in METRIC_TYPES:
      try:
        key_prefix, key_root = key.rsplit(METRIC_DELIM,1)
      except ValueError, e:
        pass
    if not key_root in METRIC_TYPES:
      continue

    key_root, val_type = METRIC_TYPES[key_root]
    key_name = METRIC_DELIM.join([key_prefix, key_root])
    logger('verb', "key_name %s" % (key_name))
    val = collectd.Values(plugin=NAME, type=val_type)
    val.type_instance = key_name
    val.values = [ value ]
    val.dispatch()

# logging function
def logger(t, msg):
    if t == 'err':
        collectd.error('%s: %s' % (NAME, msg))
    elif t == 'warn':
        collectd.warning('%s: %s' % (NAME, msg))
    elif t == 'verb':
        if VERBOSE_LOGGING:
            collectd.info('%s: %s' % (NAME, msg))
    else:
        collectd.notice('%s: %s' % (NAME, msg))
# main
collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<commit_msg>sanitize zones names<commit_after># collectd-cloudstack - cloudstack.py
#
# Author : Antoine Coetsier @ exoscale
# Description : This is a collectd python module to gather stats from cloudstack
# inspired by collectd-haproxy from Michael Leinartas - https://github.com/mleinart/collectd-haproxy

import collectd
import urllib2
import urllib
import json
import hmac
import base64
import hashlib
import re
import CloudStack

NAME = 'cloudstack'

DEFAULT_API = 'http://localh:8096/client/api'
DEFAULT_AUTH = False
DEFAULT_APIKEY = ''
DEFAULT_SECRET = ''
VERBOSE_LOGGING = False

METRIC_TYPES = {
  'memoryused': ('h_memory_used', 'memory'),
  'memorytotal': ('h_memory_total', 'memory'),
  'memoryallocated': ('h_memory_allocated', 'memory'),
  'activeviewersessions': ('console_active_sessions', 'current')
}

METRIC_DELIM = '.'

hypervisors = []

def get_stats():
  stats = dict()

  logger('verb', "get_stats calls API %s KEY %s SECRET %s" % (API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS))
  cloudstack = CloudStack.Client(API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS)
  try:
 	hypervisors = cloudstack.listHosts({
                        'type': 'Routing'
                })
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)
  for  h in hypervisors:
	metricnameMemUsed = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memoryused' ])
	metricnameMemTotal = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memorytotal' ])
	metricnameMemAlloc = METRIC_DELIM.join([ h['name'].lower(), h['podname'].lower(), h['zonename'].lower(), 'memoryallocated' ])
	try:
        	stats[metricnameMemUsed] = h['memoryused']
        	stats[metricnameMemTotal] = h['memorytotal']
        	stats[metricnameMemAlloc] = h['memoryallocated']
  		logger('verb', "readings :  %s memory used %s " % (h['name'], h['memoryused']))
	except (TypeError, ValueError), e:
        	pass
  try:
	systemvms = cloudstack.listSystemVms({
		'systemvmtype': 'consoleproxy'
		})
  except:
     	logger('warn', "status err Unable to connect to CloudStack URL at %s" % API_MONITORS)

  for systemvm in systemvms:
	metricnameSessions = METRIC_DELIM.join([ systemvm['name'].lower(), h['podid'].lower(), re.sub(r"\s+", '-', h['zonename'].lower()), 'activeviewersessions' ])
	if 'activeviewersessions' in systemvm:
		stats[metricnameSessions] = systemvm['activeviewersessions']

  return stats

# callback configuration for module
def configure_callback(conf):
  global API_MONITORS, APIKEY_MONITORS, SECRET_MONITORS, AUTH_MONITORS, VERBOSE_LOGGING
  API_MONITORS = ''
  APIKEY_MONITORS = ''
  SECRET_MONITORS = ''
  AUTH_MONITORS = DEFAULT_AUTH
  VERBOSE_LOGGING = False

  for node in conf.children:
    if node.key == "Api":
      API_MONITORS = node.values[0]
    elif node.key == "ApiKey":
      APIKEY_MONITORS = node.values[0]
    elif node.key == "Secret":
      SECRET_MONITORS = node.values[0]
    elif node.key == "Auth":
      AUTH_MONITORS = node.values[0]
    elif node.key == "Verbose":
      VERBOSE_LOGGING = bool(node.values[0])
    else:
      logger('warn', 'Unknown config key: %s' % node.key)

  if not API_MONITORS:
    API_MONITORS += DEFAULT_API

def read_callback():
  logger('verb', "beginning read_callback")
  info = get_stats()

  if not info:
    logger('warn', "%s: No data received" % NAME)
    return

  for key,value in info.items():
    key_prefix = ''
    key_root = key
    logger('verb', "read_callback key %s" % (key))
    logger('verb', "read_callback value %s" % (value))
    if not value in METRIC_TYPES:
      try:
        key_prefix, key_root = key.rsplit(METRIC_DELIM,1)
      except ValueError, e:
        pass
    if not key_root in METRIC_TYPES:
      continue

    key_root, val_type = METRIC_TYPES[key_root]
    key_name = METRIC_DELIM.join([key_prefix, key_root])
    logger('verb', "key_name %s" % (key_name))
    val = collectd.Values(plugin=NAME, type=val_type)
    val.type_instance = key_name
    val.values = [ value ]
    val.dispatch()

# logging function
def logger(t, msg):
    if t == 'err':
        collectd.error('%s: %s' % (NAME, msg))
    elif t == 'warn':
        collectd.warning('%s: %s' % (NAME, msg))
    elif t == 'verb':
        if VERBOSE_LOGGING:
            collectd.info('%s: %s' % (NAME, msg))
    else:
        collectd.notice('%s: %s' % (NAME, msg))
# main
collectd.register_config(configure_callback)
collectd.register_read(read_callback)
<|endoftext|>


<commit_before>import select, socket, threading, sys, time, traceback
from OpenSSL import SSL
import certutils

SSLv2_METHOD = SSL.SSLv2_METHOD
SSLv3_METHOD = SSL.SSLv3_METHOD
SSLv23_METHOD = SSL.SSLv23_METHOD
TLSv1_METHOD = SSL.TLSv1_METHOD

OP_ALL = SSL.OP_ALL
OP_CIPHER_SERVER_PREFERENCE = SSL.OP_CIPHER_SERVER_PREFERENCE
OP_COOKIE_EXCHANGE = SSL.OP_COOKIE_EXCHANGE
OP_DONT_INSERT_EMPTY_FRAGMENTS = SSL.OP_DONT_INSERT_EMPTY_FRAGMENTS
OP_EPHEMERAL_RSA = SSL.OP_EPHEMERAL_RSA
OP_MICROSOFT_BIG_SSLV3_BUFFER = SSL.OP_MICROSOFT_BIG_SSLV3_BUFFER
OP_MICROSOFT_SESS_ID_BUG = SSL.OP_MICROSOFT_SESS_ID_BUG
OP_MSIE_SSLV2_RSA_PADDING = SSL.OP_MSIE_SSLV2_RSA_PADDING
OP_NETSCAPE_CA_DN_BUG = SSL.OP_NETSCAPE_CA_DN_BUG
OP_NETSCAPE_CHALLENGE_BUG = SSL.OP_NETSCAPE_CHALLENGE_BUG
OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG = SSL.OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG
OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG = SSL.OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG
OP_NO_QUERY_MTU = SSL.OP_NO_QUERY_MTU
OP_NO_SSLv2 = SSL.OP_NO_SSLv2
OP_NO_SSLv3 = SSL.OP_NO_SSLv3
OP_NO_TICKET = SSL.OP_NO_TICKET
OP_NO_TLSv1 = SSL.OP_NO_TLSv1
OP_PKCS1_CHECK_1 = SSL.OP_PKCS1_CHECK_1
OP_PKCS1_CHECK_2 = SSL.OP_PKCS1_CHECK_2
OP_SINGLE_DH_USE = SSL.OP_SINGLE_DH_USE
OP_SSLEAY_080_CLIENT_DH_BUG = SSL.OP_SSLEAY_080_CLIENT_DH_BUG
OP_SSLREF2_REUSE_CERT_TYPE_BUG = SSL.OP_SSLREF2_REUSE_CERT_TYPE_BUG
OP_TLS_BLOCK_PADDING_BUG = SSL.OP_TLS_BLOCK_PADDING_BUG
OP_TLS_D5_BUG = SSL.OP_TLS_D5_BUG
OP_TLS_ROLLBACK_BUG = SSL.OP_TLS_ROLLBACK_BUG

class NetLibError(Exception): pass
class NetLibDisconnect(NetLibError): pass
class NetLibTimeout(NetLibError): pass

class _FileLike:
    BLOCKSIZE = 1024 * 32
    def __init__(self, o):
        self.o = o
        self._log = None
        self.first_byte_timestamp = None

    def set_descriptor(self, o):
        self.o = o

    def __getattr__(self, attr):
        return getattr(self.o, attr)

    def start_log(self):
        """
            Starts or resets the log.

            This will store all bytes read or written.
        """
        self._log = []

    def stop_log(self):
        """
            Stops the log.
        """
        self._log = None

    def is_logging(self):
        return self._log is not None

    def get_log(self):
        """
            Returns the log as a string.
        """
        if not self.is_logging():
            raise ValueError("Not logging!")
        return "".join(self._log)

    def add_log(self, v):
        if self.is_logging():
            self._log.append(v)

    def reset_timestamps(self):
        self.first_byte_timestamp = None

class Writer(_FileLike):
    def flush(self):
        """
            May raise NetLibDisconnect
        """
        if hasattr(self.o, "flush"):
            try:
                self.o.flush()
            except (socket.error, IOError), v:
                raise NetLibDisconnect(str(v))

    def write(self, v):
        """
            May raise NetLibDisconnect
        """
        if v:
            try:
                if hasattr(self.o, "sendall"):
                    self.add_log(v)
                    return self.o.sendall(v)
                else:
                    r = self.o.write(v)
                    self.add_log(v[:r])
                    return r
            except (SSL.Error, socket.error), v:
                raise NetLibDisconnect(str(v))

class Reader(_FileLike):
    def read(self, length):
        """
            If length is -1, we read until connection closes.
        """
        result = ''
        start = time.time()
        while length == -1 or length > 0:
            if length == -1 or length > self.BLOCKSIZE:
                rlen = self.BLOCKSIZE
            else:
                rlen = length
            try:
                data = self.o.read(rlen)
            except SSL.ZeroReturnError:
                break
            except SSL.WantReadError:
                if (time.time() - start) < self.o.gettimeout():
                    time.sleep(0.1)
                    continue
                else:
                    raise NetLibTimeout
            except socket.timeout:
                raise NetLibTimeout
            except socket.error:
                raise NetLibDisconnect
            except SSL.SysCallError:
                raise NetLibDisconnect
            self.first_byte_timestamp = self.first_byte_timestamp or time.time()
            if not data:
                break
            result += data
            if length != -1:
                length -= len(data)
        self.add_log(result)
        return result

    def readline(self, size = None):
        result = ''
        bytes_read = 0
        while True:
            if size is not None and bytes_read >= size:
                break
            try:
                ch = self.read(1)
            except NetLibDisconnect:
                break
            bytes_read += 1
            if not ch:
                break
            else:
                result += ch
                if ch == '\n':
                    break
        return result

class TCPClient:
    rbufsize = -1
    wbufsize = -1
    def __init__(self, host, port, source_address=None):
        self.host, self.port = host, port
        self.connection, self.rfile, self.wfile = None, None, None
        self.cert = None
        self.ssl_established = False
        self.source_address = source_address

    def convert_to_ssl(self, cert=None, sni=None, method=TLSv1_METHOD, options=None):
        """
            cert: Path to a file containing both client cert and private key.
        """
        context = SSL.Context(method)
        if options is not None:
            context.set_options(options)
        if cert:
            try:
                context.use_privatekey_file(cert)
                context.use_certificate_file(cert)
            except SSL.Error, v:
                raise NetLibError("SSL client certificate error: %s"%str(v))
        self.connection = SSL.Connection(context, self.connection)
        self.ssl_established = True
        if sni:
            self.connection.set_tlsext_host_name(sni)
        self.connection.set_connect_state()
        try:
            self.connection.do_handshake()
        except SSL.Error, v:
            raise NetLibError("SSL handshake error: %s"%str(v))
        self.cert = certutils.SSLCert(self.connection.get_peer_certificate())
        self.rfile.set_descriptor(self.connection)
        self.wfile.set_descriptor(self.connection)

    def connect(self):
        try:
            addr = socket.gethostbyname(self.host)
            connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if self.source_address:
                connection.bind(self.source_address)
            connection.connect((addr, self.port))
            self.rfile = Reader(connection.makefile('rb', self.rbufsize))
            self.wfile = Writer(connection.makefile('wb', self.wbufsize))
        except (socket.error, IOError), err:
            raise NetLibError('Error connecting to "%s": %s' % (self.host, err))
        self.connection = connection

    def settimeout(self, n):
        self.connection.settimeout(n)

    def gettimeout(self):
        return self.connection.gettimeout()

    def close(self):
        """
            Does a hard close of the socket, i.e. a shutdown, followed by a close.
        """
        try:
            if self.ssl_established:
                self.connection.shutdown()
            else:
                self.connection.shutdown(socket.SHUT_WR)
            #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent.
            #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html
            while self.connection.recv(4096):
                pass
            self.connection.close()
        except (socket.error, SSL.Error, IOError):
            # Socket probably already closed
            pass

class BaseHandler:
    """
        The instantiator is expected to call the handle() and finish() methods.

    """
    rbufsize = -1
    wbufsize = -1
    def __init__(self, connection, client_address, server):
        self.connection = connection
        self.rfile = Reader(self.connection.makefile('rb', self.rbufsize))
        self.wfile = Writer(self.connection.makefile('wb', self.wbufsize))

        self.client_address = client_address
        self.server = server
        self.finished = False
        self.ssl_established = False

        self.clientcert = None

    def convert_to_ssl(self, cert, key, method=SSLv23_METHOD, options=None, handle_sni=None, request_client_cert=False):
        """
            method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
            handle_sni: SNI handler, should take a connection object. Server
            name can be retrieved like this:

                            connection.get_servername()

                        And you can specify the connection keys as follows:

                            new_context = Context(TLSv1_METHOD)
                            new_context.use_privatekey(key)
                            new_context.use_certificate(cert)
                            connection.set_context(new_context)

            The request_client_cert argument requires some explanation. We're
            supposed to be able to do this with no negative effects - if the
            client has no cert to present, we're notified and proceed as usual.
            Unfortunately, Android seems to have a bug (tested on 4.2.2) - when
            an Android client is asked to present a certificate it does not
            have, it hangs up, which is frankly bogus. Some time down the track
            we may be able to make the proper behaviour the default again, but
            until then we're conservative.
        """
        ctx = SSL.Context(method)
        if not options is None:
            ctx.set_options(options)
        if handle_sni:
            # SNI callback happens during do_handshake()
            ctx.set_tlsext_servername_callback(handle_sni)
        ctx.use_privatekey_file(key)
        ctx.use_certificate_file(cert)
        if request_client_cert:
            def ver(*args):
                self.clientcert = certutils.SSLCert(args[1])
            ctx.set_verify(SSL.VERIFY_PEER, ver)
        self.connection = SSL.Connection(ctx, self.connection)
        self.ssl_established = True
        self.connection.set_accept_state()
        try:
            self.connection.do_handshake()
        except SSL.Error, v:
            raise NetLibError("SSL handshake error: %s"%str(v))
        self.rfile.set_descriptor(self.connection)
        self.wfile.set_descriptor(self.connection)

    def finish(self):
        self.finished = True
        try:
            if not getattr(self.wfile, "closed", False):
                self.wfile.flush()
            self.close()
            self.wfile.close()
            self.rfile.close()
        except (socket.error, NetLibDisconnect):
            # Remote has disconnected
            pass

    def handle(self): # pragma: no cover
        raise NotImplementedError

    def settimeout(self, n):
        self.connection.settimeout(n)

    def close(self):
        """
            Does a hard close of the socket, i.e. a shutdown, followed by a close.
        """
        try:
            if self.ssl_established:
                self.connection.shutdown()
            else:
                self.connection.shutdown(socket.SHUT_WR)
            #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent.
            #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html
            while self.connection.recv(4096):
                pass
        except (socket.error, SSL.Error):
            # Socket probably already closed
            pass

        self.connection.close()

class TCPServer:
    request_queue_size = 20
    def __init__(self, server_address):
        self.server_address = server_address
        self.__is_shut_down = threading.Event()
        self.__shutdown_request = False
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind(self.server_address)
        self.server_address = self.socket.getsockname()
        self.port = self.server_address[1]
        self.socket.listen(self.request_queue_size)

    def request_thread(self, request, client_address):
        try:
            self.handle_connection(request, client_address)
            request.close()
        except:
            self.handle_error(request, client_address)
            request.close()

    def serve_forever(self, poll_interval=0.1):
        self.__is_shut_down.clear()
        try:
            while not self.__shutdown_request:
                r, w, e = select.select([self.socket], [], [], poll_interval)
                if self.socket in r:
                    request, client_address = self.socket.accept()
                    t = threading.Thread(
                            target = self.request_thread,
                            args = (request, client_address)
                        )
                    t.setDaemon(1)
                    t.start()
        finally:
            self.__shutdown_request = False
            self.__is_shut_down.set()

    def shutdown(self):
        self.__shutdown_request = True
        self.__is_shut_down.wait()
        self.socket.close()
        self.handle_shutdown()

    def handle_error(self, request, client_address, fp=sys.stderr):
        """
            Called when handle_connection raises an exception.
        """
        # If a thread has persisted after interpreter exit, the module might be
        # none.
        if traceback:
            exc = traceback.format_exc()
            print >> fp, '-'*40
            print >> fp, "Error in processing of request from %s:%s"%client_address
            print >> fp, exc
            print >> fp, '-'*40

    def handle_connection(self, request, client_address): # pragma: no cover
        """
            Called after client connection.
        """
        raise NotImplementedError

    def handle_shutdown(self):
        """
            Called after server shutdown.
        """
        pass
<commit_msg>Better handling of cert errors.<commit_after>import select, socket, threading, sys, time, traceback
from OpenSSL import SSL
import certutils

SSLv2_METHOD = SSL.SSLv2_METHOD
SSLv3_METHOD = SSL.SSLv3_METHOD
SSLv23_METHOD = SSL.SSLv23_METHOD
TLSv1_METHOD = SSL.TLSv1_METHOD

OP_ALL = SSL.OP_ALL
OP_CIPHER_SERVER_PREFERENCE = SSL.OP_CIPHER_SERVER_PREFERENCE
OP_COOKIE_EXCHANGE = SSL.OP_COOKIE_EXCHANGE
OP_DONT_INSERT_EMPTY_FRAGMENTS = SSL.OP_DONT_INSERT_EMPTY_FRAGMENTS
OP_EPHEMERAL_RSA = SSL.OP_EPHEMERAL_RSA
OP_MICROSOFT_BIG_SSLV3_BUFFER = SSL.OP_MICROSOFT_BIG_SSLV3_BUFFER
OP_MICROSOFT_SESS_ID_BUG = SSL.OP_MICROSOFT_SESS_ID_BUG
OP_MSIE_SSLV2_RSA_PADDING = SSL.OP_MSIE_SSLV2_RSA_PADDING
OP_NETSCAPE_CA_DN_BUG = SSL.OP_NETSCAPE_CA_DN_BUG
OP_NETSCAPE_CHALLENGE_BUG = SSL.OP_NETSCAPE_CHALLENGE_BUG
OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG = SSL.OP_NETSCAPE_DEMO_CIPHER_CHANGE_BUG
OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG = SSL.OP_NETSCAPE_REUSE_CIPHER_CHANGE_BUG
OP_NO_QUERY_MTU = SSL.OP_NO_QUERY_MTU
OP_NO_SSLv2 = SSL.OP_NO_SSLv2
OP_NO_SSLv3 = SSL.OP_NO_SSLv3
OP_NO_TICKET = SSL.OP_NO_TICKET
OP_NO_TLSv1 = SSL.OP_NO_TLSv1
OP_PKCS1_CHECK_1 = SSL.OP_PKCS1_CHECK_1
OP_PKCS1_CHECK_2 = SSL.OP_PKCS1_CHECK_2
OP_SINGLE_DH_USE = SSL.OP_SINGLE_DH_USE
OP_SSLEAY_080_CLIENT_DH_BUG = SSL.OP_SSLEAY_080_CLIENT_DH_BUG
OP_SSLREF2_REUSE_CERT_TYPE_BUG = SSL.OP_SSLREF2_REUSE_CERT_TYPE_BUG
OP_TLS_BLOCK_PADDING_BUG = SSL.OP_TLS_BLOCK_PADDING_BUG
OP_TLS_D5_BUG = SSL.OP_TLS_D5_BUG
OP_TLS_ROLLBACK_BUG = SSL.OP_TLS_ROLLBACK_BUG

class NetLibError(Exception): pass
class NetLibDisconnect(NetLibError): pass
class NetLibTimeout(NetLibError): pass
class NetLibSSLError(NetLibError): pass

class _FileLike:
    BLOCKSIZE = 1024 * 32
    def __init__(self, o):
        self.o = o
        self._log = None
        self.first_byte_timestamp = None

    def set_descriptor(self, o):
        self.o = o

    def __getattr__(self, attr):
        return getattr(self.o, attr)

    def start_log(self):
        """
            Starts or resets the log.

            This will store all bytes read or written.
        """
        self._log = []

    def stop_log(self):
        """
            Stops the log.
        """
        self._log = None

    def is_logging(self):
        return self._log is not None

    def get_log(self):
        """
            Returns the log as a string.
        """
        if not self.is_logging():
            raise ValueError("Not logging!")
        return "".join(self._log)

    def add_log(self, v):
        if self.is_logging():
            self._log.append(v)

    def reset_timestamps(self):
        self.first_byte_timestamp = None

class Writer(_FileLike):
    def flush(self):
        """
            May raise NetLibDisconnect
        """
        if hasattr(self.o, "flush"):
            try:
                self.o.flush()
            except (socket.error, IOError), v:
                raise NetLibDisconnect(str(v))

    def write(self, v):
        """
            May raise NetLibDisconnect
        """
        if v:
            try:
                if hasattr(self.o, "sendall"):
                    self.add_log(v)
                    return self.o.sendall(v)
                else:
                    r = self.o.write(v)
                    self.add_log(v[:r])
                    return r
            except (SSL.Error, socket.error), v:
                raise NetLibDisconnect(str(v))

class Reader(_FileLike):
    def read(self, length):
        """
            If length is -1, we read until connection closes.
        """
        result = ''
        start = time.time()
        while length == -1 or length > 0:
            if length == -1 or length > self.BLOCKSIZE:
                rlen = self.BLOCKSIZE
            else:
                rlen = length
            try:
                data = self.o.read(rlen)
            except SSL.ZeroReturnError:
                break
            except SSL.Error, v:
                raise NetLibSSLError(v.message)
            except SSL.WantReadError:
                if (time.time() - start) < self.o.gettimeout():
                    time.sleep(0.1)
                    continue
                else:
                    raise NetLibTimeout
            except socket.timeout:
                raise NetLibTimeout
            except socket.error:
                raise NetLibDisconnect
            except SSL.SysCallError:
                raise NetLibDisconnect
            self.first_byte_timestamp = self.first_byte_timestamp or time.time()
            if not data:
                break
            result += data
            if length != -1:
                length -= len(data)
        self.add_log(result)
        return result

    def readline(self, size = None):
        result = ''
        bytes_read = 0
        while True:
            if size is not None and bytes_read >= size:
                break
            try:
                ch = self.read(1)
            except NetLibDisconnect:
                break
            bytes_read += 1
            if not ch:
                break
            else:
                result += ch
                if ch == '\n':
                    break
        return result

class TCPClient:
    rbufsize = -1
    wbufsize = -1
    def __init__(self, host, port, source_address=None):
        self.host, self.port = host, port
        self.connection, self.rfile, self.wfile = None, None, None
        self.cert = None
        self.ssl_established = False
        self.source_address = source_address

    def convert_to_ssl(self, cert=None, sni=None, method=TLSv1_METHOD, options=None):
        """
            cert: Path to a file containing both client cert and private key.
        """
        context = SSL.Context(method)
        if options is not None:
            context.set_options(options)
        if cert:
            try:
                context.use_privatekey_file(cert)
                context.use_certificate_file(cert)
            except SSL.Error, v:
                raise NetLibError("SSL client certificate error: %s"%str(v))
        self.connection = SSL.Connection(context, self.connection)
        self.ssl_established = True
        if sni:
            self.connection.set_tlsext_host_name(sni)
        self.connection.set_connect_state()
        try:
            self.connection.do_handshake()
        except SSL.Error, v:
            raise NetLibError("SSL handshake error: %s"%str(v))
        self.cert = certutils.SSLCert(self.connection.get_peer_certificate())
        self.rfile.set_descriptor(self.connection)
        self.wfile.set_descriptor(self.connection)

    def connect(self):
        try:
            addr = socket.gethostbyname(self.host)
            connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            if self.source_address:
                connection.bind(self.source_address)
            connection.connect((addr, self.port))
            self.rfile = Reader(connection.makefile('rb', self.rbufsize))
            self.wfile = Writer(connection.makefile('wb', self.wbufsize))
        except (socket.error, IOError), err:
            raise NetLibError('Error connecting to "%s": %s' % (self.host, err))
        self.connection = connection

    def settimeout(self, n):
        self.connection.settimeout(n)

    def gettimeout(self):
        return self.connection.gettimeout()

    def close(self):
        """
            Does a hard close of the socket, i.e. a shutdown, followed by a close.
        """
        try:
            if self.ssl_established:
                self.connection.shutdown()
            else:
                self.connection.shutdown(socket.SHUT_WR)
            #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent.
            #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html
            while self.connection.recv(4096):
                pass
            self.connection.close()
        except (socket.error, SSL.Error, IOError):
            # Socket probably already closed
            pass

class BaseHandler:
    """
        The instantiator is expected to call the handle() and finish() methods.

    """
    rbufsize = -1
    wbufsize = -1
    def __init__(self, connection, client_address, server):
        self.connection = connection
        self.rfile = Reader(self.connection.makefile('rb', self.rbufsize))
        self.wfile = Writer(self.connection.makefile('wb', self.wbufsize))

        self.client_address = client_address
        self.server = server
        self.finished = False
        self.ssl_established = False

        self.clientcert = None

    def convert_to_ssl(self, cert, key, method=SSLv23_METHOD, options=None, handle_sni=None, request_client_cert=False):
        """
            method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, or TLSv1_METHOD
            handle_sni: SNI handler, should take a connection object. Server
            name can be retrieved like this:

                            connection.get_servername()

                        And you can specify the connection keys as follows:

                            new_context = Context(TLSv1_METHOD)
                            new_context.use_privatekey(key)
                            new_context.use_certificate(cert)
                            connection.set_context(new_context)

            The request_client_cert argument requires some explanation. We're
            supposed to be able to do this with no negative effects - if the
            client has no cert to present, we're notified and proceed as usual.
            Unfortunately, Android seems to have a bug (tested on 4.2.2) - when
            an Android client is asked to present a certificate it does not
            have, it hangs up, which is frankly bogus. Some time down the track
            we may be able to make the proper behaviour the default again, but
            until then we're conservative.
        """
        ctx = SSL.Context(method)
        if not options is None:
            ctx.set_options(options)
        if handle_sni:
            # SNI callback happens during do_handshake()
            ctx.set_tlsext_servername_callback(handle_sni)
        ctx.use_privatekey_file(key)
        ctx.use_certificate_file(cert)
        if request_client_cert:
            def ver(*args):
                self.clientcert = certutils.SSLCert(args[1])
            ctx.set_verify(SSL.VERIFY_PEER, ver)
        self.connection = SSL.Connection(ctx, self.connection)
        self.ssl_established = True
        self.connection.set_accept_state()
        try:
            self.connection.do_handshake()
        except SSL.Error, v:
            raise NetLibError("SSL handshake error: %s"%str(v))
        self.rfile.set_descriptor(self.connection)
        self.wfile.set_descriptor(self.connection)

    def finish(self):
        self.finished = True
        try:
            if not getattr(self.wfile, "closed", False):
                self.wfile.flush()
            self.close()
            self.wfile.close()
            self.rfile.close()
        except (socket.error, NetLibDisconnect):
            # Remote has disconnected
            pass

    def handle(self): # pragma: no cover
        raise NotImplementedError

    def settimeout(self, n):
        self.connection.settimeout(n)

    def close(self):
        """
            Does a hard close of the socket, i.e. a shutdown, followed by a close.
        """
        try:
            if self.ssl_established:
                self.connection.shutdown()
            else:
                self.connection.shutdown(socket.SHUT_WR)
            #Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending readable data could lead to an immediate RST being sent.
            #http://ia600609.us.archive.org/22/items/TheUltimateSo_lingerPageOrWhyIsMyTcpNotReliable/the-ultimate-so_linger-page-or-why-is-my-tcp-not-reliable.html
            while self.connection.recv(4096):
                pass
        except (socket.error, SSL.Error):
            # Socket probably already closed
            pass

        self.connection.close()

class TCPServer:
    request_queue_size = 20
    def __init__(self, server_address):
        self.server_address = server_address
        self.__is_shut_down = threading.Event()
        self.__shutdown_request = False
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.socket.bind(self.server_address)
        self.server_address = self.socket.getsockname()
        self.port = self.server_address[1]
        self.socket.listen(self.request_queue_size)

    def request_thread(self, request, client_address):
        try:
            self.handle_connection(request, client_address)
            request.close()
        except:
            self.handle_error(request, client_address)
            request.close()

    def serve_forever(self, poll_interval=0.1):
        self.__is_shut_down.clear()
        try:
            while not self.__shutdown_request:
                r, w, e = select.select([self.socket], [], [], poll_interval)
                if self.socket in r:
                    request, client_address = self.socket.accept()
                    t = threading.Thread(
                            target = self.request_thread,
                            args = (request, client_address)
                        )
                    t.setDaemon(1)
                    t.start()
        finally:
            self.__shutdown_request = False
            self.__is_shut_down.set()

    def shutdown(self):
        self.__shutdown_request = True
        self.__is_shut_down.wait()
        self.socket.close()
        self.handle_shutdown()

    def handle_error(self, request, client_address, fp=sys.stderr):
        """
            Called when handle_connection raises an exception.
        """
        # If a thread has persisted after interpreter exit, the module might be
        # none.
        if traceback:
            exc = traceback.format_exc()
            print >> fp, '-'*40
            print >> fp, "Error in processing of request from %s:%s"%client_address
            print >> fp, exc
            print >> fp, '-'*40

    def handle_connection(self, request, client_address): # pragma: no cover
        """
            Called after client connection.
        """
        raise NotImplementedError

    def handle_shutdown(self):
        """
            Called after server shutdown.
        """
        pass
<|endoftext|>


<commit_before>import math

objects = []
springs = []

def add_object(x, halfsize, rotation=0):
  objects.append([x, halfsize, rotation])

def add_spring(a, b, offset_a, offset_b, length, stiffness):
  springs.append([a, b, offset_a, offset_b, length, stiffness])

def robotA():
  add_object(x=[0.3, 0.25], halfsize=[0.15, 0.03])
  add_object(x=[0.2, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.3, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.3], halfsize=[0.005, 0.03])

  l = 0.12
  s = 15
  add_spring(0, 1, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 1, [-0.1, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.1, 0.00], [0.0, 0.0], l, s)
  # -1 means the spring is a joint
  add_spring(0, 4, [0.1, 0], [0, -0.05], -1, s)

  return objects, springs

def robotB():
  add_object(x=[0.3, 0.25], halfsize=[0.15, 0.03])
  add_object(x=[0.2, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.3, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.15], halfsize=[0.03, 0.02])

  l = 0.12
  s = 15
  add_spring(0, 1, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 1, [-0.1, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.1, 0.00], [0.0, 0.0], l, s)

  return objects, springs

l_thigh_init_ang = 45
l_calf_init_ang = -10
r_thigh_init_ang = -10
r_calf_init_ang = -10
initHeight = 0.15

hip_pos = [0.3, 0.5 + initHeight]
thigh_half_length = 0.11
calf_half_length = 0.11

foot_half_length = 0.08

def rotAlong(half_length, deg, center):
  ang = math.radians(deg)
  return [half_length*math.sin(ang) + center[0], -half_length*math.cos(ang) + center[1]]

def robotLeg():
  #hip
  add_object(hip_pos, halfsize=[0.06, 0.11])
  hip_end = [hip_pos[0], hip_pos[1] - 0.1]

  #left
  l_thigh_center = rotAlong(thigh_half_length, l_thigh_init_ang, hip_end)
  l_thigh_end = rotAlong(thigh_half_length * 2.0, l_thigh_init_ang, hip_end)
  add_object(l_thigh_center, halfsize=[0.02, thigh_half_length], rotation=math.radians(l_thigh_init_ang))
  add_object(rotAlong(calf_half_length, l_calf_init_ang, l_thigh_end), halfsize=[0.02, calf_half_length], rotation = math.radians(l_calf_init_ang))
  l_calf_end = rotAlong(2.0 * calf_half_length, l_calf_init_ang, l_thigh_end)
  add_object([l_calf_end[0] + foot_half_length, l_calf_end[1]], halfsize=[foot_half_length, 0.02])

  #right
  add_object(rotAlong(thigh_half_length, r_thigh_init_ang, hip_end), halfsize=[0.02, thigh_half_length], rotation=math.radians(r_thigh_init_ang))
  r_thigh_end = rotAlong(thigh_half_length * 2.0, r_thigh_init_ang, hip_end)
  add_object(rotAlong(calf_half_length, r_calf_init_ang, r_thigh_end), halfsize=[0.02, calf_half_length], rotation = math.radians(r_calf_init_ang))
  r_calf_end = rotAlong(2.0 * calf_half_length, r_calf_init_ang, r_thigh_end)
  add_object([r_calf_end[0] + foot_half_length, r_calf_end[1]], halfsize=[foot_half_length, 0.02])

  l = 0.12
  s = 15

  #left springs
  add_spring(0, 1, [0,0.1], [0,-thigh_half_length], thigh_half_length + 0.1, 15)
  add_spring(1, 2, [0,thigh_half_length], [0,-thigh_half_length], 2.0* thigh_half_length, 15)
  add_spring(2, 3, [0,0], [foot_half_length,0], thigh_half_length, 15)

  add_spring(0, 1, [0, -0.1], [0.0, thigh_half_length], -1, s)
  add_spring(1, 2, [0, -thigh_half_length], [0.0, thigh_half_length], -1, s)
  add_spring(2, 3, [0, -thigh_half_length], [-foot_half_length,0], -1, s)

  #right springs
  add_spring(0, 4, [0,0.1], [0,-thigh_half_length], thigh_half_length + 0.1, 15)
  add_spring(4, 5, [0,thigh_half_length], [0,-thigh_half_length], 2.0* thigh_half_length, 15)
  add_spring(5, 6, [0,0], [foot_half_length,0], thigh_half_length, 15)

  add_spring(0, 4, [0, -0.1], [0.0, thigh_half_length], -1, s)
  add_spring(4, 5, [0, -thigh_half_length], [0.0, thigh_half_length], -1, s)
  add_spring(5, 6, [0, -thigh_half_length], [-foot_half_length,0], -1, s)

  return objects, springs

#robots = [robotA, robotB]
robots = [robotLeg]<commit_msg>still tweaking<commit_after>import math

objects = []
springs = []

def add_object(x, halfsize, rotation=0):
  objects.append([x, halfsize, rotation])

def add_spring(a, b, offset_a, offset_b, length, stiffness):
  springs.append([a, b, offset_a, offset_b, length, stiffness])

def robotA():
  add_object(x=[0.3, 0.25], halfsize=[0.15, 0.03])
  add_object(x=[0.2, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.3, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.3], halfsize=[0.005, 0.03])

  l = 0.12
  s = 15
  add_spring(0, 1, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 1, [-0.1, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.1, 0.00], [0.0, 0.0], l, s)
  # -1 means the spring is a joint
  add_spring(0, 4, [0.1, 0], [0, -0.05], -1, s)

  return objects, springs

def robotB():
  add_object(x=[0.3, 0.25], halfsize=[0.15, 0.03])
  add_object(x=[0.2, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.3, 0.15], halfsize=[0.03, 0.02])
  add_object(x=[0.4, 0.15], halfsize=[0.03, 0.02])

  l = 0.12
  s = 15
  add_spring(0, 1, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 1, [-0.1, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [-0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 2, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.03, 0.00], [0.0, 0.0], l, s)
  add_spring(0, 3, [0.1, 0.00], [0.0, 0.0], l, s)

  return objects, springs

l_thigh_init_ang = 45
l_calf_init_ang = -10
r_thigh_init_ang = -10
r_calf_init_ang = -10
initHeight = 0.15

hip_pos = [0.3, 0.5 + initHeight]
thigh_half_length = 0.11
calf_half_length = 0.11

foot_half_length = 0.08

def rotAlong(half_length, deg, center):
  ang = math.radians(deg)
  return [half_length*math.sin(ang) + center[0], -half_length*math.cos(ang) + center[1]]

def robotLeg():
  #hip
  add_object(hip_pos, halfsize=[0.06, 0.11])
  hip_end = [hip_pos[0], hip_pos[1] - 0.1]

  #left
  l_thigh_center = rotAlong(thigh_half_length, l_thigh_init_ang, hip_end)
  l_thigh_end = rotAlong(thigh_half_length * 2.0, l_thigh_init_ang, hip_end)
  add_object(l_thigh_center, halfsize=[0.02, thigh_half_length], rotation=math.radians(l_thigh_init_ang))
  add_object(rotAlong(calf_half_length, l_calf_init_ang, l_thigh_end), halfsize=[0.02, calf_half_length], rotation = math.radians(l_calf_init_ang))
  l_calf_end = rotAlong(2.0 * calf_half_length, l_calf_init_ang, l_thigh_end)
  add_object([l_calf_end[0] + foot_half_length, l_calf_end[1]], halfsize=[foot_half_length, 0.02])

  #right
  add_object(rotAlong(thigh_half_length, r_thigh_init_ang, hip_end), halfsize=[0.02, thigh_half_length], rotation=math.radians(r_thigh_init_ang))
  r_thigh_end = rotAlong(thigh_half_length * 2.0, r_thigh_init_ang, hip_end)
  add_object(rotAlong(calf_half_length, r_calf_init_ang, r_thigh_end), halfsize=[0.02, calf_half_length], rotation = math.radians(r_calf_init_ang))
  r_calf_end = rotAlong(2.0 * calf_half_length, r_calf_init_ang, r_thigh_end)
  add_object([r_calf_end[0] + foot_half_length, r_calf_end[1]], halfsize=[foot_half_length, 0.02])

  s = 200

  #left springs
  add_spring(0, 1, [0,0.1], [0,-thigh_half_length], 2.0*thigh_half_length + 0.22, 50)
  add_spring(1, 2, [0,thigh_half_length], [0,-thigh_half_length], 4.0* thigh_half_length, 50)
  add_spring(2, 3, [0,0], [foot_half_length,0], math.sqrt(pow(thigh_half_length,2)+pow(2.0*foot_half_length,2)), 50)

  add_spring(0, 1, [0, -0.1], [0.0, thigh_half_length], -1, s)
  add_spring(1, 2, [0, -thigh_half_length], [0.0, thigh_half_length], -1, s)
  add_spring(2, 3, [0, -thigh_half_length], [-foot_half_length,0], -1, s)

  #right springs
  add_spring(0, 4, [0,0.1], [0,-thigh_half_length], 2.0*thigh_half_length + 0.22, 50)
  add_spring(4, 5, [0,thigh_half_length], [0,-thigh_half_length], 4.0* thigh_half_length, 50)
  add_spring(5, 6, [0,0], [foot_half_length,0], math.sqrt(pow(thigh_half_length,2)+pow(2.0*foot_half_length,2)), 50)

  add_spring(0, 4, [0, -0.1], [0.0, thigh_half_length], -1, s)
  add_spring(4, 5, [0, -thigh_half_length], [0.0, thigh_half_length], -1, s)
  add_spring(5, 6, [0, -thigh_half_length], [-foot_half_length,0], -1, s)

  return objects, springs

#robots = [robotA, robotB]
robots = [robotLeg]<|endoftext|>


<commit_before>import os
import uuid
from datetime import datetime

import settings
from casexml.apps.case.xform import get_all_extensions_to_close, CaseProcessingResult
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.interfaces.processor import FormProcessorInterface, ProcessedForms
from corehq.form_processor.models import XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
from corehq.form_processor.submission_post import CaseStockProcessingResult
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.utils.general import set_local_domain_sql_backend_override
from couchforms.models import XFormInstance, doc_types
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider

def do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    set_local_domain_sql_backend_override(domain)
    CouchSqlDomainMigrator(domain).migrate()
    # (optional) compare the information collected to the information at the beginning

class CouchSqlDomainMigrator(object):
    def __init__(self, domain):
        from corehq.apps.tzmigration.planning import DiffDB

        assert should_use_sql_backend(domain)
        self.domain = domain
        db_filepath = get_diff_db_filepath(domain)
        self.diff_db = DiffDB.init(db_filepath)

    def migrate(self):
        self._process_main_forms()
        self._copy_unprocessed_forms()

    def _process_main_forms(self):
        last_received_on = datetime.min
        # process main forms (including cases and ledgers)
        for change in _get_main_form_iterator(self.domain).iter_all_changes():
            form = change.get_document()
            wrapped_form = XFormInstance.wrap(form)
            form_received = wrapped_form.received_on
            assert last_received_on <= form_received
            last_received_on = form_received
            print 'processing form {}: {}'.format(form['_id'], form_received)
            self._migrate_form_and_associated_models(wrapped_form)

    def _migrate_form_and_associated_models(self, couch_form):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        sql_form = _migrate_form(self.domain, couch_form)
        _migrate_form_attachments(sql_form, couch_form)
        _migrate_form_operations(sql_form, couch_form)

        self.diff_db.add_diffs(
            'form', couch_form.form_id,
            json_diff(couch_form.to_json(), sql_form.to_json())
        )

        case_stock_result = _get_case_and_ledger_updates(self.domain, sql_form)
        _save_migrated_models(sql_form, case_stock_result)

    def _copy_unprocessed_forms(self):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        for change in _get_unprocessed_form_iterator(self.domain).iter_all_changes():
            couch_form_json = change.get_document()
            couch_form = _wrap_form(couch_form_json)
            print 'copying unprocessed {} {}: {}'.format(couch_form.doc_type, couch_form.form_id, couch_form.received_on)
            sql_form = XFormInstanceSQL(
                form_id=couch_form.form_id,
                xmlns=couch_form.xmlns,
                user_id=couch_form.user_id,
            )
            _copy_form_properties(self.domain, sql_form, couch_form)
            _migrate_form_attachments(sql_form, couch_form)
            _migrate_form_operations(sql_form, couch_form)

            self.diff_db.add_diffs(
                'form', couch_form.form_id,
                json_diff(couch_form.to_json(), sql_form.to_json())
            )

            _save_migrated_models(sql_form)

def _wrap_form(doc):
    if doc['doc_type'] in doc_types():
        return doc_types()[doc['doc_type']].wrap(doc)
    if doc['doc_type'] in ("XFormInstance-Deleted", "HQSubmission"):
        return XFormInstance.wrap(doc)

def _migrate_form(domain, couch_form):
    """
    This copies the couch form into a new sql form but does not save it.

    See form_processor.parsers.form._create_new_xform
    and SubmissionPost._set_submission_properties for what this should do.
    """
    interface = FormProcessorInterface(domain)

    form_data = couch_form.form
    # todo: timezone migration if we want here
    # adjust_datetimes(form_data)
    sql_form = interface.new_xform(form_data)
    return _copy_form_properties(domain, sql_form, couch_form)

def _copy_form_properties(domain, sql_form, couch_form):
    assert isinstance(sql_form, XFormInstanceSQL)
    sql_form.domain = domain

    # submission properties
    sql_form.auth_context = couch_form.auth_context
    sql_form.submit_ip = couch_form.submit_ip

    # todo: this property appears missing from sql forms - do we need it?
    # sql_form.path = couch_form.path

    sql_form.openrosa_headers = couch_form.openrosa_headers
    sql_form.last_sync_token = couch_form.last_sync_token
    sql_form.received_on = couch_form.received_on
    sql_form.date_header = couch_form.date_header
    sql_form.app_id = couch_form.app_id
    sql_form.build_id = couch_form.build_id
    # export_tag intentionally removed
    # sql_form.export_tag = ["domain", "xmlns"]
    sql_form.partial_submission = couch_form.partial_submission
    sql_form.initial_processing_complete = couch_form.initial_processing_complete
    return sql_form

def _migrate_form_attachments(sql_form, couch_form):
    """Copy over attachement meta - includes form.xml"""
    for name, blob in couch_form.blobs.iteritems():
        sql_form.track_create(XFormAttachmentSQL(
            name=name,
            form=sql_form,
            attachment_id=uuid.uuid4().hex,
            content_type=blob.content_type,
            content_length=blob.content_length,
            blob_id=blob.id,
            md5=blob.info.md5_hash
        ))

def _migrate_form_operations(sql_form, couch_form):
    for couch_form_op in couch_form.history:
        sql_form.track_create(XFormOperationSQL(
            form=sql_form,
            user_id=couch_form_op.user,
            date=couch_form_op.date,
            operation=couch_form_op.operation
        ))

def _get_case_and_ledger_updates(domain, sql_form):
    """
    Get a CaseStockProcessingResult with the appropriate cases and ledgers to
    be saved.

    See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
    section of the form-processing code.
    """
    from casexml.apps.case.xform import get_and_check_xform_domain
    from corehq.apps.commtrack.processing import process_stock

    interface = FormProcessorInterface(domain)

    get_and_check_xform_domain(sql_form)
    xforms = [sql_form]

    with interface.casedb_cache(domain=domain, lock=False, deleted_ok=True, xforms=xforms) as case_db:
        touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)
        extensions_to_close = get_all_extensions_to_close(domain, touched_cases.values())
        case_result = CaseProcessingResult(
            domain,
            [update.case for update in touched_cases.values()],
            [],  # ignore dirtiness_flags,
            extensions_to_close
        )
        for case in case_result.cases:
            case_db.mark_changed(case)

        stock_result = process_stock(xforms, case_db)
        cases = case_db.get_cases_for_saving(sql_form.received_on)
        stock_result.populate_models()

    return CaseStockProcessingResult(
        case_result=case_result,
        case_models=cases,
        stock_result=stock_result,
    )

def _save_migrated_models(sql_form, case_stock_result=None):
    """
    See SubmissionPost.save_processed_models for ~what this should do.
    However, note that that function does some things that this one shouldn't,
    e.g. process ownership cleanliness flags.
    """
    forms_tuple = ProcessedForms(sql_form, None)
    stock_result = case_stock_result.stock_result if case_stock_result else None
    if stock_result:
        assert stock_result.populated
    return FormProcessorSQL.save_processed_models(
        forms_tuple,
        cases=case_stock_result.case_models if case_stock_result else None,
        stock_result=stock_result,
        publish_to_kafka=False
    )

def _get_main_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=['XFormInstance'],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
    )

def _get_unprocessed_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=[
            'XFormArchived',
            'XFormError',
            'XFormDeprecated',
            'XFormDuplicate',
            # todo: need to figure out which of these we plan on supporting
            'XFormInstance-Deleted',
            'HQSubmission',
            'SubmissionErrorLog',
        ],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
    )

def get_diff_db_filepath(domain):
    return os.path.join(settings.SHARED_DRIVE_CONF.restore_dir,
                        '{}-tzmigration.db'.format(domain))

def get_diff_db(domain):
    from corehq.apps.tzmigration.planning import DiffDB

    db_filepath = get_diff_db_filepath(domain)
    return DiffDB.open(db_filepath)

def delete_diff_db(domain):
    db_filepath = get_diff_db_filepath(domain)
    try:
        os.remove(db_filepath)
    except OSError as e:
        # continue if the file didn't exist to begin with
        # reraise on any other error
        if e.errno != 2:
            raise
<commit_msg>fix attachments migration<commit_after>import os
import uuid
from datetime import datetime

import settings
from casexml.apps.case.xform import get_all_extensions_to_close, CaseProcessingResult
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.interfaces.processor import FormProcessorInterface, ProcessedForms
from corehq.form_processor.models import XFormInstanceSQL, XFormOperationSQL, XFormAttachmentSQL
from corehq.form_processor.submission_post import CaseStockProcessingResult
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.utils.general import set_local_domain_sql_backend_override
from couchforms.models import XFormInstance, doc_types
from fluff.management.commands.ptop_reindexer_fluff import ReindexEventHandler
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider

def do_couch_to_sql_migration(domain):
    # (optional) collect some information about the domain's cases and forms for cross-checking
    set_local_domain_sql_backend_override(domain)
    CouchSqlDomainMigrator(domain).migrate()
    # (optional) compare the information collected to the information at the beginning

class CouchSqlDomainMigrator(object):
    def __init__(self, domain):
        from corehq.apps.tzmigration.planning import DiffDB

        assert should_use_sql_backend(domain)
        self.domain = domain
        db_filepath = get_diff_db_filepath(domain)
        self.diff_db = DiffDB.init(db_filepath)

    def migrate(self):
        self._process_main_forms()
        self._copy_unprocessed_forms()

    def _process_main_forms(self):
        last_received_on = datetime.min
        # process main forms (including cases and ledgers)
        for change in _get_main_form_iterator(self.domain).iter_all_changes():
            form = change.get_document()
            wrapped_form = XFormInstance.wrap(form)
            form_received = wrapped_form.received_on
            assert last_received_on <= form_received
            last_received_on = form_received
            print 'processing form {}: {}'.format(form['_id'], form_received)
            self._migrate_form_and_associated_models(wrapped_form)

    def _migrate_form_and_associated_models(self, couch_form):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        sql_form = _migrate_form(self.domain, couch_form)
        _migrate_form_attachments(sql_form, couch_form)
        _migrate_form_operations(sql_form, couch_form)

        self.diff_db.add_diffs(
            'form', couch_form.form_id,
            json_diff(couch_form.to_json(), sql_form.to_json())
        )

        case_stock_result = _get_case_and_ledger_updates(self.domain, sql_form)
        _save_migrated_models(sql_form, case_stock_result)

    def _copy_unprocessed_forms(self):
        from corehq.apps.tzmigration.timezonemigration import json_diff

        for change in _get_unprocessed_form_iterator(self.domain).iter_all_changes():
            couch_form_json = change.get_document()
            couch_form = _wrap_form(couch_form_json)
            print 'copying unprocessed {} {}: {}'.format(couch_form.doc_type, couch_form.form_id, couch_form.received_on)
            sql_form = XFormInstanceSQL(
                form_id=couch_form.form_id,
                xmlns=couch_form.xmlns,
                user_id=couch_form.user_id,
            )
            _copy_form_properties(self.domain, sql_form, couch_form)
            _migrate_form_attachments(sql_form, couch_form)
            _migrate_form_operations(sql_form, couch_form)

            self.diff_db.add_diffs(
                'form', couch_form.form_id,
                json_diff(couch_form.to_json(), sql_form.to_json())
            )

            _save_migrated_models(sql_form)

def _wrap_form(doc):
    if doc['doc_type'] in doc_types():
        return doc_types()[doc['doc_type']].wrap(doc)
    if doc['doc_type'] in ("XFormInstance-Deleted", "HQSubmission"):
        return XFormInstance.wrap(doc)

def _migrate_form(domain, couch_form):
    """
    This copies the couch form into a new sql form but does not save it.

    See form_processor.parsers.form._create_new_xform
    and SubmissionPost._set_submission_properties for what this should do.
    """
    interface = FormProcessorInterface(domain)

    form_data = couch_form.form
    # todo: timezone migration if we want here
    # adjust_datetimes(form_data)
    sql_form = interface.new_xform(form_data)
    return _copy_form_properties(domain, sql_form, couch_form)

def _copy_form_properties(domain, sql_form, couch_form):
    assert isinstance(sql_form, XFormInstanceSQL)
    sql_form.domain = domain

    # submission properties
    sql_form.auth_context = couch_form.auth_context
    sql_form.submit_ip = couch_form.submit_ip

    # todo: this property appears missing from sql forms - do we need it?
    # sql_form.path = couch_form.path

    sql_form.openrosa_headers = couch_form.openrosa_headers
    sql_form.last_sync_token = couch_form.last_sync_token
    sql_form.received_on = couch_form.received_on
    sql_form.date_header = couch_form.date_header
    sql_form.app_id = couch_form.app_id
    sql_form.build_id = couch_form.build_id
    # export_tag intentionally removed
    # sql_form.export_tag = ["domain", "xmlns"]
    sql_form.partial_submission = couch_form.partial_submission
    sql_form.initial_processing_complete = couch_form.initial_processing_complete
    return sql_form

def _migrate_form_attachments(sql_form, couch_form):
    """Copy over attachement meta - includes form.xml"""
    attachments = []
    for name, blob in couch_form.blobs.iteritems():
        attachments.append(XFormAttachmentSQL(
            name=name,
            form=sql_form,
            attachment_id=uuid.uuid4().hex,
            content_type=blob.content_type,
            content_length=blob.content_length,
            blob_id=blob.id,
            md5=blob.info.md5_hash
        ))
    sql_form.unsaved_attachments = attachments

def _migrate_form_operations(sql_form, couch_form):
    for couch_form_op in couch_form.history:
        sql_form.track_create(XFormOperationSQL(
            form=sql_form,
            user_id=couch_form_op.user,
            date=couch_form_op.date,
            operation=couch_form_op.operation
        ))

def _get_case_and_ledger_updates(domain, sql_form):
    """
    Get a CaseStockProcessingResult with the appropriate cases and ledgers to
    be saved.

    See SubmissionPost.process_xforms_for_cases and methods it calls for the equivalent
    section of the form-processing code.
    """
    from casexml.apps.case.xform import get_and_check_xform_domain
    from corehq.apps.commtrack.processing import process_stock

    interface = FormProcessorInterface(domain)

    get_and_check_xform_domain(sql_form)
    xforms = [sql_form]

    with interface.casedb_cache(domain=domain, lock=False, deleted_ok=True, xforms=xforms) as case_db:
        touched_cases = FormProcessorInterface(domain).get_cases_from_forms(case_db, xforms)
        extensions_to_close = get_all_extensions_to_close(domain, touched_cases.values())
        case_result = CaseProcessingResult(
            domain,
            [update.case for update in touched_cases.values()],
            [],  # ignore dirtiness_flags,
            extensions_to_close
        )
        for case in case_result.cases:
            case_db.mark_changed(case)

        stock_result = process_stock(xforms, case_db)
        cases = case_db.get_cases_for_saving(sql_form.received_on)
        stock_result.populate_models()

    return CaseStockProcessingResult(
        case_result=case_result,
        case_models=cases,
        stock_result=stock_result,
    )

def _save_migrated_models(sql_form, case_stock_result=None):
    """
    See SubmissionPost.save_processed_models for ~what this should do.
    However, note that that function does some things that this one shouldn't,
    e.g. process ownership cleanliness flags.
    """
    forms_tuple = ProcessedForms(sql_form, None)
    stock_result = case_stock_result.stock_result if case_stock_result else None
    if stock_result:
        assert stock_result.populated
    return FormProcessorSQL.save_processed_models(
        forms_tuple,
        cases=case_stock_result.case_models if case_stock_result else None,
        stock_result=stock_result,
        publish_to_kafka=False
    )

def _get_main_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=['XFormInstance'],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
    )

def _get_unprocessed_form_iterator(domain):
    return CouchDomainDocTypeChangeProvider(
        couch_db=XFormInstance.get_db(),
        domains=[domain],
        doc_types=[
            'XFormArchived',
            'XFormError',
            'XFormDeprecated',
            'XFormDuplicate',
            # todo: need to figure out which of these we plan on supporting
            'XFormInstance-Deleted',
            'HQSubmission',
            'SubmissionErrorLog',
        ],
        event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
    )

def get_diff_db_filepath(domain):
    return os.path.join(settings.SHARED_DRIVE_CONF.restore_dir,
                        '{}-tzmigration.db'.format(domain))

def get_diff_db(domain):
    from corehq.apps.tzmigration.planning import DiffDB

    db_filepath = get_diff_db_filepath(domain)
    return DiffDB.open(db_filepath)

def delete_diff_db(domain):
    db_filepath = get_diff_db_filepath(domain)
    try:
        os.remove(db_filepath)
    except OSError as e:
        # continue if the file didn't exist to begin with
        # reraise on any other error
        if e.errno != 2:
            raise
<|endoftext|>


<commit_before>import sys, pygame, math
from Ball import *
pygame.init()

class Ball():
    def __init__(self, images, speed, pos=[0,0]):
        self.speedx = speed[0]
        self.speedy = speed[1]
        self.speed = [self.speedx, self.speedy]

        self.images = []
        for image in images:
            self.images += [pygame.image.load(image)]
        self.image = self.images[0]
        self.rect = self.image.get_rect()
        self.radius = self.rect.width/2 - 2

        self.frame = 0
        self.maxFrame = len(self.images)-1
        self.timer = 0
        self.timerMax = .15 * 60

        self.didBounceX = False
        self.didBounceY = False

        self.rect = self.rect.move(pos)

        self.living = True

    def die(self):
        self.living = False

    def move(self):
        self.speed = [self.speedx, self.speedy]
        self.rect = self.rect.move(self.speed)
        self.didBounceX = False
        self.didBounceY = False

    def collideScreen(self, size):
        width = size[0]
        height = size[1]
        if not self.didBounceX:
            if self.rect.left < 0 or self.rect.right > width:
                self.die()
                if self.rect.left < 0:
                    return "left"
                else:
                    return "right"
        if not self.didBounceY:
            if self.rect.top < 0 or self.rect.bottom > height:
                self.speedy = -self.speedy
                selfdidBounceY = True
                self.move()
                return "none"
        return "none"

    def update(self, size):
        self.move()
        self.animate()

    def animate(self):
        if self.timer < self.timerMax:
            self.timer += 1
        else:
            self.timer = 0
            if self.frame < self.maxFrame:
                self.frame += 1
            else:
                self.frame = 0
        self.image = self.images[self.frame]

    def collideBall(self, other):
        if self.rect.right > other.rect.left and self.rect.left < other.rect.right:
            if self.rect.bottom > other.rect.top and self.rect.top < other.rect.bottom:
                if self.radius + other.radius > self.distanceToPt(other.rect.center):
                    if not self.didBounceX:
                        if ((self.rect.center[0] < other.rect.center[0] and self.speedx > 0) or
                            (self.rect.center[0] > other.rect.center[0] and self.speedx < 0)):
                            self.speedx = -self.speedx
                            self.didBounceX = True
                            self.move()
                    if not self.didBounceY:
                        if ((self.rect.center[1] < other.rect.center[1] and self.speedy > 0) or
                            (self.rect.center[1] > other.rect.center[1] and self.speedy < 0)):
                            self.speedy = -self.speedy
                            self.didBounceY = True
                            self.move()
                    return True
        return False

    def collidePaddle(self, other):
        if self.rect.right > other.rect.left and self.rect.left < other.rect.right:
            if self.rect.bottom > other.rect.top and self.rect.top < other.rect.bottom:
                if self.rect.centery > other.rect.top and self.rect.centery < other.rect.bottom: #front
                    self.speedx = -self.speedx
                    self.move()
                    return True
                else:
                    self.speedx = -self.speedx
                    self.speedy = -self.speedy
                    self.move()
                    self.move()
                    return True

        return False

    def distanceToPt(self, pt):
        x1 = self.rect.center[0]
        y1 = self.rect.center[1]
        x2 = pt[0]
        y2 = pt[1]
        return math.sqrt((x1-x2)**2+(y1-y2)**2)

<commit_msg>Screen fix<commit_after>import sys, pygame, math
from Ball import *
pygame.init()

class Ball():
    def __init__(self, images, speed, pos=[0,0]):
        self.speedx = speed[0]
        self.speedy = speed[1]
        self.speed = [self.speedx, self.speedy]

        self.images = []
        for image in images:
            self.images += [pygame.image.load(image)]
        self.image = self.images[0]
        self.rect = self.image.get_rect()
        self.radius = self.rect.width/2 - 2

        self.frame = 0
        self.maxFrame = len(self.images)-1
        self.timer = 0
        self.timerMax = .15 * 60

        self.didBounceX = False
        self.didBounceY = False

        self.rect = self.rect.move(pos)

        self.living = True

    def die(self):
        self.living = False

    def move(self):
        self.speed = [self.speedx, self.speedy]
        self.rect = self.rect.move(self.speed)
        self.didBounceX = False
        self.didBounceY = False

    def collideScreen(self, size):
        width = size[0]
        height = size[1]
        if not self.didBounceX:
            if self.rect.left < 0 or self.rect.right > width:
                self.die()
                if self.rect.left < 0:
                    return "left"
                else:
                    return "right"
        if not self.didBounceY:
            if self.rect.top < 0 or self.rect.bottom > height:
                self.speedy = -self.speedy
                selfdidBounceY = True
                self.move()
                return "none"
        return "none"

    def update(self, size):
        self.move()
        self.animate()
        self.collideScreen(size)

    def animate(self):
        if self.timer < self.timerMax:
            self.timer += 1
        else:
            self.timer = 0
            if self.frame < self.maxFrame:
                self.frame += 1
            else:
                self.frame = 0
        self.image = self.images[self.frame]

    def collideBall(self, other):
        if self.rect.right > other.rect.left and self.rect.left < other.rect.right:
            if self.rect.bottom > other.rect.top and self.rect.top < other.rect.bottom:
                if self.radius + other.radius > self.distanceToPt(other.rect.center):
                    if not self.didBounceX:
                        if ((self.rect.center[0] < other.rect.center[0] and self.speedx > 0) or
                            (self.rect.center[0] > other.rect.center[0] and self.speedx < 0)):
                            self.speedx = -self.speedx
                            self.didBounceX = True
                            self.move()
                    if not self.didBounceY:
                        if ((self.rect.center[1] < other.rect.center[1] and self.speedy > 0) or
                            (self.rect.center[1] > other.rect.center[1] and self.speedy < 0)):
                            self.speedy = -self.speedy
                            self.didBounceY = True
                            self.move()
                    return True
        return False

    def collidePaddle(self, other):
        if self.rect.right > other.rect.left and self.rect.left < other.rect.right:
            if self.rect.bottom > other.rect.top and self.rect.top < other.rect.bottom:
                if self.rect.centery > other.rect.top and self.rect.centery < other.rect.bottom: #front
                    self.speedx = -self.speedx
                    self.move()
                    return True
                else:
                    self.speedx = -self.speedx
                    self.speedy = -self.speedy
                    self.move()
                    self.move()
                    return True

        return False

    def distanceToPt(self, pt):
        x1 = self.rect.center[0]
        y1 = self.rect.center[1]
        x2 = pt[0]
        y2 = pt[1]
        return math.sqrt((x1-x2)**2+(y1-y2)**2)

<|endoftext|>