<commit_before># -*- coding: utf-8 -*-
"""
Copyright (c) Microsoft Open Technologies (Shanghai) Co. Ltd.  All rights reserved.

The MIT License (MIT)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import sys

sys.path.append('..')

import os
from os.path import dirname, realpath, abspath, isfile
import commands

from hackathon.hazure.cloud_service_adapter import CloudServiceAdapter
from hackathon.hmongo.models import Hackathon, AzureKey, Experiment

from hackathon import RequiredFeature, Component, Context, internal_server_error
from hackathon.hackathon_response import ok, bad_request
from hackathon.constants import FILE_TYPE

__all__ = ["AzureCertManager"]

class AzureCertManager(Component):
    """Manage azure/mooncake certificates for the hackathon

    Note that certificate is actually certificate pair: a cer file and a pem file.
    The cer file is for user to download from our site and then upload to the azure/mooncake account.
    The pem file is for azure python SDK. Everytime request from OHP to azure should include the pem cert.

    cer file will be saved in different places according to the implementation of storage. But pem file will be saved
    """

    storage = RequiredFeature("storage")

    def create_certificate(self, subscription_id, management_host, hackathon):
        """Create certificate for specific subscription and hackathon

        1. check certificate dir
        2. generate pem file
        3. generate cert file
        4. add azure key to db
        5. add hackathon azure key to db
        :param subscription_id:
        :param management_host:
        :param hackathon:
        :return:
        """

        base_url = '%s/%s' % (self.CERT_BASE, subscription_id)
        pem_url = base_url + '.pem'
        cert_url = base_url + '.cer'

        # avoid duplicate pem generation
        if not os.path.isfile(pem_url):
            pem_command = 'openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout %s -out %s -batch' % \
                          (pem_url, pem_url)
            commands.getstatusoutput(pem_command)
        else:
            self.log.debug('%s exists' % pem_url)

        # avoid duplicate cert generation
        if not os.path.isfile(cert_url):
            cert_command = 'openssl x509 -inform pem -in %s -outform der -out %s' % (pem_url, cert_url)
            commands.getstatusoutput(cert_command)
        else:
            self.log.debug('%s exists' % cert_url)

        azure_key = AzureKey.objects(subscription_id=subscription_id, management_host=management_host).first()

        if azure_key is None:
            azure_key = AzureKey(
                cert_url=base_url + '.cer',
                pem_url=base_url + '.pem',
                subscription_id=subscription_id,
                management_host=management_host,
                verified=False
            )

            azure_key.save()

            hackathon.azure_keys.append(azure_key)
            hackathon.save()
        else:
            self.log.debug('azure key exists')

        if not (azure_key in hackathon.azure_keys):
            hackathon.azure_keys.append(azure_key)
            hackathon.save()
        else:
            self.log.debug('hackathon azure key exists')

        # store cer file
        cer_context = Context(
            hackathon_name=hackathon.name,
            file_name=subscription_id + '.cer',
            file_type=FILE_TYPE.AZURE_CERT,
            content=file(cert_url)
        )
        self.log.debug("saving cerf file [%s] to azure" % cer_context.file_name)
        cer_context = self.storage.save(cer_context)
        azure_key.cert_url = cer_context.url

        # store pem file
        # encrypt certification file before upload to storage
        encrypted_pem_url = self.__encrypt_content(pem_url)
        pem_contex = Context(
            hackathon_name=hackathon.name,
            file_name=subscription_id + '.pem',
            file_type=FILE_TYPE.AZURE_CERT,
            content=file(encrypted_pem_url)
        )
        self.log.debug("saving pem file [%s] to azure" % pem_contex.file_name)
        pem_contex = self.storage.save(pem_contex)
        os.remove(encrypted_pem_url)
        azure_key.pem_url = pem_contex.url

        azure_key.save()

        return azure_key.dic()

    def get_certificates_by_expr(self, expr_id):
        """Get certificates by experiment id
        """
        # expr = self.db.get_object(Experiment, expr_id)
        expr = Experiment.objects(id=expr_id)
        # hak = self.db.find_all_objects_by(HackathonAzureKey, hackathon_id=expr.hackathon_id)
        hak = Hackathon.objects(id=expr.hackathon_id).first().azure_keys[0]
        if not hak:
            raise Exception("no azure key configured")

        return map(lambda key: self.db.get_object(AzureKey, key.azure_key_id), hak)

    def get_certificates(self, hackathon):
        """Get certificates by hackathon

        :type hackathon: Hackathon
        :param hackathon: instance of hackathon to search certificates

        :rtype list
        :return a list of AzureKey
        """

        hackathon_azure_keys = [a.dic() for a in hackathon.azure_keys]

        if len(hackathon_azure_keys) == 0:
            # if no certificates added before, return 404
            return []

        return hackathon_azure_keys

    def delete_certificate(self, certificate_id, hackathon):
        """Delete certificate by azureKey.id and hackathon

        Delete the hackathon-azureKey relationship first. If the auzreKey is not needed any more, delete the azureKey too

        :type certificate_id: int
        :param certificate_id: id of AzureKey

        :type hackathon: Hackathon
        :param hackathon: instance of Hackathon
        """
        # delete all hackathon-azureKey relationships first

        azure_key = AzureKey.objects(id=certificate_id).first()

        # if no relations left, delete the azureKey itself
        if azure_key in hackathon.azure_keys:
            try:
                if isfile(azure_key.cert_url):
                    os.remove(azure_key.cert_url)
                else:
                    self.storage.delete(azure_key.cert_url)
                if isfile(azure_key.pem_url):
                    os.remove(azure_key.pem_url)
                else:
                    self.storage.delete(azure_key.pem_url)
            except Exception as e:
                self.log.error(e)

            hackathon.azure_keys.remove(azure_key)
            hackathon.save()

        return ok(True)

    def check_sub_id(self, subscription_id):

        azure_key = AzureKey.objects(subscription_id=subscription_id).first()

        if self.util.is_local():
            if azure_key is not None:
                azure_key.verified = True
                azure_key.save()
            return ok("success")

        if azure_key is None:
            return internal_server_error("No available azure key on the server side.")

        sms = CloudServiceAdapter(azure_key.subscription_id,
                                  azure_key.get_local_pem_url(),
                                  host=azure_key.management_host)
        if sms.ping():
            azure_key.verified = True
            azure_key.save()
        else:
            return bad_request("Subscription id is not valid")

        return ok("success")

    def __init__(self):
        self.CERT_BASE = self.util.get_config('azure.cert_base')
        self.__ensure_certificates_dir()

    def __ensure_certificates_dir(self):
        """Ensure that the directory to store azure certificate exists"""
        if not self.CERT_BASE:
            self.CERT_BASE = abspath("%s/../certificates" % dirname(realpath(__file__)))

        if not os.path.exists(self.CERT_BASE):
            os.makedirs(self.CERT_BASE)

    def __encrypt_content(self, pem_url):
        encrypted_pem_url = pem_url + ".encrypted"
        cryptor = RequiredFeature("cryptor")
        cryptor.encrypt(pem_url, encrypted_pem_url)
        return encrypted_pem_url

    def get_local_pem_url(self, pem_url):
        local_pem_url = self.CERT_BASE + "/" + pem_url.split("/")[-1]
        if not isfile(local_pem_url):
            self.log.debug("Recover local %s.pem file from azure storage %s" % (local_pem_url, pem_url))
            cryptor = RequiredFeature("cryptor")
            cryptor.recover_local_file(pem_url, local_pem_url)
        return local_pem_url

# recover a pem file from azure
def get_local_pem_url(azureKey):
    azure_cert_manager = RequiredFeature("azure_cert_manager")
    return azure_cert_manager.get_local_pem_url(azureKey.pem_url)

AzureKey.get_local_pem_url = get_local_pem_url
<commit_msg>modify the return value of API checksubid<commit_after># -*- coding: utf-8 -*-
"""
Copyright (c) Microsoft Open Technologies (Shanghai) Co. Ltd.  All rights reserved.

The MIT License (MIT)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import sys

sys.path.append('..')

import os
from os.path import dirname, realpath, abspath, isfile
import commands

from hackathon.hazure.cloud_service_adapter import CloudServiceAdapter
from hackathon.hmongo.models import Hackathon, AzureKey, Experiment

from hackathon import RequiredFeature, Component, Context, internal_server_error
from hackathon.hackathon_response import ok, bad_request
from hackathon.constants import FILE_TYPE

__all__ = ["AzureCertManager"]

class AzureCertManager(Component):
    """Manage azure/mooncake certificates for the hackathon

    Note that certificate is actually certificate pair: a cer file and a pem file.
    The cer file is for user to download from our site and then upload to the azure/mooncake account.
    The pem file is for azure python SDK. Everytime request from OHP to azure should include the pem cert.

    cer file will be saved in different places according to the implementation of storage. But pem file will be saved
    """

    storage = RequiredFeature("storage")

    def create_certificate(self, subscription_id, management_host, hackathon):
        """Create certificate for specific subscription and hackathon

        1. check certificate dir
        2. generate pem file
        3. generate cert file
        4. add azure key to db
        5. add hackathon azure key to db
        :param subscription_id:
        :param management_host:
        :param hackathon:
        :return:
        """

        base_url = '%s/%s' % (self.CERT_BASE, subscription_id)
        pem_url = base_url + '.pem'
        cert_url = base_url + '.cer'

        # avoid duplicate pem generation
        if not os.path.isfile(pem_url):
            pem_command = 'openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout %s -out %s -batch' % \
                          (pem_url, pem_url)
            commands.getstatusoutput(pem_command)
        else:
            self.log.debug('%s exists' % pem_url)

        # avoid duplicate cert generation
        if not os.path.isfile(cert_url):
            cert_command = 'openssl x509 -inform pem -in %s -outform der -out %s' % (pem_url, cert_url)
            commands.getstatusoutput(cert_command)
        else:
            self.log.debug('%s exists' % cert_url)

        azure_key = AzureKey.objects(subscription_id=subscription_id, management_host=management_host).first()

        if azure_key is None:
            azure_key = AzureKey(
                cert_url=base_url + '.cer',
                pem_url=base_url + '.pem',
                subscription_id=subscription_id,
                management_host=management_host,
                verified=False
            )

            azure_key.save()

            hackathon.azure_keys.append(azure_key)
            hackathon.save()
        else:
            self.log.debug('azure key exists')

        if not (azure_key in hackathon.azure_keys):
            hackathon.azure_keys.append(azure_key)
            hackathon.save()
        else:
            self.log.debug('hackathon azure key exists')

        # store cer file
        cer_context = Context(
            hackathon_name=hackathon.name,
            file_name=subscription_id + '.cer',
            file_type=FILE_TYPE.AZURE_CERT,
            content=file(cert_url)
        )
        self.log.debug("saving cerf file [%s] to azure" % cer_context.file_name)
        cer_context = self.storage.save(cer_context)
        azure_key.cert_url = cer_context.url

        # store pem file
        # encrypt certification file before upload to storage
        encrypted_pem_url = self.__encrypt_content(pem_url)
        pem_contex = Context(
            hackathon_name=hackathon.name,
            file_name=subscription_id + '.pem',
            file_type=FILE_TYPE.AZURE_CERT,
            content=file(encrypted_pem_url)
        )
        self.log.debug("saving pem file [%s] to azure" % pem_contex.file_name)
        pem_contex = self.storage.save(pem_contex)
        os.remove(encrypted_pem_url)
        azure_key.pem_url = pem_contex.url

        azure_key.save()

        return azure_key.dic()

    def get_certificates_by_expr(self, expr_id):
        """Get certificates by experiment id
        """
        # expr = self.db.get_object(Experiment, expr_id)
        expr = Experiment.objects(id=expr_id)
        # hak = self.db.find_all_objects_by(HackathonAzureKey, hackathon_id=expr.hackathon_id)
        hak = Hackathon.objects(id=expr.hackathon_id).first().azure_keys[0]
        if not hak:
            raise Exception("no azure key configured")

        return map(lambda key: self.db.get_object(AzureKey, key.azure_key_id), hak)

    def get_certificates(self, hackathon):
        """Get certificates by hackathon

        :type hackathon: Hackathon
        :param hackathon: instance of hackathon to search certificates

        :rtype list
        :return a list of AzureKey
        """

        hackathon_azure_keys = [a.dic() for a in hackathon.azure_keys]

        if len(hackathon_azure_keys) == 0:
            # if no certificates added before, return 404
            return []

        return hackathon_azure_keys

    def delete_certificate(self, certificate_id, hackathon):
        """Delete certificate by azureKey.id and hackathon

        Delete the hackathon-azureKey relationship first. If the auzreKey is not needed any more, delete the azureKey too

        :type certificate_id: int
        :param certificate_id: id of AzureKey

        :type hackathon: Hackathon
        :param hackathon: instance of Hackathon
        """
        # delete all hackathon-azureKey relationships first

        azure_key = AzureKey.objects(id=certificate_id).first()

        # if no relations left, delete the azureKey itself
        if azure_key in hackathon.azure_keys:
            try:
                if isfile(azure_key.cert_url):
                    os.remove(azure_key.cert_url)
                else:
                    self.storage.delete(azure_key.cert_url)
                if isfile(azure_key.pem_url):
                    os.remove(azure_key.pem_url)
                else:
                    self.storage.delete(azure_key.pem_url)
            except Exception as e:
                self.log.error(e)

            hackathon.azure_keys.remove(azure_key)
            hackathon.save()

        return ok(True)

    def check_sub_id(self, subscription_id):

        azure_key = AzureKey.objects(subscription_id=subscription_id).first()

        if self.util.is_local():
            if azure_key is not None:
                azure_key.verified = True
                azure_key.save()
            return ok("success")

        if azure_key is None:
            return internal_server_error("No available azure key on the server side.")

        sms = CloudServiceAdapter(azure_key.subscription_id,
                                  azure_key.get_local_pem_url(),
                                  host=azure_key.management_host)
        if sms.ping():
            azure_key.verified = True
            azure_key.save()
        else:
            return bad_request("Subscription id is not valid, check whether subscription id is valid and upload the right cer file to azure")

        return ok("success")

    def __init__(self):
        self.CERT_BASE = self.util.get_config('azure.cert_base')
        self.__ensure_certificates_dir()

    def __ensure_certificates_dir(self):
        """Ensure that the directory to store azure certificate exists"""
        if not self.CERT_BASE:
            self.CERT_BASE = abspath("%s/../certificates" % dirname(realpath(__file__)))

        if not os.path.exists(self.CERT_BASE):
            os.makedirs(self.CERT_BASE)

    def __encrypt_content(self, pem_url):
        encrypted_pem_url = pem_url + ".encrypted"
        cryptor = RequiredFeature("cryptor")
        cryptor.encrypt(pem_url, encrypted_pem_url)
        return encrypted_pem_url

    def get_local_pem_url(self, pem_url):
        local_pem_url = self.CERT_BASE + "/" + pem_url.split("/")[-1]
        if not isfile(local_pem_url):
            self.log.debug("Recover local %s.pem file from azure storage %s" % (local_pem_url, pem_url))
            cryptor = RequiredFeature("cryptor")
            cryptor.recover_local_file(pem_url, local_pem_url)
        return local_pem_url

# recover a pem file from azure
def get_local_pem_url(azureKey):
    azure_cert_manager = RequiredFeature("azure_cert_manager")
    return azure_cert_manager.get_local_pem_url(azureKey.pem_url)

AzureKey.get_local_pem_url = get_local_pem_url
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright 2013, Big Switch Networks, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.core.urlresolvers import reverse
from django.utils import http
from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import tables

from openstack_dashboard import api

import logging

LOG = logging.getLogger(__name__)

class AddPoolLink(tables.LinkAction):
    name = "addpool"
    verbose_name = _("Add Pool")
    url = "horizon:project:loadbalancers:addpool"
    classes = ("ajax-modal", "btn-create",)

class AddVipLink(tables.LinkAction):
    name = "addvip"
    verbose_name = _("Add VIP")
    classes = ("ajax-modal", "btn-create",)

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:addvip",
                           kwargs={'pool_id': pool.id})
        return base_url

    def allowed(self, request, datum=None):
        if datum and datum.vip_id:
            return False
        return True

class AddMemberLink(tables.LinkAction):
    name = "addmember"
    verbose_name = _("Add Member")
    url = "horizon:project:loadbalancers:addmember"
    classes = ("ajax-modal", "btn-create",)

class AddMonitorLink(tables.LinkAction):
    name = "addmonitor"
    verbose_name = _("Add Monitor")
    url = "horizon:project:loadbalancers:addmonitor"
    classes = ("ajax-modal", "btn-create",)

class DeleteVipLink(tables.DeleteAction):
    name = "deletevip"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("VIP")
    data_type_plural = _("VIPs")

    def allowed(self, request, datum=None):
        if datum and not datum.vip_id:
            return False
        return True

class DeletePoolLink(tables.DeleteAction):
    name = "deletepool"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Pool")
    data_type_plural = _("Pools")

class DeleteMonitorLink(tables.DeleteAction):
    name = "deletemonitor"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Monitor")
    data_type_plural = _("Monitors")

class DeleteMemberLink(tables.DeleteAction):
    name = "deletemember"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Member")
    data_type_plural = _("Members")

class UpdatePoolLink(tables.LinkAction):
    name = "updatepool"
    verbose_name = _("Edit Pool")
    classes = ("btn-update",)

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:updatepool",
                           kwargs={'pool_id': pool.id})
        return base_url

class UpdateVipLink(tables.LinkAction):
    name = "updatevip"
    verbose_name = _("Edit VIP")

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:updatevip",
                           kwargs={'vip_id': pool.vip_id})
        return base_url

    def allowed(self, request, datum=None):
        if datum and not datum.vip_id:
            return False
        return True

class UpdateMemberLink(tables.LinkAction):
    name = "updatemember"
    verbose_name = _("Edit Member")

    def get_link_url(self, member):
        base_url = reverse("horizon:project:loadbalancers:updatemember",
                           kwargs={'member_id': member.id})
        return base_url

class UpdateMonitorLink(tables.LinkAction):
    name = "updatemonitor"
    verbose_name = _("Edit Monitor")

    def get_link_url(self, monitor):
        base_url = reverse("horizon:project:loadbalancers:updatemonitor",
                           kwargs={'monitor_id': monitor.id})
        return base_url

def get_vip_link(pool):
    return reverse("horizon:project:loadbalancers:vipdetails",
                   args=(http.urlquote(pool.vip_id),))

class AddPMAssociationLink(tables.LinkAction):
    name = "addassociation"
    verbose_name = _("Add Health Monitor")
    url = "horizon:project:loadbalancers:addassociation"

    def allowed(self, request, datum=None):
        try:
            monitors = api.lbaas.pool_health_monitors_get(request)
            for m in monitors:
                if m.id not in datum['health_monitors']:
                    return True
        except Exception:
            exceptions.handle(request,
                              _('Failed to retrieve health monitors.'))
        return False

class DeletePMAssociationLink(tables.LinkAction):
    name = "deleteassociation"
    verbose_name = _("Delete Health Monitor")
    url = "horizon:project:loadbalancers:deleteassociation"
    classes = ("btn-delete", "btn-danger")

    def allowed(self, request, datum=None):
        if datum and not datum['health_monitors']:
            return False
        return True

class PoolsTable(tables.DataTable):
    name = tables.Column("name",
                       verbose_name=_("Name"),
                       link="horizon:project:loadbalancers:pooldetails")
    description = tables.Column('description', verbose_name=_("Description"))
    subnet_name = tables.Column('subnet_name', verbose_name=_("Subnet"))
    protocol = tables.Column('protocol', verbose_name=_("Protocol"))
    vip_name = tables.Column('vip_name', verbose_name=_("VIP"),
                             link=get_vip_link)

    class Meta:
        name = "poolstable"
        verbose_name = _("Pools")
        table_actions = (AddPoolLink, DeletePoolLink)
        row_actions = (UpdatePoolLink, AddVipLink, UpdateVipLink,
                       DeleteVipLink, AddPMAssociationLink,
                       DeletePMAssociationLink, DeletePoolLink)

def get_pool_link(member):
    return reverse("horizon:project:loadbalancers:pooldetails",
                   args=(http.urlquote(member.pool_id),))

def get_member_link(member):
    return reverse("horizon:project:loadbalancers:memberdetails",
                   args=(http.urlquote(member.id),))

class MembersTable(tables.DataTable):
    address = tables.Column('address',
                            verbose_name=_("IP Address"),
                            link=get_member_link,
                            attrs={'data-type': "ip"})
    protocol_port = tables.Column('protocol_port',
                                  verbose_name=_("Protocol Port"))
    pool_name = tables.Column("pool_name",
                            verbose_name=_("Pool"), link=get_pool_link)

    class Meta:
        name = "memberstable"
        verbose_name = _("Members")
        table_actions = (AddMemberLink, DeleteMemberLink)
        row_actions = (UpdateMemberLink, DeleteMemberLink)

class MonitorsTable(tables.DataTable):
    id = tables.Column("id",
                       verbose_name=_("ID"),
                       link="horizon:project:loadbalancers:monitordetails")
    monitorType = tables.Column('type', verbose_name=_("Monitor Type"))

    class Meta:
        name = "monitorstable"
        verbose_name = _("Monitors")
        table_actions = (AddMonitorLink, DeleteMonitorLink)
        row_actions = (UpdateMonitorLink, DeleteMonitorLink)
<commit_msg>Fix "Add Association" and "Delete Association" displaying<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright 2013, Big Switch Networks, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.core.urlresolvers import reverse
from django.utils import http
from django.utils.translation import ugettext_lazy as _

from horizon import exceptions
from horizon import tables

from openstack_dashboard import api

import logging

LOG = logging.getLogger(__name__)

class AddPoolLink(tables.LinkAction):
    name = "addpool"
    verbose_name = _("Add Pool")
    url = "horizon:project:loadbalancers:addpool"
    classes = ("ajax-modal", "btn-create",)

class AddVipLink(tables.LinkAction):
    name = "addvip"
    verbose_name = _("Add VIP")
    classes = ("ajax-modal", "btn-create",)

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:addvip",
                           kwargs={'pool_id': pool.id})
        return base_url

    def allowed(self, request, datum=None):
        if datum and datum.vip_id:
            return False
        return True

class AddMemberLink(tables.LinkAction):
    name = "addmember"
    verbose_name = _("Add Member")
    url = "horizon:project:loadbalancers:addmember"
    classes = ("ajax-modal", "btn-create",)

class AddMonitorLink(tables.LinkAction):
    name = "addmonitor"
    verbose_name = _("Add Monitor")
    url = "horizon:project:loadbalancers:addmonitor"
    classes = ("ajax-modal", "btn-create",)

class DeleteVipLink(tables.DeleteAction):
    name = "deletevip"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("VIP")
    data_type_plural = _("VIPs")

    def allowed(self, request, datum=None):
        if datum and not datum.vip_id:
            return False
        return True

class DeletePoolLink(tables.DeleteAction):
    name = "deletepool"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Pool")
    data_type_plural = _("Pools")

class DeleteMonitorLink(tables.DeleteAction):
    name = "deletemonitor"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Monitor")
    data_type_plural = _("Monitors")

class DeleteMemberLink(tables.DeleteAction):
    name = "deletemember"
    action_present = _("Delete")
    action_past = _("Scheduled deletion of")
    data_type_singular = _("Member")
    data_type_plural = _("Members")

class UpdatePoolLink(tables.LinkAction):
    name = "updatepool"
    verbose_name = _("Edit Pool")
    classes = ("btn-update",)

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:updatepool",
                           kwargs={'pool_id': pool.id})
        return base_url

class UpdateVipLink(tables.LinkAction):
    name = "updatevip"
    verbose_name = _("Edit VIP")

    def get_link_url(self, pool):
        base_url = reverse("horizon:project:loadbalancers:updatevip",
                           kwargs={'vip_id': pool.vip_id})
        return base_url

    def allowed(self, request, datum=None):
        if datum and not datum.vip_id:
            return False
        return True

class UpdateMemberLink(tables.LinkAction):
    name = "updatemember"
    verbose_name = _("Edit Member")

    def get_link_url(self, member):
        base_url = reverse("horizon:project:loadbalancers:updatemember",
                           kwargs={'member_id': member.id})
        return base_url

class UpdateMonitorLink(tables.LinkAction):
    name = "updatemonitor"
    verbose_name = _("Edit Monitor")

    def get_link_url(self, monitor):
        base_url = reverse("horizon:project:loadbalancers:updatemonitor",
                           kwargs={'monitor_id': monitor.id})
        return base_url

def get_vip_link(pool):
    return reverse("horizon:project:loadbalancers:vipdetails",
                   args=(http.urlquote(pool.vip_id),))

class AddPMAssociationLink(tables.LinkAction):
    name = "addassociation"
    verbose_name = _("Add Health Monitor")
    url = "horizon:project:loadbalancers:addassociation"
    classes = ("ajax-modal", "btn-create",)

    def allowed(self, request, datum=None):
        try:
            monitors = api.lbaas.pool_health_monitors_get(request)
            for m in monitors:
                if m.id not in datum['health_monitors']:
                    return True
        except Exception:
            exceptions.handle(request,
                              _('Failed to retrieve health monitors.'))
        return False

class DeletePMAssociationLink(tables.LinkAction):
    name = "deleteassociation"
    verbose_name = _("Delete Health Monitor")
    url = "horizon:project:loadbalancers:deleteassociation"
    classes = ("ajax-modal", "btn-delete", "btn-danger")

    def allowed(self, request, datum=None):
        if datum and not datum['health_monitors']:
            return False
        return True

class PoolsTable(tables.DataTable):
    name = tables.Column("name",
                       verbose_name=_("Name"),
                       link="horizon:project:loadbalancers:pooldetails")
    description = tables.Column('description', verbose_name=_("Description"))
    subnet_name = tables.Column('subnet_name', verbose_name=_("Subnet"))
    protocol = tables.Column('protocol', verbose_name=_("Protocol"))
    vip_name = tables.Column('vip_name', verbose_name=_("VIP"),
                             link=get_vip_link)

    class Meta:
        name = "poolstable"
        verbose_name = _("Pools")
        table_actions = (AddPoolLink, DeletePoolLink)
        row_actions = (UpdatePoolLink, AddVipLink, UpdateVipLink,
                       DeleteVipLink, AddPMAssociationLink,
                       DeletePMAssociationLink, DeletePoolLink)

def get_pool_link(member):
    return reverse("horizon:project:loadbalancers:pooldetails",
                   args=(http.urlquote(member.pool_id),))

def get_member_link(member):
    return reverse("horizon:project:loadbalancers:memberdetails",
                   args=(http.urlquote(member.id),))

class MembersTable(tables.DataTable):
    address = tables.Column('address',
                            verbose_name=_("IP Address"),
                            link=get_member_link,
                            attrs={'data-type': "ip"})
    protocol_port = tables.Column('protocol_port',
                                  verbose_name=_("Protocol Port"))
    pool_name = tables.Column("pool_name",
                            verbose_name=_("Pool"), link=get_pool_link)

    class Meta:
        name = "memberstable"
        verbose_name = _("Members")
        table_actions = (AddMemberLink, DeleteMemberLink)
        row_actions = (UpdateMemberLink, DeleteMemberLink)

class MonitorsTable(tables.DataTable):
    id = tables.Column("id",
                       verbose_name=_("ID"),
                       link="horizon:project:loadbalancers:monitordetails")
    monitorType = tables.Column('type', verbose_name=_("Monitor Type"))

    class Meta:
        name = "monitorstable"
        verbose_name = _("Monitors")
        table_actions = (AddMonitorLink, DeleteMonitorLink)
        row_actions = (UpdateMonitorLink, DeleteMonitorLink)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import json
import requests
from datetime import datetime, timedelta
from util.global_def import NA, get_data_home, show, info, error
from util.network import reachable as network_reachable
from util.serialize import save, load
from util.select import RankHolder, get_weighted_random_dict_key

TARGET_SEARCH_RESULT_SIZE = 60  # this in only our set one-time search size.
G_SEARCH_PER_REQ_SIZE = 10  # use the maximum possible value google allowed in one request

class Crawler(object):
    """crawl graph url by querying google search api"""

    def __init__(self, need_save=True):
        self.__need_save = need_save
        self.__network_reachable = network_reachable()
        self.__has_write = False
        self.__url_map = {}
        self.__cache_file = get_data_home() + "url.pickle"
        is_exist, url_map = load(self.__cache_file)
        if is_exist:
            self.__url_map = url_map

    def __del__(self):
        if self.__need_save and self.__has_write:
            save(self.__cache_file, self.__url_map)

    @staticmethod
    def get_dice(size_list, size_ratio):
        assert size_list
        dice = {}
        for size in size_list:
            adopted_ratio = size_ratio[size] if size_ratio and size in size_ratio else 1
            dice[size] = RankHolder(adopted_ratio)
        return dice

    _HAS_SHOW_NO_SEARCH_MSG = False

    def crawl(self, pattern, size_list, option="", print_url=False):
        """output: urls, is_new_result"""
        show("查詢標的：", "\"" + pattern + "\"")
        key = Crawler.get_search_key(pattern, option)
        urls, size_ratio = self.get_recent_result(key)
        if urls:
            return urls, False
        if not self.__network_reachable:
            return None, False
        assert size_list and (not size_ratio or isinstance(size_ratio, dict))
        dice = Crawler.get_dice(size_list, size_ratio)
        urls = []
        next_size_ratio = {size: 0 for size in size_list}  # key: size, value: number of new result (initial with 0)
        start = {size: 1 for size in size_list}  # key: size, value: next start offset to search (start from 1, from google)
        tried_size = 0
        while tried_size < TARGET_SEARCH_RESULT_SIZE:
            chosen_size = get_weighted_random_dict_key(dice)
            this_urls, success = Crawler.crawl_by_asking_google_search(pattern, start[chosen_size], chosen_size, option)
            if not success:
                break
            urls += this_urls
            new_result = self.get_this_time_new_result_num(key, this_urls)
            next_size_ratio[chosen_size] += (new_result if NA != new_result else len(this_urls))
            start[chosen_size] += G_SEARCH_PER_REQ_SIZE
            tried_size += G_SEARCH_PER_REQ_SIZE
        # 'set' to filter out duplicated item (though not expected, but we found g-search may give duplicated result)
        urls = list(set(urls))
        if not Crawler._HAS_SHOW_NO_SEARCH_MSG:
            info('獲得url筆數：', len(urls))
        if print_url:
            for url in urls:
                show(url)
        if success:
            next_size_ratio = {size: 1 if 0 == next_size_ratio[size] else next_size_ratio[size]
                               for size in next_size_ratio}
            self.cache_url(key, urls, next_size_ratio)
        return urls, success

    def get_recent_result(self, key):
        """output: urls, size_ratio"""
        if key not in self.__url_map:
            return None, None
        [retrieved_date, new_result, urls, size_ratio] = self.__url_map[key]
        if not self.__network_reachable:
            show("使用上一次的搜尋結果（由於無網路連線）")
            # though size_ratio can be valid, we do not return it for caller usage is not expected
            return urls, None
        # spec.: we will execute a new search when there is enough new result on previous search
        #       => if previous new result is n, all result is m, we will have a new search after m/n days
        #       => if all previous result is new, then after 1 day we will have a search
        #       => if no previous result is new, then we will have a search after 'TARGET_SEARCH_RESULT_SIZE' days
        valid_day_size = len(urls) / new_result if new_result > 0 else \
            1 if NA is new_result else \
            TARGET_SEARCH_RESULT_SIZE  # new_result = 0 => no new result before
        from util.global_def import get_search_latency
        valid_day_size *= get_search_latency()
        current_date = datetime.today()
        date_diff = current_date - retrieved_date
        if self.__network_reachable and date_diff > timedelta(days=valid_day_size):  # valid_day_size天為查詢結果的“有效期”
            return None, size_ratio
        to_next_query = timedelta(days=valid_day_size) - date_diff
        hours, remainder = divmod(to_next_query.seconds, 3600)
        minutes, seconds = divmod(remainder, 60)
        show("距離下次搜尋：",
             to_next_query.days, "天",
             hours, "時",
             minutes, "分",
             seconds, "秒", ", 現有", len(urls), "筆url")
        # though size_ratio can be valid, we do not return it for caller usage is not expected
        return urls, None

    def get_this_time_new_result_num(self, key, urls):
        if key not in self.__url_map:
            return NA  # means this is a new query, all results are new
        new_result = 0
        [_, _, cached_urls, _] = self.__url_map[key]
        for url in urls:
            if url not in cached_urls:
                new_result += 1
        return new_result

    def cache_url(self, key, urls, next_size_ratio):
        self.__has_write = True
        self.__url_map[key] = [datetime.today(), self.get_this_time_new_result_num(key, urls), urls, next_size_ratio]

    @staticmethod
    def crawl_by_asking_google_search(pattern, start, size, option=""):
        assert type(pattern) in [str, unicode]
        from util.global_def import get_api_key, get_cx
        api_key = get_api_key()
        cx = get_cx()
        if not api_key or not cx:
            if not Crawler._HAS_SHOW_NO_SEARCH_MSG:
                Crawler._HAS_SHOW_NO_SEARCH_MSG = True
                info("沒有指定Google custom search需要的api_key及cx，將不進行圖片搜尋")
            return [], False
        size_option = "&imgSize=" + size if size else ""
        full_option = size_option + (option if option else "")
        base_url = 'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&searchType=image&num=%d' \
                   '&q=' + pattern + '&start=%d' + full_option
        request_str = base_url % (api_key, cx, G_SEARCH_PER_REQ_SIZE, start)
        urls = []
        success = True
        try:
            r = requests.get(request_str)
            res = json.loads(r.text)
            if "error" in res:
                Crawler.print_error(res["error"])
                return urls, False
            for image_info in res['items']:
                assert 'link' in image_info
                url = image_info['link']
                urls.append(url)
        except TypeError as e:  # for unhandled error...
            info("無法擷取最新的圖片網址：", str(e))
            success = False
        return urls, success

    @staticmethod
    def get_search_key(pattern, option):
        return pattern  # TODO:

    @staticmethod
    def print_error(data):
        assert isinstance(data, dict) and "message" in data
        error("搜尋引擎傳回錯誤訊息：", data["message"])

if __name__ == '__main__':
    from util.global_def import config_action
    config_action()
    # name '_' before the 'obj' to let python not free imported module before __del__ is called
    # (or we will have something like 'NoneType' object has no attribute 'dump' for cPickle.dump)
    _obj = Crawler(need_save=True)
    size = ["large", "xlarge", "xxlarge", "huge"]
    _obj.crawl("Inside Out", size_list=size, print_url=True)
<commit_msg>show pattern when a search report msg is issued<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import json
import requests
from datetime import datetime, timedelta
from util.global_def import NA, get_data_home, show, info, error
from util.network import reachable as network_reachable
from util.serialize import save, load
from util.select import RankHolder, get_weighted_random_dict_key

TARGET_SEARCH_RESULT_SIZE = 60  # this in only our set one-time search size.
G_SEARCH_PER_REQ_SIZE = 10  # use the maximum possible value google allowed in one request

class Crawler(object):
    """crawl graph url by querying google search api"""

    def __init__(self, need_save=True):
        self.__need_save = need_save
        self.__network_reachable = network_reachable()
        self.__has_write = False
        self.__url_map = {}
        self.__cache_file = get_data_home() + "url.pickle"
        is_exist, url_map = load(self.__cache_file)
        if is_exist:
            self.__url_map = url_map

    def __del__(self):
        if self.__need_save and self.__has_write:
            save(self.__cache_file, self.__url_map)

    @staticmethod
    def get_dice(size_list, size_ratio):
        assert size_list
        dice = {}
        for size in size_list:
            adopted_ratio = size_ratio[size] if size_ratio and size in size_ratio else 1
            dice[size] = RankHolder(adopted_ratio)
        return dice

    _HAS_SHOW_NO_SEARCH_MSG = False

    def crawl(self, pattern, size_list, option="", print_url=False):
        """output: urls, is_new_result"""
        show("查詢標的：", "\"" + pattern + "\"")
        key = Crawler.get_search_key(pattern, option)
        urls, size_ratio = self.get_recent_result(key)
        if urls:
            return urls, False
        if not self.__network_reachable:
            return None, False
        assert size_list and (not size_ratio or isinstance(size_ratio, dict))
        dice = Crawler.get_dice(size_list, size_ratio)
        urls = []
        next_size_ratio = {size: 0 for size in size_list}  # key: size, value: number of new result (initial with 0)
        start = {size: 1 for size in size_list}  # key: size, value: next start offset to search (start from 1, from google)
        tried_size = 0
        while tried_size < TARGET_SEARCH_RESULT_SIZE:
            chosen_size = get_weighted_random_dict_key(dice)
            this_urls, success = Crawler.crawl_by_asking_google_search(pattern, start[chosen_size], chosen_size, option)
            if not success:
                break
            urls += this_urls
            new_result = self.get_this_time_new_result_num(key, this_urls)
            next_size_ratio[chosen_size] += (new_result if NA != new_result else len(this_urls))
            start[chosen_size] += G_SEARCH_PER_REQ_SIZE
            tried_size += G_SEARCH_PER_REQ_SIZE
        # 'set' to filter out duplicated item (though not expected, but we found g-search may give duplicated result)
        urls = list(set(urls))
        if not Crawler._HAS_SHOW_NO_SEARCH_MSG:
            info('標的：%s, 獲得url筆數：%i' %(pattern, len(urls)))
        if print_url:
            for url in urls:
                show(url)
        if success:
            next_size_ratio = {size: 1 if 0 == next_size_ratio[size] else next_size_ratio[size]
                               for size in next_size_ratio}
            self.cache_url(key, urls, next_size_ratio)
        return urls, success

    def get_recent_result(self, key):
        """output: urls, size_ratio"""
        if key not in self.__url_map:
            return None, None
        [retrieved_date, new_result, urls, size_ratio] = self.__url_map[key]
        if not self.__network_reachable:
            show("使用上一次的搜尋結果（由於無網路連線）")
            # though size_ratio can be valid, we do not return it for caller usage is not expected
            return urls, None
        # spec.: we will execute a new search when there is enough new result on previous search
        #       => if previous new result is n, all result is m, we will have a new search after m/n days
        #       => if all previous result is new, then after 1 day we will have a search
        #       => if no previous result is new, then we will have a search after 'TARGET_SEARCH_RESULT_SIZE' days
        valid_day_size = len(urls) / new_result if new_result > 0 else \
            1 if NA is new_result else \
            TARGET_SEARCH_RESULT_SIZE  # new_result = 0 => no new result before
        from util.global_def import get_search_latency
        valid_day_size *= get_search_latency()
        current_date = datetime.today()
        date_diff = current_date - retrieved_date
        if self.__network_reachable and date_diff > timedelta(days=valid_day_size):  # valid_day_size天為查詢結果的“有效期”
            return None, size_ratio
        to_next_query = timedelta(days=valid_day_size) - date_diff
        hours, remainder = divmod(to_next_query.seconds, 3600)
        minutes, seconds = divmod(remainder, 60)
        show("距離下次搜尋：",
             to_next_query.days, "天",
             hours, "時",
             minutes, "分",
             seconds, "秒", ", 現有", len(urls), "筆url")
        # though size_ratio can be valid, we do not return it for caller usage is not expected
        return urls, None

    def get_this_time_new_result_num(self, key, urls):
        if key not in self.__url_map:
            return NA  # means this is a new query, all results are new
        new_result = 0
        [_, _, cached_urls, _] = self.__url_map[key]
        for url in urls:
            if url not in cached_urls:
                new_result += 1
        return new_result

    def cache_url(self, key, urls, next_size_ratio):
        self.__has_write = True
        self.__url_map[key] = [datetime.today(), self.get_this_time_new_result_num(key, urls), urls, next_size_ratio]

    @staticmethod
    def crawl_by_asking_google_search(pattern, start, size, option=""):
        assert type(pattern) in [str, unicode]
        from util.global_def import get_api_key, get_cx
        api_key = get_api_key()
        cx = get_cx()
        if not api_key or not cx:
            if not Crawler._HAS_SHOW_NO_SEARCH_MSG:
                Crawler._HAS_SHOW_NO_SEARCH_MSG = True
                info("沒有指定Google custom search需要的api_key及cx，將不進行圖片搜尋")
            return [], False
        size_option = "&imgSize=" + size if size else ""
        full_option = size_option + (option if option else "")
        base_url = 'https://www.googleapis.com/customsearch/v1?key=%s&cx=%s&searchType=image&num=%d' \
                   '&q=' + pattern + '&start=%d' + full_option
        request_str = base_url % (api_key, cx, G_SEARCH_PER_REQ_SIZE, start)
        urls = []
        success = True
        try:
            r = requests.get(request_str)
            res = json.loads(r.text)
            if "error" in res:
                Crawler.print_error(res["error"])
                return urls, False
            for image_info in res['items']:
                assert 'link' in image_info
                url = image_info['link']
                urls.append(url)
        except TypeError as e:  # for unhandled error...
            info("無法擷取最新的圖片網址：", str(e))
            success = False
        return urls, success

    @staticmethod
    def get_search_key(pattern, option):
        return pattern  # TODO:

    @staticmethod
    def print_error(data):
        assert isinstance(data, dict) and "message" in data
        error("搜尋引擎傳回錯誤訊息：", data["message"])

if __name__ == '__main__':
    from util.global_def import config_action
    config_action()
    # name '_' before the 'obj' to let python not free imported module before __del__ is called
    # (or we will have something like 'NoneType' object has no attribute 'dump' for cPickle.dump)
    _obj = Crawler(need_save=True)
    size = ["large", "xlarge", "xxlarge", "huge"]
    _obj.crawl("Inside Out", size_list=size, print_url=True)
<|endoftext|>


<commit_before>from decimal import Decimal
from django.contrib.sites.models import Site
from livesettings import config_value
from l10n.utils import moneyfmt
from product.models import Option, ProductPriceLookup, OptionGroup, Discount, Product, split_option_unique_id
from satchmo_utils.numbers import round_decimal
import datetime
import logging
import types
import string

log = logging.getLogger('product.utils')

def calc_discounted_by_percentage(price, percentage):
    if not percentage:
        return price
    if percentage > 1:
        log.warn("Correcting discount percentage, should be less than 1, is %s", percentage)
        percentage = percentage/100

    work = price * (1-percentage)
    cents = Decimal("0.01")
    return work.quantize(cents)

def find_auto_discounts(product):
    if not type(product) in (types.ListType, types.TupleType):
        product = (product,)
    today = datetime.date.today()
    discs = Discount.objects.filter(automatic=True, active=True, startDate__lte=today, endDate__gt=today)
    return discs.filter(valid_products__in=product).order_by('-percentage')

def find_best_auto_discount(product):
    discs = find_auto_discounts(product)
    if len(discs) > 0:
        return discs[0]
    else:
        return None

def productvariation_details(product, include_tax, user, create=False):
    """Build the product variation details, for conversion to javascript.

    Returns variation detail dictionary built like so:
    details = {
        "OPTION_KEY" : {
            "SLUG": "Variation Slug",
            "PRICE" : {"qty" : "$price", [...]},
            "SALE" : {"qty" : "$price", [...]},
            "TAXED" : "$taxed price",   # omitted if no taxed price requested
            "QTY" : 1
        },
        [...]
    }
    """

    ignore_stock = config_value('PRODUCT','NO_STOCK_CHECKOUT')
    discount = find_best_auto_discount(product)
    use_discount = discount and discount.percentage > 0

    if include_tax:
        from tax.utils import get_tax_processor
        taxer = get_tax_processor(user=user)
        tax_class = product.taxClass

    details = {'SALE' : use_discount}

    variations = ProductPriceLookup.objects.filter(parentid=product.id).order_by("-price")
    if variations.count() == 0:
        if create:
            log.debug('Creating price lookup for %s', product)
            ProductPriceLookup.objects.smart_create_for_product(product)
            variations = ProductPriceLookup.objects.filter(parentid=product.id).order_by("-price")
        else:
            log.warning('You must run satchmo_rebuild_pricing and add it to a cron-job to run every day, or else the product details will not work for product detail pages.')
    for detl in variations:
        key = detl.key
        if details.has_key(key):
            detail = details[key]
            qty = detl.quantity
        else:
            detail = {}
            detail['SLUG'] = detl.productslug

            if not detl.active:
                qty = round_decimal('-1.0')
            elif ignore_stock:
                qty = round_decimal('10000.0')
            else:
                qty = round_decimal(detl.items_in_stock)

            detail['QTY'] = round_decimal(qty)

            detail['PRICE'] = {}

            if use_discount:
                detail['SALE'] = {}

            if include_tax:
                detail['TAXED'] = {}
                if use_discount:
                    detail['TAXED_SALE'] = {}

            details[key] = detail

        qtykey = "%d" % detl.quantity

        price = detl.dynamic_price

        detail['PRICE'][qtykey] = moneyfmt(price)
        if use_discount:
            detail['SALE'][qtykey] = moneyfmt(calc_discounted_by_percentage(price, discount.percentage))

        if include_tax:
            tax_price = taxer.by_price(tax_class, price) + price
            detail['TAXED'][qtykey] = moneyfmt(tax_price)
            if use_discount:
                detail['TAXED_SALE'][qtykey] = moneyfmt(calc_discounted_by_percentage(tax_price, discount.percentage))

    return details

def rebuild_pricing():
    site = Site.objects.get_current()
    for lookup in ProductPriceLookup.objects.filter(siteid=site.id):
        lookup.delete()

    products = Product.objects.active_by_site(site=site, variations=False)

    productct = products.count()
    pricect = 0

    for product in products:
        prices = ProductPriceLookup.objects.smart_create_for_product(product)
        pricect += len(prices)

    return productct, pricect

def serialize_options(product, selected_options=()):
    """
    Return a list of optiongroups and options for display to the customer.
    Only returns options that are actually used by members of this product.

    Return Value:
    [
    {
    name: 'group name',
    id: 'group id',
    items: [{
        name: 'opt name',
        value: 'opt value',
        price_change: 'opt price',
        selected: False,
        },{..}]
    },
    {..}
    ]

    Note: This doesn't handle the case where you have multiple options and
    some combinations aren't available. For example, you have option_groups
    color and size, and you have a yellow/large, a yellow/small, and a
    white/small, but you have no white/large - the customer will still see
    the options white and large.
    """
    all_options = product.get_valid_options()
    group_sortmap = OptionGroup.objects.get_sortmap()

    # first get all objects
    # right now we only have a list of option.unique_ids, and there are
    # probably a lot of dupes, so first list them uniquely
    values = []

    if all_options != [[]]:
        vals = {}
        groups = {}
        opts = {}
        serialized = {}

        for options in all_options:
            for option in options:
                if not opts.has_key(option):
                    k, v = split_option_unique_id(option)
                    vals[v] = False
                    groups[k] = False
                    opts[option] = None

        for option in Option.objects.filter(option_group__id__in = groups.keys(), value__in = vals.keys()):
            uid = option.unique_id
            if opts.has_key(uid):
                opts[uid] = option

        # now we have all the objects in our "opts" dictionary, so build the serialization dict

        for option in opts.values():
            if not serialized.has_key(option.option_group_id):
                serialized[option.option_group.id] = {
                    'name': option.option_group.translated_name(),
                    'description': option.option_group.translated_description(),
                    'id': option.option_group.id,
                    'items': [],
                }
            if not option in serialized[option.option_group_id]['items']:
                serialized[option.option_group_id]['items'] += [option]
                option.selected = option.unique_id in selected_options

        # first sort the option groups
        for k, v in serialized.items():
            values.append((group_sortmap[k], v))

        if values:
            values.sort()
            values = zip(*values)[1]

        #now go back and make sure option items are sorted properly.
        for v in values:
            v['items'] = _sort_options(v['items'])

    log.debug('serialized: %s', values)

    log.debug('Serialized Options %s: %s', product.product.slug, values)
    return values

def _sort_options(lst):
    work = [(opt.sort_order, opt) for opt in lst]
    work.sort()
    return zip(*work)[1]

# All the functions below are used to validate custom attributes
# associated with a product or category.
# Custom ones can be added to the list via the admin setting ATTRIBUTE_VALIDATION

def validation_simple(value, obj=None):
    """
    Validates that at least one character has been entered.
    Not change is made to the value.
    """
    if len(value) > 1:
        return True, value
    else:
        return False, value

def validation_integer(value, obj=None):
    """
   Validates that value is an integer number.
   No change is made to the value
    """
    try:
        check = int(value)
        return True, value
    except:
        return False, value

def validation_yesno(value, obj=None):
    """
    Validates that yes or no is entered.
    Converts the yes or no to capitalized version
    """
    if string.upper(value) in ["YES","NO"]:
        return True, string.capitalize(value)
    else:
        return False, value

def validation_decimal(value, obj=None):
    """
    Validates that the number can be converted to a decimal
    """
    try:
        check = Decimal(value)
        return True, value
    except:
        return False, value

def validate_attribute_value(attribute, value, obj):
    """
    Helper function for forms that wish to validation a value for an
    AttributeOption.
    """
    function_name = attribute.validation.split('.')[-1]
    import_name = '.'.join(attribute.validation.split('.')[:-1])
    import_module = __import__(import_name, fromlist=[function_name])
    validation_function = getattr(import_module, function_name)
    return validation_function(value, obj)
<commit_msg>product.utils.validate_attribute_value: use non-kwarg form of __import__<commit_after>from decimal import Decimal
from django.contrib.sites.models import Site
from livesettings import config_value
from l10n.utils import moneyfmt
from product.models import Option, ProductPriceLookup, OptionGroup, Discount, Product, split_option_unique_id
from satchmo_utils.numbers import round_decimal
import datetime
import logging
import types
import string

log = logging.getLogger('product.utils')

def calc_discounted_by_percentage(price, percentage):
    if not percentage:
        return price
    if percentage > 1:
        log.warn("Correcting discount percentage, should be less than 1, is %s", percentage)
        percentage = percentage/100

    work = price * (1-percentage)
    cents = Decimal("0.01")
    return work.quantize(cents)

def find_auto_discounts(product):
    if not type(product) in (types.ListType, types.TupleType):
        product = (product,)
    today = datetime.date.today()
    discs = Discount.objects.filter(automatic=True, active=True, startDate__lte=today, endDate__gt=today)
    return discs.filter(valid_products__in=product).order_by('-percentage')

def find_best_auto_discount(product):
    discs = find_auto_discounts(product)
    if len(discs) > 0:
        return discs[0]
    else:
        return None

def productvariation_details(product, include_tax, user, create=False):
    """Build the product variation details, for conversion to javascript.

    Returns variation detail dictionary built like so:
    details = {
        "OPTION_KEY" : {
            "SLUG": "Variation Slug",
            "PRICE" : {"qty" : "$price", [...]},
            "SALE" : {"qty" : "$price", [...]},
            "TAXED" : "$taxed price",   # omitted if no taxed price requested
            "QTY" : 1
        },
        [...]
    }
    """

    ignore_stock = config_value('PRODUCT','NO_STOCK_CHECKOUT')
    discount = find_best_auto_discount(product)
    use_discount = discount and discount.percentage > 0

    if include_tax:
        from tax.utils import get_tax_processor
        taxer = get_tax_processor(user=user)
        tax_class = product.taxClass

    details = {'SALE' : use_discount}

    variations = ProductPriceLookup.objects.filter(parentid=product.id).order_by("-price")
    if variations.count() == 0:
        if create:
            log.debug('Creating price lookup for %s', product)
            ProductPriceLookup.objects.smart_create_for_product(product)
            variations = ProductPriceLookup.objects.filter(parentid=product.id).order_by("-price")
        else:
            log.warning('You must run satchmo_rebuild_pricing and add it to a cron-job to run every day, or else the product details will not work for product detail pages.')
    for detl in variations:
        key = detl.key
        if details.has_key(key):
            detail = details[key]
            qty = detl.quantity
        else:
            detail = {}
            detail['SLUG'] = detl.productslug

            if not detl.active:
                qty = round_decimal('-1.0')
            elif ignore_stock:
                qty = round_decimal('10000.0')
            else:
                qty = round_decimal(detl.items_in_stock)

            detail['QTY'] = round_decimal(qty)

            detail['PRICE'] = {}

            if use_discount:
                detail['SALE'] = {}

            if include_tax:
                detail['TAXED'] = {}
                if use_discount:
                    detail['TAXED_SALE'] = {}

            details[key] = detail

        qtykey = "%d" % detl.quantity

        price = detl.dynamic_price

        detail['PRICE'][qtykey] = moneyfmt(price)
        if use_discount:
            detail['SALE'][qtykey] = moneyfmt(calc_discounted_by_percentage(price, discount.percentage))

        if include_tax:
            tax_price = taxer.by_price(tax_class, price) + price
            detail['TAXED'][qtykey] = moneyfmt(tax_price)
            if use_discount:
                detail['TAXED_SALE'][qtykey] = moneyfmt(calc_discounted_by_percentage(tax_price, discount.percentage))

    return details

def rebuild_pricing():
    site = Site.objects.get_current()
    for lookup in ProductPriceLookup.objects.filter(siteid=site.id):
        lookup.delete()

    products = Product.objects.active_by_site(site=site, variations=False)

    productct = products.count()
    pricect = 0

    for product in products:
        prices = ProductPriceLookup.objects.smart_create_for_product(product)
        pricect += len(prices)

    return productct, pricect

def serialize_options(product, selected_options=()):
    """
    Return a list of optiongroups and options for display to the customer.
    Only returns options that are actually used by members of this product.

    Return Value:
    [
    {
    name: 'group name',
    id: 'group id',
    items: [{
        name: 'opt name',
        value: 'opt value',
        price_change: 'opt price',
        selected: False,
        },{..}]
    },
    {..}
    ]

    Note: This doesn't handle the case where you have multiple options and
    some combinations aren't available. For example, you have option_groups
    color and size, and you have a yellow/large, a yellow/small, and a
    white/small, but you have no white/large - the customer will still see
    the options white and large.
    """
    all_options = product.get_valid_options()
    group_sortmap = OptionGroup.objects.get_sortmap()

    # first get all objects
    # right now we only have a list of option.unique_ids, and there are
    # probably a lot of dupes, so first list them uniquely
    values = []

    if all_options != [[]]:
        vals = {}
        groups = {}
        opts = {}
        serialized = {}

        for options in all_options:
            for option in options:
                if not opts.has_key(option):
                    k, v = split_option_unique_id(option)
                    vals[v] = False
                    groups[k] = False
                    opts[option] = None

        for option in Option.objects.filter(option_group__id__in = groups.keys(), value__in = vals.keys()):
            uid = option.unique_id
            if opts.has_key(uid):
                opts[uid] = option

        # now we have all the objects in our "opts" dictionary, so build the serialization dict

        for option in opts.values():
            if not serialized.has_key(option.option_group_id):
                serialized[option.option_group.id] = {
                    'name': option.option_group.translated_name(),
                    'description': option.option_group.translated_description(),
                    'id': option.option_group.id,
                    'items': [],
                }
            if not option in serialized[option.option_group_id]['items']:
                serialized[option.option_group_id]['items'] += [option]
                option.selected = option.unique_id in selected_options

        # first sort the option groups
        for k, v in serialized.items():
            values.append((group_sortmap[k], v))

        if values:
            values.sort()
            values = zip(*values)[1]

        #now go back and make sure option items are sorted properly.
        for v in values:
            v['items'] = _sort_options(v['items'])

    log.debug('serialized: %s', values)

    log.debug('Serialized Options %s: %s', product.product.slug, values)
    return values

def _sort_options(lst):
    work = [(opt.sort_order, opt) for opt in lst]
    work.sort()
    return zip(*work)[1]

# All the functions below are used to validate custom attributes
# associated with a product or category.
# Custom ones can be added to the list via the admin setting ATTRIBUTE_VALIDATION

def validation_simple(value, obj=None):
    """
    Validates that at least one character has been entered.
    Not change is made to the value.
    """
    if len(value) > 1:
        return True, value
    else:
        return False, value

def validation_integer(value, obj=None):
    """
   Validates that value is an integer number.
   No change is made to the value
    """
    try:
        check = int(value)
        return True, value
    except:
        return False, value

def validation_yesno(value, obj=None):
    """
    Validates that yes or no is entered.
    Converts the yes or no to capitalized version
    """
    if string.upper(value) in ["YES","NO"]:
        return True, string.capitalize(value)
    else:
        return False, value

def validation_decimal(value, obj=None):
    """
    Validates that the number can be converted to a decimal
    """
    try:
        check = Decimal(value)
        return True, value
    except:
        return False, value

def validate_attribute_value(attribute, value, obj):
    """
    Helper function for forms that wish to validation a value for an
    AttributeOption.
    """
    function_name = attribute.validation.split('.')[-1]
    import_name = '.'.join(attribute.validation.split('.')[:-1])

    # The below __import__() call is from python docs, and is equivalent to:
    #
    #   from import_name import function_name
    #
    import_module = __import__(import_name, globals(), locals(), [function_name])

    validation_function = getattr(import_module, function_name)
    return validation_function(value, obj)
<|endoftext|>


<commit_before>#! /usr/bin/env python
#
# Copyright (c) 2011 SEOmoz
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import redis
from sauron import logger
from sauron.metrics import Metric, MetricException

class RedisMetric(Metric):
  '''
  Pls refer to https://pypi.python.org/pypi/redis for argument which should be provided

  Attributes:
     host (string): https://pypi.python.org/pypi/redis
     port (int): ()
     db (string):
     password (string):
     charset (string):
     errors (string):
     unix_socket_path (string):
  '''

    @staticmethod
    def parseMemory(x):
        try:
            if 'G' in x:
                return (x.replace('G', ''), 'Gigabytes')
            elif 'M' in x:
                return (x.replace('M', ''), 'Megabytes')
            elif 'K' in x:
                return (x.replace('K', ''), 'Kilobytes')
            else:
                return (x, 'Bytes')
        except:
            return (x, 'Bytes')

    infoUnits = {
        'redis_version'             : lambda x: (int(x.replace('.', '')), 'None'),
        'redis_git_sha1'            : lambda x: (int(x, 16), 'None'),
        'redis_dig_dirty'           : lambda x: (x, 'None'),
        'arch_bits'                 : lambda x: (x, 'Count'),
        'process_id'                : lambda x: (x, 'None'),
        'uptime_in_seconds'         : lambda x: (x, 'Seconds'),
        'uptime_in_days'            : lambda x: (x, 'None'),
        'lru_clock'                 : lambda x: (x, 'Seconds'),
        'used_cpu_sys'              : lambda x: (x, 'Seconds'),
        'used_cpu_user'             : lambda x: (x, 'Seconds'),
        'used_cpu_sys_children'     : lambda x: (x, 'Seconds'),
        'used_cpu_user_children'    : lambda x: (x, 'Seconds'),
        'connected_clients'         : lambda x: (x, 'Count'),
        'connected_slaves'          : lambda x: (x, 'Count'),
        'client_longest_output_list': lambda x: (x, 'Count'),
        'client_biggest_input_buf'  : lambda x: (x, 'Bytes'),
        'blocked_clients'           : lambda x: (x, 'Count'),
        'used_memory'               : lambda x: RedisMetric.parseMemory(x),
        'used_memory_human'         : lambda x: RedisMetric.parseMemory(x),
        'used_memory_rss'           : lambda x: RedisMetric.parseMemory(x),
        'used_memroy_peak'          : lambda x: RedisMetric.parseMemory(x),
        'used_memory_peak_human'    : lambda x: RedisMetric.parseMemory(x),
        'mem_fragmentation_ratio'   : lambda x: (x, 'None'),
        'loading'                   : lambda x: (x, 'None'),
        'aof_enabled'               : lambda x: (x, 'None'),
        'changes_since_last_save'   : lambda x: (x, 'Count'),
        'bgsave_in_progress'        : lambda x: (x, 'None'),
        'last_save_time'            : lambda x: (x, 'Seconds'),
        'bgrewriteaof_in_progress'  : lambda x: (x, 'None'),
        'total_connections_received': lambda x: (x, 'Count'),
        'total_commands_processed'  : lambda x: (x, 'Count'),
        'expired_keys'              : lambda x: (x, 'Count'),
        'evicted_keys'              : lambda x: (x, 'Count'),
        'keyspace_hits'             : lambda x: (x, 'Count'),
        'keyspace_misses'           : lambda x: (x, 'Count'),
        'pubsub_channels'           : lambda x: (x, 'Count'),
        'pubsub_patterns'           : lambda x: (x, 'Count'),
        'latest_fork_usec'          : lambda x: (x, 'Microseconds'),
        'vm_enabled'                : lambda x: (x, 'None'),
        'aof_current_size'          : lambda x: (x, 'Bytes'),
        'aof_base_size'             : lambda x: (x, 'Bytes'),
        'aof_pending_rewrite'       : lambda x: (x, 'None'),
    }

    def reconfig(self, **kwargs):
        # These are a selection of argument names. If they're
        # present, then we'll use them, otherwise, we'll use
        # the default provided by the redis module itself
        redisArgs = {}
        for arg in ['host', 'port', 'db', 'password', 'charset', 'errors', 'unix_socket_path']:
            try:
                redisArgs[arg] = kwargs[arg]
            except KeyError:
                pass
        self.redis  = redis.Redis(**redisArgs)
        # The keys we should save from the 'info' command in redis
        self.info   = kwargs.get('info'  , [])
        # The keys we should get and interpret as numbers
        self.get    = kwargs.get('get'   , [])
        # The keys we should get, and report their length
        self.llen   = kwargs.get('llen', [])
        # The keys we should get and report the hash length
        self.hlen   = kwargs.get('hlen', [])
        # The keys we should get and report the particular key from
        self.hget   = kwargs.get('hget', {})
        # The keys we should get and report the cardinality of
        self.scard  = kwargs.get('scard', [])
        # The keys we should get and report the zcardinality of
        self.zcard  = kwargs.get('zcard', [])
        # The patterns we should count the number of keys of
        self.patterns = kwargs.get('patterns', [])

    def values(self):
        try:
            results = {}
            info = self.redis.info()
            for i in self.info:
                try:
                    results[i] = RedisMetric.infoUnits[i](info[i])
                except Exception as e:
                    print repr(e)
                    results[i] = (info[i], 'None')

            both = list(self.get)
            both.extend(self.llen)
            both.extend(self.hlen)
            both.extend(['%s-%s' % (k, v) for k,v in self.hget.items()])
            both.extend(self.scard)
            both.extend(self.zcard)
            both.extend(self.patterns)
            with self.redis.pipeline() as pipe:
                for g in self.get:
                    logger.debug('get %s' % g)
                    pipe.get(g)
                for l in self.llen:
                    logger.debug('llen %s' % l)
                    pipe.llen(l)
                for h in self.hlen:
                    logger.debug('hlen %s' % h)
                    pipe.hlen(h)
                for k,v in self.hget.items():
                    logger.debug('hget %s %s' % (k, v))
                    pipe.hget(k, v)
                for s in self.scard:
                    logger.debug('scard %s' % s)
                    pipe.scard(s)
                for z in self.zcard:
                    logger.debug('zcard %s' % z)
                    pipe.zcard(z)
                for pattern in self.patterns:
                    logger.debug('keys %s' % pattern)
                    pipe.keys(pattern)
                fetched = pipe.execute()
                for k, f in zip(both, fetched):
                    if isinstance(f, list):
                        results[k] = (len(f), 'Count')
                    else:
                        results[k] = (f, 'Count')

            return {'results': results}
        except redis.RedisError as e:
            raise MetricException(e)
<commit_msg>fix indent<commit_after>#! /usr/bin/env python
#
# Copyright (c) 2011 SEOmoz
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import redis
from sauron import logger
from sauron.metrics import Metric, MetricException

class RedisMetric(Metric):
  '''
  Pls refer to https://pypi.python.org/pypi/redis for argument which should be provided

  Attributes:
     host (string): https://pypi.python.org/pypi/redis
     port (int): ()
     db (string):
     password (string):
     charset (string):
     errors (string):
     unix_socket_path (string):
  '''
  @staticmethod
  def parseMemory(x):
      try:
          if 'G' in x:
              return (x.replace('G', ''), 'Gigabytes')
          elif 'M' in x:
              return (x.replace('M', ''), 'Megabytes')
          elif 'K' in x:
              return (x.replace('K', ''), 'Kilobytes')
          else:
              return (x, 'Bytes')
      except:
          return (x, 'Bytes')

  infoUnits = {
      'redis_version'             : lambda x: (int(x.replace('.', '')), 'None'),
      'redis_git_sha1'            : lambda x: (int(x, 16), 'None'),
      'redis_dig_dirty'           : lambda x: (x, 'None'),
      'arch_bits'                 : lambda x: (x, 'Count'),
      'process_id'                : lambda x: (x, 'None'),
      'uptime_in_seconds'         : lambda x: (x, 'Seconds'),
      'uptime_in_days'            : lambda x: (x, 'None'),
      'lru_clock'                 : lambda x: (x, 'Seconds'),
      'used_cpu_sys'              : lambda x: (x, 'Seconds'),
      'used_cpu_user'             : lambda x: (x, 'Seconds'),
      'used_cpu_sys_children'     : lambda x: (x, 'Seconds'),
      'used_cpu_user_children'    : lambda x: (x, 'Seconds'),
      'connected_clients'         : lambda x: (x, 'Count'),
      'connected_slaves'          : lambda x: (x, 'Count'),
      'client_longest_output_list': lambda x: (x, 'Count'),
      'client_biggest_input_buf'  : lambda x: (x, 'Bytes'),
      'blocked_clients'           : lambda x: (x, 'Count'),
      'used_memory'               : lambda x: RedisMetric.parseMemory(x),
      'used_memory_human'         : lambda x: RedisMetric.parseMemory(x),
      'used_memory_rss'           : lambda x: RedisMetric.parseMemory(x),
      'used_memroy_peak'          : lambda x: RedisMetric.parseMemory(x),
      'used_memory_peak_human'    : lambda x: RedisMetric.parseMemory(x),
      'mem_fragmentation_ratio'   : lambda x: (x, 'None'),
      'loading'                   : lambda x: (x, 'None'),
      'aof_enabled'               : lambda x: (x, 'None'),
      'changes_since_last_save'   : lambda x: (x, 'Count'),
      'bgsave_in_progress'        : lambda x: (x, 'None'),
      'last_save_time'            : lambda x: (x, 'Seconds'),
      'bgrewriteaof_in_progress'  : lambda x: (x, 'None'),
      'total_connections_received': lambda x: (x, 'Count'),
      'total_commands_processed'  : lambda x: (x, 'Count'),
      'expired_keys'              : lambda x: (x, 'Count'),
      'evicted_keys'              : lambda x: (x, 'Count'),
      'keyspace_hits'             : lambda x: (x, 'Count'),
      'keyspace_misses'           : lambda x: (x, 'Count'),
      'pubsub_channels'           : lambda x: (x, 'Count'),
      'pubsub_patterns'           : lambda x: (x, 'Count'),
      'latest_fork_usec'          : lambda x: (x, 'Microseconds'),
      'vm_enabled'                : lambda x: (x, 'None'),
      'aof_current_size'          : lambda x: (x, 'Bytes'),
      'aof_base_size'             : lambda x: (x, 'Bytes'),
      'aof_pending_rewrite'       : lambda x: (x, 'None'),
  }

  def reconfig(self, **kwargs):
      # These are a selection of argument names. If they're
      # present, then we'll use them, otherwise, we'll use
      # the default provided by the redis module itself
      redisArgs = {}
      for arg in ['host', 'port', 'db', 'password', 'charset', 'errors', 'unix_socket_path']:
          try:
              redisArgs[arg] = kwargs[arg]
          except KeyError:
              pass
      self.redis  = redis.Redis(**redisArgs)
      # The keys we should save from the 'info' command in redis
      self.info   = kwargs.get('info'  , [])
      # The keys we should get and interpret as numbers
      self.get    = kwargs.get('get'   , [])
      # The keys we should get, and report their length
      self.llen   = kwargs.get('llen', [])
      # The keys we should get and report the hash length
      self.hlen   = kwargs.get('hlen', [])
      # The keys we should get and report the particular key from
      self.hget   = kwargs.get('hget', {})
      # The keys we should get and report the cardinality of
      self.scard  = kwargs.get('scard', [])
      # The keys we should get and report the zcardinality of
      self.zcard  = kwargs.get('zcard', [])
      # The patterns we should count the number of keys of
      self.patterns = kwargs.get('patterns', [])

  def values(self):
      try:
          results = {}
          info = self.redis.info()
          for i in self.info:
              try:
                  results[i] = RedisMetric.infoUnits[i](info[i])
              except Exception as e:
                  print repr(e)
                  results[i] = (info[i], 'None')

          both = list(self.get)
          both.extend(self.llen)
          both.extend(self.hlen)
          both.extend(['%s-%s' % (k, v) for k,v in self.hget.items()])
          both.extend(self.scard)
          both.extend(self.zcard)
          both.extend(self.patterns)
          with self.redis.pipeline() as pipe:
              for g in self.get:
                  logger.debug('get %s' % g)
                  pipe.get(g)
              for l in self.llen:
                  logger.debug('llen %s' % l)
                  pipe.llen(l)
              for h in self.hlen:
                  logger.debug('hlen %s' % h)
                  pipe.hlen(h)
              for k,v in self.hget.items():
                  logger.debug('hget %s %s' % (k, v))
                  pipe.hget(k, v)
              for s in self.scard:
                  logger.debug('scard %s' % s)
                  pipe.scard(s)
              for z in self.zcard:
                  logger.debug('zcard %s' % z)
                  pipe.zcard(z)
              for pattern in self.patterns:
                  logger.debug('keys %s' % pattern)
                  pipe.keys(pattern)
              fetched = pipe.execute()
              for k, f in zip(both, fetched):
                  if isinstance(f, list):
                      results[k] = (len(f), 'Count')
                  else:
                      results[k] = (f, 'Count')

          return {'results': results}
      except redis.RedisError as e:
          raise MetricException(e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Scrapy settings for saymedia project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'saymedia'

SPIDER_MODULES = ['saymedia.spiders']
NEWSPIDER_MODULE = 'saymedia.spiders'
ROBOTSTXT_OBEY = True

DOWNLOADER_MIDDLEWARES = {
    'saymedia.middleware.MysqlDownloaderMiddleware': 1,
    'saymedia.middleware.TimerDownloaderMiddleware': 999,
    # 'saymedia.middleware.ErrorConverterMiddleware': 999,
}

SPIDER_MIDDLEWARES = {
    'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': None,
}

ITEM_PIPELINES = {
    'saymedia.pipelines.DatabaseWriterPipeline': 0,
}

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'SEO Spider (+http://www.saymedia.com)'

DATABASE = {
    'USER': 'seo',
    'PASS': 'oes',
}

FIREBASE_URL = "https://incandescent-torch-9049.firebaseio.com"<commit_msg>Settings cleanup<commit_after># -*- coding: utf-8 -*-

# Scrapy settings for saymedia project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'saymedia'

SPIDER_MODULES = ['saymedia.spiders']
NEWSPIDER_MODULE = 'saymedia.spiders'
ROBOTSTXT_OBEY = True

DOWNLOADER_MIDDLEWARES = {
    'saymedia.middleware.MysqlDownloaderMiddleware': 1,
    'saymedia.middleware.TimerDownloaderMiddleware': 999,
    # 'saymedia.middleware.ErrorConverterMiddleware': 999,
}

SPIDER_MIDDLEWARES = {
    'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': None,
}

ITEM_PIPELINES = {
    'saymedia.pipelines.DatabaseWriterPipeline': 0,
}

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'SEO Spider (+http://www.saymedia.com)'

DATABASE = {
    'USER': 'YOUR_DATABASE_USER',
    'PASS': 'YOUR_DATABASE_PASS',
}

FIREBASE_URL = "YOUR_FIREBASE_URL"<|endoftext|>


<commit_before>import json
import sys
import pyelliptic as ec

from p2p import PeerConnection, TransportLayer
from multiprocessing import Process
import traceback

from protocol import hello_request, hello_response, proto_response_pubkey
import obelisk

# Check if user specified a crypto file to load for their marketplace
# This is located in the ppl/ folder
if len(sys.argv) < 2:
    print >> sys.stderr, "[Error] You need to specify a user crypto file in `ppl/` folder"
    sys.exit(-1)

# Return data array with details from the crypto file
# TODO: This needs to be protected better; potentially encrypted file or DB
def load_crypto_details():
    print sys.argv[1]
    with open(sys.argv[1]) as f:
        data = json.loads(f.read())
    assert "nickname" in data
    assert "secret" in data
    assert "pubkey" in data
    assert len(data["secret"]) == 2 * 32
    assert len(data["pubkey"]) == 2 * 33
    return data["nickname"], data["secret"].decode("hex"), data["pubkey"].decode("hex")

# Look in crypto file for market details
NICKNAME, SECRET, PUBKEY = load_crypto_details()

class CryptoPeerConnection(PeerConnection):

    def __init__(self, address, transport, pub):
        self._transport = transport
        self._priv = transport._myself
        self._pub = pub
        PeerConnection.__init__(self, address)

    def encrypt(self, data):
        return self._priv.encrypt(data, self._pub)

    def send(self, data):
        self.send_raw(self.encrypt(json.dumps(data)))

    def on_message(self, msg):
        # this are just acks
        pass

class CryptoTransportLayer(TransportLayer):

    def __init__(self, port=None):
        TransportLayer.__init__(self, port)
        self._myself = ec.ECC(curve='secp256k1')
        self.nick_mapping = {}

    def get_profile(self):
        peers = {}
        for uri, peer in self._peers.iteritems():
            if peer._pub:
                peers[uri] = peer._pub.encode('hex')
        return {'uri': self._uri, 'pub': self._myself.get_pubkey().encode('hex'), 'peers': peers}

    def respond_pubkey_if_mine(self, nickname, ident_pubkey):

        if ident_pubkey != PUBKEY:
            print "Public key does not match your identity"
            return

        # Return signed pubkey
        pubkey = self._myself.get_pubkey()
        ec_key = obelisk.EllipticCurveKey()
        ec_key.set_secret(SECRET)
        digest = obelisk.Hash(pubkey)
        signature = ec_key.sign(digest)

        # Send array of nickname, pubkey, signature to transport layer
        self.send(proto_response_pubkey(nickname, pubkey, signature))

    def create_peer(self, uri, pub):
        if pub:
            self.log("Creating peer " + uri + " " + pub[0:16] + "...", '*')
            pub = pub.decode('hex')
        else:
            self.log("Creating peer [seed] " + uri, '*')

        # Create the peer
        self._peers[uri] = CryptoPeerConnection(uri, self, pub)

        # Call 'peer' callbacks on listeners
        self.trigger_callbacks('peer', self._peers[uri])

    def send_enc(self, uri, msg):
        peer = self._peers[uri]
        pub = peer._pub

        # Now send a hello message to the peer
        if pub:
            self.log("Sending encrypted [" + msg['type']  + "] message to %s" % uri)
            peer.send(msg)
        else:
            # Will send clear profile on initial if no pub
            self.log("Sending unencrypted [" + msg['type']  + "] message to %s" % uri)
            self._peers[uri].send_raw(json.dumps(msg))

    def init_peer(self, msg):
        print "Initialize Peer: ", msg
        uri = msg['uri']
        pub = msg.get('pub')
        msg_type = msg.get('type')

        if not uri in self._peers:
            print 'Create New Peer: ',uri
            self.create_peer(uri, pub)
        elif pub and not self._peers[uri]._pub:
            self.log("Setting public key for seed node")
            self._peers[uri]._pub = pub.decode('hex')
        elif pub and (self._peers[uri]._pub != pub.decode('hex')):
            self.log("Adjusting public key for node")
            self._peers[uri]._pub = pub.decode('hex')

        if not msg_type:
            self.send_enc(uri, hello_request(self.get_profile()))
        elif msg_type == 'hello_request':
            self.send_enc(uri, hello_response(self.get_profile()))

    def on_raw_message(self, serialized):

        try:
            msg = json.loads(serialized)
            self.log("receive [%s]" % msg.get('type', 'unknown'))
        except ValueError:
            try:
                msg = json.loads(self._myself.decrypt(serialized))
                self.log("Decrypted raw message [%s]" % msg.get('type', 'unknown'))
            except:
                self.log("incorrect msg ! %s..." % self._myself.decrypt(serialized))
                traceback.print_exc()
                return

        msg_type = msg.get('type')

        if msg_type.startswith('hello') and msg.get('uri') :
            self.init_peer(msg)
            for uri, pub in msg.get('peers', {}).iteritems():
                # Do not add yourself as a peer
                if uri != self._uri:
                    self.init_peer({'uri': uri, 'pub': pub})
            self.log("Update peer table [%s peers]" % len(self._peers))
        else:
            self.on_message(msg)

<commit_msg>Fixes infinite hello messages in networks with >2 nodes.<commit_after>import json
import sys
import pyelliptic as ec

from p2p import PeerConnection, TransportLayer
from multiprocessing import Process
import traceback

from protocol import hello_request, hello_response, proto_response_pubkey
import obelisk

# Check if user specified a crypto file to load for their marketplace
# This is located in the ppl/ folder
if len(sys.argv) < 2:
    print >> sys.stderr, "[Error] You need to specify a user crypto file in `ppl/` folder"
    sys.exit(-1)

# Return data array with details from the crypto file
# TODO: This needs to be protected better; potentially encrypted file or DB
def load_crypto_details():
    print sys.argv[1]
    with open(sys.argv[1]) as f:
        data = json.loads(f.read())
    assert "nickname" in data
    assert "secret" in data
    assert "pubkey" in data
    assert len(data["secret"]) == 2 * 32
    assert len(data["pubkey"]) == 2 * 33
    return data["nickname"], data["secret"].decode("hex"), data["pubkey"].decode("hex")

# Look in crypto file for market details
NICKNAME, SECRET, PUBKEY = load_crypto_details()

class CryptoPeerConnection(PeerConnection):

    def __init__(self, address, transport, pub):
        self._transport = transport
        self._priv = transport._myself
        self._pub = pub
        PeerConnection.__init__(self, address)

    def encrypt(self, data):
        return self._priv.encrypt(data, self._pub)

    def send(self, data):
        self.send_raw(self.encrypt(json.dumps(data)))

    def on_message(self, msg):
        # this are just acks
        pass

class CryptoTransportLayer(TransportLayer):

    def __init__(self, port=None):
        TransportLayer.__init__(self, port)
        self._myself = ec.ECC(curve='secp256k1')
        self.nick_mapping = {}

    def get_profile(self):
        peers = {}
        for uri, peer in self._peers.iteritems():
            if peer._pub:
                peers[uri] = peer._pub.encode('hex')
        return {'uri': self._uri, 'pub': self._myself.get_pubkey().encode('hex'), 'peers': peers}

    def respond_pubkey_if_mine(self, nickname, ident_pubkey):

        if ident_pubkey != PUBKEY:
            print "Public key does not match your identity"
            return

        # Return signed pubkey
        pubkey = self._myself.get_pubkey()
        ec_key = obelisk.EllipticCurveKey()
        ec_key.set_secret(SECRET)
        digest = obelisk.Hash(pubkey)
        signature = ec_key.sign(digest)

        # Send array of nickname, pubkey, signature to transport layer
        self.send(proto_response_pubkey(nickname, pubkey, signature))

    def create_peer(self, uri, pub):
        if pub:
            self.log("Creating peer " + uri + " " + pub[0:16] + "...", '*')
            pub = pub.decode('hex')
        else:
            self.log("Creating peer [seed] " + uri, '*')

        # Create the peer
        self._peers[uri] = CryptoPeerConnection(uri, self, pub)

        # Call 'peer' callbacks on listeners
        self.trigger_callbacks('peer', self._peers[uri])

    def send_enc(self, uri, msg):
        peer = self._peers[uri]
        pub = peer._pub

        # Now send a hello message to the peer
        if pub:
            self.log("Sending encrypted [" + msg['type']  + "] message to %s" % uri)
            peer.send(msg)
        else:
            # Will send clear profile on initial if no pub
            self.log("Sending unencrypted [" + msg['type']  + "] message to %s" % uri)
            self._peers[uri].send_raw(json.dumps(msg))

    def init_peer(self, msg):
        print "Initialize Peer: ", msg
        uri = msg['uri']
        pub = msg.get('pub')
        msg_type = msg.get('type')

        if not uri in self._peers:
            # unknown peer
            print 'Create New Peer: ',uri
            self.create_peer(uri, pub)

            if not msg_type:
                self.send_enc(uri, hello_request(self.get_profile()))
            elif msg_type == 'hello_request':
                self.send_enc(uri, hello_response(self.get_profile()))

        else:
            # known peer
            if pub:
                # test if we have to update the pubkey
                if not self._peers[uri]._pub:
                    self.log("Setting public key for seed node")
                    self._peers[uri]._pub = pub.decode('hex')
                if (self._peers[uri]._pub != pub.decode('hex')):
                    self.log("Updating public key for node")
                    self._peers[uri]._pub = pub.decode('hex')

            if msg_type == 'hello_request':
                # reply only if necessary
                self.send_enc(uri, hello_response(self.get_profile()))

    def on_raw_message(self, serialized):

        try:
            msg = json.loads(serialized)
            self.log("receive [%s]" % msg.get('type', 'unknown'))
        except ValueError:
            try:
                msg = json.loads(self._myself.decrypt(serialized))
                self.log("Decrypted raw message [%s]" % msg.get('type', 'unknown'))
            except:
                self.log("incorrect msg ! %s..." % self._myself.decrypt(serialized))
                traceback.print_exc()
                return

        msg_type = msg.get('type')

        if msg_type.startswith('hello') and msg.get('uri') :
            self.init_peer(msg)
            for uri, pub in msg.get('peers', {}).iteritems():
                # Do not add yourself as a peer
                if uri != self._uri:
                    self.init_peer({'uri': uri, 'pub': pub})
            self.log("Update peer table [%s peers]" % len(self._peers))
        else:
            self.on_message(msg)

<|endoftext|>