<commit_before>import logging
import time
from collections import defaultdict
from .cassandra import CassandraServer
from ..resources import WorkerResource
from hiispider.servers.mixins import JobQueueMixin, PageCacheQueueMixin, JobGetterMixin, JobHistoryMixin
from hiispider.requestqueuer import QueueTimeoutException
from twisted.internet import reactor, task
from twisted.web import server
from twisted.internet.defer import inlineCallbacks, returnValue
import pprint
from traceback import format_exc
from hiispider.exceptions import *
from uuid import UUID
import cPickle
from zlib import decompress

PRETTYPRINTER = pprint.PrettyPrinter(indent=4)
logger = logging.getLogger(__name__)

class WorkerServer(CassandraServer, JobQueueMixin, PageCacheQueueMixin, JobGetterMixin, JobHistoryMixin):

    public_ip = None
    local_ip = None
    network_information = {}
    simultaneous_jobs = 19
    uuids_dequeued = 0
    jobs_complete = 0
    job_failures = 0
    job_queue = []
    jobsloop = None
    dequeueloop = None
    logloop = None
    pending_uuid_reqs = 0
    uuid_queue = []
    uuid_dequeueing = False
    uuid_queue_size = 500
    job_queue_size = 500
    uncached_uuid_queue = []
    uncached_uuid_dequeueing = False
    user_account_queue = []
    user_account_dequeueing = False
    service_credential_dequeueing = False

    def __init__(self, config, port=None):
        super(WorkerServer, self).__init__(config)
        self.t0 = time.time()
        self.setupJobQueue(config)
        self.setupPageCacheQueue(config)
        self.setupJobGetter(config)
        self.setupJobHistory(config)
        # HTTP interface
        resource = WorkerResource(self)
        if port is None:
            port = config["worker_server_port"]
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        self.scheduler_server = config["scheduler_server"]
        self.scheduler_server_port = config["scheduler_server_port"]
        self.config = config
        # setup manhole
        manhole_namespace = {
            'service': self,
            'globals': globals(),
        }
        reactor.listenTCP(config["manhole_worker_port"], self.getManholeFactory(manhole_namespace, admin=config["manhole_password"]))

    def start(self):
        start_deferred = super(WorkerServer, self).start()
        start_deferred.addCallback(self._workerStart)
        return start_deferred

    @inlineCallbacks
    def _workerStart(self, started=None):
        logger.debug("Starting worker components.")
        yield self.startJobQueue()
        yield self.startPageCacheQueue()
        yield self.setupJobHistory(self.config)
        self.jobsloop = task.LoopingCall(self.executeJobs)
        self.jobsloop.start(0.5)
        self.dequeueloop = task.LoopingCall(self.dequeue)
        self.dequeueloop.start(10)
        self.logloop = task.LoopingCall(self.logStatus)
        self.logloop.start(5)

    @inlineCallbacks
    def shutdown(self):
        self.uuid_queue = []
        self.uncached_uuid_queue = []
        self.user_account_queue = []
        self.jobsloop.stop()
        self.dequeueloop.stop()
        self.logloop.stop()
        yield self.stopJobQueue()
        yield super(WorkerServer, self).shutdown()
        yield self.stopPageCacheQueue()

    def dequeue(self):
        self.dequeuejobs()
        if self.uncached_uuid_queue:
            self.lookupjobs()

    def dequeuejobs(self):
        if self.uuid_dequeueing:
            return
        if len(self.job_queue) > self.job_queue_size or len(self.uuid_queue) > self.uuid_queue_size:
            return
        self.uuid_dequeueing = True
        self._dequeuejobs()

    def _dequeuejobs(self):
        if len(self.uuid_queue) < self.uuid_queue_size * 4 and len(self.job_queue) < self.job_queue_size * 4:
            d = self.jobs_rabbit_queue.get()
            d.addCallback(self._dequeuejobsCallback)
            d.addErrback(self._dequeuejobsErrback)
        else:
            self.uuid_dequeueing = False

    def _dequeuejobsCallback(self, msg):
        self.jobs_chan.basic_ack(msg.delivery_tag)
        self.uuid_queue.append(UUID(bytes=msg.content.body).hex)
        self.uuids_dequeued += 1
        if len(self.uuid_queue) > 200:
            uuids, self.uuid_queue = self.uuid_queue, []
            d = self.redis_client.mget(*uuids)
            d.addCallback(self._dequeuejobsCallback2, uuids)
            d.addErrback(self._dequeuejobsErrback)
        else:
            self._dequeuejobs()

    def _dequeuejobsErrback(self, error):
        self._dequeuejobs()

    def _dequeuejobsCallback2(self, data, uuids):
        results = zip(uuids, data)
        for row in results:
            if row[1]:
                job = cPickle.loads(decompress(row[1]))
                logger.debug('Found uuid in Redis: %s' % row[0])
                self.job_queue.append(job)
            else:
                logger.debug('Could not find uuids %s in Redis.' % row[0])
                self.uncached_uuid_queue.append(row[0])
        self._dequeuejobs()

    @inlineCallbacks
    def lookupjobs(self):
        while self.uncached_uuid_queue:
            uuids, self.uncached_uuid_queue = self.uncached_uuid_queue[0:100], self.uncached_uuid_queue[100:]
            sql = """SELECT uuid, content_userprofile.user_id as user_id, username, host, account_id, type
                FROM spider_service, auth_user, content_userprofile
                WHERE uuid IN ('%s')
                AND auth_user.id=spider_service.user_id
                AND auth_user.id=content_userprofile.user_id
                """ % "','".join(uuids)
            try:
                data = yield self.mysql.runQuery(sql)
            except:
                logger.error("Could not find users in '%s'" % "','".join(uuids))
                continue
            self.user_account_queue.extend(data)
        accounts_by_type = defaultdict(list)
        for user_account in self.user_account_queue:
            accounts_by_type[user_account["type"]].append(user_account)
        self.user_account_queue = []
        for service_type in accounts_by_type:
            accounts_by_id = defaultdict(list)
            for user_accounts in accounts_by_type[service_type]:
                accounts_by_id[user_accounts["account_id"]].append(user_account)
            sql = "SELECT * FROM content_%saccount WHERE account_id IN (%s)" % (service_type, ",".join(accounts_by_id.keys()))
            try:
                data = yield self.mysql.runQuery(sql)
            except Exception, e:
                logger.error("Could not find service %s, %s" % (service_type, sql))
                continue
            for service_credentials in data:
                user_account = accounts_by_id[service_credentials["account_id"]]
                job = Job(
                    function_name=user_account['type'],
                    service_credentials=service_credentials,
                    user_account=user_account,
                    uuid=user_account['uuid'])
                self.mapJob(job) # Do this now so mapped values are cached.
                self._setJobCache(job)
                self.job_queue.append(job)

    def executeJobs(self):
        for i in range(0, self.simultaneous_jobs - len(self.active_jobs)):
            try:
                job = self.job_queue.pop(0)
                d = super(WorkerServer, self).executeJob(job)
                d.addCallback(self._executeJobCallback, job)
                d.addErrback(self._executeJobErrback, job)
            except:
                return

    def _executeJobCallback(self, data, job):
        self.saveJobHistory(job, True)
        self.jobs_complete += 1
        self.clearPageCache(job)

    def _executeJobErrback(self, error, job):
        try:
            error.raiseException()
        except DeleteReservationException:
            self.jobs_complete += 1
            self.deleteReservation(job.uuid)
        except StaleContentException:
            self.jobs_complete += 1
        except QueueTimeoutException, e:
            self.job_failures += 1
            logger.error("Queue timeout for %s" % job.subservice)
            dotted_function = '.'.join(job.function_name.split('/'))
            self.stats.increment('job.%s.queuetimeout' % dotted_function)
            self.stats.increment('pg.queuetimeout.hit', 0.05)
            self.saveJobHistory(job, False)
        except NegativeCacheException, e:
            self.jobs_complete += 1
            logger.error("Negative cache for %s" % job.subservice)
            dotted_function = '.'.join(job.function_name.split('/'))
            if isinstance(e, NegativeReqCacheException):
                self.stats.increment('job.%s.negreqcache' % dotted_function)
            else:
                self.stats.increment('job.%s.negcache' % dotted_function)
            self.saveJobHistory(job, False)
        except Exception, e:
            self.job_failures += 1
            plugin = job.function_name.split('/')[0]
            plugl = logging.getLogger(plugin)
            plugl.error("Error executing job:\n%s\n%s" % (job, format_exc()))
            self.stats.increment('job.exceptions', 0.1)
            self.saveJobHistory(job, False)

    @inlineCallbacks
    def getFastCache(self, uuid):
        try:
            data = yield self.redis_client.get("fastcache:%s" % uuid)
        except:
            logger.debug("Could not get Fast Cache for %s" % uuid)
        returnValue(data)

    @inlineCallbacks
    def setFastCache(self, uuid, data):
        if not isinstance(data, str):
            raise Exception("FastCache must be a string.")
        if uuid is None:
            return
        try:
            yield self.redis_client.set("fastcache:%s" % uuid, data)
            logger.debug("Successfully set fast cache for %s" % uuid)
        except Exception, e:
            logger.error("Could not set fast cache: %s" % e)

    def logStatus(self):
        logger.critical('UUIDs: %d' % len(self.uuid_queue))
        logger.critical('Uncached UUIDs: %d' % len(self.uncached_uuid_queue))
        logger.critical('User accounts: %d' % len(self.user_account_queue))
        logger.critical('Queued Jobs: %d' % len(self.job_queue))
        logger.critical('Active Jobs: %d' % len(self.active_jobs))
        logger.critical('Completed Jobs: %d' % self.jobs_complete)
        logger.critical('Job failures: %d' % self.job_failures)
        logger.critical('Lost jobs: %d' % (self.uuids_dequeued - self.jobs_complete - self.job_failures - len(self.uuid_queue) - len(self.uncached_uuid_queue) - len(self.user_account_queue) - len(self.job_queue) - len(self.active_jobs)))
<commit_msg>Unskip feed/fetch<commit_after>import logging
import time
from collections import defaultdict
from .cassandra import CassandraServer
from ..resources import WorkerResource
from hiispider.servers.mixins import JobQueueMixin, PageCacheQueueMixin, JobGetterMixin, JobHistoryMixin
from hiispider.requestqueuer import QueueTimeoutException
from twisted.internet import reactor, task
from twisted.web import server
from twisted.internet.defer import inlineCallbacks, returnValue
import pprint
from traceback import format_exc
from hiispider.exceptions import *
from uuid import UUID
import cPickle
from zlib import decompress

PRETTYPRINTER = pprint.PrettyPrinter(indent=4)
logger = logging.getLogger(__name__)

class WorkerServer(CassandraServer, JobQueueMixin, PageCacheQueueMixin, JobGetterMixin, JobHistoryMixin):

    public_ip = None
    local_ip = None
    network_information = {}
    simultaneous_jobs = 20
    uuids_dequeued = 0
    jobs_complete = 0
    job_failures = 0
    job_queue = []
    jobsloop = None
    dequeueloop = None
    logloop = None
    pending_uuid_reqs = 0
    uuid_queue = []
    uuid_dequeueing = False
    uuid_queue_size = 500
    job_queue_size = 500
    uncached_uuid_queue = []
    uncached_uuid_dequeueing = False
    user_account_queue = []
    user_account_dequeueing = False
    service_credential_dequeueing = False

    def __init__(self, config, port=None):
        super(WorkerServer, self).__init__(config)
        self.t0 = time.time()
        self.setupJobQueue(config)
        self.setupPageCacheQueue(config)
        self.setupJobGetter(config)
        self.setupJobHistory(config)
        # HTTP interface
        resource = WorkerResource(self)
        if port is None:
            port = config["worker_server_port"]
        self.site_port = reactor.listenTCP(port, server.Site(resource))
        self.scheduler_server = config["scheduler_server"]
        self.scheduler_server_port = config["scheduler_server_port"]
        self.config = config
        # setup manhole
        manhole_namespace = {
            'service': self,
            'globals': globals(),
        }
        reactor.listenTCP(config["manhole_worker_port"], self.getManholeFactory(manhole_namespace, admin=config["manhole_password"]))

    def start(self):
        start_deferred = super(WorkerServer, self).start()
        start_deferred.addCallback(self._workerStart)
        return start_deferred

    @inlineCallbacks
    def _workerStart(self, started=None):
        logger.debug("Starting worker components.")
        yield self.startJobQueue()
        yield self.startPageCacheQueue()
        yield self.setupJobHistory(self.config)
        self.jobsloop = task.LoopingCall(self.executeJobs)
        self.jobsloop.start(0.2)
        self.dequeueloop = task.LoopingCall(self.dequeue)
        self.dequeueloop.start(10)
        self.logloop = task.LoopingCall(self.logStatus)
        self.logloop.start(5)

    @inlineCallbacks
    def shutdown(self):
        self.uuid_queue = []
        self.uncached_uuid_queue = []
        self.user_account_queue = []
        self.jobsloop.stop()
        self.dequeueloop.stop()
        self.logloop.stop()
        yield self.stopJobQueue()
        yield super(WorkerServer, self).shutdown()
        yield self.stopPageCacheQueue()

    def dequeue(self):
        self.dequeuejobs()
        if self.uncached_uuid_queue:
            self.lookupjobs()

    def dequeuejobs(self):
        if self.uuid_dequeueing:
            return
        if len(self.job_queue) > self.job_queue_size or len(self.uuid_queue) > self.uuid_queue_size:
            return
        self.uuid_dequeueing = True
        self._dequeuejobs()

    def _dequeuejobs(self):
        if len(self.uuid_queue) < self.uuid_queue_size * 4 and len(self.job_queue) < self.job_queue_size * 4:
            d = self.jobs_rabbit_queue.get()
            d.addCallback(self._dequeuejobsCallback)
            d.addErrback(self._dequeuejobsErrback)
        else:
            self.uuid_dequeueing = False

    def _dequeuejobsCallback(self, msg):
        self.jobs_chan.basic_ack(msg.delivery_tag)
        self.uuid_queue.append(UUID(bytes=msg.content.body).hex)
        self.uuids_dequeued += 1
        if len(self.uuid_queue) > 200:
            uuids, self.uuid_queue = self.uuid_queue, []
            d = self.redis_client.mget(*uuids)
            d.addCallback(self._dequeuejobsCallback2, uuids)
            d.addErrback(self._dequeuejobsErrback)
        else:
            self._dequeuejobs()

    def _dequeuejobsErrback(self, error):
        self._dequeuejobs()

    def _dequeuejobsCallback2(self, data, uuids):
        results = zip(uuids, data)
        for row in results:
            if row[1]:
                job = cPickle.loads(decompress(row[1]))
                logger.debug('Found uuid in Redis: %s' % row[0])
                self.job_queue.append(job)
            else:
                logger.debug('Could not find uuids %s in Redis.' % row[0])
                self.uncached_uuid_queue.append(row[0])
        self._dequeuejobs()

    @inlineCallbacks
    def lookupjobs(self):
        while self.uncached_uuid_queue:
            uuids, self.uncached_uuid_queue = self.uncached_uuid_queue[0:100], self.uncached_uuid_queue[100:]
            sql = """SELECT uuid, content_userprofile.user_id as user_id, username, host, account_id, type
                FROM spider_service, auth_user, content_userprofile
                WHERE uuid IN ('%s')
                AND auth_user.id=spider_service.user_id
                AND auth_user.id=content_userprofile.user_id
                """ % "','".join(uuids)
            try:
                data = yield self.mysql.runQuery(sql)
            except:
                logger.error("Could not find users in '%s'" % "','".join(uuids))
                continue
            self.user_account_queue.extend(data)
        accounts_by_type = defaultdict(list)
        for user_account in self.user_account_queue:
            accounts_by_type[user_account["type"]].append(user_account)
        self.user_account_queue = []
        for service_type in accounts_by_type:
            accounts_by_id = defaultdict(list)
            for user_accounts in accounts_by_type[service_type]:
                accounts_by_id[user_accounts["account_id"]].append(user_account)
            sql = "SELECT * FROM content_%saccount WHERE account_id IN (%s)" % (service_type, ",".join(accounts_by_id.keys()))
            try:
                data = yield self.mysql.runQuery(sql)
            except Exception, e:
                logger.error("Could not find service %s, %s" % (service_type, sql))
                continue
            for service_credentials in data:
                user_account = accounts_by_id[service_credentials["account_id"]]
                job = Job(
                    function_name=user_account['type'],
                    service_credentials=service_credentials,
                    user_account=user_account,
                    uuid=user_account['uuid'])
                self.mapJob(job) # Do this now so mapped values are cached.
                self._setJobCache(job)
                self.job_queue.append(job)

    def executeJobs(self):
        for i in range(0, self.simultaneous_jobs - len(self.active_jobs)):
            try:
                job = self.job_queue.pop(0)
                d = super(WorkerServer, self).executeJob(job)
                d.addCallback(self._executeJobCallback, job)
                d.addErrback(self._executeJobErrback, job)
            except:
                return

    def _executeJobCallback(self, data, job):
        self.saveJobHistory(job, True)
        self.jobs_complete += 1
        self.clearPageCache(job)

    def _executeJobErrback(self, error, job):
        try:
            error.raiseException()
        except DeleteReservationException:
            self.jobs_complete += 1
            self.deleteReservation(job.uuid)
        except StaleContentException:
            self.jobs_complete += 1
        except QueueTimeoutException, e:
            self.job_failures += 1
            logger.error("Queue timeout for %s" % job.subservice)
            dotted_function = '.'.join(job.function_name.split('/'))
            self.stats.increment('job.%s.queuetimeout' % dotted_function)
            self.stats.increment('pg.queuetimeout.hit', 0.05)
            self.saveJobHistory(job, False)
        except NegativeCacheException, e:
            self.jobs_complete += 1
            logger.error("Negative cache for %s" % job.subservice)
            dotted_function = '.'.join(job.function_name.split('/'))
            if isinstance(e, NegativeReqCacheException):
                self.stats.increment('job.%s.negreqcache' % dotted_function)
            else:
                self.stats.increment('job.%s.negcache' % dotted_function)
            self.saveJobHistory(job, False)
        except Exception, e:
            self.job_failures += 1
            plugin = job.function_name.split('/')[0]
            plugl = logging.getLogger(plugin)
            plugl.error("Error executing job:\n%s\n%s" % (job, format_exc()))
            self.stats.increment('job.exceptions', 0.1)
            self.saveJobHistory(job, False)

    @inlineCallbacks
    def getFastCache(self, uuid):
        try:
            data = yield self.redis_client.get("fastcache:%s" % uuid)
        except:
            logger.debug("Could not get Fast Cache for %s" % uuid)
        returnValue(data)

    @inlineCallbacks
    def setFastCache(self, uuid, data):
        if not isinstance(data, str):
            raise Exception("FastCache must be a string.")
        if uuid is None:
            return
        try:
            yield self.redis_client.set("fastcache:%s" % uuid, data)
            logger.debug("Successfully set fast cache for %s" % uuid)
        except Exception, e:
            logger.error("Could not set fast cache: %s" % e)

    def logStatus(self):
        logger.critical('UUIDs: %d' % len(self.uuid_queue))
        logger.critical('Uncached UUIDs: %d' % len(self.uncached_uuid_queue))
        logger.critical('User accounts: %d' % len(self.user_account_queue))
        logger.critical('Queued Jobs: %d' % len(self.job_queue))
        logger.critical('Active Jobs: %d' % len(self.active_jobs))
        logger.critical('Completed Jobs: %d' % self.jobs_complete)
        logger.critical('Job failures: %d' % self.job_failures)
        logger.critical('Lost jobs: %d' % (self.uuids_dequeued - self.jobs_complete - self.job_failures - len(self.uuid_queue) - len(self.uncached_uuid_queue) - len(self.user_account_queue) - len(self.job_queue) - len(self.active_jobs)))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import with_statement
from cms.api import create_page
from cms.models import Page
from cms.test_utils.testcases import CMSTestCase
from cms.test_utils.util.context_managers import SettingsOverride
from django.contrib.auth.models import User
from django.contrib.sites.models import Site

class SiteTestCase(CMSTestCase):
    """Site framework specific test cases.

    All stuff which is changing settings.SITE_ID for tests should come here.
    """
    def setUp(self):
        self.assertEqual(Site.objects.all().count(), 1)
        with SettingsOverride(SITE_ID=1):

            u = User(username="test", is_staff=True, is_active=True, is_superuser=True)
            u.set_password("test")
            u.save()

            # setup sites
            self.site2 = Site.objects.create(domain="sample2.com", name="sample2.com", pk=2)
            self.site3 = Site.objects.create(domain="sample3.com", name="sample3.com", pk=3)

        self._login_context = self.login_user_context(u)
        self._login_context.__enter__()

    def tearDown(self):
        self._login_context.__exit__(None, None, None)

    def test_site_framework(self):
        #Test the site framework, and test if it's possible to disable it
        with SettingsOverride(SITE_ID=self.site2.pk):
            create_page("page_2a", "nav_playground.html", "de", site=self.site2)

            response = self.client.get("/en/admin/cms/page/?site__exact=%s" % self.site3.pk)
            self.assertEqual(response.status_code, 200)
            create_page("page_3b", "nav_playground.html", "de", site=self.site3)

        with SettingsOverride(SITE_ID=self.site3.pk):
            create_page("page_3a", "nav_playground.html", "nl", site=self.site3)

            # with param
            self.assertEqual(Page.objects.on_site(self.site2.pk).count(), 1)
            self.assertEqual(Page.objects.on_site(self.site3.pk).count(), 2)

            self.assertEqual(Page.objects.drafts().on_site().count(), 2)

        with SettingsOverride(SITE_ID=self.site2.pk):
            # without param
            self.assertEqual(Page.objects.drafts().on_site().count(), 1)

<commit_msg>Added tentative test for bug #1881<commit_after># -*- coding: utf-8 -*-
from __future__ import with_statement
from django.test import Client
from cms.api import create_page
from cms.models import Page
from cms.views import details
from cms.test_utils.testcases import CMSTestCase
from cms.test_utils.util.context_managers import SettingsOverride
from django.contrib.auth.models import User
from django.contrib.sites.models import Site

class SiteTestCase(CMSTestCase):
    """Site framework specific test cases.

    All stuff which is changing settings.SITE_ID for tests should come here.
    """
    def setUp(self):
        self.assertEqual(Site.objects.all().count(), 1)
        with SettingsOverride(SITE_ID=1):

            u = User(username="test", is_staff=True, is_active=True, is_superuser=True)
            u.set_password("test")
            u.save()

            # setup sites
            self.site2 = Site.objects.create(domain="sample2.com", name="sample2.com", pk=2)
            self.site3 = Site.objects.create(domain="sample3.com", name="sample3.com", pk=3)

        self._login_context = self.login_user_context(u)
        self._login_context.__enter__()

    def tearDown(self):
        self._login_context.__exit__(None, None, None)

    def test_site_framework(self):
        #Test the site framework, and test if it's possible to disable it
        with SettingsOverride(SITE_ID=self.site2.pk):
            create_page("page_2a", "nav_playground.html", "de", site=self.site2)

            response = self.client.get("/en/admin/cms/page/?site__exact=%s" % self.site3.pk)
            self.assertEqual(response.status_code, 200)
            create_page("page_3b", "nav_playground.html", "de", site=self.site3)

        with SettingsOverride(SITE_ID=self.site3.pk):
            create_page("page_3a", "nav_playground.html", "nl", site=self.site3)

            # with param
            self.assertEqual(Page.objects.on_site(self.site2.pk).count(), 1)
            self.assertEqual(Page.objects.on_site(self.site3.pk).count(), 2)

            self.assertEqual(Page.objects.drafts().on_site().count(), 2)

        with SettingsOverride(SITE_ID=self.site2.pk):
            # without param
            self.assertEqual(Page.objects.drafts().on_site().count(), 1)

    def test_site_publish(self):
        pages = {"2": range(0, 5), "3": range(0, 5)}
        with SettingsOverride(LANGUAGE_CODE="de"):
            with SettingsOverride(SITE_ID=self.site2.pk):
                pages["2"][0] = create_page("page_2", "nav_playground.html", "de",
                                            site=self.site2)
                pages["2"][0].publish()
                pages["2"][1] = create_page("page_2_1", "nav_playground.html", "de",
                                            parent=pages["2"][0], site=self.site2)
                pages["2"][2] = create_page("page_2_2", "nav_playground.html", "de",
                                            parent=pages["2"][0], site=self.site2)
                pages["2"][3] = create_page("page_2_1_1", "nav_playground.html", "de",
                                            parent=pages["2"][1], site=self.site2)
                pages["2"][4] = create_page("page_2_1_2", "nav_playground.html", "de",
                                            parent=pages["2"][1], site=self.site2)

                for page in pages["2"]:
                    page.publish()
                for page in pages["2"]:
                    if page.is_home():
                        page_url = "/de/"
                    else:
                        page_url = page.get_absolute_url(language='de')
                    response = self.client.get(page_url)
                    self.assertEqual(response.status_code, 200)

            with SettingsOverride(SITE_ID=self.site3.pk):
                pages["3"][0] = create_page("page_3", "nav_playground.html", "de",
                                            site=self.site3)
                pages["3"][0].publish()
                pages["3"][1] = create_page("page_3_1", "nav_playground.html", "de",
                                            parent=pages["3"][0], site=self.site3)
                pages["3"][2] = create_page("page_3_2", "nav_playground.html", "de",
                                            parent=pages["3"][0], site=self.site3)
                pages["3"][3] = create_page("page_3_1_1", "nav_playground.html", "de",
                                            parent=pages["3"][1], site=self.site3)
                pages["3"][4] = create_page("page_3_1_2", "nav_playground.html", "de",
                                            parent=pages["3"][1], site=self.site3)

                for page in pages["3"]:
                    page.publish()
                for page in pages["3"]:
                    if page.is_home():
                        page_url = "/de/"
                    else:
                        page_url = page.get_absolute_url(language='de')
                    response = self.client.get(page_url)
                    self.assertEqual(response.status_code, 200)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import inspect, time
from twisted.internet.defer import inlineCallbacks, returnValue as returnD
from twisted.internet.task import LoopingCall
from txmongo import MongoConnection, _MongoFactory, collection
from txmongo.filter import sort as mongosort, ASCENDING, DESCENDING
from gazouilleur.config import DEBUG, MONGODB
from gazouilleur.lib.log import loggerr
_MongoFactory.noisy = False

class MongoConn(object):

    def __init__(self, retries=3, timeout=0):
        self.retries = retries
        self.timeout = max(30, timeout)
        self.supervisor = LoopingCall(self.__check_timeout__)
        self.conn = None
        self.db = None
        self.coll = None
        self.method = None
        # inject all methods defined in the MongoCollection class
        for m in inspect.getmembers(collection.Collection, predicate=inspect.ismethod):
            setattr(self, m[0], self.__get_proxy__(m[0]))

    @inlineCallbacks
    def close(self):
        if self.supervisor.running:
            self.supervisor.stop()
        if self.conn:
            try:
                yield self.conn.disconnect()
            except Exception as e:
                pass
            del self.conn
            self.conn = None
        self.db = None
        self.coll = None
        self.method = None

    def logerr(self, action, message=""):
        if self.coll:
            action += " %s" % self.coll
        if self.method:
            action += " %s" % self.method
        loggerr("%s. %s" % (action, message), action="mongodb")

    def __get_proxy__(self, method):
        @inlineCallbacks
        def __proxy(coll, *args, **kwargs):
            self.timedout = time.time() + self.timeout
            self.supervisor.start(self.timeout)
            res = yield self.__run__(coll, method, *args, **kwargs)
            if time.time() > self.timedout:
                self.logerr("YEAH REACHED %s AFTER TIMEOUT %s!!! %s %s" % (time.time() - self.timedout, self.timeout, self.coll, self.method))

            if self.supervisor.running:
                self.supervisor.stop()
            returnD(res)
        return __proxy

    def __check_timeout__(self):
        due = time.time() - self.timedout
        if due > 0:
            self.logerr("MONGO TIMEOUT (%s)!!! %s %s since %s" % (self.timeout, self.coll, self.method, due))
            if due > self.timeout:
                self.logerr("MONGO DOUBLE TIMEOUT (%s) closing now %s %s" % (int(due + self.timeout), self.coll, self.method))
                self.close()

    @inlineCallbacks
    def __run__(self, coll, method, *args, **kwargs):
        attempts_left = self.retries
        result = []
        lasttry = False
        if 'lasttry' in kwargs:
            lasttry = True
            del kwargs['lasttry']
        while True:
            try:
                self.coll = coll
                self.method = method
                if not self.conn and not self.db:
                    status = "Connec"
                    self.conn = yield MongoConnection(MONGODB['HOST'], MONGODB['PORT'], reconnect=False)
                    self.db = self.conn[MONGODB['DATABASE']]
                    status = "Authentica"
                    yield self.db.authenticate(MONGODB['USER'], MONGODB['PSWD'])
                status = "Communica"
                result = yield getattr(self.db[coll], method)(*args, **kwargs)
            except Exception as e:
                if not lasttry:
                    if attempts_left > 0:
                        attempts_left -= 1
                        if DEBUG:
                            self.logerr("%sting" % status, "Retry #%d" % (self.retries-attempts_left))
                        try:
                            yield self.conn.disconnect()
                        except:
                            pass
                        continue
                    if DEBUG:
                        self.logerr("%sting" % status, "HARD RETRY %s %s" % (type(e), str(e)))
                    result = yield Mongo(coll, method, *args, lasttry=True, timeout=self.timeout, **kwargs)
                yield self.close()
            if self.supervisor.running:
                self.supervisor.stop()
            returnD(result)

@inlineCallbacks
def Mongo(coll, method, *args, **kwargs):
    timeout = kwargs.pop('timeout', 30)
    db = MongoConn(timeout=timeout)
    res = yield getattr(db, method)(coll, *args, **kwargs)
    yield db.close()
    del db
    returnD(res)

def sortasc(field):
    return mongosort(ASCENDING(field))

def sortdesc(field):
    return mongosort(DESCENDING(field))

@inlineCallbacks
def save_lasttweet_id(channel, tweet_id):
    yield Mongo('lasttweets', 'update', {'channel': channel}, {'channel': channel, 'tweet_id': tweet_id}, upsert=True)

@inlineCallbacks
def find_stats(query, **kwargs):
    res = yield Mongo('stats', 'find', query, **kwargs)
    returnD(res)

@inlineCallbacks
def ensure_indexes():
    db = MongoConn()
    yield db.ensure_index('logs', sortasc('channel') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('logs', sortasc('channel') + sortasc('user') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tasks', sortasc('channel') + sortasc('timestamp'), background=True)
    yield db.ensure_index('feeds', sortasc('database') + sortasc('timestamp'), background=True)
    yield db.ensure_index('feeds', sortasc('channel') + sortasc('database') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('filters', sortasc('channel') + sortasc('keyword') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('news', sortasc('channel') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('news', sortasc('channel') + sortasc('source') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortasc('id') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortasc('user') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortdesc('id'), background=True)
    yield db.ensure_index('tweets', sortdesc('id') + sortasc('channel') + sortasc('uniq_rt_hash'), background=True)
    yield db.ensure_index('stats', sortdesc('timestamp') + sortasc('user'), background=True)
    yield db.ensure_index('lasttweets', sortasc('channel'), background=True)
    yield db.close()
    del db

<commit_msg>add useful mongo indexes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import inspect, time
from twisted.internet.defer import inlineCallbacks, returnValue as returnD
from twisted.internet.task import LoopingCall
from txmongo import MongoConnection, _MongoFactory, collection
from txmongo.filter import sort as mongosort, ASCENDING, DESCENDING
from gazouilleur.config import DEBUG, MONGODB
from gazouilleur.lib.log import loggerr
_MongoFactory.noisy = False

class MongoConn(object):

    def __init__(self, retries=3, timeout=0):
        self.retries = retries
        self.timeout = max(30, timeout)
        self.supervisor = LoopingCall(self.__check_timeout__)
        self.conn = None
        self.db = None
        self.coll = None
        self.method = None
        # inject all methods defined in the MongoCollection class
        for m in inspect.getmembers(collection.Collection, predicate=inspect.ismethod):
            setattr(self, m[0], self.__get_proxy__(m[0]))

    @inlineCallbacks
    def close(self):
        if self.supervisor.running:
            self.supervisor.stop()
        if self.conn:
            try:
                yield self.conn.disconnect()
            except Exception as e:
                pass
            del self.conn
            self.conn = None
        self.db = None
        self.coll = None
        self.method = None

    def logerr(self, action, message=""):
        if self.coll:
            action += " %s" % self.coll
        if self.method:
            action += " %s" % self.method
        loggerr("%s. %s" % (action, message), action="mongodb")

    def __get_proxy__(self, method):
        @inlineCallbacks
        def __proxy(coll, *args, **kwargs):
            self.timedout = time.time() + self.timeout
            self.supervisor.start(self.timeout)
            res = yield self.__run__(coll, method, *args, **kwargs)
            if time.time() > self.timedout:
                self.logerr("YEAH REACHED %s AFTER TIMEOUT %s!!! %s %s" % (time.time() - self.timedout, self.timeout, self.coll, self.method))

            if self.supervisor.running:
                self.supervisor.stop()
            returnD(res)
        return __proxy

    def __check_timeout__(self):
        due = time.time() - self.timedout
        if due > 0:
            self.logerr("MONGO TIMEOUT (%s)!!! %s %s since %s" % (self.timeout, self.coll, self.method, due))
            if due > self.timeout:
                self.logerr("MONGO DOUBLE TIMEOUT (%s) closing now %s %s" % (int(due + self.timeout), self.coll, self.method))
                self.close()

    @inlineCallbacks
    def __run__(self, coll, method, *args, **kwargs):
        attempts_left = self.retries
        result = []
        lasttry = False
        if 'lasttry' in kwargs:
            lasttry = True
            del kwargs['lasttry']
        while True:
            try:
                self.coll = coll
                self.method = method
                if not self.conn and not self.db:
                    status = "Connec"
                    self.conn = yield MongoConnection(MONGODB['HOST'], MONGODB['PORT'], reconnect=False)
                    self.db = self.conn[MONGODB['DATABASE']]
                    status = "Authentica"
                    yield self.db.authenticate(MONGODB['USER'], MONGODB['PSWD'])
                status = "Communica"
                result = yield getattr(self.db[coll], method)(*args, **kwargs)
            except Exception as e:
                if not lasttry:
                    if attempts_left > 0:
                        attempts_left -= 1
                        if DEBUG:
                            self.logerr("%sting" % status, "Retry #%d" % (self.retries-attempts_left))
                        try:
                            yield self.conn.disconnect()
                        except:
                            pass
                        continue
                    if DEBUG:
                        self.logerr("%sting" % status, "HARD RETRY %s %s" % (type(e), str(e)))
                    result = yield Mongo(coll, method, *args, lasttry=True, timeout=self.timeout, **kwargs)
                yield self.close()
            if self.supervisor.running:
                self.supervisor.stop()
            returnD(result)

@inlineCallbacks
def Mongo(coll, method, *args, **kwargs):
    timeout = kwargs.pop('timeout', 30)
    db = MongoConn(timeout=timeout)
    res = yield getattr(db, method)(coll, *args, **kwargs)
    yield db.close()
    del db
    returnD(res)

def sortasc(field):
    return mongosort(ASCENDING(field))

def sortdesc(field):
    return mongosort(DESCENDING(field))

@inlineCallbacks
def save_lasttweet_id(channel, tweet_id):
    yield Mongo('lasttweets', 'update', {'channel': channel}, {'channel': channel, 'tweet_id': tweet_id}, upsert=True)

@inlineCallbacks
def find_stats(query, **kwargs):
    res = yield Mongo('stats', 'find', query, **kwargs)
    returnD(res)

@inlineCallbacks
def ensure_indexes():
    db = MongoConn()
    yield db.ensure_index('logs', sortasc('channel') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('logs', sortasc('channel') + sortasc('user') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tasks', sortasc('channel') + sortasc('timestamp'), background=True)
    yield db.ensure_index('feeds', sortasc('database') + sortasc('timestamp'), background=True)
    yield db.ensure_index('feeds', sortasc('channel') + sortasc('database'), background=True)
    yield db.ensure_index('feeds', sortasc('channel') + sortasc('database') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('filters', sortasc('channel'), background=True)
    yield db.ensure_index('filters', sortasc('channel') + sortasc('keyword') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('news', sortasc('channel') + sortdesc('_id'), background=True)
    yield db.ensure_index('news', sortasc('channel') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('news', sortasc('channel') + sortasc('source') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('dms', sortasc('id') + sortasc('channel'), background=True)
    yield db.ensure_index('tweets', sortasc('id'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortdesc('id'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortasc('id') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tweets', sortasc('channel') + sortasc('user') + sortdesc('timestamp'), background=True)
    yield db.ensure_index('tweets', sortdesc('id') + sortasc('channel') + sortasc('uniq_rt_hash'), background=True)
    yield db.ensure_index('stats', sortdesc('timestamp') + sortasc('user'), background=True)
    yield db.ensure_index('lasttweets', sortasc('channel'), background=True)
    yield db.close()
    del db

<|endoftext|>


<commit_before>from flask import Blueprint, render_template, current_app, request, url_for, redirect
from domain.usuario import Usuario, DTOUsuario, SenhaCriptografada

view_usuarios = Blueprint('usuarios', __name__)

@view_usuarios.route("/")
def index():
    usuarios = current_app.crud_usuario.listar()
    print(usuarios)
    return render_template("usuarios/index.html", usuarios=usuarios)

@view_usuarios.route("/novo")
def novo():
    return render_template("usuarios/novo.html", dto_usuario=DTOUsuario())

@view_usuarios.route("/<id_usuario>")
def detalhes(id_usuario):
    return render_template("usuarios/detalhes.html", id_usuario=id_usuario)

@view_usuarios.route("/<id_usuario>/editar")
def editar(id_usuario):
    usuario = current_app.crud_usuario.obter(id)

    #a senha do usuário está criptografada, não dá pra recuperar em texto
    #plano. Usando o digest para comparar
    dto = DTOUsuario(usuario.nome, usuario.email, None, usuario.nivelAcesso)
    digest = usuario.senhaCriptografada.digest
    return render_template("usuarios/editar.html", id_usuario=id_usuario, dto_usuario=dto, digest=digest)

@view_usuarios.route("/<id_usuario>", methods=["POST"])
def alterar(id_usuario):
    dto = DTOUsuario(request.form["nome"], request.form["email"], request.form["senha"], request.form["nivelAcesso"])
    current_app.crud_usuario.alterar(id_usuario, dto)
    return redirect(url_for('recursos.index'))

@view_usuarios.route("/", methods=["POST"])
def criar():
    dto = DTOUsuario(request.form["nome"], request.form["email"], request.form["senha"], request.form["nivelAcesso"])
    current_app.crud_recurso.criar(dto)
    return redirect(url_for('recursos.index'))
<commit_msg>fixes nas routes<commit_after>from flask import Blueprint, render_template, current_app, request, url_for, redirect
from domain.usuario import Usuario, DTOUsuario, SenhaCriptografada

view_usuarios = Blueprint('usuarios', __name__)

@view_usuarios.route("/")
def index():
    usuarios = current_app.crud_usuario.listar()
    print(usuarios)
    return render_template("usuarios/index.html", usuarios=usuarios)

@view_usuarios.route("/novo")
def novo():
    return render_template("usuarios/novo.html", dto_usuario=DTOUsuario())

@view_usuarios.route("/<id_usuario>")
def detalhes(id_usuario):
    return render_template("usuarios/detalhes.html", id_usuario=id_usuario)

@view_usuarios.route("/<id_usuario>/editar")
def editar(id_usuario):
    usuario = current_app.crud_usuario.obter(id)

    #usando digest para comparar senhas
    dto = DTOUsuario(usuario.nome, usuario.email, None, usuario.nivelAcesso)
    digest = usuario.senhaCriptografada.digest
    return render_template("usuarios/editar.html", id_usuario=id_usuario, dto_usuario=dto, digest=digest)

@view_usuarios.route("/<id_usuario>", methods=["POST"])
def alterar(id_usuario):
    dto = DTOUsuario(request.form["nome"], request.form["email"], request.form["senha"], request.form["nivelAcesso"])
    current_app.crud_usuario.alterar(id_usuario, dto)
    return redirect(url_for('usuarios.index'))

@view_usuarios.route("/", methods=["POST"])
def criar():
    dto = DTOUsuario(request.form["nome"], request.form["email"], request.form["senha"], request.form["nivelAcesso"])
    current_app.crud_usuario.criar(dto)
    return redirect(url_for('usuarios.index'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from django.db.models import Q
import re

APPEND_TO_SLUG = "-copy"
COPY_SLUG_REGEX = re.compile(r'^.*-copy(?:-(\d)*)?$')

def is_valid_page_slug(page, parent, lang, slug, site):
    """Validates given slug depending on settings.
    """
    from cms.models import Title
    # Exclude the page with the publisher_state == page.PUBLISHER_STATE_DELETE
    qs = Title.objects.filter(page__site=site, slug=slug).exclude(
        Q(page=page) |
        Q(page=page.publisher_public) |
        Q(page__publisher_state=page.PUBLISHER_STATE_DELETE)
    )

    if settings.i18n_installed:
        qs = qs.filter(language=lang)

    if not settings.CMS_FLAT_URLS:
        if parent and not parent.is_home():
            qs = qs.filter(page__parent=parent)
        else:
            qs = qs.filter(page__parent__isnull=True)

    if page.pk:
        qs = qs.exclude(language=lang, page=page)
    if qs.count():
        return False
    return True

def get_available_slug(title, new_slug=None):
    """Smart function generates slug for title if current title slug cannot be
    used. Appends APPEND_TO_SLUG to slug and checks it again.

    (Used in page copy function)

    Returns: slug
    """
    slug = new_slug or title.slug
    if is_valid_page_slug(title.page, title.page.parent, title.language, slug, title.page.site):
        return slug

    # add nice copy attribute, first is -copy, then -copy-2, -copy-3, ....
    match = COPY_SLUG_REGEX.match(slug)
    if match:
        try:
            next = int(match.groups()[0]) + 1
            slug = "-".join(slug.split('-')[:-1]) + "-%d" % next
        except TypeError:
            slug = slug + "-2"

    else:
        slug = slug + APPEND_TO_SLUG
    return get_available_slug(title, slug)

def check_title_slugs(page):
    """Checks page title slugs for duplicity if required, used after page move/
    cut/paste.
    """
    for title in page.title_set.all():
        old_slug = title.slug
        title.slug = get_available_slug(title)
        if title.slug != old_slug:
            title.save()
<commit_msg>Thanks yml<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from django.db.models import Q
import re

APPEND_TO_SLUG = "-copy"
COPY_SLUG_REGEX = re.compile(r'^.*-copy(?:-(\d)*)?$')

def is_valid_page_slug(page, parent, lang, slug, site):
    """Validates given slug depending on settings.
    """
    from cms.models import Title
    # Exclude the page with the publisher_state == page.PUBLISHER_STATE_DELETE
    qs = Title.objects.filter(page__site=site, slug=slug).exclude(
        Q(page=page) &
        Q(page=page.publisher_public) &
        Q(page__publisher_state=page.PUBLISHER_STATE_DELETE)
    )

    if settings.i18n_installed:
        qs = qs.filter(language=lang)

    if not settings.CMS_FLAT_URLS:
        if parent and not parent.is_home():
            qs = qs.filter(page__parent=parent)
        else:
            qs = qs.filter(page__parent__isnull=True)

    if page.pk:
        qs = qs.exclude(language=lang, page=page)
    if qs.count():
        return False
    return True

def get_available_slug(title, new_slug=None):
    """Smart function generates slug for title if current title slug cannot be
    used. Appends APPEND_TO_SLUG to slug and checks it again.

    (Used in page copy function)

    Returns: slug
    """
    slug = new_slug or title.slug
    if is_valid_page_slug(title.page, title.page.parent, title.language, slug, title.page.site):
        return slug

    # add nice copy attribute, first is -copy, then -copy-2, -copy-3, ....
    match = COPY_SLUG_REGEX.match(slug)
    if match:
        try:
            next = int(match.groups()[0]) + 1
            slug = "-".join(slug.split('-')[:-1]) + "-%d" % next
        except TypeError:
            slug = slug + "-2"

    else:
        slug = slug + APPEND_TO_SLUG
    return get_available_slug(title, slug)

def check_title_slugs(page):
    """Checks page title slugs for duplicity if required, used after page move/
    cut/paste.
    """
    for title in page.title_set.all():
        old_slug = title.slug
        title.slug = get_available_slug(title)
        if title.slug != old_slug:
            title.save()
<|endoftext|>


<commit_before>import web3.utils.utils as utils
import web3.utils.abi as abi
from web3.solidity.coder import coder
import web3.web3.formatters as formatters
from web3.utils.crypto import sha3

class SolidityFunction(object):

    def __init__(self, eth, json, address):
        self._eth = eth
        self._inputTypes = [i["type"] for i in json["inputs"]]
        self._outputTypes = [o["type"] for o in json["outputs"]]
        self._constant = json["constant"]
        self._name = abi.transformToFullName(json)
        self._address = address

    def extractDefaultBlock(self, args):
        if (len(args) > len(self._inputTypes) and utils.isObject(args[-1])):
            return formatters.inputDefaultBlockNumberFormatter(args.pop())

    def toPayload(self, args):
        """
        Should be used to create payload from arguments
        """
        options = {}
        if len(args) > len(self._inputTypes) and utils.isObject(args[-1]):
            options = args[-1]
        options["to"] = self._address
        options["data"] = "0x" + \
            self.signature() + coder.encodeParams(self._inputTypes, args)
        return options

    def signature(self):
        """
        Should be used to get function signature
        """
        return sha3(self._name)[:8]

    def unpackOutput(self, output):
        if not output:
            return

        if len(output) >= 2:
            output = output[2:]

        result = coder.decodeParams(self._outputTypes, output)

        if len(result) == 1:
            return result[0]

        return result

    def call(self, *arguments):
        """
        Calls a contract function.
        """
        args = list(arguments)
        defaultBlock = self.extractDefaultBlock(args)
        payload = self.toPayload(args)

        output = self._eth.call(payload, defaultBlock)
        return self.unpackOutput(output)

    def sendTransaction(self, *arguments):
        """
        Should be used to sendTransaction to solidity function
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args)

        return self._eth.sendTransaction(payload)

    def estimateGas(self, *arguments):
        """
        Should be used to estimateGas of solidity function
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args)

        return self._eth.estimateGas(payload)

    def getData(self, *arguments):
        """
        Return the encoded data of the call
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args)

        return payload["data"]

    def displayName(self):
        """
        Should be used to get function display name
        """
        return abi.extractDisplayName(self._name)

    def typeName(self):
        """
        Should be used to get function type name
        """
        return abi.extractTypeName(self._name)

    def request(self, *arguments):
        """
        Should be called to get rpc requests from solidity function
        """
        return {
            "method": "eth_call" if self._constant else "eth_sendTransaction",
            "params": [self.toPayload(arguments)],
            "format": self.unpackOutput
        }

    def execute(self, *arguments):
        transaction = not self._constant

        if transaction:
            return self.sendTransaction(*arguments)
        else:
            return self.call(*arguments)

    def attachToContract(self, contract):
        execute = self.execute
        """
        displayName = self.displayName()
        # still missing arguments here
        print(displayName, self.typeName())
        try:
            getattr(contract, displayName)
        except AttributeError:
            setattr(contract, displayName, execute)
        m = getattr(contract, displayName)
        setattr(m, self.typeName(), execute)
        """
        setattr(contract, self.displayName(), execute)
        # setattr(contract, self.typeName(), execute)
<commit_msg>Allow kwargs parameters for contract transactions ([gas,gasPrice,value,data,nonce,from,to]=)<commit_after>import web3.utils.utils as utils
import web3.utils.abi as abi
from web3.solidity.coder import coder
import web3.web3.formatters as formatters
from web3.utils.crypto import sha3

class SolidityFunction(object):

    def __init__(self, eth, json, address):
        self._eth = eth
        self._inputTypes = [i["type"] for i in json["inputs"]]
        self._outputTypes = [o["type"] for o in json["outputs"]]
        self._constant = json["constant"]
        self._name = abi.transformToFullName(json)
        self._address = address

    def extractDefaultBlock(self, args):
        if (len(args) > len(self._inputTypes) and utils.isObject(args[-1])):
            return formatters.inputDefaultBlockNumberFormatter(args.pop())

    def toPayload(self, args, **kwargs):
        """
        Should be used to create payload from arguments
        """
        options = {}
        if len(args) > len(self._inputTypes) and utils.isObject(args[-1]):
            options = args[-1]
        options["to"] = self._address
        options["data"] = "0x" + \
            self.signature() + coder.encodeParams(self._inputTypes, args)

        for key in kwargs:
            options[key] = kwargs[key]

        return options

    def signature(self):
        """
        Should be used to get function signature
        """
        return sha3(self._name)[:8]

    def unpackOutput(self, output):
        if not output:
            return

        if len(output) >= 2:
            output = output[2:]

        result = coder.decodeParams(self._outputTypes, output)

        if len(result) == 1:
            return result[0]

        return result

    def call(self, *arguments):
        """
        Calls a contract function.
        """
        args = list(arguments)
        defaultBlock = self.extractDefaultBlock(args)
        payload = self.toPayload(args)

        output = self._eth.call(payload, defaultBlock)
        return self.unpackOutput(output)

    def sendTransaction(self, *arguments, **kwargs):
        """
        Should be used to sendTransaction to solidity function
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args, **kwargs)

        return self._eth.sendTransaction(payload)

    def estimateGas(self, *arguments):
        """
        Should be used to estimateGas of solidity function
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args)

        return self._eth.estimateGas(payload)

    def getData(self, *arguments):
        """
        Return the encoded data of the call
        """
        args = [a for a in arguments if a]
        payload = self.toPayload(args)

        return payload["data"]

    def displayName(self):
        """
        Should be used to get function display name
        """
        return abi.extractDisplayName(self._name)

    def typeName(self):
        """
        Should be used to get function type name
        """
        return abi.extractTypeName(self._name)

    def request(self, *arguments):
        """
        Should be called to get rpc requests from solidity function
        """
        return {
            "method": "eth_call" if self._constant else "eth_sendTransaction",
            "params": [self.toPayload(arguments)],
            "format": self.unpackOutput
        }

    def execute(self, *arguments, **kwargs):
        transaction = not self._constant

        if transaction:
            return self.sendTransaction(*arguments, **kwargs)
        else:
            return self.call(*arguments)

    def attachToContract(self, contract):
        execute = self.execute
        """
        displayName = self.displayName()
        # still missing arguments here
        print(displayName, self.typeName())
        try:
            getattr(contract, displayName)
        except AttributeError:
            setattr(contract, displayName, execute)
        m = getattr(contract, displayName)
        setattr(m, self.typeName(), execute)
        """
        setattr(contract, self.displayName(), execute)
        # setattr(contract, self.typeName(), execute)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Created by zhangzhuo@360.cn on 17/6/20
from . import blueprint
from .. import WF, kael_client
import time
import uuid

@blueprint.route("/hi/", methods=['GET'], versions=[1])
def hello():
    r = WF.get_last_version()
    return r

@blueprint.route("/stream/", versions=[1], stream=True)
def subscribe():
    while True:
        result = time.time()
        time.sleep(2)
        yield result

@blueprint.route("/pull/<string:topic>", versions=[1], stream=True)
def pull(topic):
    id = str(uuid.uuid4())
    WF.create_queue(id, exclusive=True, auto_delete=True, )
    while True:
        for i in WF.pull_msg(id, topic):
            yield i[-1]
        time.sleep(2)

@blueprint.route("/<string:namespace>/status", versions=[1], stream=True)
def server_status(namespace):
    client = kael_client(namespace)
    while True:
        time.sleep(2)
        r = client.command("get_pkg_version", pkg_type='service')
        data = client.get_response(r)
        yield dict(service=data)
        time.sleep(2)
        r = client.command("get_pkg_version", pkg_type='crontab')
        data = client.get_response(r)
        yield dict(crontab=data)

@blueprint.route("/<namespace>/update/<pkg_type>/<service>", versions=[1])
def server_update(namespace, pkg_type, service):
    client = kael_client(namespace)
    res = client._update_pkg_client_helper(service, pkg_type)
    _restart_command(client, pkg_type, res)
    return res

@blueprint.route("/<namespace>/install/<pkg_type>/<service>/<path:install_path>", versions=[1])
def server_install(namespace, pkg_type, service, install_path):
    client = kael_client(namespace)
    res = client._install_pkg_client_helper(service, pkg_type, install_path)
    _restart_command(client, pkg_type, res)
    return res

def _restart_command(client, pkg_type, res_msg):
    if pkg_type == 'crontab':
        client.command('restart_crontab')
    elif pkg_type == 'service':
        for server, msg in res_msg.iteritems():
            if msg.split('.')[0] == 'Update OK':
                client.command('restart_service', id=server)
<commit_msg>web: add install and update<commit_after># -*- coding: utf-8 -*-
# Created by zhangzhuo@360.cn on 17/6/20
from flask import request

from . import blueprint
from .. import WF, kael_client
import time
import uuid

@blueprint.route("/hi/", methods=['GET'], versions=[1])
def hello():
    r = WF.get_last_version()
    return r

@blueprint.route("/stream/", versions=[1], stream=True)
def subscribe():
    while True:
        result = time.time()
        time.sleep(2)
        yield result

@blueprint.route("/pull/<string:topic>", versions=[1], stream=True)
def pull(topic):
    id = str(uuid.uuid4())
    WF.create_queue(id, exclusive=True, auto_delete=True, )
    while True:
        for i in WF.pull_msg(id, topic):
            yield i[-1]
        time.sleep(2)

@blueprint.route("/<string:namespace>/status", versions=[1], stream=True)
def server_status(namespace):
    client = kael_client(namespace)
    while True:
        time.sleep(2)
        r = client.command("get_pkg_version", pkg_type='service')
        data = client.get_response(r)
        yield dict(service=data)
        time.sleep(2)
        r = client.command("get_pkg_version", pkg_type='crontab')
        data = client.get_response(r)
        yield dict(crontab=data)

@blueprint.route("/<namespace>/operation", methods=['POST'], versions=[1])
def server_update_install(namespace):
    """更新安装功能，全参数POST"""
    kwargs = request.get_json()
    # 必选
    operation = kwargs.get('operation')  # install update
    pkg_type = kwargs.get('pkg_type')  # service crontab
    package = kwargs.get('package')
    install_path = kwargs.get('install_path')
    # 可选
    version = kwargs.get('version')
    server_id = kwargs.get('server_id')  # 只向某机器发送
    server_not_id = kwargs.get('server_not_id')  # 不向一些机器发送

    if pkg_type not in ('crontab', 'service'):
        return 'No pkg_type %s' % pkg_type
    if operation not in ('update', 'install'):
        return 'No operation %s' % operation
    if not package:
        return 'No package'

    if operation == 'install':
        if not install_path:
            return 'No install_path'
        client = kael_client(namespace)
        res = client._install_pkg_client_helper(package, pkg_type, version=version, id=server_id, not_id=server_not_id)
        _restart_command(client, pkg_type, res)

    else:
        client = kael_client(namespace)
        res = client._update_pkg_client_helper(package, pkg_type, version=version, id=server_id, not_id=server_not_id)
        _restart_command(client, pkg_type, res)
    return res

@blueprint.route("/<namespace>/update/<pkg_type>/<package>", versions=[1])
def server_update(namespace, pkg_type, package):
    client = kael_client(namespace)
    res = client._update_pkg_client_helper(package, pkg_type)
    _restart_command(client, pkg_type, res)
    return res

@blueprint.route("/<namespace>/install/<pkg_type>/<package>/<path:install_path>", versions=[1])
def server_install(namespace, pkg_type, package, install_path):
    client = kael_client(namespace)
    res = client._install_pkg_client_helper(package, pkg_type, install_path)
    _restart_command(client, pkg_type, res)
    return res

def _restart_command(client, pkg_type, res_msg):
    if pkg_type == 'crontab':
        client.command('restart_crontab')
    elif pkg_type == 'service':
        for server, msg in res_msg.iteritems():
            if msg.split('.')[0] == 'Update OK':
                client.command('restart_service', id=server)
<|endoftext|>


<commit_before>from django.db.models import Q
from django.db import connection
import json
from gems.core.models import ContactGroupMember
import logging

logger = logging.getLogger(__name__)

class FieldFilter:
    def __init__(self, operator, value, field, loperator=None):
        self.operator = operator
        self.value = value

        if loperator is None:
            self.loperator = 'and'
        else:
            self.loperator = loperator

        self.field = field
        self.q = None

        if field.type == 'N':

            if field.name == 'survey':
                field.name = 'survey__name'

            if field.name == 'series':
                field.name = 'survey__series'

            if field.name == 'contact':
                field.name = 'contact__msisdn'

            # These are the normal fields
            if operator == 'eq':
                kwargs = {
                    '{0}__iexact'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'ex':
                kwargs = {
                    '{0}__exact'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'gt' \
                    or operator == 'gte' \
                    or operator == 'lt' \
                    or operator == 'lte':
                kwargs = {
                    '{0}__{1}'.format(field.name, operator): value
                }
                self.q = Q(**kwargs)
            elif operator == 'co':
                kwargs = {
                    '{0}__icontains'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'neq':
                kwargs = {
                    '{0}'.format(field.name): value
                }
                self.q = ~Q(**kwargs)
            elif operator == 'nco':
                kwargs = {
                    '{0}__icontains'.format(field.name): value
                }
                self.q = ~Q(**kwargs)
        else:
            # These are the hstore fields
            # Special hack!!!
            # * contains is a sql ==
            # * icontains is a sql like
            if operator == 'eq':
                self.q = Q(answer__contains={field.name: value})
            elif operator == 'ex':
                self.q = Q(answer__contains={field.name: value})
            elif operator == 'gt':
                self.q = Q(answer__gt={field.name: value})
            elif operator == 'gte':
                self.q = Q(answer__gte={field.name: value})
            elif operator == 'lt':
                self.q = Q(answer__lt={field.name: value})
            elif operator == 'lte':
                self.q = Q(answer__lte={field.name: value})
            elif operator == 'co':
                self.q = Q(answer__icontains={field.name: "%" + value + "%"})
            elif operator == 'neq':
                self.q = ~Q(answer__contains={field.name: value})
            elif operator == 'nco':
                self.q = ~Q(answer__icontains={field.name: "%" + value + "%"})

    @staticmethod
    def decode(payload, field):
        op = payload['operator']
        value = payload['value']
        lop = None
        if 'loperator' in payload:
            lop = payload['loperator']

        return FieldFilter(op, value, field, lop)

class Filter:
    def __init__(self, field, filters=None, loperator=None):
        self.field = field
        self.filters = filters

        if self.filters is None:
            self.filters = []

        if loperator is None:
            self.loperator = 'and'
        else:
            self.loperator = loperator

    @staticmethod
    def decode(payload):
        field = UIField.decode(payload['field'])
        lop = None

        if 'loperator' in payload:
            lop = payload['loperator']

        local_filters = []
        temp_filters = payload['filters']
        for temp_filter in temp_filters:
            local_filters.append(FieldFilter.decode(temp_filter, field))

        return Filter(field, local_filters, lop)

class UIField:
    def __init__(self, name, field_type):
        if type(name) is tuple:
            self.name = name[0]
        else:
            self.name = name
        self.type = field_type

    @staticmethod
    def decode(payload):
        return UIField(payload['name'], payload['type'])

class UIFieldEncoder(json.JSONEncoder):
    def default(self, obj):
        """
        :type obj: UIField
        """
        if isinstance(obj, UIField):
            retval = {
                'name': obj.name,
                'type': obj.type
            }
            return retval
        else:
            return json.JSONEncoder.default(self, obj)

def get_surveyresult_hstore_keys(ui_field=True):
    """
    :rtype: List of UIField objects
    :return: Unique set of keys from the answer hstore field in the
             surveyresult table
    """
    sql = 'select distinct hKey ' \
          'from (select skeys(answer) as hKey ' \
          'from core_surveyresult) as dt'
    cursor = connection.cursor()
    answer_keys = []

    cursor.execute(sql)

    for answer_key in cursor.fetchall():
        if ui_field:
            answer_keys.append(UIField(answer_key, 'H'))
        else:
            if type(answer_key) is tuple:
                answer_keys.append(answer_key[0])
            else:
                answer_keys.append(answer_key)

    return answer_keys

def process_group_member(api, member, group):
    # if for some reason we don't have the vumi key in the db for this contact, fetch the contact from vumi
    if member.vkey is None or member.vkey == '':
        try:
            contact = api.get_contact(msisdn=member.msisdn)
            member.vkey = contact['key']
            member.save()
        except Exception:
            logger.info('Contact: %s not found in vumi' % member)
            return

    try:
        api.update_contact(member.vkey, {u'groups': (group.group_key, )})
    except Exception:
        logger.info('Contact: %s update failed' % member)
        return

    try:
        group_member, created = ContactGroupMember.objects.get_or_create(group=group, contact=member)
        group_member.synced = True
        group_member.save()
    except Exception:
        logger.exception('Failed to add %s contact to %s group' % (contact.msisdn, group.name))

def remove_group_member(api, member, group):
    try:
        contact = api.get_contact(msisdn=member.msisdn)
        if member.vkey is None or member.vkey == '':
            member.vkey = contact['key']
            member.save()
    except Exception:
        logger.info('Contact: %s not found in vumi' % member)
        return
    groups = contact['groups']
    if group.group_key in groups:
        updated_groups = groups.remove(group.group_key)

        if updated_groups is None:
            updated_groups = ''
        try:
            api.update_contact(member.vkey, {u'groups': updated_groups})
        except Exception:
            logger.info('Contact: %s update failed' % member)
            return

    ContactGroupMember.objects.filter(group=group, contact=member).delete()<commit_msg>Added try catch<commit_after>from django.db.models import Q
from django.db import connection
import json
from gems.core.models import ContactGroupMember
import logging

logger = logging.getLogger(__name__)

class FieldFilter:
    def __init__(self, operator, value, field, loperator=None):
        self.operator = operator
        self.value = value

        if loperator is None:
            self.loperator = 'and'
        else:
            self.loperator = loperator

        self.field = field
        self.q = None

        if field.type == 'N':

            if field.name == 'survey':
                field.name = 'survey__name'

            if field.name == 'series':
                field.name = 'survey__series'

            if field.name == 'contact':
                field.name = 'contact__msisdn'

            # These are the normal fields
            if operator == 'eq':
                kwargs = {
                    '{0}__iexact'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'ex':
                kwargs = {
                    '{0}__exact'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'gt' \
                    or operator == 'gte' \
                    or operator == 'lt' \
                    or operator == 'lte':
                kwargs = {
                    '{0}__{1}'.format(field.name, operator): value
                }
                self.q = Q(**kwargs)
            elif operator == 'co':
                kwargs = {
                    '{0}__icontains'.format(field.name): value
                }
                self.q = Q(**kwargs)
            elif operator == 'neq':
                kwargs = {
                    '{0}'.format(field.name): value
                }
                self.q = ~Q(**kwargs)
            elif operator == 'nco':
                kwargs = {
                    '{0}__icontains'.format(field.name): value
                }
                self.q = ~Q(**kwargs)
        else:
            # These are the hstore fields
            # Special hack!!!
            # * contains is a sql ==
            # * icontains is a sql like
            if operator == 'eq':
                self.q = Q(answer__contains={field.name: value})
            elif operator == 'ex':
                self.q = Q(answer__contains={field.name: value})
            elif operator == 'gt':
                self.q = Q(answer__gt={field.name: value})
            elif operator == 'gte':
                self.q = Q(answer__gte={field.name: value})
            elif operator == 'lt':
                self.q = Q(answer__lt={field.name: value})
            elif operator == 'lte':
                self.q = Q(answer__lte={field.name: value})
            elif operator == 'co':
                self.q = Q(answer__icontains={field.name: "%" + value + "%"})
            elif operator == 'neq':
                self.q = ~Q(answer__contains={field.name: value})
            elif operator == 'nco':
                self.q = ~Q(answer__icontains={field.name: "%" + value + "%"})

    @staticmethod
    def decode(payload, field):
        op = payload['operator']
        value = payload['value']
        lop = None
        if 'loperator' in payload:
            lop = payload['loperator']

        return FieldFilter(op, value, field, lop)

class Filter:
    def __init__(self, field, filters=None, loperator=None):
        self.field = field
        self.filters = filters

        if self.filters is None:
            self.filters = []

        if loperator is None:
            self.loperator = 'and'
        else:
            self.loperator = loperator

    @staticmethod
    def decode(payload):
        field = UIField.decode(payload['field'])
        lop = None

        if 'loperator' in payload:
            lop = payload['loperator']

        local_filters = []
        temp_filters = payload['filters']
        for temp_filter in temp_filters:
            local_filters.append(FieldFilter.decode(temp_filter, field))

        return Filter(field, local_filters, lop)

class UIField:
    def __init__(self, name, field_type):
        if type(name) is tuple:
            self.name = name[0]
        else:
            self.name = name
        self.type = field_type

    @staticmethod
    def decode(payload):
        return UIField(payload['name'], payload['type'])

class UIFieldEncoder(json.JSONEncoder):
    def default(self, obj):
        """
        :type obj: UIField
        """
        if isinstance(obj, UIField):
            retval = {
                'name': obj.name,
                'type': obj.type
            }
            return retval
        else:
            return json.JSONEncoder.default(self, obj)

def get_surveyresult_hstore_keys(ui_field=True):
    """
    :rtype: List of UIField objects
    :return: Unique set of keys from the answer hstore field in the
             surveyresult table
    """
    sql = 'select distinct hKey ' \
          'from (select skeys(answer) as hKey ' \
          'from core_surveyresult) as dt'
    cursor = connection.cursor()
    answer_keys = []

    cursor.execute(sql)

    for answer_key in cursor.fetchall():
        if ui_field:
            answer_keys.append(UIField(answer_key, 'H'))
        else:
            if type(answer_key) is tuple:
                answer_keys.append(answer_key[0])
            else:
                answer_keys.append(answer_key)

    return answer_keys

def process_group_member(api, member, group):
    # if for some reason we don't have the vumi key in the db for this contact, fetch the contact from vumi
    if member.vkey is None or member.vkey == '':
        try:
            contact = api.get_contact(msisdn=member.msisdn)
            member.vkey = contact['key']
            member.save()
        except Exception:
            logger.info('Contact: %s not found in vumi' % member)
            return

    try:
        api.update_contact(member.vkey, {u'groups': (group.group_key, )})
    except Exception:
        logger.info('Contact: %s update failed' % member)
        return

    try:
        group_member, created = ContactGroupMember.objects.get_or_create(group=group, contact=member)
        group_member.synced = True
        group_member.save()
    except Exception:
        logger.exception('Failed to add %s contact to %s group' % (contact.msisdn, group.name))

def remove_group_member(api, member, group):
    try:
        contact = api.get_contact(msisdn=member.msisdn)
        if member.vkey is None or member.vkey == '':
            member.vkey = contact['key']
            member.save()
    except Exception:
        logger.info('Contact: %s not found in vumi' % member)
        return
    groups = contact['groups']
    if group.group_key in groups:
        updated_groups = groups.remove(group.group_key)

        if updated_groups is None:
            updated_groups = ''
        try:
            api.update_contact(member.vkey, {u'groups': updated_groups})
        except Exception:
            logger.info('Contact: %s update failed' % member)
            return

    try:
        ContactGroupMember.objects.filter(group=group, contact=member).delete()
    except Exception:
        logger.exception('Failed to delete %s contact in %s group' % (member.msisdn, group.name))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) Spyder Project Developers
#
# Licensed under the terms of the MIT License
# (see LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Tests for memoryprofiler.py."""

# Standard library imports
import os

# Third party imports
from pytestqt import qtbot
from qtpy.QtCore import Qt

from spyder.utils.qthelpers import qapplication
MAIN_APP = qapplication()

# Local imports
from spyder_memory_profiler.widgets.memoryprofiler import MemoryProfilerWidget

try:
    from unittest.mock import Mock
except ImportError:
    from mock import Mock  # Python 2

TEST_SCRIPT = \
"""@profile
def foo():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    del b
    return a
foo()"""

def test_profile_and_display_results(qtbot, tmpdir, monkeypatch):
    """Run profiler on simple script and check that results are okay."""
    os.chdir(tmpdir.strpath)
    testfilename = tmpdir.join('test_foo.py').strpath

    with open(testfilename, 'w') as f:
        f.write(TEST_SCRIPT)

    MockQMessageBox = Mock()
    monkeypatch.setattr('spyder_memory_profiler.widgets.memoryprofiler.QMessageBox',
                        MockQMessageBox)

    widget = MemoryProfilerWidget(None)
    qtbot.addWidget(widget)
    widget.analyze(testfilename)
    qtbot.wait(2000) # wait for tests to run

    MockQMessageBox.assert_not_called()
    dt = widget.datatree
    assert dt.topLevelItemCount() == 1  # number of functions profiled

    top = dt.topLevelItem(0)
    assert top.data(0, Qt.DisplayRole).startswith('foo ')
    assert top.childCount() == 6
    for i in range(6):
        assert top.child(i).data(0, Qt.DisplayRole) == i + 1  # line no

    # column 2 has increment (in Mib); displayed as 'xxx MiB' so need to strip
    # last 4 characters
    assert float(top.child(2).data(2, Qt.DisplayRole)[:-4]) >= 7  # increment (MiB)
    assert float(top.child(2).data(2, Qt.DisplayRole)[:-4]) <= 8
    assert float(top.child(3).data(2, Qt.DisplayRole)[:-4]) >= 150
    assert float(top.child(3).data(2, Qt.DisplayRole)[:-4]) <= 160
    assert float(top.child(4).data(2, Qt.DisplayRole)[:-4]) >= -160
    assert float(top.child(4).data(2, Qt.DisplayRole)[:-4]) <= -150
    assert float(top.child(5).data(2, Qt.DisplayRole)[:-4]) == 0
<commit_msg>Use signal in test to determine when profiling is finished<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) Spyder Project Developers
#
# Licensed under the terms of the MIT License
# (see LICENSE.txt for details)
# -----------------------------------------------------------------------------
"""Tests for memoryprofiler.py."""

# Standard library imports
import os

# Third party imports
from pytestqt import qtbot
from qtpy.QtCore import Qt

from spyder.utils.qthelpers import qapplication
MAIN_APP = qapplication()

# Local imports
from spyder_memory_profiler.widgets.memoryprofiler import MemoryProfilerWidget

try:
    from unittest.mock import Mock
except ImportError:
    from mock import Mock  # Python 2

TEST_SCRIPT = \
"""@profile
def foo():
    a = [1] * (10 ** 6)
    b = [2] * (2 * 10 ** 7)
    del b
    return a
foo()"""

def test_profile_and_display_results(qtbot, tmpdir, monkeypatch):
    """Run profiler on simple script and check that results are okay."""
    os.chdir(tmpdir.strpath)
    testfilename = tmpdir.join('test_foo.py').strpath

    with open(testfilename, 'w') as f:
        f.write(TEST_SCRIPT)

    MockQMessageBox = Mock()
    monkeypatch.setattr('spyder_memory_profiler.widgets.memoryprofiler.QMessageBox',
                        MockQMessageBox)

    widget = MemoryProfilerWidget(None)
    qtbot.addWidget(widget)
    with qtbot.waitSignal(widget.sig_finished, timeout=10000, raising=True):
        widget.analyze(testfilename)

    MockQMessageBox.assert_not_called()
    dt = widget.datatree
    assert dt.topLevelItemCount() == 1  # number of functions profiled

    top = dt.topLevelItem(0)
    assert top.data(0, Qt.DisplayRole).startswith('foo ')
    assert top.childCount() == 6
    for i in range(6):
        assert top.child(i).data(0, Qt.DisplayRole) == i + 1  # line no

    # column 2 has increment (in Mib); displayed as 'xxx MiB' so need to strip
    # last 4 characters
    assert float(top.child(2).data(2, Qt.DisplayRole)[:-4]) >= 7  # increment (MiB)
    assert float(top.child(2).data(2, Qt.DisplayRole)[:-4]) <= 8
    assert float(top.child(3).data(2, Qt.DisplayRole)[:-4]) >= 150
    assert float(top.child(3).data(2, Qt.DisplayRole)[:-4]) <= 160
    assert float(top.child(4).data(2, Qt.DisplayRole)[:-4]) >= -160
    assert float(top.child(4).data(2, Qt.DisplayRole)[:-4]) <= -150
    assert float(top.child(5).data(2, Qt.DisplayRole)[:-4]) == 0
<|endoftext|>


<commit_before>from geneagrapher import *

def ggrapher():
	geneagrapher = Geneagrapher()
	try:
		geneagrapher.parseInput()
	except SyntaxError, e:
		print geneagrapher.parser.get_usage()
		print e
	geneagrapher.buildGraph()
	geneagrapher.generateDotFile()
<commit_msg>Removed "import *" and modified code accordingly.<commit_after>import geneagrapher

def ggrapher():
	ggrapher = geneagrapher.Geneagrapher()
	try:
		ggrapher.parseInput()
	except SyntaxError, e:
		print ggrapher.parser.get_usage()
		print e
	ggrapher.buildGraph()
	ggrapher.generateDotFile()
<|endoftext|>


<commit_before>import os
import csv
import re
import collections
import logging
from cStringIO import StringIO
import sys
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE,SIG_DFL)

import AsciiDammit
import dedupe

def preProcess(column):
    """
    Do a little bit of data cleaning with the help of
    [AsciiDammit](https://github.com/tnajdek/ASCII--Dammit) and
    Regex. Things like casing, extra spaces, quotes and new lines can
    be ignored.
    """
    column = AsciiDammit.asciiDammit(column)
    column = re.sub('  +', ' ', column)
    column = re.sub('\n', ' ', column)
    column = column.strip().strip('"').strip("'").lower().strip()
    return column

def readData(input_file, field_names):
    """
    Read in our data from a CSV file and create a dictionary of records,
    where the key is a unique record ID and each value is a
    [frozendict](http://code.activestate.com/recipes/414283-frozen-dictionaries/)
    (hashable dictionary) of the row fields.

    **Currently, dedupe depends upon records' unique ids being integers
    with no integers skipped. The smallest valued unique id must be 0 or
    1. Expect this requirement will likely be relaxed in the future.**
    """

    data = {}
    reader = csv.DictReader(StringIO(input_file))
    for i, row in enumerate(reader):
        clean_row = [(k, preProcess(v)) for (k, v) in row.items()]
        row_id = i
        data[row_id] = dedupe.core.frozendict(clean_row)

    return data

# ## Writing results
def writeResults(clustered_dupes, input_file, output_file):

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    logging.info('saving results to: %s' % output_file)

    cluster_membership = collections.defaultdict(lambda : 'x')
    for (cluster_id, cluster) in enumerate(clustered_dupes):
        for record_id in cluster:
            cluster_membership[record_id] = cluster_id

    with open(output_file, 'w') as f:
        writer = csv.writer(f)

        reader = csv.reader(StringIO(input_file))

        heading_row = reader.next()
        heading_row.insert(0, 'Cluster ID')
        writer.writerow(heading_row)

        for i, row in enumerate(reader):
            row_id = i
            cluster_id = cluster_membership[row_id]
            row.insert(0, cluster_id)
            writer.writerow(row)

# ## Printing results to stdout
def printResults(clustered_dupes, input_file):

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    cluster_membership = collections.defaultdict(lambda : 'x')
    for (cluster_id, cluster) in enumerate(clustered_dupes):
        for record_id in cluster:
            cluster_membership[record_id] = cluster_id

    reader = csv.reader(StringIO(input_file))

    heading_row = reader.next()
    heading_row.insert(0, 'Cluster ID')
    _printClusterRow(heading_row)

    for i, row in enumerate(reader):
        row_id = i
        cluster_id = cluster_membership[row_id]
        row.insert(0, cluster_id)
        _printClusterRow(row)

def _printClusterRow(row):
    result_str = ''
    for value in row:
        result_str += (str(value) + ',')
    result_str += '\n'

    sys.stdout.write(result_str)
<commit_msg>make sure that helper function to pipe rows to STDOUT is actually quoting where appropriate on the way out Closes #23<commit_after>import os
import csv
import re
import collections
import logging
from cStringIO import StringIO
import sys
from signal import signal, SIGPIPE, SIG_DFL
signal(SIGPIPE,SIG_DFL)

import AsciiDammit
import dedupe

def preProcess(column):
    """
    Do a little bit of data cleaning with the help of
    [AsciiDammit](https://github.com/tnajdek/ASCII--Dammit) and
    Regex. Things like casing, extra spaces, quotes and new lines can
    be ignored.
    """
    column = AsciiDammit.asciiDammit(column)
    column = re.sub('  +', ' ', column)
    column = re.sub('\n', ' ', column)
    column = column.strip().strip('"').strip("'").lower().strip()
    return column

def readData(input_file, field_names):
    """
    Read in our data from a CSV file and create a dictionary of records,
    where the key is a unique record ID and each value is a
    [frozendict](http://code.activestate.com/recipes/414283-frozen-dictionaries/)
    (hashable dictionary) of the row fields.

    **Currently, dedupe depends upon records' unique ids being integers
    with no integers skipped. The smallest valued unique id must be 0 or
    1. Expect this requirement will likely be relaxed in the future.**
    """

    data = {}
    reader = csv.DictReader(StringIO(input_file))
    for i, row in enumerate(reader):
        clean_row = [(k, preProcess(v)) for (k, v) in row.items()]
        row_id = i
        data[row_id] = dedupe.core.frozendict(clean_row)

    return data

# ## Writing results
def writeResults(clustered_dupes, input_file, output_file):

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    logging.info('saving results to: %s' % output_file)

    cluster_membership = collections.defaultdict(lambda : 'x')
    for (cluster_id, cluster) in enumerate(clustered_dupes):
        for record_id in cluster:
            cluster_membership[record_id] = cluster_id

    with open(output_file, 'w') as f:
        writer = csv.writer(f, quoting=csv.QUOTE_NONNUMERIC)
        reader = csv.reader(StringIO(input_file), quoting=csv.QUOTE_NONNUMERIC)

        heading_row = reader.next()
        heading_row.insert(0, 'Cluster ID')
        writer.writerow(heading_row)

        for i, row in enumerate(reader):
            row_id = i
            cluster_id = cluster_membership[row_id]
            row.insert(0, cluster_id)
            writer.writerow(row)

# ## Printing results to stdout
def printResults(clustered_dupes, input_file):

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    cluster_membership = collections.defaultdict(lambda : 'x')
    for (cluster_id, cluster) in enumerate(clustered_dupes):
        for record_id in cluster:
            cluster_membership[record_id] = cluster_id

    reader = csv.reader(StringIO(input_file))

    heading_row = reader.next()
    heading_row.insert(0, 'Cluster ID')
    _printClusterRow(heading_row)

    for i, row in enumerate(reader):
        row_id = i
        cluster_id = cluster_membership[row_id]
        row.insert(0, cluster_id)
        _printClusterRow(row)

def _printClusterRow(row):
    result_str = ''
    for value in row:
        if type(value) == int or type(value) == float:
            result_str += '%s,' % (value)
        else:
            result_str += '"%s",' % (value)
    result_str += '\n'

    sys.stdout.write(result_str)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import print_function
import subprocess
import sys
import os
import getopt

def usage():
    print("genee-puppet init")
    print("genee-puppet run [-n name] [-p port]")
    print("    default: name=puppetmaster, port=8140")

def puppet_init():
    os.mkdir("conf.d")
    os.mkdir("ssl.d")

def puppet_run():

    try:
        opts, args = getopt.getopt(sys.argv[2:], "n:p:")
    except getopt.GetoptError as e:
        print(str(e))

    name = 'puppetmaster'
    port = 8140

    for o, a in opts:
        if o == "-n":
            name = a
        elif o == "-p":
            port = int(a)

    if not os.path.exists('conf.d') or not os.path.exists('ssl.d'):
        print('Run "genee-puppet init" first.')
        sys.exit(0)

    try:
        cmd = ("docker run --name %s -d --restart=always -v %s:/opt/puppet -v %s:/var/lib/puppet/ssl -v /dev/log:/dev/log -p %d:8140 genee/puppetmaster" % (name, os.getcwd() + '/conf.d', os.getcwd() + '/ssl.d', port))
        print(cmd + "\n")
        print(subprocess.check_output(cmd, shell=True, universal_newlines=True))
    except subprocess.CalledProcessError as e:
        return

def main():

    if len(sys.argv) < 2:
        usage()
        sys.exit(1)

    if sys.argv[1] == 'init':
        puppet_init()
    elif sys.argv[1] == 'run':
        puppet_run()
    else:
        usage()
        sys.exit(1)

if __name__ == "__main__":
    main()
<commit_msg>Clone repo instead of make a dummy dir.<commit_after># -*- coding: utf-8 -*-

from __future__ import print_function
import subprocess
import sys
import os
import getopt

def usage():
    print("genee-puppet init")
    print("genee-puppet run [-n name] [-p port]")
    print("    default: name=puppetmaster, port=8140")

def puppet_init():
    if not os.path.exists("ssl.d"):
        os.mkdir("ssl.d")
    if not os.path.exists("conf.d"):
        subprocess.check_call("git clone https://bitbucket.org/genee/genee-puppet.git conf.d", shell=True)

def puppet_run():

    try:
        opts, args = getopt.getopt(sys.argv[2:], "n:p:")
    except getopt.GetoptError as e:
        print(str(e))

    name = 'puppetmaster'
    port = 8140

    for o, a in opts:
        if o == "-n":
            name = a
        elif o == "-p":
            port = int(a)

    if not os.path.exists('conf.d') or not os.path.exists('ssl.d'):
        print('Run "genee-puppet init" first.')
        sys.exit(0)

    try:
        cmd = ("docker run --name %s -d --restart=always -v %s:/opt/puppet -v %s:/var/lib/puppet/ssl -v /dev/log:/dev/log -p %d:8140 genee/puppetmaster" % (name, os.getcwd() + '/conf.d', os.getcwd() + '/ssl.d', port))
        print(cmd + "\n")
        print(subprocess.check_output(cmd, shell=True, universal_newlines=True))
    except subprocess.CalledProcessError as e:
        return

def main():

    if len(sys.argv) < 2:
        usage()
        sys.exit(1)

    if sys.argv[1] == 'init':
        puppet_init()
    elif sys.argv[1] == 'run':
        puppet_run()
    else:
        usage()
        sys.exit(1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import csv
import mapper
import utils

class CSVParser(object):
	"""CSV Parser capable of parsing against a pre-defined mapper file"""
	def __init__(self, csvFile, fmapper=None, hasHeader=False):
		super(CSVParser, self).__init__()
		# the csv file
		self.csvFile = csvFile
		# the mapper object
		self.fmapper = fmapper
		# whether the csv file contains columns in the first row
		self.hasHeader = hasHeader

	# get records from mapper file
	def getRecords(self):
		if self.fmapper == None:
			if self.hasHeader == False:
				raise Exception('No mapper specified, set hasHeader=True if csv file contains headers')
			else:
				self.fmapper = mapper.FieldMapper(self.csvData[0]) # first row of csv
		return self.fmapper.getRecords()

	# parses a CSV file
	def parseCSV(self):
		with open(self.csvFile, 'rU') as csvfile:
			rdr = csv.reader(csvfile, delimiter='\t', quotechar='|')
			x = []
			for row in rdr:
				x.append(row[0].split(','))
			self.csvData = x

	# convert type
	def convertType(self,to,val):
		if to == '':
			return val
		i = ''
		exec('i = %s(%s)' %(to,val))
		return i

	# convert csv data to record
	def toDict(self, cdat, rec):
		d = {}
		i = 0
		for j in rec:
			a = cdat[i]
			if 'type' in j:
				a = self.convertType(j['type'], a)
			d[j['name']] = a
			i= i+1
		return d

	def getIndex(self, x, l):
		if x > (l-1):
			return x%l
		return x

	# csv against mapper as dict instance
	def buildDict(self, onAppend=None,popHeader=True):
		if hasattr(self, 'csvData') == False:
			self.parseCSV()

		recs = self.getRecords()
		if self.hasHeader and popHeader:
			self.csvData.pop(0) # remove the header line

		l = len(recs)
		dicts = []
		for x in range(0, len(self.csvData)):
			i = self.getIndex(x, l)
			dic = self.toDict(self.csvData[x], recs[i])
			if onAppend == None:
				dicts.append(dic)
			else:
				dicts.append(onAppend(dic))
		return dicts

	# as object instance
	def buildObject(self):
		return self.buildDict(utils.CSVObject)

# Parser instance to CSV
class CSVWriter():
	"""Generate CSV Output"""
	def __init__(self, dic):
		self.dic = dic

	# write to csv file
	def write(self, fileName):
		isDict = type(self.dic[0]) == dict
		with open(fileName, 'wb') as fs:
			headings = None
			if isDict:
				headings = self.dic[0].keys()
			else:
				headings = self.dic[0].attribs()

			w = csv.DictWriter(fs, headings)
			w.writeheader()
			if isDict:
				for row in self.dic:
					w.writerow(row)
			else:
				for row in self.dic:
					w.writerow(row.__dict__)
<commit_msg>Python 2.6 Support<commit_after>import csv
import mapper
import utils

class CSVParser(object):
	"""CSV Parser capable of parsing against a pre-defined mapper file"""
	def __init__(self, csvFile, fmapper=None, hasHeader=False):
		super(CSVParser, self).__init__()
		# the csv file
		self.csvFile = csvFile
		# the mapper object
		self.fmapper = fmapper
		# whether the csv file contains columns in the first row
		self.hasHeader = hasHeader

	# get records from mapper file
	def getRecords(self):
		if self.fmapper == None:
			if self.hasHeader == False:
				raise Exception('No mapper specified, set hasHeader=True if csv file contains headers')
			else:
				self.fmapper = mapper.FieldMapper(self.csvData[0]) # first row of csv
		return self.fmapper.getRecords()

	# parses a CSV file
	def parseCSV(self):
		with open(self.csvFile, 'rU') as csvfile:
			rdr = csv.reader(csvfile, delimiter='\t', quotechar='|')
			x = []
			for row in rdr:
				x.append(row[0].split(','))
			self.csvData = x

	# convert type
	def convertType(self,to,val):
		if to == '':
			return val
		i = ''
		exec('i = %s(%s)' %(to,val))
		return i

	# convert csv data to record
	def toDict(self, cdat, rec):
		d = {}
		i = 0
		for j in rec:
			a = cdat[i]
			if 'type' in j:
				a = self.convertType(j['type'], a)
			d[j['name']] = a
			i= i+1
		return d

	def getIndex(self, x, l):
		if x > (l-1):
			return x%l
		return x

	# csv against mapper as dict instance
	def buildDict(self, onAppend=None,popHeader=True):
		if hasattr(self, 'csvData') == False:
			self.parseCSV()

		recs = self.getRecords()
		if self.hasHeader and popHeader:
			self.csvData.pop(0) # remove the header line

		l = len(recs)
		dicts = []
		for x in range(0, len(self.csvData)):
			i = self.getIndex(x, l)
			dic = self.toDict(self.csvData[x], recs[i])
			if onAppend == None:
				dicts.append(dic)
			else:
				dicts.append(onAppend(dic))
		return dicts

	# as object instance
	def buildObject(self):
		return self.buildDict(utils.CSVObject)

# Parser instance to CSV
class CSVWriter():
	"""Generate CSV Output"""
	def __init__(self, dic):
		self.dic = dic

	# write to csv file
	def write(self, fileName):
		isDict = type(self.dic[0]) == dict
		with open(fileName, 'wb') as fs:
			headings = None
			if isDict:
				headings = self.dic[0].keys()
			else:
				headings = self.dic[0].attribs()
			w = csv.DictWriter(fs, headings)
                        w.writerow( dict((fn,fn) for fn in headings))
			if isDict:
				for row in self.dic:
					w.writerow(row)
			else:
				for row in self.dic:
					w.writerow(row.__dict__)
<|endoftext|>