<commit_before># -*- coding: utf-8 -*-
# Import python libs
import logging
import os
import signal
import time
import sys
import multiprocessing
import signal

import threading
import Queue

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

HAS_PSUTIL = False
try:
    import psutil
    HAS_PSUTIL = True
except ImportError:
    pass

try:
    import systemd.daemon
    HAS_PYTHON_SYSTEMD = True
except ImportError:
    HAS_PYTHON_SYSTEMD = False

def set_pidfile(pidfile, user):
    '''
    Save the pidfile
    '''
    pdir = os.path.dirname(pidfile)
    if not os.path.isdir(pdir) and pdir:
        os.makedirs(pdir)
    try:
        with salt.utils.fopen(pidfile, 'w+') as ofile:
            ofile.write(str(os.getpid()))
    except IOError:
        pass

    log.debug(('Created pidfile: {0}').format(pidfile))
    if salt.utils.is_windows():
        return True

    import pwd  # after confirming not running Windows
    #import grp
    try:
        pwnam = pwd.getpwnam(user)
        uid = pwnam[2]
        gid = pwnam[3]
        #groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem]
    except IndexError:
        sys.stderr.write(
            'Failed to set the pid to user: {0}. The user is not '
            'available.\n'.format(
                user
            )
        )
        sys.exit(os.EX_NOUSER)

    if os.getuid() == uid:
        # The current user already owns the pidfile. Return!
        return

    try:
        os.chown(pidfile, uid, gid)
    except OSError as err:
        msg = (
            'Failed to set the ownership of PID file {0} to user {1}.'.format(
                pidfile, user
            )
        )
        log.debug('{0} Traceback follows:\n'.format(msg), exc_info=True)
        sys.stderr.write('{0}\n'.format(msg))
        sys.exit(err.errno)
    log.debug('Chowned pidfile: {0} to user: {1}'.format(pidfile, user))

def clean_proc(proc, wait_for_kill=10):
    '''
    Generic method for cleaning up multiprocessing procs
    '''
    # NoneType and other fun stuff need not apply
    if not proc:
        return
    try:
        waited = 0
        while proc.is_alive():
            proc.terminate()
            waited += 1
            time.sleep(0.1)
            if proc.is_alive() and (waited >= wait_for_kill):
                log.error(
                    'Process did not die with terminate(): {0}'.format(
                        proc.pid
                    )
                )
                os.kill(proc.pid, signal.SIGKILL)
    except (AssertionError, AttributeError):
        # Catch AssertionError when the proc is evaluated inside the child
        # Catch AttributeError when the process dies between proc.is_alive()
        # and proc.terminate() and turns into a NoneType
        pass

def os_is_running(pid):
    '''
    Use OS facilities to determine if a process is running
    '''
    if HAS_PSUTIL:
        return psutil.pid_exists(pid)
    else:
        try:
            os.kill(pid, 0)  # SIG 0 is the "are you alive?" signal
            return True
        except OSError:
            return False

class ThreadPool(object):
    '''
    This is a very VERY basic threadpool implementation
    This was made instead of using multiprocessing ThreadPool because
    we want to set max queue size and we want to daemonize threads (neither
    is exposed in the stdlib version).

    Since there isn't much use for this class as of right now this implementation
    Only supports daemonized threads and will *not* return results

    TODO: if this is found to be more generally useful it would be nice to pull
    in the majority of code from upstream or from http://bit.ly/1wTeJtM
    '''
    def __init__(self,
                 num_threads=None,
                 queue_size=0):
        # if no count passed, default to number of CPUs
        if num_threads is None:
            num_threads = multiprocessing.cpu_count()
        self.num_threads = num_threads

        # create a task queue of queue_size
        self._job_queue = Queue.Queue(queue_size)

        self._workers = []

        # create worker threads
        for idx in xrange(num_threads):
            thread = threading.Thread(target=self._thread_target)
            thread.daemon = True
            thread.start()
            self._workers.append(thread)

    # intentionally not called "apply_async"  since we aren't keeping track of
    # the return at all, if we want to make this API compatible with multiprocessing
    # threadpool we can in the future, and we won't have to worry about name collision
    def fire_async(self, func, args=None, kwargs=None):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}
        try:
            self._job_queue.put((func, args, kwargs), False)
            return True
        except Queue.Full:
            return False

    def _thread_target(self):
        while True:
            # 1s timeout so that if the parent dies this thread will die after 1s
            try:
                func, args, kwargs = self._job_queue.get(timeout=1)
            except Queue.Empty:
                continue
            try:
                func(*args, **kwargs)
            except Exception as err:
                log.debug(err, exc_info=True)

class ProcessManager(object):
    '''
    A class which will manage processes that should be running
    '''
    def __init__(self, name=None, wait_for_kill=1):
        # pid -> {tgt: foo, Process: object, args: args, kwargs: kwargs}
        self._process_map = {}

        self.name = name
        if self.name is None:
            self.name = self.__class__.__name__

        self.wait_for_kill = wait_for_kill

        # store some pointers for the SIGTERM handler
        self._pid = os.getpid()
        self._sigterm_handler = signal.getsignal(signal.SIGTERM)

    def add_process(self, tgt, args=None, kwargs=None):
        '''
        Create a processes and args + kwargs
        This will deterimine if it is a Process class, otherwise it assumes
        it is a function
        '''
        if args is None:
            args = []

        if kwargs is None:
            kwargs = {}

        if type(multiprocessing.Process) == type(tgt) and issubclass(tgt, multiprocessing.Process):
            p = tgt(*args, **kwargs)
        else:
            p = multiprocessing.Process(target=tgt, args=args, kwargs=kwargs)

        p.start()
        log.debug("Started '{0}'(*{1}, **{2} with pid {3}".format(tgt,
                                                                  args,
                                                                  kwargs,
                                                                  p.pid))
        self._process_map[p.pid] = {'tgt': tgt,
                                    'args': args,
                                    'kwargs': kwargs,
                                    'Process': p}

    def restart_process(self, pid):
        '''
        Create new process (assuming this one is dead), then remove the old one
        '''
        log.info(('Process {0} ({1}) died with exit status {2},'
                  ' restarting...').format(self._process_map[pid]['tgt'],
                                           pid,
                                           self._process_map[pid]['Process'].exitcode))
        self._process_map[pid]['Process'].join(1)

        self.add_process(self._process_map[pid]['tgt'],
                         self._process_map[pid]['args'],
                         self._process_map[pid]['kwargs'])

        del self._process_map[pid]

    def run(self):
        '''
        Load and start all available api modules
        '''
        salt.utils.appendproctitle(self.name)

        # make sure to kill the subprocesses if the parent is killed
        signal.signal(signal.SIGTERM, self.kill_children)

        try:
            if HAS_PYTHON_SYSTEMD and systemd.daemon.booted():
                systemd.daemon.notify('READY=1')
        except SystemError:
            # Daemon wasn't started by systemd
            pass

        while True:
            try:
                # in case someone died while we were waiting...
                self.check_children()

                pid, exit_status = os.wait()
                if pid not in self._process_map:
                    log.debug(('Process of pid {0} died, not a known'
                               ' process, will not restart').format(pid))
                    continue
                self.restart_process(pid)
            except OSError:
                break

    def check_children(self):
        '''
        Check the children once
        '''
        for pid, mapping in self._process_map.iteritems():
            if not mapping['Process'].is_alive():
                self.restart_process(pid)

    def kill_children(self, *args):
        '''
        Kill all of the children
        '''
        # check that this is the correct process, children inherit this
        # handler, if we are in a child lets just run the original handler
        if os.getpid() != self._pid:
            if callable(self._sigterm_handler):
                return self._sigterm_handler(*args)
            elif self._sigterm_handler is not None:
                return signal.default_int_handler(signal.SIGTERM)(*args)
            else:
                return

        for pid, p_map in self._process_map.items():
            p_map['Process'].terminate()

        end_time = time.time() + self.wait_for_kill  # when to die

        while self._process_map and time.time() < end_time:
            for pid, p_map in self._process_map.items():
                p_map['Process'].join(0)

                # This is a race condition if a signal was passed to all children
                try:
                    del self._process_map[pid]
                except KeyError:
                    pass
        # if anyone is done after
        for pid in self._process_map:
            try:
                os.kill(signal.SIGKILL, pid)
            # in case the process has since decided to die, os.kill returns OSError
            except OSError:
                pass
<commit_msg>Clarify comment<commit_after># -*- coding: utf-8 -*-
# Import python libs
import logging
import os
import signal
import time
import sys
import multiprocessing
import signal

import threading
import Queue

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

HAS_PSUTIL = False
try:
    import psutil
    HAS_PSUTIL = True
except ImportError:
    pass

try:
    import systemd.daemon
    HAS_PYTHON_SYSTEMD = True
except ImportError:
    HAS_PYTHON_SYSTEMD = False

def set_pidfile(pidfile, user):
    '''
    Save the pidfile
    '''
    pdir = os.path.dirname(pidfile)
    if not os.path.isdir(pdir) and pdir:
        os.makedirs(pdir)
    try:
        with salt.utils.fopen(pidfile, 'w+') as ofile:
            ofile.write(str(os.getpid()))
    except IOError:
        pass

    log.debug(('Created pidfile: {0}').format(pidfile))
    if salt.utils.is_windows():
        return True

    import pwd  # after confirming not running Windows
    #import grp
    try:
        pwnam = pwd.getpwnam(user)
        uid = pwnam[2]
        gid = pwnam[3]
        #groups = [g.gr_gid for g in grp.getgrall() if user in g.gr_mem]
    except IndexError:
        sys.stderr.write(
            'Failed to set the pid to user: {0}. The user is not '
            'available.\n'.format(
                user
            )
        )
        sys.exit(os.EX_NOUSER)

    if os.getuid() == uid:
        # The current user already owns the pidfile. Return!
        return

    try:
        os.chown(pidfile, uid, gid)
    except OSError as err:
        msg = (
            'Failed to set the ownership of PID file {0} to user {1}.'.format(
                pidfile, user
            )
        )
        log.debug('{0} Traceback follows:\n'.format(msg), exc_info=True)
        sys.stderr.write('{0}\n'.format(msg))
        sys.exit(err.errno)
    log.debug('Chowned pidfile: {0} to user: {1}'.format(pidfile, user))

def clean_proc(proc, wait_for_kill=10):
    '''
    Generic method for cleaning up multiprocessing procs
    '''
    # NoneType and other fun stuff need not apply
    if not proc:
        return
    try:
        waited = 0
        while proc.is_alive():
            proc.terminate()
            waited += 1
            time.sleep(0.1)
            if proc.is_alive() and (waited >= wait_for_kill):
                log.error(
                    'Process did not die with terminate(): {0}'.format(
                        proc.pid
                    )
                )
                os.kill(proc.pid, signal.SIGKILL)
    except (AssertionError, AttributeError):
        # Catch AssertionError when the proc is evaluated inside the child
        # Catch AttributeError when the process dies between proc.is_alive()
        # and proc.terminate() and turns into a NoneType
        pass

def os_is_running(pid):
    '''
    Use OS facilities to determine if a process is running
    '''
    if HAS_PSUTIL:
        return psutil.pid_exists(pid)
    else:
        try:
            os.kill(pid, 0)  # SIG 0 is the "are you alive?" signal
            return True
        except OSError:
            return False

class ThreadPool(object):
    '''
    This is a very VERY basic threadpool implementation
    This was made instead of using multiprocessing ThreadPool because
    we want to set max queue size and we want to daemonize threads (neither
    is exposed in the stdlib version).

    Since there isn't much use for this class as of right now this implementation
    Only supports daemonized threads and will *not* return results

    TODO: if this is found to be more generally useful it would be nice to pull
    in the majority of code from upstream or from http://bit.ly/1wTeJtM
    '''
    def __init__(self,
                 num_threads=None,
                 queue_size=0):
        # if no count passed, default to number of CPUs
        if num_threads is None:
            num_threads = multiprocessing.cpu_count()
        self.num_threads = num_threads

        # create a task queue of queue_size
        self._job_queue = Queue.Queue(queue_size)

        self._workers = []

        # create worker threads
        for idx in xrange(num_threads):
            thread = threading.Thread(target=self._thread_target)
            thread.daemon = True
            thread.start()
            self._workers.append(thread)

    # intentionally not called "apply_async"  since we aren't keeping track of
    # the return at all, if we want to make this API compatible with multiprocessing
    # threadpool we can in the future, and we won't have to worry about name collision
    def fire_async(self, func, args=None, kwargs=None):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}
        try:
            self._job_queue.put((func, args, kwargs), False)
            return True
        except Queue.Full:
            return False

    def _thread_target(self):
        while True:
            # 1s timeout so that if the parent dies this thread will die within 1s
            try:
                func, args, kwargs = self._job_queue.get(timeout=1)
            except Queue.Empty:
                continue
            try:
                func(*args, **kwargs)
            except Exception as err:
                log.debug(err, exc_info=True)

class ProcessManager(object):
    '''
    A class which will manage processes that should be running
    '''
    def __init__(self, name=None, wait_for_kill=1):
        # pid -> {tgt: foo, Process: object, args: args, kwargs: kwargs}
        self._process_map = {}

        self.name = name
        if self.name is None:
            self.name = self.__class__.__name__

        self.wait_for_kill = wait_for_kill

        # store some pointers for the SIGTERM handler
        self._pid = os.getpid()
        self._sigterm_handler = signal.getsignal(signal.SIGTERM)

    def add_process(self, tgt, args=None, kwargs=None):
        '''
        Create a processes and args + kwargs
        This will deterimine if it is a Process class, otherwise it assumes
        it is a function
        '''
        if args is None:
            args = []

        if kwargs is None:
            kwargs = {}

        if type(multiprocessing.Process) == type(tgt) and issubclass(tgt, multiprocessing.Process):
            p = tgt(*args, **kwargs)
        else:
            p = multiprocessing.Process(target=tgt, args=args, kwargs=kwargs)

        p.start()
        log.debug("Started '{0}'(*{1}, **{2} with pid {3}".format(tgt,
                                                                  args,
                                                                  kwargs,
                                                                  p.pid))
        self._process_map[p.pid] = {'tgt': tgt,
                                    'args': args,
                                    'kwargs': kwargs,
                                    'Process': p}

    def restart_process(self, pid):
        '''
        Create new process (assuming this one is dead), then remove the old one
        '''
        log.info(('Process {0} ({1}) died with exit status {2},'
                  ' restarting...').format(self._process_map[pid]['tgt'],
                                           pid,
                                           self._process_map[pid]['Process'].exitcode))
        self._process_map[pid]['Process'].join(1)

        self.add_process(self._process_map[pid]['tgt'],
                         self._process_map[pid]['args'],
                         self._process_map[pid]['kwargs'])

        del self._process_map[pid]

    def run(self):
        '''
        Load and start all available api modules
        '''
        salt.utils.appendproctitle(self.name)

        # make sure to kill the subprocesses if the parent is killed
        signal.signal(signal.SIGTERM, self.kill_children)

        try:
            if HAS_PYTHON_SYSTEMD and systemd.daemon.booted():
                systemd.daemon.notify('READY=1')
        except SystemError:
            # Daemon wasn't started by systemd
            pass

        while True:
            try:
                # in case someone died while we were waiting...
                self.check_children()

                pid, exit_status = os.wait()
                if pid not in self._process_map:
                    log.debug(('Process of pid {0} died, not a known'
                               ' process, will not restart').format(pid))
                    continue
                self.restart_process(pid)
            except OSError:
                break

    def check_children(self):
        '''
        Check the children once
        '''
        for pid, mapping in self._process_map.iteritems():
            if not mapping['Process'].is_alive():
                self.restart_process(pid)

    def kill_children(self, *args):
        '''
        Kill all of the children
        '''
        # check that this is the correct process, children inherit this
        # handler, if we are in a child lets just run the original handler
        if os.getpid() != self._pid:
            if callable(self._sigterm_handler):
                return self._sigterm_handler(*args)
            elif self._sigterm_handler is not None:
                return signal.default_int_handler(signal.SIGTERM)(*args)
            else:
                return

        for pid, p_map in self._process_map.items():
            p_map['Process'].terminate()

        end_time = time.time() + self.wait_for_kill  # when to die

        while self._process_map and time.time() < end_time:
            for pid, p_map in self._process_map.items():
                p_map['Process'].join(0)

                # This is a race condition if a signal was passed to all children
                try:
                    del self._process_map[pid]
                except KeyError:
                    pass
        # if anyone is done after
        for pid in self._process_map:
            try:
                os.kill(signal.SIGKILL, pid)
            # in case the process has since decided to die, os.kill returns OSError
            except OSError:
                pass
<|endoftext|>


<commit_before># Copyright 2016 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
from tspapi import Source
from tspapi import Sender
import requests
import json

def event_create_good_response(status_code):
    """
    Determines what status codes represent a good response from an API call.
    """
    return status_code == requests.codes.created or status_code == requests.codes.accepted

class BaseEvent(object):
    def __init__(self, *args, **kwargs):
        self._created_at = kwargs['created_at'] if 'created_at' in kwargs else None
        self._event_id = kwargs['event_id'] if 'event_id' in kwargs else None
        self._fingerprint_fields = kwargs['fingerprint_fields'] if 'fingerprint_fields' in kwargs else None
        self._id = kwargs['id'] if 'id' in kwargs else None
        self._message = kwargs['message'] if 'message' in kwargs else None
        self._properties = kwargs['properties'] if 'properties' in kwargs else None
        self._source = kwargs['source'] if 'source' in kwargs else None
        self._sender = kwargs['sender'] if 'sender' in kwargs else None
        self._severity = kwargs['severity'] if 'severity' in kwargs else None
        self._status = kwargs['status'] if 'status' in kwargs else None
        self._tags = kwargs['tags'] if 'tags' in kwargs else None
        self._tenant_id = kwargs['tenant_id'] if 'tenant_id' in kwargs else None
        self._title = kwargs['title'] if 'title' in kwargs else None

        self._received_at = kwargs['received_at'] if 'received_at' in kwargs else None

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "{0}(created_at={1}" \
               ", event_id='{2}'" \
               ", fingerprint_fields='{3}'" \
               ", id='{4}'" \
               ", message='{5}'" \
               ", properties={6}" \
               ", {7}" \
               ", sender='{8}'" \
               ", severity='{9}'" \
               ", status='{10}'" \
               ", tags='{11}'" \
               ", tenant_id={12}" \
               ", title={13}" \
               ")".format(
                self.__class__.__name__,
                self._created_at,
                self._event_id,
                self._fingerprint_fields,
                self._id,
                self._message,
                self._properties,
                self._source.__repr__() if self._source is not None else None,
                self._sender,
                self._severity,
                self._status,
                self._tags,
                self._tenant_id,
                self._title)

    @property
    def created_at(self):
        return self._created_at

    @property
    def event_id(self):
        return self._event_id

    @property
    def fingerprint_fields(self):
        return self._fingerprint_fields

    @property
    def id(self):
        return self._id

    @property
    def message(self):
        return self._message

    @property
    def properties(self):
        return self._properties

    @property
    def received_at(self):
        return self._received_at

    @property
    def title(self):
        return self._title

    @property
    def sender(self):
        return self._sender

    @property
    def severity(self):
        return self._severity

    @property
    def source(self):
        return self._source

    @property
    def status(self):
        return self._status

    @property
    def tags(self):
        return self._tags

    @property
    def tenant_id(self):
        return self._tenant_id

class RawEvent(BaseEvent):
    def __init__(self, *args, **kwargs):
        super(RawEvent, self).__init__(*args, **kwargs)

class Event(BaseEvent):
    def __init__(self, *args, **kwargs):
        super(Event, self).__init__(*args, **kwargs)
        self._id = kwargs['id'] if 'id' in kwargs else None
        self._times_seen = kwargs['times_seen'] if 'times_seen' in kwargs else None

    @property
    def id(self):
        return self._id

    @property
    def times_seen(self):
        return self._times_seen

def serialize_instance(obj):
    d = {}
    if obj.created_at is not None:
        d['createdAt'] = obj.created_at
    if obj.title is not None:
        d['title'] = obj.title
    if obj.source is not None:
        d['source'] = obj.source.serialize()
    return d

def event_create_handle_results(api_result, context=None):
    # Only process if we get HTTP result of 200
    result = None
    if (api_result.status_code == requests.codes.created or
        api_result.status_code == requests.codes.accepted) and len(api_result.text) > 0:
        result = json.loads(api_result.text)
    return result

def event_get_handle_results(api_result, context=None):
    logging.debug("event_get_handle_results")
    events = None
    # Only process if we get HTTP result of 200
    if api_result.status_code == requests.codes.ok:
        print(api_result.text)
        results = json.loads(api_result.text)
        events = []
        for event in results['items']:
            source = Source.dict_to_source(event['source'])
            sender = None
            if 'sender' in event:
                sender = Source.dict_to_source(event['sender'])
            status = None
            if 'status' in event:
                status = event['status']
            properties = None
            if 'properties' in event:
                properties = event['properties']
            severity = None
            if 'severity' in event:
                severity = event['severity']
            events.append(Event(fingerprint_fields=event['fingerprintFields'],
                                first_seen_at=event['firstSeenAt'],
                                id=event['id'],
                                last_seen_at=event['lastSeenAt'],
                                properties=properties,
                                sender=sender,
                                severity=severity,
                                source=source,
                                status=status,
                                tenant_id=['tenantId'],
                                times_seen=event['timesSeen'],
                                title=event['title']))

    return events
<commit_msg>Change for backward capability with production TrueSight Pulse API<commit_after># Copyright 2016 BMC Software, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
from pprint import pprint
from tspapi import Source
from tspapi import Sender
import requests
import json

def event_create_good_response(status_code):
    """
    Determines what status codes represent a good response from an API call.
    """
    return status_code == requests.codes.created or status_code == requests.codes.accepted

class BaseEvent(object):
    def __init__(self, *args, **kwargs):
        self._created_at = kwargs['created_at'] if 'created_at' in kwargs else None
        self._event_id = kwargs['event_id'] if 'event_id' in kwargs else None
        self._fingerprint_fields = kwargs['fingerprint_fields'] if 'fingerprint_fields' in kwargs else None
        self._id = kwargs['id'] if 'id' in kwargs else None
        self._message = kwargs['message'] if 'message' in kwargs else None
        self._properties = kwargs['properties'] if 'properties' in kwargs else None
        self._source = kwargs['source'] if 'source' in kwargs else None
        self._sender = kwargs['sender'] if 'sender' in kwargs else None
        self._severity = kwargs['severity'] if 'severity' in kwargs else None
        self._status = kwargs['status'] if 'status' in kwargs else None
        self._tags = kwargs['tags'] if 'tags' in kwargs else None
        self._tenant_id = kwargs['tenant_id'] if 'tenant_id' in kwargs else None
        self._title = kwargs['title'] if 'title' in kwargs else None

        self._received_at = kwargs['received_at'] if 'received_at' in kwargs else None

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "{0}(created_at={1}" \
               ", event_id='{2}'" \
               ", fingerprint_fields='{3}'" \
               ", id='{4}'" \
               ", message='{5}'" \
               ", properties={6}" \
               ", {7}" \
               ", sender='{8}'" \
               ", severity='{9}'" \
               ", status='{10}'" \
               ", tags='{11}'" \
               ", tenant_id={12}" \
               ", title={13}" \
               ")".format(
                self.__class__.__name__,
                self._created_at,
                self._event_id,
                self._fingerprint_fields,
                self._id,
                self._message,
                self._properties,
                self._source.__repr__() if self._source is not None else None,
                self._sender,
                self._severity,
                self._status,
                self._tags,
                self._tenant_id,
                self._title)

    @property
    def created_at(self):
        return self._created_at

    @property
    def event_id(self):
        return self._event_id

    @property
    def fingerprint_fields(self):
        return self._fingerprint_fields

    @property
    def id(self):
        return self._id

    @property
    def message(self):
        return self._message

    @property
    def properties(self):
        return self._properties

    @property
    def received_at(self):
        return self._received_at

    @property
    def title(self):
        return self._title

    @property
    def sender(self):
        return self._sender

    @property
    def severity(self):
        return self._severity

    @property
    def source(self):
        return self._source

    @property
    def status(self):
        return self._status

    @property
    def tags(self):
        return self._tags

    @property
    def tenant_id(self):
        return self._tenant_id

class RawEvent(BaseEvent):
    def __init__(self, *args, **kwargs):
        super(RawEvent, self).__init__(*args, **kwargs)

class Event(BaseEvent):
    def __init__(self, *args, **kwargs):
        super(Event, self).__init__(*args, **kwargs)
        self._id = kwargs['id'] if 'id' in kwargs else None
        self._times_seen = kwargs['times_seen'] if 'times_seen' in kwargs else None

    @property
    def id(self):
        return self._id

    @property
    def times_seen(self):
        return self._times_seen

def serialize_instance(obj):
    d = {}
    if obj.created_at is not None:
        d['createdAt'] = obj.created_at
    if obj.title is not None:
        d['title'] = obj.title
    if obj.source is not None:
        d['source'] = obj.source.serialize()
    return d

def event_create_handle_results(api_result, context=None):
    # Only process if we get HTTP result of 200
    result = None
    if (api_result.status_code == requests.codes.created or
        api_result.status_code == requests.codes.accepted) and len(api_result.text) > 0:
        result = json.loads(api_result.text)
    return result

def event_get_handle_results(api_result, context=None):
    logging.debug("event_get_handle_results")
    events = None
    # Only process if we get HTTP result of 200
    if api_result.status_code == requests.codes.ok:
        print(api_result.text)
        results = json.loads(api_result.text)
        events = []

        # Regression in API changed 'results' to 'items'
        # check to handle both
        if 'items' in results:
            results_key = 'items'
        else:
            results_key = 'results'

        for event in results[results_key]:
            source = Source.dict_to_source(event['source'])
            sender = None
            if 'sender' in event:
                sender = Source.dict_to_source(event['sender'])
            status = None
            if 'status' in event:
                status = event['status']
            properties = None
            if 'properties' in event:
                properties = event['properties']
            severity = None
            if 'severity' in event:
                severity = event['severity']
            events.append(Event(fingerprint_fields=event['fingerprintFields'],
                                first_seen_at=event['firstSeenAt'],
                                id=event['id'],
                                last_seen_at=event['lastSeenAt'],
                                properties=properties,
                                sender=sender,
                                severity=severity,
                                source=source,
                                status=status,
                                tenant_id=['tenantId'],
                                times_seen=event['timesSeen'],
                                title=event['title']))

    return events
<|endoftext|>


<commit_before>#!/usr/bin/env python

import json
import awsiot
import logging
import sys
import time
import xmlrpclib
import supervisor.xmlrpc

def callback(client, user_data, message):
    try:
        msg = json.loads(message.payload)
    except ValueError:
        msg = None
    logging.debug("received {} {}".format(message.topic, msg))
    for x in args.topic:
        if message.topic.startswith(x):
            commands = filter(None, message.topic.replace(x, '').split('/'))
            if len(commands) > 0:
                cmd = commands.pop(0)
                if cmd == 'getAllProcessInfo':
                    logging.debug("command: {}".format(cmd))
                    results = proxy.supervisor.getAllProcessInfo()
                    logging.info("getAllProcessInfo {}".format(results))
                    if args.thing:
                        supervised = []
                        for s in results:
                            supervised.append('{}({})'.format(s['name'], s['statename']))
                        publisher = awsiot.Publisher(args.endpoint, args.rootCA, args.cert, args.key, args.thing,
                                                     args.groupCA)
                        publisher.publish(awsiot.iot_thing_topic(args.thing),
                                          awsiot.iot_payload(awsiot.REPORTED, {'supervised': ','.join(supervised)}))

                else:
                    logging.warning('Unrecognized command: {}'.format(cmd))
            else:
                logging.warning("No commands: {}".format(message.topic))

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("-s", "--socket_path", help="socket path", default='/var/run/supervisor.sock')
    args = parser.parse_args()

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    subscriber = awsiot.Subscriber(args.endpoint, args.rootCA, args.cert, args.key, args.thing, args.groupCA)

    proxy = xmlrpclib.ServerProxy(
        'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport(
            None, None, serverurl='unix://{}'.format(args.socket_path)))

    if args.topic is not None and len(args.topic) > 0:
        for t in args.topic:
            subscriber.subscribe("{}/#".format(t), callback)
            time.sleep(2)  # pause

    # Loop forever
    try:
        while True:
            time.sleep(0.5)  # sleep needed because CPU race
    except (KeyboardInterrupt, SystemExit):
        sys.exit()
<commit_msg>add supervisor subscriber<commit_after>#!/usr/bin/env python

import json
import awsiot
import logging
import sys
import time
import xmlrpclib
import supervisor.xmlrpc

def callback(client, user_data, message):
    try:
        msg = json.loads(message.payload)
    except ValueError:
        msg = None
    logging.debug("received {} {}".format(message.topic, msg))
    for x in args.topic:
        if message.topic.startswith(x):
            commands = filter(None, message.topic.replace(x, '').split('/'))
            if len(commands) > 0:
                cmd = commands.pop(0)
                if cmd == 'getAllProcessInfo':
                    logging.debug("command: {}".format(cmd))
                    results = proxy.supervisor.getAllProcessInfo()
                    logging.info("getAllProcessInfo {}".format(results))
                    if args.thing:
                        supervised = []
                        for s in results:
                            supervised.append('{}({})'.format(s['name'], s['statename']))
                        publisher = awsiot.Publisher(args.endpoint, args.rootCA, args.cert, args.key, args.thing,
                                                     args.groupCA)
                        publisher.publish(awsiot.iot_thing_topic(args.thing),
                                          awsiot.iot_payload(awsiot.REPORTED, {'supervised': ','.join(supervised)}))

                else:
                    logging.warning('Unrecognized command: {}'.format(cmd))
            else:
                logging.warning("No commands: {}".format(message.topic))

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("--socket_path", help="socket path", default='/var/run/supervisor.sock')
    args = parser.parse_args()

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    subscriber = awsiot.Subscriber(args.endpoint, args.rootCA, args.cert, args.key, args.thing, args.groupCA)

    proxy = xmlrpclib.ServerProxy(
        'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport(
            None, None, serverurl='unix://{}'.format(args.socket_path)))

    if args.topic is not None and len(args.topic) > 0:
        for t in args.topic:
            subscriber.subscribe("{}/#".format(t), callback)
            time.sleep(2)  # pause

    # Loop forever
    try:
        while True:
            time.sleep(0.5)  # sleep needed because CPU race
    except (KeyboardInterrupt, SystemExit):
        sys.exit()
<|endoftext|>


<commit_before>class TimetableGenerator:

    def __init__(self):
        self.generated = []
        self.generated2 = []
        self.total_combinations = 0

    def store_timetable(self, tt):
        self.total_combinations += 1
        tt.heuristic = tt.total_time()
        tt.heuristic2 = tt.total_time2()

        if len(self.generated) <= 99:
            self.generated.append(tt)
            self.generated2.append(tt)
        else:
            if tt.heuristic < self.generated[99].heuristic:  # tt is better
                self.generated.pop(99)
                self.generated.append(tt)
                self.generated.sort(key=lambda tt: tt.heuristic)

            if tt.heuristic2 < self.generated2[99].heuristic2:  # tt is better
                self.generated2.pop(99)
                self.generated2.append(tt)
                self.generated2.sort(key=lambda tt: tt.heuristic2)

    def generate_timetables(self, lesson_blocks):
        self.generate(Timetable(), lesson_blocks)

        for i in xrange(1, len(self.generated)+1):
            self.generated[len(self.generated) - i].score += i

        for i in xrange(1, len(self.generated2)+1):
            self.generated2[len(self.generated2) - i].score += i

        self.generated.sort(key=lambda tt: tt.score)

    def generate(self, timetable, lesson_blocks):
        if not lesson_blocks:
            self.store_timetable(timetable)
        else:
            next_lesson_block = lesson_blocks[0]
            for shift in next_lesson_block.shifts:
                #if timetable.supports(shift):
                self.generate(timetable.append_shift(shift), lesson_blocks[1:])

class Timetable:

    def __init__(self):
        self.lessons = []
        self.score = 0

    def append_shift(self, shift):
        new_timetable = Timetable()
        new_timetable.lessons.extend(self.lessons)
        new_timetable.lessons.extend(shift.slots)
        return new_timetable

    def supports(self, shift):
        for slot in shift.slots:
            for existing in self.lessons:
                if (slot.overlaps_with(existing)):
                    return False
        return True

    #def is_feasible(self):
    #    for slot in self.lessons:
    #        for other in self.lessons:
    #            if (slot is not other) and (slot.overlaps_with(other)):
    #                return False
    #    return True

    # heuristics for selecting timetables
    def total_time2(self):
        result = 0
        for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
            daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
            if daily_lessons:
                earliest_start = min([slot.start.minutes for slot in daily_lessons])
                latest_end = max([slot.end.minutes for slot in daily_lessons])
                interval = latest_end - earliest_start
        return result

    def total_time(self):
        result = 0
        for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
            daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
            if daily_lessons:
                if len(daily_lessons) > 1:
                    daily_lessons.sort(key=lambda x: x.start.minutes)
                    prev = daily_lessons[0].end.minutes
                    for slot in daily_lessons[1:]:
                        if slot.start.minutes > prev:
                            result = result + (slot.start.minutes - prev)
                        else:
                            result = result + 120

                        prev = slot.end.minutes

        return result

class Course(object):

    def __init__(self, name):
        self.name = name
        self.lesson_blocks = []

    def add_lesson_block(self, lesson_block):
        self.lesson_blocks.append(lesson_block)
        lesson_block.parent_course = self

    def get_block_by_category(self, category):
        for block in self.lesson_blocks:
            if block.category == category:
                return block

class LessonBlock:

    def __init__(self, category):
        self.category = category
        self.shifts = []

    def add_shift(self, shift):
        self.shifts.append(shift)
        shift.parent_lesson_block = self

class Shift:

    def __init__(self, name):
        self.name = name
        self.slots = []

    def add_lesson_slot(self, lesson_slot):
        self.slots.append(lesson_slot)
        lesson_slot.parent_shift = self

class LessonSlot:

    def __init__(self, day, start, end, room):
        self.day = day
        self.start = start
        self.end = end
        self.room = room

    def course_name(self):
        return self.parent_shift.parent_lesson_block.parent_course.name

    def lesson_category(self):
        return self.parent_shift.parent_lesson_block.category

	def overlaps_with_group(self, group):
		for other in group:
			if self != other:
				if self.overlaps_with(other) == True:
					return True
		return False

    def overlaps_with(self, other):
        return self.day == other.day and \
               (self.start.is_before(other.end) or self.end.is_after(other.start))

class Weekday:
    MONDAY = 0
    TUESDAY = 1
    WEDNESDAY = 2
    THURSDAY = 3
    FRIDAY = 4
    SATURDAY = 5
    SUNDAY = 6

class Time:

    def __init__(self, hour, minute):
        self.minutes = hour * 60 + minute

    def is_before(self, other):
        return self.minutes < other.minutes

    def is_after(self, other):
        return self.minutes > other.minutes

    def __str__(self):
        return "%d:%02d" % (self.minutes/60, self.minutes%60)
<commit_msg>correcting effin overlaps<commit_after>class TimetableGenerator:

	def __init__(self):
		self.generated = []
		self.generated2 = []
		self.total_combinations = 0

	def store_timetable(self, tt):
		self.total_combinations += 1
		tt.heuristic = tt.total_time()
		tt.heuristic2 = tt.total_time2()

		if len(self.generated) <= 99:
			self.generated.append(tt)
			self.generated2.append(tt)
		else:
			if tt.heuristic < self.generated[99].heuristic:  # tt is better
				self.generated.pop(99)
				self.generated.append(tt)
				self.generated.sort(key=lambda tt: tt.heuristic)

			if tt.heuristic2 < self.generated2[99].heuristic2:  # tt is better
				self.generated2.pop(99)
				self.generated2.append(tt)
				self.generated2.sort(key=lambda tt: tt.heuristic2)

	def generate_timetables(self, lesson_blocks):
		self.generate(Timetable(), lesson_blocks)

		for i in xrange(1, len(self.generated)+1):
			self.generated[len(self.generated) - i].score += i

		for i in xrange(1, len(self.generated2)+1):
			self.generated2[len(self.generated2) - i].score += i

		self.generated.sort(key=lambda tt: tt.score)

	def generate(self, timetable, lesson_blocks):
		if not lesson_blocks:
			self.store_timetable(timetable)
		else:
			next_lesson_block = lesson_blocks[0]
			for shift in next_lesson_block.shifts:
				#if timetable.supports(shift):
				self.generate(timetable.append_shift(shift), lesson_blocks[1:])

class Timetable:

	def __init__(self):
		self.lessons = []
		self.score = 0

	def append_shift(self, shift):
		new_timetable = Timetable()
		new_timetable.lessons.extend(self.lessons)
		new_timetable.lessons.extend(shift.slots)
		return new_timetable

	def supports(self, shift):
		for slot in shift.slots:
			for existing in self.lessons:
				if (slot.overlaps_with(existing)):
					return False
		return True

	#def is_feasible(self):
	#	for slot in self.lessons:
	#		for other in self.lessons:
	#			if (slot is not other) and (slot.overlaps_with(other)):
	#				return False
	#	return True

	# heuristics for selecting timetables
	def total_time2(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				earliest_start = min([slot.start.minutes for slot in daily_lessons])
				latest_end = max([slot.end.minutes for slot in daily_lessons])
				interval = latest_end - earliest_start
		return result

	def total_time(self):
		result = 0
		for weekday in range(Weekday.MONDAY, Weekday.SUNDAY):
			daily_lessons = [slot for slot in self.lessons if slot.day == weekday]
			if daily_lessons:
				if len(daily_lessons) > 1:
					daily_lessons.sort(key=lambda x: x.start.minutes)
					prev = daily_lessons[0].end.minutes
					for slot in daily_lessons[1:]:
						if slot.start.minutes > prev:
							result = result + (slot.start.minutes - prev)
						else:
							result = result + 120

						prev = slot.end.minutes

		return result

class Course(object):

	def __init__(self, name):
		self.name = name
		self.lesson_blocks = []

	def add_lesson_block(self, lesson_block):
		self.lesson_blocks.append(lesson_block)
		lesson_block.parent_course = self

	def get_block_by_category(self, category):
		for block in self.lesson_blocks:
			if block.category == category:
				return block

class LessonBlock:

	def __init__(self, category):
		self.category = category
		self.shifts = []

	def add_shift(self, shift):
		self.shifts.append(shift)
		shift.parent_lesson_block = self

class Shift:

	def __init__(self, name):
		self.name = name
		self.slots = []

	def add_lesson_slot(self, lesson_slot):
		self.slots.append(lesson_slot)
		lesson_slot.parent_shift = self

class LessonSlot:

	def __init__(self, day, start, end, room):
		self.day = day
		self.start = start
		self.end = end
		self.room = room

	def course_name(self):
		return self.parent_shift.parent_lesson_block.parent_course.name

	def lesson_category(self):
		return self.parent_shift.parent_lesson_block.category

	def overlaps_with_group(self, group):
		for other in group:
			if self != other:
				if self.overlaps_with(other) == True:
					return True
		return False

	def overlaps_with(self, other):
		return self.day == other.day and \
			   (self.start.is_before(other.end) or self.end.is_after(other.start))

class Weekday:
	MONDAY = 0
	TUESDAY = 1
	WEDNESDAY = 2
	THURSDAY = 3
	FRIDAY = 4
	SATURDAY = 5
	SUNDAY = 6

class Time:

	def __init__(self, hour, minute):
		self.minutes = hour * 60 + minute

	def is_before(self, other):
		return self.minutes < other.minutes

	def is_after(self, other):
		return self.minutes > other.minutes

	def __str__(self):
		return "%d:%02d" % (self.minutes/60, self.minutes%60)
<|endoftext|>


<commit_before># Tweepy
# Copyright 2009-2020 Joshua Roesslein
# See LICENSE for details.

import six

class TweepError(Exception):
    """Tweepy exception"""

    def __init__(self, reason, response=None, api_code=None):
        self.reason = six.text_type(reason)
        self.response = response
        self.api_code = api_code
        super(TweepError, self).__init__(reason)

    def __str__(self):
        return self.reason

def is_rate_limit_error_message(message):
    """Check if the supplied error message belongs to a rate limit error."""
    return isinstance(message, list) \
        and len(message) > 0 \
        and 'code' in message[0] \
        and message[0]['code'] == 88

class RateLimitError(TweepError):
    """Exception for Tweepy hitting the rate limit."""
    # RateLimitError has the exact same properties and inner workings
    # as TweepError for backwards compatibility reasons.
    pass
<commit_msg>Improve formatting in is_rate_limit_error_message function<commit_after># Tweepy
# Copyright 2009-2020 Joshua Roesslein
# See LICENSE for details.

import six

class TweepError(Exception):
    """Tweepy exception"""

    def __init__(self, reason, response=None, api_code=None):
        self.reason = six.text_type(reason)
        self.response = response
        self.api_code = api_code
        super(TweepError, self).__init__(reason)

    def __str__(self):
        return self.reason

def is_rate_limit_error_message(message):
    """Check if the supplied error message belongs to a rate limit error."""
    return (isinstance(message, list) and len(message) > 0 and
            'code' in message[0] and message[0]['code'] == 88)

class RateLimitError(TweepError):
    """Exception for Tweepy hitting the rate limit."""
    # RateLimitError has the exact same properties and inner workings
    # as TweepError for backwards compatibility reasons.
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.contrib.gis.db import models
from django.contrib.postgres.fields import HStoreField

class Tweet(models.Model):

    id = models.BigIntegerField(primary_key=True)
    author = models.ForeignKey('User', related_name='tweets')

    created_at = models.DateTimeField()

    text = models.TextField()

    favorited = models.BooleanField(default=False)
    retweeted = models.BooleanField(default=False)
    truncated = models.BooleanField(default=False)

    source = models.CharField(max_length=100)
    source_url = models.URLField(null=True)

    lang = models.CharField(max_length=10)

    favorite_count = models.PositiveIntegerField()
    retweet_count = models.PositiveIntegerField()
    replies_count = models.PositiveIntegerField(null=True)

    in_reply_to_status = models.ForeignKey('Tweet', null=True, related_name='replies')
    in_reply_to_user = models.ForeignKey('User', null=True, related_name='replies')

    favorites_users = models.ManyToManyField('User', related_name='favorites')
    retweeted_status = models.ForeignKey('Tweet', null=True, related_name='retweets')

    possibly_sensitive = models.NullBooleanField()
    is_quote_status = models.BooleanField(default=False)

    contributors = HStoreField(null=True)

    place = models.ForeignKey("Place", null=True)

    coordinates = models.PointField(null=True)
    geo = models.PointField(null=True)

    entities = HStoreField(null=True)

    objects = models.GeoManager()

    def __unicode__(self):
        return u'%s: %s' % (self.author, self.text)

    class Meta:
        ordering = ["-created_at"]
        index_together = ["created_at", "coordinates"]

class Place(models.Model):
    id = models.CharField(max_length=20, primary_key=True)
    full_name = models.CharField(max_length=200)
    name = models.CharField(max_length=50)
    country = models.CharField(max_length=100)
    country_code = models.CharField(max_length=4)
    bounding_box = models.PolygonField()
    place_type = models.CharField(max_length=50)
    url = models.URLField()
    attributes = HStoreField(null=True)

    objects = models.GeoManager()

class User(models.Model):

    id = models.BigIntegerField(primary_key=True)
    screen_name = models.CharField(u'Screen name', max_length=50, unique=True)

    name = models.CharField(u'Name', max_length=100)
    description = models.TextField(u'Description')
    location = models.CharField(u'Location', max_length=100)
    time_zone = models.CharField(u'Time zone', max_length=100, null=True)
    lang = models.CharField(max_length=10)

    created_at = models.DateTimeField()

    contributors_enabled = models.NullBooleanField(u'Contributors enabled', default=False)
    default_profile = models.NullBooleanField(u'Default profile', default=False)
    default_profile_image = models.NullBooleanField(u'Default profile image', default=False)
    follow_request_sent = models.NullBooleanField(u'Follow request sent', default=False)
    following = models.NullBooleanField(u'Following', default=False)
    geo_enabled = models.NullBooleanField(u'Geo enabled', default=False)
    is_translator = models.NullBooleanField(u'Is translator', default=False)
    is_translation_enabled = models.NullBooleanField(u'Is translation enable', default=False)
    has_extended_profile = models.NullBooleanField(u'Has extended profile', default=False)
    notifications = models.NullBooleanField(u'Notifications', default=False)
    profile_use_background_image = models.NullBooleanField(u'Profile use background image', default=False)
    protected = models.NullBooleanField(u'Protected', default=False)
    verified = models.NullBooleanField(u'Verified', default=False)

    profile_background_image_url = models.URLField(max_length=300)
    profile_background_image_url_https = models.URLField(max_length=300)
    profile_background_tile = models.NullBooleanField(default=False)
    profile_background_color = models.CharField(max_length=6)
    profile_banner_url = models.URLField(max_length=300)
    profile_image_url = models.URLField(max_length=300)
    profile_image_url_https = models.URLField(max_length=300)
    url = models.URLField(max_length=300, null=True)

    profile_link_color = models.CharField(max_length=6)
    profile_sidebar_border_color = models.CharField(max_length=6)
    profile_sidebar_fill_color = models.CharField(max_length=6)
    profile_text_color = models.CharField(max_length=6)

    favourites_count = models.PositiveIntegerField()
    followers_count = models.PositiveIntegerField()
    friends_count = models.PositiveIntegerField()
    listed_count = models.PositiveIntegerField()
    statuses_count = models.PositiveIntegerField()
    utc_offset = models.IntegerField(null=True)

    entities = HStoreField(null=True)

    followers = models.ManyToManyField('User')

    def __unicode__(self):
        return self.name<commit_msg>missing contained_within in Place model.<commit_after># -*- coding: utf-8 -*-
from django.contrib.gis.db import models
from django.contrib.postgres.fields import HStoreField

class Tweet(models.Model):

    id = models.BigIntegerField(primary_key=True)
    author = models.ForeignKey('User', related_name='tweets')

    created_at = models.DateTimeField()

    text = models.TextField()

    favorited = models.BooleanField(default=False)
    retweeted = models.BooleanField(default=False)
    truncated = models.BooleanField(default=False)

    source = models.CharField(max_length=100)
    source_url = models.URLField(null=True)

    lang = models.CharField(max_length=10)

    favorite_count = models.PositiveIntegerField()
    retweet_count = models.PositiveIntegerField()
    replies_count = models.PositiveIntegerField(null=True)

    in_reply_to_status = models.ForeignKey('Tweet', null=True, related_name='replies')
    in_reply_to_user = models.ForeignKey('User', null=True, related_name='replies')

    favorites_users = models.ManyToManyField('User', related_name='favorites')
    retweeted_status = models.ForeignKey('Tweet', null=True, related_name='retweets')

    possibly_sensitive = models.NullBooleanField()
    is_quote_status = models.BooleanField(default=False)

    contributors = HStoreField(null=True)

    place = models.ForeignKey("Place", null=True)

    coordinates = models.PointField(null=True)
    geo = models.PointField(null=True)

    entities = HStoreField(null=True)

    objects = models.GeoManager()

    def __unicode__(self):
        return u'%s: %s' % (self.author, self.text)

    class Meta:
        ordering = ["-created_at"]
        index_together = ["created_at", "coordinates"]

class Place(models.Model):
    id = models.CharField(max_length=20, primary_key=True)
    full_name = models.CharField(max_length=200)
    name = models.CharField(max_length=50)
    country = models.CharField(max_length=100)
    country_code = models.CharField(max_length=4)
    bounding_box = models.PolygonField()
    place_type = models.CharField(max_length=50)
    url = models.URLField()
    contained_within = HStoreField(null=True)
    attributes = HStoreField(null=True)

    objects = models.GeoManager()

class User(models.Model):

    id = models.BigIntegerField(primary_key=True)
    screen_name = models.CharField(u'Screen name', max_length=50, unique=True)

    name = models.CharField(u'Name', max_length=100)
    description = models.TextField(u'Description')
    location = models.CharField(u'Location', max_length=100)
    time_zone = models.CharField(u'Time zone', max_length=100, null=True)
    lang = models.CharField(max_length=10)

    created_at = models.DateTimeField()

    contributors_enabled = models.NullBooleanField(u'Contributors enabled', default=False)
    default_profile = models.NullBooleanField(u'Default profile', default=False)
    default_profile_image = models.NullBooleanField(u'Default profile image', default=False)
    follow_request_sent = models.NullBooleanField(u'Follow request sent', default=False)
    following = models.NullBooleanField(u'Following', default=False)
    geo_enabled = models.NullBooleanField(u'Geo enabled', default=False)
    is_translator = models.NullBooleanField(u'Is translator', default=False)
    is_translation_enabled = models.NullBooleanField(u'Is translation enable', default=False)
    has_extended_profile = models.NullBooleanField(u'Has extended profile', default=False)
    notifications = models.NullBooleanField(u'Notifications', default=False)
    profile_use_background_image = models.NullBooleanField(u'Profile use background image', default=False)
    protected = models.NullBooleanField(u'Protected', default=False)
    verified = models.NullBooleanField(u'Verified', default=False)

    profile_background_image_url = models.URLField(max_length=300)
    profile_background_image_url_https = models.URLField(max_length=300)
    profile_background_tile = models.NullBooleanField(default=False)
    profile_background_color = models.CharField(max_length=6)
    profile_banner_url = models.URLField(max_length=300)
    profile_image_url = models.URLField(max_length=300)
    profile_image_url_https = models.URLField(max_length=300)
    url = models.URLField(max_length=300, null=True)

    profile_link_color = models.CharField(max_length=6)
    profile_sidebar_border_color = models.CharField(max_length=6)
    profile_sidebar_fill_color = models.CharField(max_length=6)
    profile_text_color = models.CharField(max_length=6)

    favourites_count = models.PositiveIntegerField()
    followers_count = models.PositiveIntegerField()
    friends_count = models.PositiveIntegerField()
    listed_count = models.PositiveIntegerField()
    statuses_count = models.PositiveIntegerField()
    utc_offset = models.IntegerField(null=True)

    entities = HStoreField(null=True)

    followers = models.ManyToManyField('User')

    def __unicode__(self):
        return self.name<|endoftext|>


<commit_before>from __future__ import absolute_import

from django.contrib import admin

from .models import Tweet

@admin.register(Tweet)
class TweetAdmin(admin.ModelAdmin):
    list_display = ('modified', 'tweeted', 'text')
    fields = ('text', 'tweeted')
    search_fields = ('text',)
    ordering = ('created',)
<commit_msg>Updates to admin fields<commit_after>from __future__ import absolute_import

from django.contrib import admin

from .models import Tweet

@admin.register(Tweet)
class TweetAdmin(admin.ModelAdmin):
    list_display = ('created', 'tweeted', 'text')
    fields = ('text',)
    search_fields = ('text',)
    ordering = ('created',)
<|endoftext|>


<commit_before>#!/usr/bin/python

__author__ = 'ay'
import numpy as np
from numpy import random as rand
import argparse, h5py, math, code

parser=argparse.ArgumentParser(description='Simulate Phenotypes with Particular Parameters.h')

parser.add_argument('genofiles',type=str,help='Text file with location of genotypes for chr i on line i')
parser.add_argument('phenofile',type=str,help='Location of hdf5 file to output phenotypes to')
parser.add_argument('nphen',type=int,help='Number of phenotypes to simulate')
parser.add_argument('nc',type=int,help='Number of Causal Variants in Model')
parser.add_argument('nd',type=int,help='Number of causal variants with dominance effects')
parser.add_argument('n_interact',type=int,help='Number of pairs of causal variants that interact')
parser.add_argument('h2',type=float,help='Narrow Sense Heritability')
parser.add_argument('hd',type=float,help='Proportion of Variance from Dominance Effects')
parser.add_argument('h2_epi',type=float,help='Proportion of Variance from Interactions')
parser.add_argument('--save_causal_gts',action='store_true',default=False)
#parser.add_argument('--seed',type=int,help='Set seed for random sampling of genetic effects',
#                    default=64)
parser.add_argument('--min_maf',type=float,help='Minimum minor allele frequency of causal loci',
                    default=0.05)
parser.add_argument('--max_missing',type=float,help='Maximum percentage missing calls for causal locus',
                    default=1)
parser.add_argument('--normal_effects',action='store_true',default=False,
                    help='Normally distributed effect size distribution. Default is equal effect size but random sign.')
parser.add_argument('--variance_effect',type=float,help='Size of variance effect for loci with additive effect',default=0.0)
args=parser.parse_args()
genofiles=args.genofiles
phenout=args.phenofile
nphen=args.nphen
nc=args.nc
nd=args.nd
n_interact=args.n_interact
h2=args.h2
hd=args.hd
h2_epi=args.h2_epi

# Set random seed
#rand.seed(args.seed)

def dom_convert(g,f):
    g=int(g)
    if g==0:
        return f**2
    elif g==1:
        return -f*(1-f)
    elif g==2:
        return (1-f)**2
    else:
        print(g)
        raise(ValueError)

def equal_effect_sim(nvar):
    return 2*rand.binomial(1,0.5,size=nvar)-1

def phenosim(add_gts,dom_gts,interaction_gts,h2,hd,h2_epi,equal_effects,var_effect):
    nc=add_gts.shape[1]
    N=add_gts.shape[0]
    # Generate additive effects
    if equal_effects:
        add_effects=equal_effect_sim(nc)
    else:
        add_effects=rand.randn((nc))
    # Generate Additive Component of Model
    A=np.dot(add_gts,add_effects)
    # Scale to appropriate variance
    A=np.sqrt(h2)*A/A.std()

    # Generate dominance effects
    if dom_gts.shape[0]>0:
        nd=dom_gts.shape[1]
        if equal_effects:
            dom_effects=equal_effect_sim(nd)
        else:
            dom_effects=rand.randn((nd))
        # Generate Dominance component of model
        D=np.dot(dom_gts,dom_effects)
        # Scale to have appropriate variance
        D=np.sqrt(hd)*D/D.std()
    else:
        D=0

    # Sample interaction effects
    if h2_epi>0:
        n_interact=interaction_gts.shape[1]
        if equal_effects:
            epi_effects=equal_effect_sim(n_interact)
        else:
            epi_effects=rand.randn((n_interact))
        Epi=np.dot(interaction_gts,epi_effects)
        Epi=np.sqrt(h2_epi)*Epi/Epi.std()
    else:
        Epi=0

    # Simulate residual error
    E=rand.randn((N))
    if var_effect > 0:
        var_effects = var_effect*equal_effect_sim(nc)
        V = np.exp(np.dot(add_gts, var_effects))
    else:
        V=np.ones((nc))
    E_var=(1-h2-h2_epi-hd)*V
    E=np.array([np.random.normal(scale=E_var[x]) for x in xrange(0,N)])

    # Combine to form phenotype
    Y=A+D+Epi+E
    return(Y)

# Find out number of chromosomes and genotypes in each chr
gfiles=open(genofiles,'r')
chr_lengths=[]
genofile=gfiles.readline().split('\n')[0]
nchr=0
while len(genofile)>0:
    ghdf5=h5py.File(genofile,'r')
    genotypes=np.transpose(ghdf5['genotypes'])
    N=genotypes.shape[0]
    chr_lengths.append(genotypes.shape[1])
    nchr+=1
    ghdf5.close()
    genofile=gfiles.readline().split('\n')[0]

gfiles.close()
genome_length=sum(chr_lengths)

if nc>genome_length:
	raise(ValueError("Number of loci, "+str(genome_length)+ ", less than number of causal loci in simulation, "+str(nc)))
print('Simulating Phenotype from '+str(genome_length)+' variants from '+
      str(nchr)+' chromosomes for '+str(N)+' individuals')
print('Minimum MAF: '+str(args.min_maf))
print('Max missingness: '+str(args.max_missing)+'%')

# Store causal loci by chromosome and index within chromosome gts
causal_loci=np.zeros((nc,2),dtype=int)
# Store causal gts in array
causal_gts=np.zeros((N,nc))
n_selected=0
gfiles=open(genofiles,'r')
for i in xrange(0,nchr):
    genofile=gfiles.readline().split('\n')[0]
    genofile=h5py.File(genofile,'r')
    genotypes=np.array(genofile['genotypes']).T
    # Number of causal variants from this chromosome
    if i==nchr-1:
	nc_chr=nc-n_selected
    else:
	nc_chr=int(math.floor(float(chr_lengths[i])/float(genome_length)*nc))
    if nc_chr>0:
    	# Skip this many columns between causal variants
    	colskip=int(chr_lengths[i]/nc_chr)-1
    	# Record indices of causal variants within chromosome
    	causal_chr=np.array(range(0,chr_lengths[i],colskip),dtype=int)
    	causal_chr=causal_chr[0:nc_chr]
        # Select nearest locus with MAF greater than min_maf
        for l in xrange(0,nc_chr):
            loc_index=causal_chr[l]
            g=genotypes[:,loc_index]
            print(loc_index)
            missing=100*float(np.sum(g<0))/float(genotypes.shape[0])
            freq=freq=np.mean(g[g>0])/2.0
            # Mean imputation
            g[g<0]=freq*2.0
            if freq>0.5:
                freq=1-freq
            # Check if it passes threshold
            while freq<args.min_maf or missing>args.max_missing:
                loc_index+=1
                g=genotypes[:,loc_index]
                missing=100*float(np.sum(g<0))/float(genotypes.shape[0])
                freq=np.mean(g[g>0])/2.0
                # Mean imputation
                g[g<0]=freq*2.0
                if freq>0.5:
                    freq=1-freq
            causal_chr[l]=loc_index
            print(loc_index)
    	# Store causal variant locations
    	causal_loci[n_selected:(n_selected+nc_chr),0]=i+1
    	causal_loci[n_selected:(n_selected+nc_chr),1]=causal_chr
    	# Add causal variants from this chromosome to the matrix of causal vars
        print('Grabbing genotypes')
    	causal_gts[:,n_selected:(n_selected+nc_chr)]=genotypes[:,causal_chr]
        print('Got genotypes')
	n_selected+=nc_chr
gfiles.close()
#code.interact(local=locals())
# Record causal loci in phenofile
phenofile=h5py.File(phenout,'w')
phenofile.create_dataset('causal',data=causal_loci)

####################
# Mean normalise variables
add_gts=causal_gts-causal_gts.mean(axis=0)

# Choose SNPs to have dominance effects
if nd>0:
    dom_indices=[i for i in xrange(0,nc)]
    dom_indices=np.array(rand.choice(dom_indices,size=nd,replace=False),dtype=int)
    dom_gts=np.array(causal_gts[:,dom_indices],dtype=float)
    freqs=np.mean(dom_gts,axis=1)/2.0
    for i in xrange(0,nd):
        freq=freqs[i]
        for j in xrange(0,N):
            dom_gts[j,i]=dom_convert(dom_gts[j,i],freq)

    # Convert indices relative to additive to indices in SNP file
    dom_indices=causal_loci[dom_indices,]
    phenofile.create_dataset('dom',data=dom_indices)
else:
    dom_gts=np.array(())

## Interactions
npairs=nc*(nc-1)/2
# Array for interaction genotypes
interaction_gts=np.zeros((N,n_interact))

# Sample pairs to have interactions
interact_indices=rand.choice(npairs,size=n_interact,replace=False)
# Record which SNPs interact
interacting_snp_indices=np.zeros((n_interact,4),dtype=int)

# Fill in interaction genotypes
pair_count=0
interact_count=0
for i in xrange(0,nc):
    for j in xrange(0,i):
        if pair_count in interact_indices:
            # Record which SNPs interact
            interacting_snp_indices[interact_count,0:2]=causal_loci[i,:]
            interacting_snp_indices[interact_count,2:4]=causal_loci[j,:]
	    # Form interaction genotypes
            interaction_gts[:,interact_count]=add_gts[:,i]*add_gts[:,j]
            interact_count+=1
        pair_count+=1

phenofile.create_dataset('interactions',data=interacting_snp_indices)

# Simulate phenotypes and save to sim group of screen hdf5 file
phenotypes=phenofile.create_dataset('phenotypes',(N,nphen),dtype=float)
print('Simulating Phenotypes')
if args.normal_effects:
    equal_effects=False
else:
    equal_effects=True
for i in xrange(0,nphen):
    if nd>0:
        phenotypes[:,i]=phenosim(add_gts,dom_gts,interaction_gts,h2,hd,h2_epi,equal_effects,args.variance_effect)
    else:
        phenotypes[:,i]=phenosim(add_gts,np.array(()),interaction_gts,h2,hd,h2_epi,equal_effects,args.variance_effect)

# Save causal genotypes (including dominance and epistasis)
if args.save_causal_gts:
	phenofile.create_dataset('causal_gts',data=causal_gts)
	phenofile.create_dataset('interaction_gts',data=interaction_gts)
	phenofile.create_dataset('dom_gts',data=dom_gts)
phenofile.close()<commit_msg>interact<commit_after>#!/usr/bin/python

__author__ = 'ay'
import numpy as np
from numpy import random as rand
import argparse, h5py, math, code

def id_dict_make(ids):
    if not type(ids)==np.ndarray:
        raise(ValueError('Unsupported ID type: should be numpy nd.array'))
    id_dict={}
    for id_index in xrange(0,len(ids)):
        id_dict[tuple(ids[id_index,:])]=id_index
    return id_dict

parser=argparse.ArgumentParser(description='Simulate Phenotypes with Particular Parameters.h')

parser.add_argument('genofiles',type=str,help='Text file with location of genotypes for chr i on line i')
parser.add_argument('phenofile',type=str,help='Location of hdf5 file to output phenotypes to')
parser.add_argument('nphen',type=int,help='Number of phenotypes to simulate')
parser.add_argument('nc',type=int,help='Number of Causal Variants in Model')
parser.add_argument('nd',type=int,help='Number of causal variants with dominance effects')
parser.add_argument('n_interact',type=int,help='Number of pairs of causal variants that interact')
parser.add_argument('h2',type=float,help='Narrow Sense Heritability')
parser.add_argument('hd',type=float,help='Proportion of Variance from Dominance Effects')
parser.add_argument('h2_epi',type=float,help='Proportion of Variance from Interactions')
parser.add_argument('--save_causal_gts',action='store_true',default=False)
#parser.add_argument('--seed',type=int,help='Set seed for random sampling of genetic effects',
#                    default=64)
parser.add_argument('--min_maf',type=float,help='Minimum minor allele frequency of causal loci',
                    default=0.05)
parser.add_argument('--max_missing',type=float,help='Maximum percentage missing calls for causal locus',
                    default=1)
parser.add_argument('--normal_effects',action='store_true',default=False,
                    help='Normally distributed effect size distribution. Default is equal effect size but random sign.')
parser.add_argument('--variance_effect',type=float,help='Size of variance effect for loci with additive effect',default=0.0)
args=parser.parse_args()
genofiles=args.genofiles
phenout=args.phenofile
nphen=args.nphen
nc=args.nc
nd=args.nd
n_interact=args.n_interact
h2=args.h2
hd=args.hd
h2_epi=args.h2_epi

# Set random seed
#rand.seed(args.seed)

def dom_convert(g,f):
    g=int(g)
    if g==0:
        return f**2
    elif g==1:
        return -f*(1-f)
    elif g==2:
        return (1-f)**2
    else:
        print(g)
        raise(ValueError)

def equal_effect_sim(nvar):
    return 2*rand.binomial(1,0.5,size=nvar)-1

def phenosim(add_gts,dom_gts,interaction_gts,h2,hd,h2_epi,equal_effects,var_effect):
    nc=add_gts.shape[1]
    N=add_gts.shape[0]
    # Generate additive effects
    if equal_effects:
        add_effects=equal_effect_sim(nc)
    else:
        add_effects=rand.randn((nc))
    # Generate Additive Component of Model
    A=np.dot(add_gts,add_effects)
    # Scale to appropriate variance
    A=np.sqrt(h2)*A/A.std()

    # Generate dominance effects
    if dom_gts.shape[0]>0:
        nd=dom_gts.shape[1]
        if equal_effects:
            dom_effects=equal_effect_sim(nd)
        else:
            dom_effects=rand.randn((nd))
        # Generate Dominance component of model
        D=np.dot(dom_gts,dom_effects)
        # Scale to have appropriate variance
        D=np.sqrt(hd)*D/D.std()
    else:
        D=0

    # Sample interaction effects
    if h2_epi>0:
        n_interact=interaction_gts.shape[1]
        if equal_effects:
            epi_effects=equal_effect_sim(n_interact)
        else:
            epi_effects=rand.randn((n_interact))
        Epi=np.dot(interaction_gts,epi_effects)
        Epi=np.sqrt(h2_epi)*Epi/Epi.std()
    else:
        Epi=0

    # Simulate residual error
    E=rand.randn((N))
    if var_effect > 0:
        var_effects = var_effect*equal_effect_sim(nc)
        V = np.exp(np.dot(add_gts, var_effects))
    else:
        V=np.ones((nc))
    E_var=(1-h2-h2_epi-hd)*V
    E=np.array([np.random.normal(scale=E_var[x]) for x in xrange(0,N)])

    # Combine to form phenotype
    Y=A+D+Epi+E
    return(Y)

# Find out number of chromosomes and genotypes in each chr
gfiles=open(genofiles,'r')
chr_lengths=[]
genofile=gfiles.readline().split('\n')[0]
nchr=0
while len(genofile)>0:
    ghdf5=h5py.File(genofile,'r')
    genotypes=np.transpose(ghdf5['genotypes'])
    N=genotypes.shape[0]
    chr_lengths.append(genotypes.shape[1])
    nchr+=1
    ghdf5.close()
    genofile=gfiles.readline().split('\n')[0]

gfiles.close()
genome_length=sum(chr_lengths)

if nc>genome_length:
	raise(ValueError("Number of loci, "+str(genome_length)+ ", less than number of causal loci in simulation, "+str(nc)))
print('Simulating Phenotype from '+str(genome_length)+' variants from '+
      str(nchr)+' chromosomes for '+str(N)+' individuals')
print('Minimum MAF: '+str(args.min_maf))
print('Max missingness: '+str(args.max_missing)+'%')

# Store causal loci by chromosome and index within chromosome gts
causal_loci=np.zeros((nc,2),dtype=int)
# Store causal gts in array
causal_gts=np.zeros((N,nc))
n_selected=0
gfiles=open(genofiles,'r')
for i in xrange(0,nchr):
    genofile=gfiles.readline().split('\n')[0]
    genofile=h5py.File(genofile,'r')
    genotypes=np.array(genofile['genotypes']).T
    # Get sample IDs
    if i==0:
        sample_id=np.array(genofile['sample_id'])
        sample_id_dict=id_dict_make(sample_id)
    else:
        sample_i=np.array(geno_file['sample_id'])
        sample_i_match = np.array([sample_id_dict[tuple(x)] for x in sample_i])
        genotypes=genotypes[sample_i_match,:]
    # Number of causal variants from this chromosome
    if i==nchr-1:
	nc_chr=nc-n_selected
    else:
	nc_chr=int(math.floor(float(chr_lengths[i])/float(genome_length)*nc))
    if nc_chr>0:
    	# Skip this many columns between causal variants
    	colskip=int(chr_lengths[i]/nc_chr)-1
    	# Record indices of causal variants within chromosome
    	causal_chr=np.array(range(0,chr_lengths[i],colskip),dtype=int)
    	causal_chr=causal_chr[0:nc_chr]
        # Select nearest locus with MAF greater than min_maf
        for l in xrange(0,nc_chr):
            loc_index=causal_chr[l]
            g=genotypes[:,loc_index]
            print(loc_index)
            missing=100*float(np.sum(g<0))/float(genotypes.shape[0])
            freq=freq=np.mean(g[g>0])/2.0
            # Mean imputation
            g[g<0]=freq*2.0
            if freq>0.5:
                freq=1-freq
            # Check if it passes threshold
            while freq<args.min_maf or missing>args.max_missing:
                loc_index+=1
                g=genotypes[:,loc_index]
                missing=100*float(np.sum(g<0))/float(genotypes.shape[0])
                freq=np.mean(g[g>0])/2.0
                # Mean imputation
                g[g<0]=freq*2.0
                if freq>0.5:
                    freq=1-freq
            causal_chr[l]=loc_index
            print(loc_index)
    	# Store causal variant locations
    	causal_loci[n_selected:(n_selected+nc_chr),0]=i+1
    	causal_loci[n_selected:(n_selected+nc_chr),1]=causal_chr
    	# Add causal variants from this chromosome to the matrix of causal vars
        print('Grabbing genotypes')
    	causal_gts[:,n_selected:(n_selected+nc_chr)]=genotypes[:,causal_chr]
        print('Got genotypes')
	n_selected+=nc_chr
gfiles.close()
#code.interact(local=locals())
# Record causal loci in phenofile
phenofile=h5py.File(phenout,'w')
phenofile.create_dataset('causal',data=causal_loci)

####################
# Mean normalise variables
add_gts=causal_gts-causal_gts.mean(axis=0)

# Choose SNPs to have dominance effects
if nd>0:
    dom_indices=[i for i in xrange(0,nc)]
    dom_indices=np.array(rand.choice(dom_indices,size=nd,replace=False),dtype=int)
    dom_gts=np.array(causal_gts[:,dom_indices],dtype=float)
    freqs=np.mean(dom_gts,axis=1)/2.0
    for i in xrange(0,nd):
        freq=freqs[i]
        for j in xrange(0,N):
            dom_gts[j,i]=dom_convert(dom_gts[j,i],freq)

    # Convert indices relative to additive to indices in SNP file
    dom_indices=causal_loci[dom_indices,]
    phenofile.create_dataset('dom',data=dom_indices)
else:
    dom_gts=np.array(())

## Interactions
npairs=nc*(nc-1)/2
# Array for interaction genotypes
interaction_gts=np.zeros((N,n_interact))

# Sample pairs to have interactions
interact_indices=rand.choice(npairs,size=n_interact,replace=False)
# Record which SNPs interact
interacting_snp_indices=np.zeros((n_interact,4),dtype=int)

# Fill in interaction genotypes
pair_count=0
interact_count=0
for i in xrange(0,nc):
    for j in xrange(0,i):
        if pair_count in interact_indices:
            # Record which SNPs interact
            interacting_snp_indices[interact_count,0:2]=causal_loci[i,:]
            interacting_snp_indices[interact_count,2:4]=causal_loci[j,:]
	    # Form interaction genotypes
            interaction_gts[:,interact_count]=add_gts[:,i]*add_gts[:,j]
            interact_count+=1
        pair_count+=1

phenofile.create_dataset('interactions',data=interacting_snp_indices)

# Simulate phenotypes and save to sim group of screen hdf5 file
phenotypes=phenofile.create_dataset('phenotypes',(N,nphen),dtype=float)
print('Simulating Phenotypes')
if args.normal_effects:
    equal_effects=False
else:
    equal_effects=True
for i in xrange(0,nphen):
    if nd>0:
        phenotypes[:,i]=phenosim(add_gts,dom_gts,interaction_gts,h2,hd,h2_epi,equal_effects,args.variance_effect)
    else:
        phenotypes[:,i]=phenosim(add_gts,np.array(()),interaction_gts,h2,hd,h2_epi,equal_effects,args.variance_effect)

# Save causal genotypes (including dominance and epistasis)
if args.save_causal_gts:
	phenofile.create_dataset('causal_gts',data=causal_gts)
	phenofile.create_dataset('interaction_gts',data=interaction_gts)
	phenofile.create_dataset('dom_gts',data=dom_gts)
    phenofile.create_dataset('sample_id',sample_id)
phenofile.close()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2014 Savoir-faire Linux (<www.savoirfairelinux.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm
from openerp.tools.translate import _

class account_analytic_account(orm.Model):

    _inherit = 'account.analytic.account'

    def _get_propagatable_fields(self):
        return {
            'name',
            'code',
        }

    def _get_related_result(self, cr, uid, account_id, context=None):
        """ Return the parent result analytic id."""

        if not account_id:
            return False
        else:
            result_pool = self.pool.get('program.result')

            parent_result_id = result_pool.search(cr, uid, [
                ('account_analytic_id', '=', account_id)
            ], context=context)

            if not parent_result_id:
                return False

            return result_pool.browse(
                cr, uid, parent_result_id[0], context=context)

    def create(self, cr, uid, vals, context=None):
        """ Create an analytic account

        Create the related result if this account is under
        the root program node"""

        account_id = super(account_analytic_account, self).create(
            cr, uid, vals, context=context
        )

        if context.get('result_written', False):
            return account_id

        parent_id = vals.get('parent_id', False)

        # If there is no parent ID it cannot be under the root
        # program node.
        if not parent_id:
            return account_id

        # If the parent analytic account is related has an associated
        # result, we assume we are under the root program node.
        parent_result = self._get_related_result(cr, uid, parent_id,
                                                 context=context)

        # If there are NO parent results we don't need to create the related
        # result
        if not parent_result:
            return account_id

        result_pool = self.pool.get('program.result')

        if parent_result:
            parent_result_id = parent_result.id
        else:
            parent_result_id = False

        propagated_fields = {
            i: j
            for i, j in vals.items() if i in self._get_propagatable_fields()
        }

        result_pool.create(
            cr, uid, dict(propagated_fields.items() + {
                'name': vals.get('name', False),
                'code': vals.get('code', False),
                'account_analytic_id': account_id,
                'parent_id': parent_result_id,
            }.items()),
            context=dict(context, account_written=True)
        )
        return account_id

    def write(self, cr, uid, ids, values, context=None):

        result_pool = self.pool.get('program.result')

        if context is None:
            context = {}

        if type(ids) not in (list, ):
            ids = [ids]

        res = super(account_analytic_account, self).write(
            cr, uid, ids, values, context=context
        )

        if context.get('result_written', False):
            return res

        for account in self.browse(cr, uid, ids, context=context):
            propagated_fields = {
                i: j for i, j in values.items()
                if i in self._get_propagatable_fields()
            }

            if values.get('parent_id'):
                propagated_fields['parent_id'] = False

                related_result = self._get_related_result(
                    cr, uid, values['parent_id'], context=context
                )

                if related_result:
                    propagated_fields['parent_id'] = related_result.id

            if propagated_fields:
                related_result = result_pool.search(
                    cr, uid,
                    [('account_analytic_id', '=', account.id), ],
                    context=context
                )

                result_pool.write(
                    cr, uid, related_result, propagated_fields,
                    context=dict(context, account_written=True)
                )

        return res

    def unlink(self, cr, uid, ids, context=None):
        result_pool = self.pool.get('program.result')
        if type(ids) is not list:
            ids = [ids]
        if result_pool.search(cr, uid, [('account_analytic_id', 'in', ids)],
                              context=context):
            raise orm.except_orm(
                _('Error'),
                _('Account still associated with a Program Result'))
        return super(account_analytic_account, self).unlink(
            cr, uid, ids, context=context
        )
<commit_msg>Add missing context validation<commit_after># -*- coding: utf-8 -*-

##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2014 Savoir-faire Linux (<www.savoirfairelinux.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import orm
from openerp.tools.translate import _

class account_analytic_account(orm.Model):

    _inherit = 'account.analytic.account'

    def _get_propagatable_fields(self):
        return {
            'name',
            'code',
        }

    def _get_related_result(self, cr, uid, account_id, context=None):
        """ Return the parent result analytic id."""

        if not account_id:
            return False
        else:
            result_pool = self.pool.get('program.result')

            parent_result_id = result_pool.search(cr, uid, [
                ('account_analytic_id', '=', account_id)
            ], context=context)

            if not parent_result_id:
                return False

            return result_pool.browse(
                cr, uid, parent_result_id[0], context=context)

    def create(self, cr, uid, vals, context=None):
        """ Create an analytic account

        Create the related result if this account is under
        the root program node"""
        context = context or {}

        account_id = super(account_analytic_account, self).create(
            cr, uid, vals, context=context
        )

        if context.get('result_written', False):
            return account_id

        parent_id = vals.get('parent_id', False)

        # If there is no parent ID it cannot be under the root
        # program node.
        if not parent_id:
            return account_id

        # If the parent analytic account is related has an associated
        # result, we assume we are under the root program node.
        parent_result = self._get_related_result(cr, uid, parent_id,
                                                 context=context)

        # If there are NO parent results we don't need to create the related
        # result
        if not parent_result:
            return account_id

        result_pool = self.pool.get('program.result')

        if parent_result:
            parent_result_id = parent_result.id
        else:
            parent_result_id = False

        propagated_fields = {
            i: j
            for i, j in vals.items() if i in self._get_propagatable_fields()
        }

        result_pool.create(
            cr, uid, dict(propagated_fields.items() + {
                'name': vals.get('name', False),
                'code': vals.get('code', False),
                'account_analytic_id': account_id,
                'parent_id': parent_result_id,
            }.items()),
            context=dict(context, account_written=True)
        )
        return account_id

    def write(self, cr, uid, ids, values, context=None):

        result_pool = self.pool.get('program.result')

        if context is None:
            context = {}

        if type(ids) not in (list, ):
            ids = [ids]

        res = super(account_analytic_account, self).write(
            cr, uid, ids, values, context=context
        )

        if context.get('result_written', False):
            return res

        for account in self.browse(cr, uid, ids, context=context):
            propagated_fields = {
                i: j for i, j in values.items()
                if i in self._get_propagatable_fields()
            }

            if values.get('parent_id'):
                propagated_fields['parent_id'] = False

                related_result = self._get_related_result(
                    cr, uid, values['parent_id'], context=context
                )

                if related_result:
                    propagated_fields['parent_id'] = related_result.id

            if propagated_fields:
                related_result = result_pool.search(
                    cr, uid,
                    [('account_analytic_id', '=', account.id), ],
                    context=context
                )

                result_pool.write(
                    cr, uid, related_result, propagated_fields,
                    context=dict(context, account_written=True)
                )

        return res

    def unlink(self, cr, uid, ids, context=None):
        result_pool = self.pool.get('program.result')
        if type(ids) is not list:
            ids = [ids]
        if result_pool.search(cr, uid, [('account_analytic_id', 'in', ids)],
                              context=context):
            raise orm.except_orm(
                _('Error'),
                _('Account still associated with a Program Result'))
        return super(account_analytic_account, self).unlink(
            cr, uid, ids, context=context
        )
<|endoftext|>


<commit_before>import hexchat
import threading
import twitch.hook, twitch.channel, twitch.logger
log = twitch.logger.get()

# Don't show the "changed topic" message (but do update the topic bar)
def topic_print_cb(word, word_eol, msgtype):
	return hexchat.EAT_ALL

# work around hexchat bugs (likes to segfault when using RECV in other threads)
topic_changes = []
def topic_update_cb(userdata):
	try:
		while len(topic_changes) > 0:
			log.debug("%d updates queued" % len(topic_changes))
			topic = topic_changes.pop(0)
			log.debug("topic change: %s" % str(topic))

			cmd = "RECV :twitch.py!twitch@twitch.tv TOPIC #{0} :{1}"\
				.format(topic["channel"], topic["topic"])
			log.debug(cmd)

			hexchat.command(cmd)
			log.debug("Posted topic change OK")

	except:
		log.exception("Unhandled exception in twitch.topic_update_cb")

	finally:
		return True # keep timer running

# Thread callback to update a channel
def update_channel_thread(chan):
	try:
		if chan.update() and chan.makeTopic():
			log.debug("queue update for channel %s" % chan.name)
			topic_changes.append({
				"channel": chan.name,
				"topic"  : chan.topic,
			})
	except:
		log.exception("Unhandled exception in %s" %
			threading.current_thread().name)

# Periodically update channel info
def update_channels_cb(userdata):
	try:
		for name in twitch.channel.channels:
			log.debug("Update channel %s" % name)
			t = threading.Thread(
				target = update_channel_thread,
				args   = (twitch.channel.channels[name],),
				name   = "twitch.update_channel_thread(%s)" % name,
				daemon = True)
			t.start()
	except:
		log.exception("Unhandled exception in twitch.update_channels_cb")

	finally:
		return True # keep timer running

# Manually update a channel
def update_channel(chan):
	log.debug("manually queue update for channel %s" % chan.name)
	topic_changes.append({
		"channel": chan.name,
		"topic"  : chan.topic,
	})

def run():
	twitch.hook.prnt('Topic Change', topic_print_cb)
	twitch.hook.timer(1000, topic_update_cb)
	twitch.hook.timer(60000, update_channels_cb)
<commit_msg>topic: honor topic.refreshinterval setting<commit_after>import hexchat
import threading
import twitch.hook, twitch.channel, twitch.logger, twitch.settings
log = twitch.logger.get()

# Don't show the "changed topic" message (but do update the topic bar)
def topic_print_cb(word, word_eol, msgtype):
	return hexchat.EAT_ALL

# work around hexchat bugs (likes to segfault when using RECV in other threads)
topic_changes = []
def topic_update_cb(userdata):
	try:
		while len(topic_changes) > 0:
			log.debug("%d updates queued" % len(topic_changes))
			topic = topic_changes.pop(0)
			log.debug("topic change: %s" % str(topic))

			cmd = "RECV :twitch.py!twitch@twitch.tv TOPIC #{0} :{1}"\
				.format(topic["channel"], topic["topic"])
			log.debug(cmd)

			hexchat.command(cmd)
			log.debug("Posted topic change OK")

	except:
		log.exception("Unhandled exception in twitch.topic_update_cb")

	finally:
		return True # keep timer running

# Thread callback to update a channel
def update_channel_thread(chan):
	try:
		if chan.update() and chan.makeTopic():
			log.debug("queue update for channel %s" % chan.name)
			topic_changes.append({
				"channel": chan.name,
				"topic"  : chan.topic,
			})
	except:
		log.exception("Unhandled exception in %s" %
			threading.current_thread().name)

# Periodically update channel info
def update_channels_cb(userdata):
	try:
		for name in twitch.channel.channels:
			log.debug("Update channel %s" % name)
			t = threading.Thread(
				target = update_channel_thread,
				args   = (twitch.channel.channels[name],),
				name   = "twitch.update_channel_thread(%s)" % name,
				daemon = True)
			t.start()
	except:
		log.exception("Unhandled exception in twitch.update_channels_cb")

	finally:
		return True # keep timer running

# Manually update a channel
def update_channel(chan):
	log.debug("manually queue update for channel %s" % chan.name)
	topic_changes.append({
		"channel": chan.name,
		"topic"  : chan.topic,
	})

def run():
	# XXX if setting is changed, reset the timer
	twitch.hook.prnt('Topic Change', topic_print_cb)
	twitch.hook.timer(1000, topic_update_cb)
	twitch.hook.timer(twitch.settings.get('topic.refreshinterval') * 1000,
		update_channels_cb)
<|endoftext|>