<commit_before>import logging

from seabird.plugin import Plugin

from .isupport import ISupportPlugin, prefix_parse, status_prefix_parse, mode_parse

LOG = logging.getLogger(__name__)

class User:
    def __init__(self, nick):
        self.nick = nick
        self.channels = {}

# Mapping of users to channels and other useful information
class UserTrack(Plugin):
    # See https://gist.github.com/belak/09edcc4f5e51056bf5bc728647659d81 for
    # more info
    def __init__(self, bot):
        super().__init__(bot)

        # Grab the ISUPPORT plugin for PREFIX info
        self.isupport = self.bot.load_plugin(ISupportPlugin)

        # This is just a simple dict which will store all users this bot knows
        # about.
        self.users = {}

        # We use multi-prefix to simplify a few operations. Because it's part
        # of the core IRCv3.1 spec, it should be supported almost everywhere.
        self.bot.cap_req('multi-prefix')

    def get_user(self, nick):
        return self.users.get(nick)

    def add_user(self, nick):
        user = self.get_user(nick)
        if user:
            return user

        LOG.info('Adding user %s', nick)

        user = User(nick)
        self.users[nick] = user
        return user

    def remove_user(self, nick):
        LOG.info('Deleting user %s', nick)

        del self.users[nick]

    # Now that the public interface is out of the way, we need to actually get
    # the tracking done.
    def irc_001(self, msg):
        self.add_user(msg.args[0])

    def irc_353(self, msg):
        # RPL_NAMREPLY
        channel = msg.args[2]
        prefix = prefix_parse(self.isupport.supported.get('PREFIX'))

        for nick in msg.args[3].split(' '):
            if not nick:
                continue

            modes, nick = status_prefix_parse(prefix, nick)

            user = self.get_user(nick)
            if not user:
                user = self.add_user(nick)

            user.channels[channel] = modes
            LOG.info('Modes for %s in %s are %s', nick, channel, modes)

    def irc_join(self, msg):
        user = self.add_user(msg.identity.name)
        user.channels[msg.params[0]] = set()

    def irc_nick(self, msg):
        oldnick = msg.identity.name
        newnick = msg.args[0]

        if not self.get_user(oldnick):
            raise ValueError('Missing renamed nick {}'.format(oldnick))

        self.users[newnick] = self.get_user(oldnick)
        self.users[newnick].nick = newnick

        del self.users[oldnick]

        LOG.info('Nick renamed %s --> %s', oldnick, newnick)

    def irc_part(self, msg):
        if msg.event == 'PART':
            user = self.get_user(msg.identity.name)
        else:
            user = self.get_user(msg.args[1])

        channel = msg.args[0]
        if not user:
            LOG.warning("Got a part/kick for nonexistent user: %s in %s",
                        msg.identity.name, msg.args[0])
            return
        elif channel not in user.channels:
            LOG.warning("Got a part/kick for user not in a channel: %s in %s",
                        msg.identity.name, msg.args[0])
            return

        user.channels.pop(channel)

        if user.nick == self.bot.current_nick:
            # We left the channel, so remove all unneeded users
            for u_nick, u_user in list(self.users.items()):
                u_user.channels.pop(channel, None)

                # Don't delete ourselves
                if u_user.nick == self.bot.current_nick:
                    continue

                if not u_user.channels:
                    self.remove_user(u_nick)
        elif not user.channels:
            # If they're not in any tracked channels, we need to forget about
            # them.
            self.remove_user(user.nick)

    # Dirty hack because I'm too lazy to re-implement this for kicks
    irc_kick = irc_part

    def irc_mode(self, msg):
        modegroups = self.isupport.supported.get('CHANMODES')
        prefix = prefix_parse(self.isupport.supported.get('PREFIX'))

        target = msg.args[0]
        modes = msg.args[1]
        params = msg.args[2:]

        # We don't care about user modes right now, so if the target isn't a
        # channel, we discard the message.
        channels = tuple(self.isupport.supported.get("CHANTYPES"))
        if not target.startswith(channels):
            return

        gen = mode_parse(modes, params, modegroups, prefix)
        prefix = prefix.mode_to_prefix
        for mode, param, adding in gen:
            # There are a bunch of other types of modes, but we only care about
            # prefix modes because they are the only ones which can be applied
            # to users in the way we want.
            if mode in prefix:
                user = self.get_user(param)
                if not user:
                    LOG.warning('User %s is not known. Skipping changing of mode %s', param, mode)
                    continue

                if adding:
                    LOG.info("Setting mode %s for %s in %s", mode, param, target)
                    user.channels[target].add(mode)
                else:
                    LOG.info("Unsetting mode %s for %s in %s", mode, param, target)
                    user.channels[target].discard(mode)

                LOG.info("Modes for %s in %s: %s", param, target, user.channels[target])

    def irc_quit(self, msg):
        self.remove_user(msg.identity.name)
<commit_msg>Fix exception on channel join<commit_after>import logging

from seabird.plugin import Plugin

from .isupport import ISupportPlugin, prefix_parse, status_prefix_parse, mode_parse

LOG = logging.getLogger(__name__)

class User:
    def __init__(self, nick):
        self.nick = nick
        self.channels = {}

# Mapping of users to channels and other useful information
class UserTrack(Plugin):
    # See https://gist.github.com/belak/09edcc4f5e51056bf5bc728647659d81 for
    # more info
    def __init__(self, bot):
        super().__init__(bot)

        # Grab the ISUPPORT plugin for PREFIX info
        self.isupport = self.bot.load_plugin(ISupportPlugin)

        # This is just a simple dict which will store all users this bot knows
        # about.
        self.users = {}

        # We use multi-prefix to simplify a few operations. Because it's part
        # of the core IRCv3.1 spec, it should be supported almost everywhere.
        self.bot.cap_req('multi-prefix')

    def get_user(self, nick):
        return self.users.get(nick)

    def add_user(self, nick):
        user = self.get_user(nick)
        if user:
            return user

        LOG.info('Adding user %s', nick)

        user = User(nick)
        self.users[nick] = user
        return user

    def remove_user(self, nick):
        LOG.info('Deleting user %s', nick)

        del self.users[nick]

    # Now that the public interface is out of the way, we need to actually get
    # the tracking done.
    def irc_001(self, msg):
        self.add_user(msg.args[0])

    def irc_353(self, msg):
        # RPL_NAMREPLY
        channel = msg.args[2]
        prefix = prefix_parse(self.isupport.supported.get('PREFIX'))

        for nick in msg.args[3].split(' '):
            if not nick:
                continue

            modes, nick = status_prefix_parse(prefix, nick)

            user = self.get_user(nick)
            if not user:
                user = self.add_user(nick)

            user.channels[channel] = modes
            LOG.info('Modes for %s in %s are %s', nick, channel, modes)

    def irc_join(self, msg):
        user = self.add_user(msg.identity.name)
        user.channels[msg.args[0]] = set()

    def irc_nick(self, msg):
        oldnick = msg.identity.name
        newnick = msg.args[0]

        if not self.get_user(oldnick):
            raise ValueError('Missing renamed nick {}'.format(oldnick))

        self.users[newnick] = self.get_user(oldnick)
        self.users[newnick].nick = newnick

        del self.users[oldnick]

        LOG.info('Nick renamed %s --> %s', oldnick, newnick)

    def irc_part(self, msg):
        if msg.event == 'PART':
            user = self.get_user(msg.identity.name)
        else:
            user = self.get_user(msg.args[1])

        channel = msg.args[0]
        if not user:
            LOG.warning("Got a part/kick for nonexistent user: %s in %s",
                        msg.identity.name, msg.args[0])
            return
        elif channel not in user.channels:
            LOG.warning("Got a part/kick for user not in a channel: %s in %s",
                        msg.identity.name, msg.args[0])
            return

        user.channels.pop(channel)

        if user.nick == self.bot.current_nick:
            # We left the channel, so remove all unneeded users
            for u_nick, u_user in list(self.users.items()):
                u_user.channels.pop(channel, None)

                # Don't delete ourselves
                if u_user.nick == self.bot.current_nick:
                    continue

                if not u_user.channels:
                    self.remove_user(u_nick)
        elif not user.channels:
            # If they're not in any tracked channels, we need to forget about
            # them.
            self.remove_user(user.nick)

    # Dirty hack because I'm too lazy to re-implement this for kicks
    irc_kick = irc_part

    def irc_mode(self, msg):
        modegroups = self.isupport.supported.get('CHANMODES')
        prefix = prefix_parse(self.isupport.supported.get('PREFIX'))

        target = msg.args[0]
        modes = msg.args[1]
        params = msg.args[2:]

        # We don't care about user modes right now, so if the target isn't a
        # channel, we discard the message.
        channels = tuple(self.isupport.supported.get("CHANTYPES"))
        if not target.startswith(channels):
            return

        gen = mode_parse(modes, params, modegroups, prefix)
        prefix = prefix.mode_to_prefix
        for mode, param, adding in gen:
            # There are a bunch of other types of modes, but we only care about
            # prefix modes because they are the only ones which can be applied
            # to users in the way we want.
            if mode in prefix:
                user = self.get_user(param)
                if not user:
                    LOG.warning('User %s is not known. Skipping changing of mode %s', param, mode)
                    continue

                if adding:
                    LOG.info("Setting mode %s for %s in %s", mode, param, target)
                    user.channels[target].add(mode)
                else:
                    LOG.info("Unsetting mode %s for %s in %s", mode, param, target)
                    user.channels[target].discard(mode)

                LOG.info("Modes for %s in %s: %s", param, target, user.channels[target])

    def irc_quit(self, msg):
        self.remove_user(msg.identity.name)
<|endoftext|>


<commit_before>#   Copyright 2011 Michael Garski (mgarski@mac.com)
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import json
import time

import tornado.httpserver
import tornado.ioloop
import tornado.web

HDR_CONTENT_TYPE = 'Content-Type'
HDR_JSON = 'application/json'

PORT = 8080

class QueryHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        sleep = self.get_argument('s', 0)
        if sleep > 0:
            time.sleep(int(sleep))

        response = {'response': self.request.arguments }
        self.set_header(HDR_CONTENT_TYPE, HDR_JSON)
        self.write(json.dumps(response))

class UpdateHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        self.write(json.dumps(self.request.arguments))

application = tornado.web.Application([
    (r"/query", QueryHandler),
    (r"/update", UpdateHandler)
])

class TestServer():
    def start(self):
        http_server = tornado.httpserver.HTTPServer(application)
        http_server.listen(PORT)
        tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    server = TestServer()
    server.start()
    print('server started')<commit_msg>Make TestServer inherit from object.<commit_after>#   Copyright 2011 Michael Garski (mgarski@mac.com)
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import json
import time

import tornado.httpserver
import tornado.ioloop
import tornado.web

HDR_CONTENT_TYPE = 'Content-Type'
HDR_JSON = 'application/json'

PORT = 8080

class QueryHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        sleep = self.get_argument('s', 0)
        if sleep > 0:
            time.sleep(int(sleep))

        response = {'response': self.request.arguments }
        self.set_header(HDR_CONTENT_TYPE, HDR_JSON)
        self.write(json.dumps(response))

class UpdateHandler(tornado.web.RequestHandler):
    def post(self, *args, **kwargs):
        self.write(json.dumps(self.request.arguments))

application = tornado.web.Application([
    (r"/query", QueryHandler),
    (r"/update", UpdateHandler)
])

class TestServer(object):
    def start(self):
        http_server = tornado.httpserver.HTTPServer(application)
        http_server.listen(PORT)
        tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    server = TestServer()
    server.start()
    print('server started')<|endoftext|>


<commit_before>"""
data_handler.py: defines the classes that serve data to the various Stile systematics tests in the
default drivers.
"""
import os
import glob

class DataHandler:
<<<<<<< HEAD
    temp_dir = None # Can (should?) be overwritten by other DataHandlers
=======
    """
    A class which contains information about the data set Stile is to be run on.  This is used for
    the default drivers, not necessarily the pipeline-specific drivers (such as HSC/LSST).

    The class needs to be able to do two things:
      - List some data given a set of requirements (DataHandler.listData()).  The requirements
        generally follow the form:
         -- object_types: a list of strings such as "star", "PSF star", "galaxy" or "galaxy random"
              describing the objects that are needed for the tests.
         -- epoch: whether this is a single/summary catalog, or a multiepoch time series. (Coadded
              catalogs with no per-epoch information count as a single/summary catalog!)
         -- extent: "CCD", "field", "patch" or "tract".  This can be ignored if you don't mind
              running some extra inappropriate tests!  "CCD" should be a CCD-type dataset, "field" a
              single pointing/field-of-view, "patch" an intermediate-size area, and "tract" a large
              area.  (These terms have specific meanings in the LSST pipeline, but are used for
              convenience here.)
         -- data_format: "image" or "catalog."  Right now no image-level tests are implemented but we
              request this kwarg for future development.
      - Take each element of the data list from DataHandler.listData() and retrieve it for use
        (DataHandler.getData()), optionally with bins defined.  (Bins can also be defined on a test-
        by-test basis, depending on which format makes the most sense for your data setup.)

      Additionally, the class can define a .getOutputPath() function to place the data in a more
      complex system than the default (all in one directory with long output path names).
      """
>>>>>>> #8
    def __init__(self):
        raise NotImplementedError()

    def listData(self,object_types,epoch,extent,data_format,required_fields=None):
        raise NotImplementedError()

    def getData(self,ident,object_types=None,epoch=None,extent=None,data_format=None,bin_list=None):
        """
        Return some data matching the `ident` for the given kwargs.  This can be a numpy array, a
        tuple (file_name, field_schema) for a file already existing on the filesystem, or a list of
        either of those things.

        If it's a tuple (file_name, field_schema), the assumption is that it can be read by a simple
        FITS or ASCII reader.  The format will be determined from the file extension.
        """
        raise NotImplementedError()

    def getOutputPath(self,extension='.dat',multi_file=False,*args):
        """
        Return a path to an output file given a list of strings that should appear in the output
        filename, taking care not to clobber other files (unless requested).
        @param args       A list of strings to appear in the file name
        @param extension  The file extension to be used
        @param multi_file Whether multiple files with the same args will be created within a single
                          run of Stile. This appends a number to the file name; if clobbering is
                          allowed, this argument also prevents Stile from writing over outputs from
                          the same systematics test during the same run.
        @returns A path to an output file meeting the input specifications.
        """
        #TODO: no-clobbering case
        sys_test_string = '_'.join(args)
        if multi_file:
            nfiles = glob.glob(os.path.join(self.output_path,sys_test_string)+'*'+extension)
            return os.path.join(self.output_path,sys_test_string+'_'+str(nfiles)+extension)
        else:
<<<<<<< HEAD
            return os.path.join(self.output_path,base_path+extension)

class HSCDataHandler():
    def __init__(self, hsc_args):
        try:
            import lsst.daf.persistence as dafPersist
        except:
            raise ImportError("Cannot import lsst.daf.persistence which is required for "+
                              "HSCDataHandler.")
        dataDir = hsc_args.dataDir
        self.butler = dafPersist.Butler(dataDir)
        self.output_path = '.'
        max_cache = hsc_args.get('max_cache',1)
        max_image_cache = hsc_args.get('max_image_cache',max_cache)
        max_catalog_cache = hsc_args.get('max_catalog_cache',max_cache)
        self.initCache(max_image_cache,max_catalog_cache)

    def _fixCovMomentsD(self,arr,name):
        pass
    def _fixFlag(self,arr,name):
        pass
    def _fixArrayD(self,arr,name):
        pass
    def _fixPointD(self,arr,name):
        pass
    def _fixCovPointD(self,arr,name):
        pass
    def _fixCoord(self,arr,name):
        pass
    def _fixPSF(self,arr,name):
        pass

    def _modifyArray(self,data,required_fields):
        posskeys_fix = [('coord',self._fixCoord)] # and others
        posskeys_ok = [('psf.flux','psf.flux')] # and others
        cols = {}
        dtypes = []
        for key,stile_key in posskeys_ok:
            if key in data.schema:
                cols[stile_key] = append(data.get(key))
                dtypes.append((stile_key,cols[-1].dtype))
        for key,func in posskeys_fix:
            if key in data.schema:
                objs = func([data_point.get(key) for data_point in data],key)
                for col, dtype in objs:
                    cols[dtype[0]] = col
                    dtypes.append(dtype)
        data = numpy.zeros(len(data),dtype=dtypes)
        for key in data.dtype.names:
            data[key] = cols[key]
        return data

    def _makePSFImage(self,psf):
        """
        Turn the PSF object into an image
        """
        raise NotImplementedError()

    def _matchReqs(self, ident, required_fields):
        """
        Make sure the catalog contains the required fields, and retrieve or measure them if not.
        """
        if not required_fields:
            return True
        else:
            raise NotImplementedError()

    def _getImageData(self,ident,bin_list=None,data_format='image',epoch=None,extent=None,
                           force=False,required_fields=None):
        """
        Get and process any image data from the butler.
        """
        if data_format!='image':
            raise ValueError('_getImageData requires "image" for the data_format')
        if self.inCache(ident,'image'):
            data = self.getCache(ident,'image')
        else:
            if epoch=='multi':
                epoch_key = 'deepCoadd_'
            else:
                epoch_key = ''
            if 'psf' in required_fields:
                data = self.butler.get(ident,epoch_key+'psf')
                data = self._makePSFImage(data)
            else:
                data = self.butler.get(ident,epoch_key+'calexp')
            self.saveCache(ident,data,'image')
        return self.bin(data,bin_list)

    def _getCatalogData(self,ident,bin_list=None,data_format='catalog',epoch=None,extent=None,
                             force=False,required_fields=None):
        """
        Get and process any catalog data from the butler.  Make sure it contains the right fields
        for required_fields, or retrieve or measure them from other places if not.
        """
        if data_format!='catalog':
            raise ValueError('_getImageData requires "catalog" for the data_format')
        if self.inCache(ident,'catalog'):
            data = self.getCache(ident,'catalog')
            if not self._matchReqs(ident,required_fields):
                data = self._updateReqs(data,ident,required_fields) # Retrieve, add necessary fields
                self.saveCache(ident,data,'catalog') # Save the extended data back to the cache
        else:
            if epoch=='multi':
                epoch_key = 'deepCoadd_'
            else:
                epoch_key = ''
            data = self.butler.get(ident,epoch_key+'src')
            if any(['psf' in rq for rq in required_fields and rq not in data]):
                data['psf'] = self.butler.get(ident,epoch_key+'psf')
            data = self._modifyArray(data,required_fields)
            self.saveCache(ident,data,'catalog')
        return self.bin(data,bin_list)

    def listData(self,pair_or_single,epoch,extent,data_format,required_fields=None):
        raise NotImplementedError()

    def getData(self,ident,pair_or_single=None,epoch=None,extent=None,data_format=None,
                random=False,bin_list=None,force=False,required_fields=None):
        if random:
            raise NotImplementedError()
        if force:
            return self.bin(ident,bin_list)
        elif format=='image':
            return self._getImageData(ident,bin_list,format,epoch,extent,force,required_fields)
        else:
            return self._getTableData(ident,bin_list,format,epoch,extent,force,required_fields)
=======
            return os.path.join(self.output_path,sys_test_string+extension)

>>>>>>> #8
<commit_msg>Whoops, forgot data_handler.py on last merge (#7)<commit_after>"""
data_handler.py: defines the classes that serve data to the various Stile systematics tests in the
default drivers.
"""
import os
import glob

class DataHandler:
    """
    A class which contains information about the data set Stile is to be run on.  This is used for
    the default drivers, not necessarily the pipeline-specific drivers (such as HSC/LSST).

    The class needs to be able to do two things:
      - List some data given a set of requirements (DataHandler.listData()).  The requirements
        generally follow the form:
         -- object_types: a list of strings such as "star", "PSF star", "galaxy" or "galaxy random"
              describing the objects that are needed for the tests.
         -- epoch: whether this is a single/summary catalog, or a multiepoch time series. (Coadded
              catalogs with no per-epoch information count as a single/summary catalog!)
         -- extent: "CCD", "field", "patch" or "tract".  This can be ignored if you don't mind
              running some extra inappropriate tests!  "CCD" should be a CCD-type dataset, "field" a
              single pointing/field-of-view, "patch" an intermediate-size area, and "tract" a large
              area.  (These terms have specific meanings in the LSST pipeline, but are used for
              convenience here.)
         -- data_format: "image" or "catalog."  Right now no image-level tests are implemented but we
              request this kwarg for future development.
      - Take each element of the data list from DataHandler.listData() and retrieve it for use
        (DataHandler.getData()), optionally with bins defined.  (Bins can also be defined on a test-
        by-test basis, depending on which format makes the most sense for your data setup.)

      Additionally, the class can define a .getOutputPath() function to place the data in a more
      complex system than the default (all in one directory with long output path names).
      """
    def __init__(self):
        raise NotImplementedError()

    def listData(self,object_types,epoch,extent,data_format,required_fields=None):
        raise NotImplementedError()

    def getData(self,ident,object_types=None,epoch=None,extent=None,data_format=None,bin_list=None):
        """
        Return some data matching the `ident` for the given kwargs.  This can be a numpy array, a
        tuple (file_name, field_schema) for a file already existing on the filesystem, or a list of
        either of those things.

        If it's a tuple (file_name, field_schema), the assumption is that it can be read by a simple
        FITS or ASCII reader.  The format will be determined from the file extension.
        """
        raise NotImplementedError()

    def getOutputPath(self,extension='.dat',multi_file=False,*args):
        """
        Return a path to an output file given a list of strings that should appear in the output
        filename, taking care not to clobber other files (unless requested).
        @param args       A list of strings to appear in the file name
        @param extension  The file extension to be used
        @param multi_file Whether multiple files with the same args will be created within a single
                          run of Stile. This appends a number to the file name; if clobbering is
                          allowed, this argument also prevents Stile from writing over outputs from
                          the same systematics test during the same run.
        @returns A path to an output file meeting the input specifications.
        """
        #TODO: no-clobbering case
        sys_test_string = '_'.join(args)
        if multi_file:
            nfiles = glob.glob(os.path.join(self.output_path,sys_test_string)+'*'+extension)
            return os.path.join(self.output_path,sys_test_string+'_'+str(nfiles)+extension)
        else:
            return os.path.join(self.output_path,sys_test_string+extension)

class HSCDataHandler():
    def __init__(self, hsc_args):
        try:
            import lsst.daf.persistence as dafPersist
        except:
            raise ImportError("Cannot import lsst.daf.persistence which is required for "+
                              "HSCDataHandler.")
        dataDir = hsc_args.dataDir
        self.butler = dafPersist.Butler(dataDir)
        self.output_path = '.'
        max_cache = hsc_args.get('max_cache',1)
        max_image_cache = hsc_args.get('max_image_cache',max_cache)
        max_catalog_cache = hsc_args.get('max_catalog_cache',max_cache)
        self.initCache(max_image_cache,max_catalog_cache)

    def _fixCovMomentsD(self,arr,name):
        pass
    def _fixFlag(self,arr,name):
        pass
    def _fixArrayD(self,arr,name):
        pass
    def _fixPointD(self,arr,name):
        pass
    def _fixCovPointD(self,arr,name):
        pass
    def _fixCoord(self,arr,name):
        pass
    def _fixPSF(self,arr,name):
        pass

    def _modifyArray(self,data,required_fields):
        posskeys_fix = [('coord',self._fixCoord)] # and others
        posskeys_ok = [('psf.flux','psf.flux')] # and others
        cols = {}
        dtypes = []
        for key,stile_key in posskeys_ok:
            if key in data.schema:
                cols[stile_key] = append(data.get(key))
                dtypes.append((stile_key,cols[-1].dtype))
        for key,func in posskeys_fix:
            if key in data.schema:
                objs = func([data_point.get(key) for data_point in data],key)
                for col, dtype in objs:
                    cols[dtype[0]] = col
                    dtypes.append(dtype)
        data = numpy.zeros(len(data),dtype=dtypes)
        for key in data.dtype.names:
            data[key] = cols[key]
        return data

    def _makePSFImage(self,psf):
        """
        Turn the PSF object into an image
        """
        raise NotImplementedError()

    def _matchReqs(self, ident, required_fields):
        """
        Make sure the catalog contains the required fields, and retrieve or measure them if not.
        """
        if not required_fields:
            return True
        else:
            raise NotImplementedError()

    def _getImageData(self,ident,bin_list=None,data_format='image',epoch=None,extent=None,
                           force=False,required_fields=None):
        """
        Get and process any image data from the butler.
        """
        if data_format!='image':
            raise ValueError('_getImageData requires "image" for the data_format')
        if self.inCache(ident,'image'):
            data = self.getCache(ident,'image')
        else:
            if epoch=='multi':
                epoch_key = 'deepCoadd_'
            else:
                epoch_key = ''
            if 'psf' in required_fields:
                data = self.butler.get(ident,epoch_key+'psf')
                data = self._makePSFImage(data)
            else:
                data = self.butler.get(ident,epoch_key+'calexp')
            self.saveCache(ident,data,'image')
        return self.bin(data,bin_list)

    def _getCatalogData(self,ident,bin_list=None,data_format='catalog',epoch=None,extent=None,
                             force=False,required_fields=None):
        """
        Get and process any catalog data from the butler.  Make sure it contains the right fields
        for required_fields, or retrieve or measure them from other places if not.
        """
        if data_format!='catalog':
            raise ValueError('_getImageData requires "catalog" for the data_format')
        if self.inCache(ident,'catalog'):
            data = self.getCache(ident,'catalog')
            if not self._matchReqs(ident,required_fields):
                data = self._updateReqs(data,ident,required_fields) # Retrieve, add necessary fields
                self.saveCache(ident,data,'catalog') # Save the extended data back to the cache
        else:
            if epoch=='multi':
                epoch_key = 'deepCoadd_'
            else:
                epoch_key = ''
            data = self.butler.get(ident,epoch_key+'src')
            if any(['psf' in rq for rq in required_fields and rq not in data]):
                data['psf'] = self.butler.get(ident,epoch_key+'psf')
            data = self._modifyArray(data,required_fields)
            self.saveCache(ident,data,'catalog')
        return self.bin(data,bin_list)

    def listData(self,pair_or_single,epoch,extent,data_format,required_fields=None):
        raise NotImplementedError()

    def getData(self,ident,pair_or_single=None,epoch=None,extent=None,data_format=None,
                random=False,bin_list=None,force=False,required_fields=None):
        if random:
            raise NotImplementedError()
        if force:
            return self.bin(ident,bin_list)
        elif format=='image':
            return self._getImageData(ident,bin_list,format,epoch,extent,force,required_fields)
        else:
            return self._getTableData(ident,bin_list,format,epoch,extent,force,required_fields)

<|endoftext|>


<commit_before># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import webnotes
from webnotes import msgprint
from webnotes.utils import cint, flt, cstr
from stock.utils import get_valuation_method
import json

# future reposting

_exceptions = []
def update_entries_after(args, verbose=1):
	"""
		update valution rate and qty after transaction
		from the current time-bucket onwards

		args = {
			"item_code": "ABC",
			"warehouse": "XYZ",
			"posting_date": "2012-12-12",
			"posting_time": "12:00"
		}
	"""
	previous_sle = get_sle_before_datetime(args)

	qty_after_transaction = flt(previous_sle.get("qty_after_transaction"))
	valuation_rate = flt(previous_sle.get("valuation_rate"))
	stock_queue = json.loads(previous_sle.get("stock_queue") or "[]")
	stock_value = 0.0

	entries_to_fix = get_sle_after_datetime(previous_sle or \
		{"item_code": args["item_code"], "warehouse": args["warehouse"]}, for_update=True)

	valuation_method = get_valuation_method(args["item_code"])

	for sle in entries_to_fix:
		if sle.serial_no or not cint(webnotes.conn.get_default("allow_negative_stock")):
			# validate negative stock for serialized items, fifo valuation
			# or when negative stock is not allowed for moving average
			if not validate_negative_stock(qty_after_transaction, sle):
				qty_after_transaction += flt(sle.actual_qty)
				continue

		if sle.serial_no:
			valuation_rate = get_serialized_values(qty_after_transaction, sle, valuation_rate)
		elif valuation_method == "Moving Average":
			valuation_rate = get_moving_average_values(qty_after_transaction, sle, valuation_rate)
		else:
			valuation_rate = get_fifo_values(qty_after_transaction, sle, stock_queue)

		qty_after_transaction += flt(sle.actual_qty)

		# get stock value
		if sle.serial_no:
			stock_value = qty_after_transaction * valuation_rate
		elif valuation_method == "Moving Average":
			stock_value = (qty_after_transaction > 0) and \
				(qty_after_transaction * valuation_rate) or 0
		else:
			stock_value = sum((flt(batch[0]) * flt(batch[1]) for batch in stock_queue))
			# print sle.posting_date, sle.actual_qty, sle.incoming_rate, stock_queue, stock_value
		# update current sle
		webnotes.conn.sql("""update `tabStock Ledger Entry`
			set qty_after_transaction=%s, valuation_rate=%s, stock_queue=%s,
			stock_value=%s where name=%s""",
			(qty_after_transaction, valuation_rate,
			json.dumps(stock_queue), stock_value, sle.name))

	if _exceptions:
		_raise_exceptions(args, verbose)

	# update bin
	if not webnotes.conn.exists({"doctype": "Bin", "item_code": args["item_code"],
			"warehouse": args["warehouse"]}):
		bin_wrapper = webnotes.bean([{
			"doctype": "Bin",
			"item_code": args["item_code"],
			"warehouse": args["warehouse"],
		}])
		bin_wrapper.ignore_permissions = 1
		bin_wrapper.insert()

	webnotes.conn.sql("""update `tabBin` set valuation_rate=%s, actual_qty=%s,
		stock_value=%s,
		projected_qty = (actual_qty + indented_qty + ordered_qty + planned_qty - reserved_qty)
		where item_code=%s and warehouse=%s""", (valuation_rate, qty_after_transaction,
		stock_value, args["item_code"], args["warehouse"]))

def get_sle_before_datetime(args, for_update=False):
	"""
		get previous stock ledger entry before current time-bucket

		Details:
		get the last sle before the current time-bucket, so that all values
		are reposted from the current time-bucket onwards.
		this is necessary because at the time of cancellation, there may be
		entries between the cancelled entries in the same time-bucket
	"""
	sle = get_stock_ledger_entries(args,
		["timestamp(posting_date, posting_time) < timestamp(%(posting_date)s, %(posting_time)s)"],
		"desc", "limit 1", for_update=for_update)

	return sle and sle[0] or webnotes._dict()

def get_sle_after_datetime(args, for_update=False):
	"""get Stock Ledger Entries after a particular datetime, for reposting"""
	# NOTE: using for update of
	return get_stock_ledger_entries(args,
		["timestamp(posting_date, posting_time) > timestamp(%(posting_date)s, %(posting_time)s)"],
		"asc", for_update=for_update)

def get_stock_ledger_entries(args, conditions=None, order="desc", limit=None, for_update=False):
	"""get stock ledger entries filtered by specific posting datetime conditions"""
	if not args.get("posting_date"):
		args["posting_date"] = "1900-01-01"
	if not args.get("posting_time"):
		args["posting_time"] = "00:00"

	return webnotes.conn.sql("""select * from `tabStock Ledger Entry`
		where item_code = %%(item_code)s
		and warehouse = %%(warehouse)s
		and ifnull(is_cancelled, 'No') = 'No'
		%(conditions)s
		order by timestamp(posting_date, posting_time) %(order)s, name %(order)s
		%(limit)s %(for_update)s""" % {
			"conditions": conditions and ("and " + " and ".join(conditions)) or "",
			"limit": limit or "",
			"for_update": for_update and "for update" or "",
			"order": order
		}, args, as_dict=1)

def validate_negative_stock(qty_after_transaction, sle):
	"""
		validate negative stock for entries current datetime onwards
		will not consider cancelled entries
	"""
	diff = qty_after_transaction + flt(sle.actual_qty)

	if diff < 0 and abs(diff) > 0.0001:
		# negative stock!
		global _exceptions
		exc = sle.copy().update({"diff": diff})
		_exceptions.append(exc)
		return False
	else:
		return True

def get_serialized_values(qty_after_transaction, sle, valuation_rate):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)
	serial_no = cstr(sle.serial_no).split("\n")

	if incoming_rate < 0:
		# wrong incoming rate
		incoming_rate = valuation_rate
	elif incoming_rate == 0 or flt(sle.actual_qty) < 0:
		# In case of delivery/stock issue, get average purchase rate
		# of serial nos of current entry
		incoming_rate = flt(webnotes.conn.sql("""select avg(ifnull(purchase_rate, 0))
			from `tabSerial No` where name in (%s)""" % (", ".join(["%s"]*len(serial_no))),
			tuple(serial_no))[0][0])

	if incoming_rate and not valuation_rate:
		valuation_rate = incoming_rate
	else:
		new_stock_qty = qty_after_transaction + actual_qty
		if new_stock_qty > 0:
			new_stock_value = qty_after_transaction * valuation_rate + actual_qty * incoming_rate
			if new_stock_value > 0:
				# calculate new valuation rate only if stock value is positive
				# else it remains the same as that of previous entry
				valuation_rate = new_stock_value / new_stock_qty

	return valuation_rate

def get_moving_average_values(qty_after_transaction, sle, valuation_rate):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)

	if not incoming_rate:
		# In case of delivery/stock issue in_rate = 0 or wrong incoming rate
		incoming_rate = valuation_rate

	elif qty_after_transaction < 0:
		# if negative stock, take current valuation rate as incoming rate
		valuation_rate = incoming_rate

	new_stock_qty = qty_after_transaction + actual_qty
	new_stock_value = qty_after_transaction * valuation_rate + actual_qty * incoming_rate

	if new_stock_qty > 0 and new_stock_value > 0:
		valuation_rate = new_stock_value / flt(new_stock_qty)
	elif new_stock_qty <= 0:
		valuation_rate = 0.0

	# NOTE: val_rate is same as previous entry if new stock value is negative

	return valuation_rate

def get_fifo_values(qty_after_transaction, sle, stock_queue):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)
	if not stock_queue:
		stock_queue.append([0, 0])

	if actual_qty > 0:
		if stock_queue[-1][0] > 0:
			stock_queue.append([actual_qty, incoming_rate])
		else:
			qty = stock_queue[-1][0] + actual_qty
			stock_queue[-1] = [qty, qty > 0 and incoming_rate or 0]
	else:
		incoming_cost = 0
		qty_to_pop = abs(actual_qty)
		while qty_to_pop:
			if not stock_queue:
				stock_queue.append([0, 0])

			batch = stock_queue[0]

			if 0 < batch[0] <= qty_to_pop:
				# if batch qty > 0
				# not enough or exactly same qty in current batch, clear batch
				incoming_cost += flt(batch[0]) * flt(batch[1])
				qty_to_pop -= batch[0]
				stock_queue.pop(0)
			else:
				# all from current batch
				incoming_cost += flt(qty_to_pop) * flt(batch[1])
				batch[0] -= qty_to_pop
				qty_to_pop = 0

	stock_value = sum((flt(batch[0]) * flt(batch[1]) for batch in stock_queue))
	stock_qty = sum((flt(batch[0]) for batch in stock_queue))

	valuation_rate = stock_qty and (stock_value / flt(stock_qty)) or 0

	return valuation_rate

def _raise_exceptions(args, verbose=1):
	deficiency = min(e["diff"] for e in _exceptions)
	msg = """Negative stock error:
		Cannot complete this transaction because stock will start
		becoming negative (%s) for Item %s in Warehouse
		%s on %s %s in Transaction %s %s.
		Total Quantity Deficiency: %s""" % \
		(_exceptions[0]["diff"], args.get("item_code"), args.get("warehouse"),
		_exceptions[0]["posting_date"], _exceptions[0]["posting_time"],
		_exceptions[0]["voucher_type"], _exceptions[0]["voucher_no"],
		abs(deficiency))
	if verbose:
		msgprint(msg, raise_exception=1)
	else:
		raise webnotes.ValidationError, msg

def get_previous_sle(args, for_update=False):
	"""
		get the last sle on or before the current time-bucket,
		to get actual qty before transaction, this function
		is called from various transaction like stock entry, reco etc

		args = {
			"item_code": "ABC",
			"warehouse": "XYZ",
			"posting_date": "2012-12-12",
			"posting_time": "12:00",
			"sle": "name of reference Stock Ledger Entry"
		}
	"""
	if not args.get("sle"): args["sle"] = ""

	sle = get_stock_ledger_entries(args, ["name != %(sle)s",
		"timestamp(posting_date, posting_time) <= timestamp(%(posting_date)s, %(posting_time)s)"],
		"desc", "limit 1", for_update=for_update)
	return sle and sle[0] or {}<commit_msg>[stock ledger][fix] message for negative qty error<commit_after># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import webnotes
from webnotes import msgprint
from webnotes.utils import cint, flt, cstr
from stock.utils import get_valuation_method
import json

# future reposting

_exceptions = []
def update_entries_after(args, verbose=1):
	"""
		update valution rate and qty after transaction
		from the current time-bucket onwards

		args = {
			"item_code": "ABC",
			"warehouse": "XYZ",
			"posting_date": "2012-12-12",
			"posting_time": "12:00"
		}
	"""
	global _exceptions
	_exceptions = []

	previous_sle = get_sle_before_datetime(args)

	qty_after_transaction = flt(previous_sle.get("qty_after_transaction"))
	valuation_rate = flt(previous_sle.get("valuation_rate"))
	stock_queue = json.loads(previous_sle.get("stock_queue") or "[]")
	stock_value = 0.0

	entries_to_fix = get_sle_after_datetime(previous_sle or \
		{"item_code": args["item_code"], "warehouse": args["warehouse"]}, for_update=True)

	valuation_method = get_valuation_method(args["item_code"])

	for sle in entries_to_fix:
		if sle.serial_no or not cint(webnotes.conn.get_default("allow_negative_stock")):
			# validate negative stock for serialized items, fifo valuation
			# or when negative stock is not allowed for moving average
			if not validate_negative_stock(qty_after_transaction, sle):
				qty_after_transaction += flt(sle.actual_qty)
				continue

		if sle.serial_no:
			valuation_rate = get_serialized_values(qty_after_transaction, sle, valuation_rate)
		elif valuation_method == "Moving Average":
			valuation_rate = get_moving_average_values(qty_after_transaction, sle, valuation_rate)
		else:
			valuation_rate = get_fifo_values(qty_after_transaction, sle, stock_queue)

		qty_after_transaction += flt(sle.actual_qty)

		# get stock value
		if sle.serial_no:
			stock_value = qty_after_transaction * valuation_rate
		elif valuation_method == "Moving Average":
			stock_value = (qty_after_transaction > 0) and \
				(qty_after_transaction * valuation_rate) or 0
		else:
			stock_value = sum((flt(batch[0]) * flt(batch[1]) for batch in stock_queue))
			# print sle.posting_date, sle.actual_qty, sle.incoming_rate, stock_queue, stock_value
		# update current sle
		webnotes.conn.sql("""update `tabStock Ledger Entry`
			set qty_after_transaction=%s, valuation_rate=%s, stock_queue=%s,
			stock_value=%s where name=%s""",
			(qty_after_transaction, valuation_rate,
			json.dumps(stock_queue), stock_value, sle.name))

	if _exceptions:
		_raise_exceptions(args, verbose)

	# update bin
	if not webnotes.conn.exists({"doctype": "Bin", "item_code": args["item_code"],
			"warehouse": args["warehouse"]}):
		bin_wrapper = webnotes.bean([{
			"doctype": "Bin",
			"item_code": args["item_code"],
			"warehouse": args["warehouse"],
		}])
		bin_wrapper.ignore_permissions = 1
		bin_wrapper.insert()

	webnotes.conn.sql("""update `tabBin` set valuation_rate=%s, actual_qty=%s,
		stock_value=%s,
		projected_qty = (actual_qty + indented_qty + ordered_qty + planned_qty - reserved_qty)
		where item_code=%s and warehouse=%s""", (valuation_rate, qty_after_transaction,
		stock_value, args["item_code"], args["warehouse"]))

def get_sle_before_datetime(args, for_update=False):
	"""
		get previous stock ledger entry before current time-bucket

		Details:
		get the last sle before the current time-bucket, so that all values
		are reposted from the current time-bucket onwards.
		this is necessary because at the time of cancellation, there may be
		entries between the cancelled entries in the same time-bucket
	"""
	sle = get_stock_ledger_entries(args,
		["timestamp(posting_date, posting_time) < timestamp(%(posting_date)s, %(posting_time)s)"],
		"desc", "limit 1", for_update=for_update)

	return sle and sle[0] or webnotes._dict()

def get_sle_after_datetime(args, for_update=False):
	"""get Stock Ledger Entries after a particular datetime, for reposting"""
	# NOTE: using for update of
	return get_stock_ledger_entries(args,
		["timestamp(posting_date, posting_time) > timestamp(%(posting_date)s, %(posting_time)s)"],
		"asc", for_update=for_update)

def get_stock_ledger_entries(args, conditions=None, order="desc", limit=None, for_update=False):
	"""get stock ledger entries filtered by specific posting datetime conditions"""
	if not args.get("posting_date"):
		args["posting_date"] = "1900-01-01"
	if not args.get("posting_time"):
		args["posting_time"] = "00:00"

	return webnotes.conn.sql("""select * from `tabStock Ledger Entry`
		where item_code = %%(item_code)s
		and warehouse = %%(warehouse)s
		and ifnull(is_cancelled, 'No') = 'No'
		%(conditions)s
		order by timestamp(posting_date, posting_time) %(order)s, name %(order)s
		%(limit)s %(for_update)s""" % {
			"conditions": conditions and ("and " + " and ".join(conditions)) or "",
			"limit": limit or "",
			"for_update": for_update and "for update" or "",
			"order": order
		}, args, as_dict=1)

def validate_negative_stock(qty_after_transaction, sle):
	"""
		validate negative stock for entries current datetime onwards
		will not consider cancelled entries
	"""
	diff = qty_after_transaction + flt(sle.actual_qty)

	if diff < 0 and abs(diff) > 0.0001:
		# negative stock!
		global _exceptions
		exc = sle.copy().update({"diff": diff})
		_exceptions.append(exc)
		return False
	else:
		return True

def get_serialized_values(qty_after_transaction, sle, valuation_rate):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)
	serial_no = cstr(sle.serial_no).split("\n")

	if incoming_rate < 0:
		# wrong incoming rate
		incoming_rate = valuation_rate
	elif incoming_rate == 0 or flt(sle.actual_qty) < 0:
		# In case of delivery/stock issue, get average purchase rate
		# of serial nos of current entry
		incoming_rate = flt(webnotes.conn.sql("""select avg(ifnull(purchase_rate, 0))
			from `tabSerial No` where name in (%s)""" % (", ".join(["%s"]*len(serial_no))),
			tuple(serial_no))[0][0])

	if incoming_rate and not valuation_rate:
		valuation_rate = incoming_rate
	else:
		new_stock_qty = qty_after_transaction + actual_qty
		if new_stock_qty > 0:
			new_stock_value = qty_after_transaction * valuation_rate + actual_qty * incoming_rate
			if new_stock_value > 0:
				# calculate new valuation rate only if stock value is positive
				# else it remains the same as that of previous entry
				valuation_rate = new_stock_value / new_stock_qty

	return valuation_rate

def get_moving_average_values(qty_after_transaction, sle, valuation_rate):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)

	if not incoming_rate:
		# In case of delivery/stock issue in_rate = 0 or wrong incoming rate
		incoming_rate = valuation_rate

	elif qty_after_transaction < 0:
		# if negative stock, take current valuation rate as incoming rate
		valuation_rate = incoming_rate

	new_stock_qty = qty_after_transaction + actual_qty
	new_stock_value = qty_after_transaction * valuation_rate + actual_qty * incoming_rate

	if new_stock_qty > 0 and new_stock_value > 0:
		valuation_rate = new_stock_value / flt(new_stock_qty)
	elif new_stock_qty <= 0:
		valuation_rate = 0.0

	# NOTE: val_rate is same as previous entry if new stock value is negative

	return valuation_rate

def get_fifo_values(qty_after_transaction, sle, stock_queue):
	incoming_rate = flt(sle.incoming_rate)
	actual_qty = flt(sle.actual_qty)
	if not stock_queue:
		stock_queue.append([0, 0])

	if actual_qty > 0:
		if stock_queue[-1][0] > 0:
			stock_queue.append([actual_qty, incoming_rate])
		else:
			qty = stock_queue[-1][0] + actual_qty
			stock_queue[-1] = [qty, qty > 0 and incoming_rate or 0]
	else:
		incoming_cost = 0
		qty_to_pop = abs(actual_qty)
		while qty_to_pop:
			if not stock_queue:
				stock_queue.append([0, 0])

			batch = stock_queue[0]

			if 0 < batch[0] <= qty_to_pop:
				# if batch qty > 0
				# not enough or exactly same qty in current batch, clear batch
				incoming_cost += flt(batch[0]) * flt(batch[1])
				qty_to_pop -= batch[0]
				stock_queue.pop(0)
			else:
				# all from current batch
				incoming_cost += flt(qty_to_pop) * flt(batch[1])
				batch[0] -= qty_to_pop
				qty_to_pop = 0

	stock_value = sum((flt(batch[0]) * flt(batch[1]) for batch in stock_queue))
	stock_qty = sum((flt(batch[0]) for batch in stock_queue))

	valuation_rate = stock_qty and (stock_value / flt(stock_qty)) or 0

	return valuation_rate

def _raise_exceptions(args, verbose=1):
	deficiency = min(e["diff"] for e in _exceptions)
	msg = """Negative stock error:
		Cannot complete this transaction because stock will start
		becoming negative (%s) for Item %s in Warehouse
		%s on %s %s in Transaction %s %s.
		Total Quantity Deficiency: %s""" % \
		(_exceptions[0]["diff"], args.get("item_code"), args.get("warehouse"),
		_exceptions[0]["posting_date"], _exceptions[0]["posting_time"],
		_exceptions[0]["voucher_type"], _exceptions[0]["voucher_no"],
		abs(deficiency))
	if verbose:
		msgprint(msg, raise_exception=1)
	else:
		raise webnotes.ValidationError, msg

def get_previous_sle(args, for_update=False):
	"""
		get the last sle on or before the current time-bucket,
		to get actual qty before transaction, this function
		is called from various transaction like stock entry, reco etc

		args = {
			"item_code": "ABC",
			"warehouse": "XYZ",
			"posting_date": "2012-12-12",
			"posting_time": "12:00",
			"sle": "name of reference Stock Ledger Entry"
		}
	"""
	if not args.get("sle"): args["sle"] = ""

	sle = get_stock_ledger_entries(args, ["name != %(sle)s",
		"timestamp(posting_date, posting_time) <= timestamp(%(posting_date)s, %(posting_time)s)"],
		"desc", "limit 1", for_update=for_update)
	return sle and sle[0] or {}<|endoftext|>


<commit_before>########################################################################################
## This file is a part of YAP package of scripts. https://github.com/shpakoo/YAP
## Distributed under the MIT license: http://www.opensource.org/licenses/mit-license.php
## Copyright (c) 2011-2012 J.Craig Venter Institute.
########################################################################################

#!/usr/bin/python
#################################################
## 	A new program
#################################################
import sys, shlex, glob, time
from optparse import OptionParser
from string import maketrans
from subprocess import *

_author="Sebastian Szpakowski"
_date="2011/01/01"
_version="Version 1"

#################################################
##		Classes
##
	#################################################
	### Iterator over input file.
	### every line is converted into a dictionary with variables referred to by their
	### header name
class 	GeneralPurposeParser:
	def	__init__(self, file, skip=0, sep="\t"):
		self.filename = file
		self.fp = open(self.filename, "r")
		self.linecounter = 0
		self.sep=sep

		while skip>0:
			skip = skip-1
			self.next()

	def	__iter__(self):
		return (self)

	def	next(self):
		for currline in self.fp:
			currline = currline.strip().split(self.sep)
			self.linecounter = self.linecounter + 1
			return(currline)
		raise StopIteration

	def	__str__(self):
		count = 0

		return ("%s\n[LINE %s]" % (self.filename, self.linecounter))

	#################################################
	### Iterator over input fata file.
	### Only reading when requested
	### Useful for very large FASTA files
	### with many sequences

class	FastaParser:
	def	__init__ (self, x):
		self.filename = x
		self.fp = open(x, "r")
		self.currline = ""
		self.currentFastaName = ""
		self.currentFastaSequence = ""
		self.lastitem=False

	def	__iter__(self):
		return(self)

		#####
	def	next(self):
		for self.currline in self.fp:
			if self.currline.startswith(">"):
				self.currline = self.currline[1:]
				if self.currentFastaName == "":
					self.currentFastaName = self.currline
				else:
					otpt = (self.currentFastaName.strip(), self.currentFastaSequence.strip())
					self.currentFastaName = self.currline
					self.currentFastaSequence = ""
					self.previoustell = self.fp.tell()
					return (otpt)

			else:
				self.addSequence(self.currline)

		if not self.lastitem:
			self.lastitem=True
			return (self.currentFastaName.strip(), self.currentFastaSequence.strip())
		else:
			raise StopIteration

       	def	addSequence(self, x):
       		self.currentFastaSequence = "%s%s" % (self.currentFastaSequence, x.strip())

	def	__str__():
		return ("reading file: %s" %self.filename)

#################################################
##		Functions
##
def	qsub(num):
	batches.append(num)
	command = "qsub -P %s -N tmp.%s.in -cwd -M sszpakow@jcvi.org -m a \"/home/sszpakow/bin/python %s -i tmp.%s.in -o tmp.%s.out -s -p REFSEQ \"" % (options.projectid, num, sys.argv[0], num, num   )

	#command = "qsub -P %s -N tmp.%s.in -cwd -l \"fast\" -M sszpakow@jcvi.org -m a \"/home/sszpakow/bin/python %s -i tmp.%s.in -o tmp.%s.out -s -p REFSEQ \"" % (options.projectid, num, sys.argv[0], num, num   )
	# 	   qsub -P 810013 -N tmp.2.in -cwd -l "fast" -M sszpakow@jcvi.org -m a "/home/sszpakow/bin/python /home/sszpakow/scripts/summarizeAlignment.py -i tmp.2.in -o tmp.2.out -s -p REFSEQ "
	#print shlex.split(command)
	p = Popen(shlex.split(command), stdout=PIPE)

	print command
#################################################
##		Arguments
##
parser = OptionParser()

parser.add_option("-p", "--pattern", dest="pattern",
                  help="the first sequence name that starts with P will be used as a reference for coordinates", metavar="P")

parser.add_option("-i", "--input", dest="infilename",
                  help="NAME of an input file (fasta format)", metavar="NAME")

parser.add_option("-o", "--output", dest="outfilename",
                  help="NAME of an output file", metavar="NAME")

parser.add_option("-t", "--tmpsize", type="int", dest="size",
                  help="N fasta sequences will be put in each temporary file [default: %default]", metavar="N", default=10000)

parser.add_option("-s", "--slave",
                  action="store_false", dest="master", default=True,
                  help="qsub worker")

parser.add_option("-P", "--project", dest="projectid", default="810013",
                  help="projectid", metavar="num")

(options, args) = parser.parse_args()

#################################################
##		Begin
##

if options.master:
	#### init
	curname = 0
	batches = list()

	#### find pattern in the large input file
	file = FastaParser(options.infilename)
	refhed = ""
	seq= ""

	print "Searching for %s's and primers..." % (options.pattern)

	references = list()
	primers = list()
	otpt = open("all.found.alignedprimercoords", "w")
	for head, seq in file:
		if head.startswith("%s" % (options.pattern)) and refhed == "":
			refhed = head
			refseq = seq
			print "\t^--found !"

		if head.startswith("_primer_"):
			print "\t^--found %s!" % (head)
			seq = seq.lower()
			primerseq = seq.lower().replace("-", "").replace(".", "")
			letters =len(primerseq)
			#### ....------ACTG-ACT-----......
 			####           S      E

			start = seq.find(primerseq[0])
			counter = 0

			while letters > 0 and (start + counter) < len(seq) and seq[start+counter] != "." :
				k = seq[start + counter]

				print k, start, counter
				if k in ['a', 'c', 'g', 't']:
					letters = letters - 1
				counter +=1

			end = start + counter

			line = "%s\t%s\t%s\t%s\t%s\t%s\n" % (head, seq[start-2: end+2], start, end, len(primerseq), seq[start: end].count("-"))
			otpt.write(line)
			print line.strip()
	otpt.close()

	#### create temporary files with the references found and submit qsub jobs
	file = FastaParser(options.infilename)

	counter=0
	otpt = open("tmp.%s.in"% (curname), "w")
	otpt.write(">REFSEQ\n%s\n" % (refseq))

	for head, seq in file:
		if counter == options.size:
			otpt.close()
			qsub(curname)
			curname +=1
			otpt = open("tmp.%s.in"% (curname), "w")
			otpt.write(">REFSEQ\n%s\n" % (refseq))
			counter=0
		otpt.write(">%s\n%s\n" % (head, seq))

		counter+=1

	otpt.close()
	qsub(curname)

	#### wait until the jobs finish, merge whatever is available

	c1 = list()
	c2 = list()
	c3 = list()

	previousnums=set()
	while len(batches)>0:
		for file in glob.glob("*.out.done"):
			print file
			cols = dict()
			for line in GeneralPurposeParser(file[:-5]):

				#line = line.strip().split("\t")

				for column, val in enumerate(line):
					if not cols.has_key(column):
						cols[column]=list()

					cols[column].append(int(val))

			#print cols.keys()

			if len(c1)==0:
				c1 = cols[0]
				c2 = cols[1]
				c3 = cols[2]
			else:
				tmp = cols[2]

				c3 = [c3[index]+val for index, val in enumerate(tmp)]

			batch = file.strip().split(".")[1]
			batches.remove(int(batch))

			for XXX in glob.glob("tmp.%s.*" % (batch)):
				print "-%s" % (XXX)
				p = Popen(shlex.split("rm %s" % (XXX)),  stdout=PIPE)

		if len(batches) in previousnums:
			pass
		else:
			print "waiting for %s more..." % (len(batches))
			previousnums.add(len(batches))

		time.sleep(5)

	#### output results
	otpt = open(options.outfilename, "w")
	for index, val in enumerate(c1):
		line = "%s\t%s\t%s\n" % (c1[index], c2[index], c3[index])
		otpt.write(line)
	otpt.close()
else:

	file = FastaParser(options.infilename)
	coords = list()
	inttab = "acgtACGTN.-"
	outtab = "11111111000"
	transtab = maketrans(inttab, outtab)
	refhed = ""

	print "Searching for %s..." % (options.pattern)
	for head, seq in file:
		if head.startswith("%s" % (options.pattern)) and refhed == "":
			refhed = head
			refseq = list(seq.translate(transtab))
			refseq = [int(y) for y in refseq]
			print "\t^--found !"
		else:
			seq = list(seq.translate(transtab))
			seq = [int(y) for y in seq]
			#print seq
			if len(coords)==0:
				coords=seq
			else:
				coords = [coords[index]+val for index, val in enumerate(seq)]

	counter=0
	otpt = open(options.outfilename, 'w')
	for index, val in enumerate(refseq):
		if val==1:
			otpt.write("%s\t%s\t%s\n" % (index+1, counter+1, coords[index] ) )
			counter+=1
	otpt.close()

	open("%s.done" % (options.outfilename), 'w')

#################################################
##		Finish
#################################################
<commit_msg>Paths fixed, email address no longer hardcoded (new parameter -M)<commit_after>########################################################################################
## This file is a part of YAP package of scripts. https://github.com/shpakoo/YAP
## Distributed under the MIT license: http://www.opensource.org/licenses/mit-license.php
## Copyright (c) 2011-2012 J.Craig Venter Institute.
########################################################################################

#!/usr/bin/python
#################################################
## 	A new program
#################################################
import sys, shlex, glob, time
from optparse import OptionParser
from string import maketrans
from subprocess import *

_author="Sebastian Szpakowski"
_date="2011/01/01"
_version="Version 1"

#################################################
##		Classes
##
	#################################################
	### Iterator over input file.
	### every line is converted into a dictionary with variables referred to by their
	### header name
class 	GeneralPurposeParser:
	def	__init__(self, file, skip=0, sep="\t"):
		self.filename = file
		self.fp = open(self.filename, "r")
		self.linecounter = 0
		self.sep=sep

		while skip>0:
			skip = skip-1
			self.next()

	def	__iter__(self):
		return (self)

	def	next(self):
		for currline in self.fp:
			currline = currline.strip().split(self.sep)
			self.linecounter = self.linecounter + 1
			return(currline)
		raise StopIteration

	def	__str__(self):
		count = 0

		return ("%s\n[LINE %s]" % (self.filename, self.linecounter))

	#################################################
	### Iterator over input fata file.
	### Only reading when requested
	### Useful for very large FASTA files
	### with many sequences

class	FastaParser:
	def	__init__ (self, x):
		self.filename = x
		self.fp = open(x, "r")
		self.currline = ""
		self.currentFastaName = ""
		self.currentFastaSequence = ""
		self.lastitem=False

	def	__iter__(self):
		return(self)

		#####
	def	next(self):
		for self.currline in self.fp:
			if self.currline.startswith(">"):
				self.currline = self.currline[1:]
				if self.currentFastaName == "":
					self.currentFastaName = self.currline
				else:
					otpt = (self.currentFastaName.strip(), self.currentFastaSequence.strip())
					self.currentFastaName = self.currline
					self.currentFastaSequence = ""
					self.previoustell = self.fp.tell()
					return (otpt)

			else:
				self.addSequence(self.currline)

		if not self.lastitem:
			self.lastitem=True
			return (self.currentFastaName.strip(), self.currentFastaSequence.strip())
		else:
			raise StopIteration

       	def	addSequence(self, x):
       		self.currentFastaSequence = "%s%s" % (self.currentFastaSequence, x.strip())

	def	__str__():
		return ("reading file: %s" %self.filename)

#################################################
##		Functions
##
def	qsub(num):
	batches.append(num)
	command = "qsub -P %s -N tmp.%s.in -cwd -M %s -m a \"python %s -i tmp.%s.in -o tmp.%s.out -s -p REFSEQ \"" % (options.projectid, num,options.mailaddress, sys.argv[0], num, num   )

	#command = "qsub -P %s -N tmp.%s.in -cwd -l \"fast\" -M sszpakow@jcvi.org -m a \"/home/sszpakow/bin/python %s -i tmp.%s.in -o tmp.%s.out -s -p REFSEQ \"" % (options.projectid, num, sys.argv[0], num, num   )
	# 	   qsub -P 810013 -N tmp.2.in -cwd -l "fast" -M sszpakow@jcvi.org -m a "/home/sszpakow/bin/python /home/sszpakow/scripts/summarizeAlignment.py -i tmp.2.in -o tmp.2.out -s -p REFSEQ "
	#print shlex.split(command)
	p = Popen(shlex.split(command), stdout=PIPE)

	print command
#################################################
##		Arguments
##
parser = OptionParser()

parser.add_option("-p", "--pattern", dest="pattern",
                  help="the first sequence name that starts with P will be used as a reference for coordinates", metavar="P")

parser.add_option("-i", "--input", dest="infilename",
                  help="NAME of an input file (fasta format)", metavar="NAME")

parser.add_option("-o", "--output", dest="outfilename",
                  help="NAME of an output file", metavar="NAME")

parser.add_option("-t", "--tmpsize", type="int", dest="size",
                  help="N fasta sequences will be put in each temporary file [default: %default]", metavar="N", default=10000)

parser.add_option("-s", "--slave",
                  action="store_false", dest="master", default=True,
                  help="qsub worker")

parser.add_option("-P", "--project", dest="projectid", default="810013",
                  help="projectid", metavar="num")

parser.add_option("-M", "--mail", dest="mailaddress", default="sszpakow@jcvi.org",
                  help="email address", metavar="address")

(options, args) = parser.parse_args()

#################################################
##		Begin
##

if options.master:
	#### init
	curname = 0
	batches = list()

	#### find pattern in the large input file
	file = FastaParser(options.infilename)
	refhed = ""
	seq= ""

	print "Searching for %s's and primers..." % (options.pattern)

	references = list()
	primers = list()
	otpt = open("all.found.alignedprimercoords", "w")
	for head, seq in file:
		if head.startswith("%s" % (options.pattern)) and refhed == "":
			refhed = head
			refseq = seq
			print "\t^--found !"

		if head.startswith("_primer_"):
			print "\t^--found %s!" % (head)
			seq = seq.lower()
			primerseq = seq.lower().replace("-", "").replace(".", "")
			letters =len(primerseq)
			#### ....------ACTG-ACT-----......
 			####           S      E

			start = seq.find(primerseq[0])
			counter = 0

			while letters > 0 and (start + counter) < len(seq) and seq[start+counter] != "." :
				k = seq[start + counter]

				print k, start, counter
				if k in ['a', 'c', 'g', 't']:
					letters = letters - 1
				counter +=1

			end = start + counter

			line = "%s\t%s\t%s\t%s\t%s\t%s\n" % (head, seq[start-2: end+2], start, end, len(primerseq), seq[start: end].count("-"))
			otpt.write(line)
			print line.strip()
	otpt.close()

	#### create temporary files with the references found and submit qsub jobs
	file = FastaParser(options.infilename)

	counter=0
	otpt = open("tmp.%s.in"% (curname), "w")
	otpt.write(">REFSEQ\n%s\n" % (refseq))

	for head, seq in file:
		if counter == options.size:
			otpt.close()
			qsub(curname)
			curname +=1
			otpt = open("tmp.%s.in"% (curname), "w")
			otpt.write(">REFSEQ\n%s\n" % (refseq))
			counter=0
		otpt.write(">%s\n%s\n" % (head, seq))

		counter+=1

	otpt.close()
	qsub(curname)

	#### wait until the jobs finish, merge whatever is available

	c1 = list()
	c2 = list()
	c3 = list()

	previousnums=set()
	while len(batches)>0:
		for file in glob.glob("*.out.done"):
			print file
			cols = dict()
			for line in GeneralPurposeParser(file[:-5]):

				#line = line.strip().split("\t")

				for column, val in enumerate(line):
					if not cols.has_key(column):
						cols[column]=list()

					cols[column].append(int(val))

			#print cols.keys()

			if len(c1)==0:
				c1 = cols[0]
				c2 = cols[1]
				c3 = cols[2]
			else:
				tmp = cols[2]

				c3 = [c3[index]+val for index, val in enumerate(tmp)]

			batch = file.strip().split(".")[1]
			batches.remove(int(batch))

			for XXX in glob.glob("tmp.%s.*" % (batch)):
				print "-%s" % (XXX)
				p = Popen(shlex.split("rm %s" % (XXX)),  stdout=PIPE)

		if len(batches) in previousnums:
			pass
		else:
			print "waiting for %s more..." % (len(batches))
			previousnums.add(len(batches))

		time.sleep(5)

	#### output results
	otpt = open(options.outfilename, "w")
	for index, val in enumerate(c1):
		line = "%s\t%s\t%s\n" % (c1[index], c2[index], c3[index])
		otpt.write(line)
	otpt.close()
else:

	file = FastaParser(options.infilename)
	coords = list()
	inttab = "acgtACGTN.-"
	outtab = "11111111000"
	transtab = maketrans(inttab, outtab)
	refhed = ""

	print "Searching for %s..." % (options.pattern)
	for head, seq in file:
		if head.startswith("%s" % (options.pattern)) and refhed == "":
			refhed = head
			refseq = list(seq.translate(transtab))
			refseq = [int(y) for y in refseq]
			print "\t^--found !"
		else:
			seq = list(seq.translate(transtab))
			seq = [int(y) for y in seq]
			#print seq
			if len(coords)==0:
				coords=seq
			else:
				coords = [coords[index]+val for index, val in enumerate(seq)]

	counter=0
	otpt = open(options.outfilename, 'w')
	for index, val in enumerate(refseq):
		if val==1:
			otpt.write("%s\t%s\t%s\n" % (index+1, counter+1, coords[index] ) )
			counter+=1
	otpt.close()

	open("%s.done" % (options.outfilename), 'w')

#################################################
##		Finish
#################################################
<|endoftext|>


<commit_before>import time
from datetime import datetime, timedelta
from hashlib import md5

from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime

from loads.db import get_database
from loads.transport.client import Client

_COUNTS = frozenset((
    'addError', 'addSuccess', 'stopTestRun', 'startTest',
    'startTestRun', 'stopTest', 'add_hit',
    'socket_open', 'socket_message', 'socket_close',
    'socket_message'
))

class Controller(object):

    def __init__(self):
        self.db = get_database(
            settings.LOADS_DATABASE['backend'],
            **settings.LOADS_DATABASE['options']
        )
        self.client = Client(settings.LOADS_BROKER)

    def stop(self, run_id):
        self.client.stop_run(run_id)
        self.get_broker_info()

    def ping_db(self):
        return self.db.ping()

    def get_broker_info(self):
        return self.client.ping()

    def get_runs(self, **filters):
        if filters == {}:
            return self.db.get_runs()

        runs = []

        for run in self.db.get_runs():
            info = self.get_run_info(run)
            for key, value in filters.items():
                if key not in info['metadata']:
                    continue
                else:
                    if info['metadata'][key] == value:
                        runs.append(run)
                        break

        return runs

    def get_run_info(self, run_id, data=True):
        result = {}
        if data:
            data = self.db.get_data(run_id, size=100)
            result['data'] = data
            errors = {}

            lines = self.db.get_data(run_id, data_type='addError', size=100)
            for line in lines:
                error, tb, tb2 = line['exc_info']
                hashed = md5(error).hexdigest()
                if hashed in errors:
                    old_count, tb = errors[hashed]
                    errors[hashed] = old_count + 1, tb
                else:
                    errors[hashed] = 1, tb + '\n' + tb2

            errors = errors.items()
            errors.sort()
            result['errors'] = errors

        counts = self.db.get_counts(run_id)
        custom = {}
        for key, value in list(counts.items()):
            if key in _COUNTS:
                continue
            custom[key] = value
            del counts[key]

        result['custom'] = custom

        metadata = self.db.get_metadata(run_id)
        started = metadata.get('started')
        ended = metadata.get('ended', time.time())
        active = metadata.get('active', False)

        # aproximative = should be set by the broker
        if started is not None:
            if active:
                elapsed = time.time() - started
            else:
                elapsed = ended - started

            hits = counts.get('add_hit', 0)
            if hits == 0:
                rps = 0
            else:
                rps = hits / elapsed

            #TODO: Move to template
            started = datetime.fromtimestamp(int(started))
            metadata['started'] = started.strftime('%Y-%m-%d %H:%M:%S')
            counts['rps'] = int(rps)
            counts['elapsed'] = naturaltime(elapsed)
            ended = started + timedelta(seconds=elapsed)
            counts['finished'] = naturaltime(ended)
            counts['success'] = counts.get('addError', 0) == 0
            metadata['style'] = counts['success'] and 'green' or 'red'
        else:
            metadata['started'] = 'N/A'
            counts['rps'] = 0
            counts['elapsed'] = 0
            counts['finished'] = 'N/A'
            counts['success'] = False
            metadata['style'] = 'red'

        if metadata.get('active', False):
            metadata['active_label'] = 'Running'
        else:
            metadata['active_label'] = 'Ended'

        result['counts'] = counts
        result['metadata'] = metadata
        return result
<commit_msg>Make filters behave more logical<commit_after>import time
from datetime import datetime, timedelta
from hashlib import md5

from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime

from loads.db import get_database
from loads.transport.client import Client

_COUNTS = frozenset((
    'addError', 'addSuccess', 'stopTestRun', 'startTest',
    'startTestRun', 'stopTest', 'add_hit',
    'socket_open', 'socket_message', 'socket_close',
    'socket_message'
))

class Controller(object):

    def __init__(self):
        self.db = get_database(
            settings.LOADS_DATABASE['backend'],
            **settings.LOADS_DATABASE['options']
        )
        self.client = Client(settings.LOADS_BROKER)

    def stop(self, run_id):
        self.client.stop_run(run_id)
        self.get_broker_info()

    def ping_db(self):
        return self.db.ping()

    def get_broker_info(self):
        return self.client.ping()

    def get_runs(self, **filters):
        if filters == {}:
            return self.db.get_runs()

        runs = []

        for run in self.db.get_runs():
            info = self.get_run_info(run)
            for key, value in filters.items():
                if key not in info['metadata']:
                    continue
                else:
                    if info['metadata'][key] == value:
                        runs.append(run)

        return runs

    def get_run_info(self, run_id, data=True):
        result = {}
        if data:
            data = self.db.get_data(run_id, size=100)
            # Make it easier to serialize so a list is better than a generator
            result['data'] = list(data)
            errors = {}

            lines = self.db.get_data(run_id, data_type='addError', size=100)
            for line in lines:
                error, tb, tb2 = line['exc_info']
                hashed = md5(error).hexdigest()
                if hashed in errors:
                    old_count, tb = errors[hashed]
                    errors[hashed] = old_count + 1, tb
                else:
                    errors[hashed] = 1, tb + '\n' + tb2

            errors = errors.items()
            errors.sort()
            result['errors'] = errors

        counts = self.db.get_counts(run_id)
        custom = {}
        for key, value in list(counts.items()):
            if key in _COUNTS:
                continue
            custom[key] = value
            del counts[key]

        result['custom'] = custom

        metadata = self.db.get_metadata(run_id)
        started = metadata.get('started')
        ended = metadata.get('ended', time.time())
        active = metadata.get('active', False)

        # aproximative = should be set by the broker
        if started is not None:
            if active:
                elapsed = time.time() - started
            else:
                elapsed = ended - started

            hits = counts.get('add_hit', 0)
            if hits == 0:
                rps = 0
            else:
                rps = hits / elapsed

            #TODO: Move to template
            started = datetime.fromtimestamp(int(started))
            metadata['started'] = started.strftime('%Y-%m-%d %H:%M:%S')
            counts['rps'] = int(rps)
            counts['elapsed'] = naturaltime(elapsed)
            ended = started + timedelta(seconds=elapsed)
            counts['finished'] = naturaltime(ended)
            counts['success'] = counts.get('addError', 0) == 0
            metadata['style'] = counts['success'] and 'green' or 'red'
        else:
            metadata['started'] = 'N/A'
            counts['rps'] = 0
            counts['elapsed'] = 0
            counts['finished'] = 'N/A'
            counts['success'] = False
            metadata['style'] = 'red'

        if metadata.get('active', False):
            metadata['active_label'] = 'Running'
        else:
            metadata['active_label'] = 'Ended'

        result['counts'] = counts
        result['metadata'] = metadata
        return result
<|endoftext|>


<commit_before># Startup file for the NodeBox OS X application
# PyObjC requires the startup file to be in the root folder.
# This just imports everything from the nodebox.gui.mac module
# and works from there

import os
#import operator
#import warnings

# for the libraries
import sgmllib
import bs4

# moved to Library
# import numpy

import sqlite3
import zipfile

import hashlib
import base64
import re
import pickle
#import UserList

# needed by matplotlib et al.
# import six
# import pyparsing
# import cycler
# import requests
# import pymongo
# import dateutil
# import copy
# import csv
# import functools
# import itertools
# import gzip
# import io
# import httplib
# import http
# import http.client
# import http.server
# import urllib
# import urlparse
# import decimal
# import gettext

# PIL / Pillow support
import PIL
import PIL.Image
import PIL.ImageFilter
import PIL.ImageChops
import PIL.ImageEnhance
import PIL.ImageOps
import PIL.ImageDraw
import PIL.ImageStat

import objc
import Foundation
import AppKit

from PyObjCTools import AppHelper

import nodebox
import nodebox.geo
import nodebox.graphics
import nodebox.gui
import nodebox.gui.mac

# pattern
# import xml.etree
# import xml.etree.cElementTree

# Do not import these here; they are in the app and are imported by scripts
# If you import here, 10.6 breaks
# import scipy
# import matplotlib
# import cairo

# Twyg is in Library
# import twyg

#import PyObjCTools.Debugging
#PyObjCTools.Debugging.installVerboseExceptionHandler()

AppHelper.runEventLoop()
<commit_msg>Updated imports for pattern.<commit_after># Startup file for the NodeBox OS X application
# PyObjC requires the startup file to be in the root folder.
# This just imports everything from the nodebox.gui.mac module
# and works from there

import os
import operator
import warnings

# for the libraries
import sgmllib
import bs4

# moved to Library
import numpy

import sqlite3
import zipfile

import hashlib
import base64
import re
import pickle

# pattern lib
import new
import csv
import functools
import itertools
import httplib
import codecs
import calendar
import types
import MySQLdb

import api
import feed
import oauth
import json
import locale

#import UserList

# needed by matplotlib et al.
# import six
# import pyparsing
# import cycler
# import requests
# import pymongo
# import dateutil
# import copy
# import gzip
# import io
# import http
# import http.client
# import http.server
# import urllib
# import urlparse
# import decimal
# import gettext

# PIL / Pillow support
import PIL
import PIL.Image
import PIL.ImageFilter
import PIL.ImageChops
import PIL.ImageEnhance
import PIL.ImageOps
import PIL.ImageDraw
import PIL.ImageStat

import objc
import Foundation
import AppKit

from PyObjCTools import AppHelper

import nodebox
import nodebox.geo
import nodebox.graphics
import nodebox.gui
import nodebox.gui.mac

# pattern
# import xml.etree
# import xml.etree.cElementTree

# Do not import these here; they are in the app and are imported by scripts
# If you import here, 10.6 breaks
# import scipy
# import matplotlib
# import cairo

# Twyg is in Library
# import twyg

#import PyObjCTools.Debugging
#PyObjCTools.Debugging.installVerboseExceptionHandler()

AppHelper.runEventLoop()
<|endoftext|>


<commit_before>import re
from corehq.apps.sms.api import send_sms_to_verified_number
from corehq.util.translation import localize

from custom.ilsgateway.tanzania.handlers.arrived import ArrivedHandler
from custom.ilsgateway.tanzania.handlers.delivered import DeliveredHandler
from custom.ilsgateway.tanzania.handlers.help import HelpHandler
from custom.ilsgateway.tanzania.handlers.la import LossAndAdjustment
from custom.ilsgateway.tanzania.handlers.language import LanguageHandler
from custom.ilsgateway.tanzania.handlers.messageinitiator import MessageInitiatior
from custom.ilsgateway.tanzania.handlers.notdelivered import NotDeliveredHandler
from custom.ilsgateway.tanzania.handlers.notsubmitted import NotSubmittedHandler
from custom.ilsgateway.tanzania.handlers.register import RegisterHandler
from custom.ilsgateway.tanzania.handlers.soh import SOHHandler
from custom.ilsgateway.tanzania.handlers.stockout import StockoutHandler
from custom.ilsgateway.tanzania.handlers.stop import StopHandler
from custom.ilsgateway.tanzania.handlers.supervision import SupervisionHandler
from custom.ilsgateway.tanzania.handlers.randr import RandrHandler
from custom.ilsgateway.tanzania.handlers.yes import YesHandler
from custom.ilsgateway.models import ILSGatewayConfig
from custom.ilsgateway.tanzania.reminders import CONTACT_SUPERVISOR

def handle(verified_contact, text, msg=None):
    user = verified_contact.owner if verified_contact else None
    domain = user.domain

    if domain and not ILSGatewayConfig.for_domain(domain):
        return False

    args = text.split()
    if not args:
        return False
    keyword = args[0]

    if keyword.startswith('#'):
        return False

    args = args[1:]
    params = {
        'user': user,
        'domain': domain,
        'args': args,
        'msg': msg,
        'verified_contact': verified_contact
    }

    def not_function(word):
        if args and re.match("del", word):
            return NotDeliveredHandler
        elif args and re.match("sub", word):
            return NotSubmittedHandler
        return None

    handlers = {
        ('soh', 'hmk'): SOHHandler,
        ('submitted', 'nimetuma'): RandrHandler,
        ('delivered', 'dlvd', 'nimepokea'): DeliveredHandler,
        ('sijapokea',): NotDeliveredHandler,
        ('sijatuma',): NotSubmittedHandler,
        ('supervision', 'usimamizi'): SupervisionHandler,
        ('arrived', 'aliwasili'): ArrivedHandler,
        ('help', 'msaada'): HelpHandler,
        ('language', 'lang', 'lugha'): LanguageHandler,
        ('stop', 'acha', 'hapo'): StopHandler,
        ('yes', 'ndio', 'ndyo'): YesHandler,
        ('la', 'um'): LossAndAdjustment,
        ('stockout', 'hakuna'): StockoutHandler,
        ('register', 'reg', 'join', 'sajili'): RegisterHandler,
        ('test',): MessageInitiatior,
        ('not',): not_function(args[0]) if args else None
    }

    def choose_handler(keyword):
        for k, v in handlers.iteritems():
            if keyword.lower() in k:
                return v
        return None

    handler_class = choose_handler(keyword)
    handler = handler_class(**params) if handler_class else None

    if handler:
        if args:
            return handler.handle()
        else:
            return handler.help()
    elif keyword != 'l':
        with localize(verified_contact.owner.get_language_code()):
            send_sms_to_verified_number(verified_contact, unicode(CONTACT_SUPERVISOR))
        return True
<commit_msg>Remove newlines from all messages<commit_after>import re
from corehq.apps.sms.api import send_sms_to_verified_number
from corehq.util.translation import localize

from custom.ilsgateway.tanzania.handlers.arrived import ArrivedHandler
from custom.ilsgateway.tanzania.handlers.delivered import DeliveredHandler
from custom.ilsgateway.tanzania.handlers.help import HelpHandler
from custom.ilsgateway.tanzania.handlers.la import LossAndAdjustment
from custom.ilsgateway.tanzania.handlers.language import LanguageHandler
from custom.ilsgateway.tanzania.handlers.messageinitiator import MessageInitiatior
from custom.ilsgateway.tanzania.handlers.notdelivered import NotDeliveredHandler
from custom.ilsgateway.tanzania.handlers.notsubmitted import NotSubmittedHandler
from custom.ilsgateway.tanzania.handlers.register import RegisterHandler
from custom.ilsgateway.tanzania.handlers.soh import SOHHandler
from custom.ilsgateway.tanzania.handlers.stockout import StockoutHandler
from custom.ilsgateway.tanzania.handlers.stop import StopHandler
from custom.ilsgateway.tanzania.handlers.supervision import SupervisionHandler
from custom.ilsgateway.tanzania.handlers.randr import RandrHandler
from custom.ilsgateway.tanzania.handlers.yes import YesHandler
from custom.ilsgateway.models import ILSGatewayConfig
from custom.ilsgateway.tanzania.reminders import CONTACT_SUPERVISOR

def handle(verified_contact, text, msg=None):
    user = verified_contact.owner if verified_contact else None
    domain = user.domain

    if domain and not ILSGatewayConfig.for_domain(domain):
        return False

    text = text.replace('\r', ' ').replace('\n', ' ').strip()
    args = text.split()
    if not args:
        return False
    keyword = args[0]

    if keyword.startswith('#'):
        return False

    args = args[1:]
    params = {
        'user': user,
        'domain': domain,
        'args': args,
        'msg': msg,
        'verified_contact': verified_contact
    }

    def not_function(word):
        if args and re.match("del", word):
            return NotDeliveredHandler
        elif args and re.match("sub", word):
            return NotSubmittedHandler
        return None

    handlers = {
        ('soh', 'hmk'): SOHHandler,
        ('submitted', 'nimetuma'): RandrHandler,
        ('delivered', 'dlvd', 'nimepokea'): DeliveredHandler,
        ('sijapokea',): NotDeliveredHandler,
        ('sijatuma',): NotSubmittedHandler,
        ('supervision', 'usimamizi'): SupervisionHandler,
        ('arrived', 'aliwasili'): ArrivedHandler,
        ('help', 'msaada'): HelpHandler,
        ('language', 'lang', 'lugha'): LanguageHandler,
        ('stop', 'acha', 'hapo'): StopHandler,
        ('yes', 'ndio', 'ndyo'): YesHandler,
        ('la', 'um'): LossAndAdjustment,
        ('stockout', 'hakuna'): StockoutHandler,
        ('register', 'reg', 'join', 'sajili'): RegisterHandler,
        ('test',): MessageInitiatior,
        ('not',): not_function(args[0]) if args else None
    }

    def choose_handler(keyword):
        for k, v in handlers.iteritems():
            if keyword.lower() in k:
                return v
        return None

    handler_class = choose_handler(keyword)
    handler = handler_class(**params) if handler_class else None

    if handler:
        if args:
            return handler.handle()
        else:
            return handler.help()
    elif keyword != 'l':
        with localize(verified_contact.owner.get_language_code()):
            send_sms_to_verified_number(verified_contact, unicode(CONTACT_SUPERVISOR))
        return True
<|endoftext|>