<commit_before>import discord
import re
from discord.ext import commands

class MCOCTools:
    '''Tools for Marvel Contest of Champions'''
    lookup_links = {
            'event': (
                'Tiny MCoC Schedule',
                '<http://simians.tk/MCOC-Sched>',
                'Josh Morris Schedule',
                'https://d2jixqqjqj5d23.cloudfront.net/assets/developer/imgs/icons/google-spreadsheet-icon.png'),
            'hook': (
                'hook/champions',
                '<http://hook.github.io/champions>',
                'hook/champions for Collector',
                'https://assets-cdn.github.com/favicon.ico'),
            'spotlight': (
                'MCoC Spotlight',
                '<http://simians.tk/MCoCspotlight>'),
            'marvelsynergy': (
                'Team Builder',
                '<http://www.marvelsynergy.com/team-builder>',
                'Marvel Synergy',
                'http://www.marvelsynergy.com/images/marvelsynergy.png'),
            'alsciende':(
                'Alsciende Mastery Tool',
                '<https://alsciende.github.io/masteries/v10.0.1/#>',
                'by u/alsciende',
                'https://assets-cdn.github.com/favicon.ico'),
            'simulator': (
                'Mastery Simulator',
                '<http://simians.tk/msimSDF>'),
            'streak': (
                'Infinite Streak',
                'http://simians.tk/-sdf-streak')
                #'http://simians.tk/SDFstreak')
    }
    mcolor = discord.Color.gold()
    icon_sdf = 'https://raw.githubusercontent.com/JasonJW/mcoc-cogs/master/mcoc/data/sdf_icon.png'

    def __init__(self, bot):
        self.bot = bot

    def present(self, lookup):
        em=discord.Embed(color=self.mcolor,title=lookup[0],description=lookup[1])
        print(len(lookup))
        if len(lookup) > 2:
            em.set_footer(text=lookup[2],icon_url=lookup[3])
        else:
            em.set_footer(text='Presented by [-SDF-]',icon_url=self.icon_sdf)
        return em

    #Gold Realm disabled
    # @commands.command()
    # async def gold(self,msg = None):
    #     '''Gold Realm Schedule'''
    #     package = '```           PST      CST      EST\nJun 5    10:00    12:00    13:00\nJun 5    19:00    21:00    22:00\nJun 6     4:00     6:00     7:00\nJun 6    13:00    15:00    16:00\nJun 6    22:00     0:00     1:00\nJun 7     7:00     9:00    10:00\nJun 7    16:00    18:00    19:00\nJun 8     1:00     3:00     4:00\nJun 8    10:00    12:00    13:00\nJun 8    19:00    21:00    22:00\nJun 9     4:00     6:00     7:00\nJun 9    13:00    15:00    16:00\nJun 10   22:00     0:00     1:00\nJun 11    7:00     9:00    10:00\nJun 11   16:00    18:00    19:00\nJun 12    1:00     3:00     4:00```'
    #     # await self.bot.say('```'+package+'```')
    #     if msg is not None:
    #         package = msg+'\n'+package
    #     em = discord.Embed(color=discord.Color.gold(),title='Gold Realm Schedule',description=package)
    #     em.set_footer(text='Presented by [-SDF-]',icon_url=self.icon_sdf)
    #     await self.bot.say(embed=em)

    @commands.command(help=lookup_links['event'][0], aliases=['events','schedule',])
    async def event(self):
        lookup = self.lookup_links['event']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['spotlight'][0],)
    async def spotlight(self):
        lookup = self.lookup_links['spotlight']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['marvelsynergy'][0])
    async def marvelsynergy(self):
        lookup = self.lookup_links['marvelsynergy']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['simulator'][0],aliases=['msim'])
    async def simulator(self):
        lookup = self.lookup_links['simulator']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['alsciende'][0], aliases=('mrig',))
    async def alsciende(self):
        lookup = self.lookup_links['alsciende']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['streak'][0])
    async def streak(self):
        lookup = self.lookup_links['streak']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['hook'][0])
    async def hook(self):
        lookup = self.lookup_links['hook']
        await self.bot.say(embed=self.present(lookup))

#### Mastery stuff###
class MCOCMastery:
    masteryColor={'Offense': discord.Color.red(),
            'Defense': discord.Color.blue(),
            'Utility': discord.Color.green()}
    tokens={'Vitality','Greater Vitality','Salve','Recovery','Energy Resistance','Physical Resistance','Block Proficiancy','Perfect Block','Stand Your Ground','Collar Tech','Serum Science','Willpower','Coagulate','Suture','Inequity','Resonate',
        'Strength','Greater Strength','Pierce','Lesser Precision','Precision','Lesser Cruelty','Cruelty','Courage','Extended Fury','Enhanced Fury','Pure Skill','Mutagenesis','Glass Canon','Recoil','Liquid Courage','Double Edge','Despair','DeepWounds','Unfazed','Assassin',
        'Wisdom','Intelligence','Limber','Parry','Stupefy','Petrify','Pacify','Dexterity','Pitance','Prosperity','Cosmic Awareness','Mystic Dispersion','Detect Cosmic','Detect Tech','Detect Mystic','Scouter Lens','Detect Mutant','Detect Science','Detect Skill'}

    @commands.group(pass_context=True, aliases=['masteries',])
    async def mastery(self, ctx):
        if ctx.invoked_subcommand is None:
            await self.bot.send_cmd_help(ctx)
            return

    @mastery.command(pass_context=True, name='cost')
    async def _cost(self,ctx):
        await self.bot.say('Dummy message for cost')

def setup(bot):
    bot.add_cog(MCOCTools(bot))
<commit_msg>no message<commit_after>import discord
import re
from discord.ext import commands

class MCOCTools:
    '''Tools for Marvel Contest of Champions'''
    lookup_links = {
            'event': (
                'Tiny MCoC Schedule',
                '<http://simians.tk/MCOC-Sched>',
                'Josh Morris Schedule',
                'https://d2jixqqjqj5d23.cloudfront.net/assets/developer/imgs/icons/google-spreadsheet-icon.png'),
            'hook': (
                'hook/champions',
                '<http://hook.github.io/champions>',
                'hook/champions for Collector',
                'https://assets-cdn.github.com/favicon.ico'),
            'spotlight': (
                'MCoC Spotlight',
                '<http://simians.tk/MCoCspotlight>'),
            'marvelsynergy': (
                'Team Builder',
                '<http://www.marvelsynergy.com/team-builder>',
                'Marvel Synergy',
                'http://www.marvelsynergy.com/images/marvelsynergy.png'),
            'alsciende':(
                'Alsciende Mastery Tool',
                '<https://alsciende.github.io/masteries/v10.0.1/#>',
                'by u/alsciende',
                'https://assets-cdn.github.com/favicon.ico'),
            'simulator': (
                'Mastery Simulator',
                '<http://simians.tk/msimSDF>'),
            'streak': (
                'Infinite Streak',
                'http://simians.tk/-sdf-streak')
                #'http://simians.tk/SDFstreak')
    }
    mcolor = discord.Color.gold()
    icon_sdf = 'https://raw.githubusercontent.com/JasonJW/mcoc-cogs/master/mcoc/data/sdf_icon.png'

    def __init__(self, bot):
        self.bot = bot

    def present(self, lookup):
        em=discord.Embed(color=self.mcolor,title=lookup[0],description=lookup[1])
        print(len(lookup))
        if len(lookup) > 2:
            em.set_footer(text=lookup[2],icon_url=lookup[3])
        else:
            em.set_footer(text='Presented by [-SDF-]',icon_url=self.icon_sdf)
        return em

    #Gold Realm disabled
    # @commands.command()
    # async def gold(self,msg = None):
    #     '''Gold Realm Schedule'''
    #     package = '```           PST      CST      EST\nJun 5    10:00    12:00    13:00\nJun 5    19:00    21:00    22:00\nJun 6     4:00     6:00     7:00\nJun 6    13:00    15:00    16:00\nJun 6    22:00     0:00     1:00\nJun 7     7:00     9:00    10:00\nJun 7    16:00    18:00    19:00\nJun 8     1:00     3:00     4:00\nJun 8    10:00    12:00    13:00\nJun 8    19:00    21:00    22:00\nJun 9     4:00     6:00     7:00\nJun 9    13:00    15:00    16:00\nJun 10   22:00     0:00     1:00\nJun 11    7:00     9:00    10:00\nJun 11   16:00    18:00    19:00\nJun 12    1:00     3:00     4:00```'
    #     # await self.bot.say('```'+package+'```')
    #     if msg is not None:
    #         package = msg+'\n'+package
    #     em = discord.Embed(color=discord.Color.gold(),title='Gold Realm Schedule',description=package)
    #     em.set_footer(text='Presented by [-SDF-]',icon_url=self.icon_sdf)
    #     await self.bot.say(embed=em)

    @commands.command(help=lookup_links['event'][0], aliases=['events','schedule',])
    async def event(self):
        lookup = self.lookup_links['event']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['spotlight'][0],)
    async def spotlight(self):
        lookup = self.lookup_links['spotlight']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['marvelsynergy'][0])
    async def marvelsynergy(self):
        lookup = self.lookup_links['marvelsynergy']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['simulator'][0],aliases=['msim'])
    async def simulator(self):
        lookup = self.lookup_links['simulator']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['alsciende'][0], aliases=('mrig',))
    async def alsciende(self):
        lookup = self.lookup_links['alsciende']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['streak'][0])
    async def streak(self):
        lookup = self.lookup_links['streak']
        await self.bot.say(embed=self.present(lookup))

    @commands.command(help=lookup_links['hook'][0])
    async def hook(self):
        lookup = self.lookup_links['hook']
        await self.bot.say(embed=self.present(lookup))

#### Mastery stuff###
# class MCOCMastery:
    masteryColor={'Offense': discord.Color.red(),
            'Defense': discord.Color.blue(),
            'Utility': discord.Color.green()}
    tokens={'Vitality','Greater Vitality','Salve','Recovery','Energy Resistance','Physical Resistance','Block Proficiancy','Perfect Block','Stand Your Ground','Collar Tech','Serum Science','Willpower','Coagulate','Suture','Inequity','Resonate',
            'Strength','Greater Strength','Pierce','Lesser Precision','Precision','Lesser Cruelty','Cruelty','Courage','Extended Fury','Enhanced Fury','Pure Skill','Mutagenesis','Glass Canon','Recoil','Liquid Courage','Double Edge','Despair','DeepWounds','Unfazed','Assassin',
            'Wisdom','Intelligence','Limber','Parry','Stupefy','Petrify','Pacify','Dexterity','Pitance','Prosperity','Cosmic Awareness','Mystic Dispersion','Detect Cosmic','Detect Tech','Detect Mystic','Scouter Lens','Detect Mutant','Detect Science','Detect Skill'}

    @commands.group(pass_context=True, aliases=['masteries',])
    async def mastery(self, ctx):
        if ctx.invoked_subcommand is None:
            await self.bot.send_cmd_help(ctx)
            return

    @mastery.command(pass_context=True, name='cost')
    async def _cost(self,ctx):
        await self.bot.say('Dummy message for cost')

def setup(bot):
    bot.add_cog(MCOCTools(bot))
<|endoftext|>


<commit_before>"""(Mostly HTTP) response classes.

Copyright 2006 John J. Lee <jjl@pobox.com>

This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
included with the distribution).

"""

import copy, mimetools
from cStringIO import StringIO
from urllib2 import HTTPError

# XXX Andrew Dalke kindly sent me a similar class in response to my request on
# comp.lang.python, which I then proceeded to lose.  I wrote this class
# instead, but I think he's released his code publicly since, could pinch the
# tests from it, at least...

# For testing seek_wrapper invariant (note that
# test_urllib2.HandlerTest.test_seekable is expected to fail when this
# invariant checking is turned on).  The invariant checking is done by module
# ipdc, which is available here:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/436834
## from ipdbc import ContractBase
## class seek_wrapper(ContractBase):
class seek_wrapper:
    """Adds a seek method to a file object.

    This is only designed for seeking on readonly file-like objects.

    Wrapped file-like object must have a read method.  The readline method is
    only supported if that method is present on the wrapped object.  The
    readlines method is always supported.  xreadlines and iteration are
    supported only for Python 2.2 and above.

    Public attribute: wrapped (the wrapped file object).

    WARNING: All other attributes of the wrapped object (ie. those that are not
    one of wrapped, read, readline, readlines, xreadlines, __iter__ and next)
    are passed through unaltered, which may or may not make sense for your
    particular file object.

    """
    # General strategy is to check that cache is full enough, then delegate to
    # the cache (self.__cache, which is a cStringIO.StringIO instance).  A seek
    # position (self.__pos) is maintained independently of the cache, in order
    # that a single cache may be shared between multiple seek_wrapper objects.
    # Copying using module copy shares the cache in this way.

    def __init__(self, wrapped):
        self.wrapped = wrapped
        self.__have_readline = hasattr(self.wrapped, "readline")
        self.__cache = StringIO()
        self.__pos = 0  # seek position

    def invariant(self):
        # The end of the cache is always at the same place as the end of the
        # wrapped file.
        return self.wrapped.tell() == len(self.__cache.getvalue())

    def __getattr__(self, name):
        wrapped = self.__dict__.get("wrapped")
        if wrapped:
            return getattr(wrapped, name)
        return getattr(self.__class__, name)

    def seek(self, offset, whence=0):
        assert whence in [0,1,2]

        # how much data, if any, do we need to read?
        if whence == 2:  # 2: relative to end of *wrapped* file
            if offset < 0: raise ValueError("negative seek offset")
            # since we don't know yet where the end of that file is, we must
            # read everything
            to_read = None
        else:
            if whence == 0:  # 0: absolute
                if offset < 0: raise ValueError("negative seek offset")
                dest = offset
            else:  # 1: relative to current position
                pos = self.__pos
                if pos < offset:
                    raise ValueError("seek to before start of file")
                dest = pos + offset
            end = len(self.__cache.getvalue())
            to_read = dest - end
            if to_read < 0:
                to_read = 0

        if to_read != 0:
            self.__cache.seek(0, 2)
            if to_read is None:
                assert whence == 2
                self.__cache.write(self.wrapped.read())
                self.__pos = self.__cache.tell() - offset
            else:
                self.__cache.write(self.wrapped.read(to_read))
                # Don't raise an exception even if we've seek()ed past the end
                # of .wrapped, since fseek() doesn't complain in that case.
                # Also like fseek(), pretend we have seek()ed past the end,
                # i.e. not:
                #self.__pos = self.__cache.tell()
                # but rather:
                self.__pos = dest
        else:
            self.__pos = dest

    def tell(self):
        return self.__pos

    def __copy__(self):
        cpy = self.__class__(self.wrapped)
        cpy.__cache = self.__cache
        return cpy

    def get_data(self):
        pos = self.__pos
        try:
            self.seek(0)
            return self.read(-1)
        finally:
            self.__pos = pos

    def read(self, size=-1):
        pos = self.__pos
        end = len(self.__cache.getvalue())
        available = end - pos

        # enough data already cached?
        if size <= available and size != -1:
            self.__cache.seek(pos)
            self.__pos = pos+size
            return self.__cache.read(size)

        # no, so read sufficient data from wrapped file and cache it
        self.__cache.seek(0, 2)
        if size == -1:
            self.__cache.write(self.wrapped.read())
        else:
            to_read = size - available
            assert to_read > 0
            self.__cache.write(self.wrapped.read(to_read))
        self.__cache.seek(pos)

        data = self.__cache.read(size)
        self.__pos = self.__cache.tell()
        assert self.__pos == pos + len(data)
        return data

    def readline(self, size=-1):
        if not self.__have_readline:
            raise NotImplementedError("no readline method on wrapped object")

        # line we're about to read might not be complete in the cache, so
        # read another line first
        pos = self.__pos
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.readline())
        self.__cache.seek(pos)

        data = self.__cache.readline()
        if size != -1:
            r = data[:size]
            self.__pos = pos+size
        else:
            r = data
            self.__pos = pos+len(data)
        return r

    def readlines(self, sizehint=-1):
        pos = self.__pos
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.read())
        self.__cache.seek(pos)
        data = self.__cache.readlines(sizehint)
        self.__pos = self.__cache.tell()
        return data

    def __iter__(self): return self
    def next(self):
        line = self.readline()
        if line == "": raise StopIteration
        return line

    xreadlines = __iter__

    def __repr__(self):
        return ("<%s at %s whose wrapped object = %r>" %
                (self.__class__.__name__, hex(id(self)), self.wrapped))

class response_seek_wrapper(seek_wrapper):

    """
    Supports copying response objects and setting response body data.

    """

    def __init__(self, wrapped):
        seek_wrapper.__init__(self, wrapped)
        self._headers = self.wrapped.info()

    def __copy__(self):
        cpy = seek_wrapper.__copy__(self)
        # copy headers from delegate
        cpy._headers = copy.copy(self.info())
        return cpy

    def info(self):
        return self._headers

    def set_data(self, data):
        self.seek(0)
        self.read()
        self.close()
        cache = self._seek_wrapper__cache = StringIO()
        cache.write(data)
        self.seek(0)

class httperror_seek_wrapper(response_seek_wrapper, HTTPError):

    # this only derives from HTTPError in order to be a subclass --
    # the HTTPError behaviour comes from delegation

    def __init__(self, wrapped):
        assert isinstance(wrapped, closeable_response), wrapped
        response_seek_wrapper.__init__(self, wrapped)
        # be compatible with undocumented HTTPError attributes :-(
        self.hdrs = wrapped._headers
        self.filename = wrapped._url

    # we don't want the HTTPError implementation of these

    def geturl(self):
        return self.wrapped.geturl()

    def close(self):
        self.wrapped.close()

class eoffile:
    # file-like object that always claims to be at end-of-file...
    def read(self, size=-1): return ""
    def readline(self, size=-1): return ""
    def __iter__(self): return self
    def next(self): return ""
    def close(self): pass

class eofresponse(eoffile):
    def __init__(self, url, headers, code, msg):
        self._url = url
        self._headers = headers
        self.code = code
        self.msg = msg
    def geturl(self): return self._url
    def info(self): return self._headers

class closeable_response:
    """Avoids unnecessarily clobbering urllib.addinfourl methods on .close().

    Only supports responses returned by mechanize.HTTPHandler.

    After .close(), the following methods are supported:

    .read()
    .readline()
    .info()
    .geturl()
    .__iter__()
    .next()
    .close()

    and the following attributes are supported:

    .code
    .msg

    Also supports pickling (but the stdlib currently does something to prevent
    it: http://python.org/sf/1144636).

    """
    # presence of this attr indicates is useable after .close()
    closeable_response = None

    def __init__(self, fp, headers, url, code, msg):
        self._set_fp(fp)
        self._headers = headers
        self._url = url
        self.code = code
        self.msg = msg

    def _set_fp(self, fp):
        self.fp = fp
        self.read = self.fp.read
        self.readline = self.fp.readline
        if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
        if hasattr(self.fp, "fileno"):
            self.fileno = self.fp.fileno
        else:
            self.fileno = lambda: None
        self.__iter__ = self.fp.__iter__
        self.next = self.fp.next

    def __repr__(self):
        return '<%s at %s whose fp = %r>' % (
            self.__class__.__name__, hex(id(self)), self.fp)

    def info(self):
        return self._headers

    def geturl(self):
        return self._url

    def close(self):
        wrapped = self.fp
        wrapped.close()
        new_wrapped = eofresponse(
            self._url, self._headers, self.code, self.msg)
        self._set_fp(new_wrapped)

    def __getstate__(self):
        # There are three obvious options here:
        # 1. truncate
        # 2. read to end
        # 3. close socket, pickle state including read position, then open
        #    again on unpickle and use Range header
        # XXXX um, 4. refuse to pickle unless .close()d.  This is better,
        #  actually ("errors should never pass silently").  Pickling doesn't
        #  work anyway ATM, because of http://python.org/sf/1144636 so fix
        #  this later

        # 2 breaks pickle protocol, because one expects the original object
        # to be left unscathed by pickling.  3 is too complicated and
        # surprising (and too much work ;-) to happen in a sane __getstate__.
        # So we do 1.

        state = self.__dict__.copy()
        new_wrapped = eofresponse(
            self._url, self._headers, self.code, self.msg)
        state["wrapped"] = new_wrapped
        return state

def make_response(data, headers, url, code, msg):
    """Convenient factory for objects implementing response interface.

    data: string containing response body data
    headers: sequence of (name, value) pairs
    url: URL of response
    code: integer response code (e.g. 200)
    msg: string response code message (e.g. "OK")

    """
    hdr_text = []
    for name_value in headers:
        hdr_text.append("%s: %s" % name_value)
    mime_headers = mimetools.Message(StringIO("\n".join(hdr_text)))
    r = closeable_response(StringIO(data), mime_headers, url, code, msg)
    return response_seek_wrapper(r)

# Horrible, but needed, at least until fork urllib2.  Even then, may want
# to preseve urllib2 compatibility.
def upgrade_response(response):
    """Return a copy of response that supports mechanize response interface.

    Accepts responses from both mechanize and urllib2 handlers.
    """
    if isinstance(response, HTTPError):
        wrapper_class = httperror_seek_wrapper
    else:
        wrapper_class = response_seek_wrapper

    if hasattr(response, "closeable_response"):
        if not hasattr(response, "seek"):
            response = wrapper_class(response)
        return copy.copy(response)

    # a urllib2 handler constructed the response, i.e. the response is an
    # urllib.addinfourl or a urllib2.HTTPError, instead of a
    # _Util.closeable_response as returned by e.g. mechanize.HTTPHandler
    try:
        code = response.code
    except AttributeError:
        code = None
    try:
        msg = response.msg
    except AttributeError:
        msg = None

    # may have already-.read() data from .seek() cache
    data = None
    get_data = getattr(response, "get_data", None)
    if get_data:
        data = get_data()

    response = closeable_response(
        response.fp, response.info(), response.geturl(), code, msg)
    response = wrapper_class(response)
    if data:
        response.set_data(data)
    return response
<commit_msg>Fix catching HTTPError subclasses while still preserving all their response behaviour, with some somewhat-evil runtime class creation trickery<commit_after>"""(Mostly HTTP) response classes.

Copyright 2006 John J. Lee <jjl@pobox.com>

This code is free software; you can redistribute it and/or modify it
under the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt
included with the distribution).

"""

import copy, mimetools
from cStringIO import StringIO
import urllib2

# XXX Andrew Dalke kindly sent me a similar class in response to my request on
# comp.lang.python, which I then proceeded to lose.  I wrote this class
# instead, but I think he's released his code publicly since, could pinch the
# tests from it, at least...

# For testing seek_wrapper invariant (note that
# test_urllib2.HandlerTest.test_seekable is expected to fail when this
# invariant checking is turned on).  The invariant checking is done by module
# ipdc, which is available here:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/436834
## from ipdbc import ContractBase
## class seek_wrapper(ContractBase):
class seek_wrapper:
    """Adds a seek method to a file object.

    This is only designed for seeking on readonly file-like objects.

    Wrapped file-like object must have a read method.  The readline method is
    only supported if that method is present on the wrapped object.  The
    readlines method is always supported.  xreadlines and iteration are
    supported only for Python 2.2 and above.

    Public attribute: wrapped (the wrapped file object).

    WARNING: All other attributes of the wrapped object (ie. those that are not
    one of wrapped, read, readline, readlines, xreadlines, __iter__ and next)
    are passed through unaltered, which may or may not make sense for your
    particular file object.

    """
    # General strategy is to check that cache is full enough, then delegate to
    # the cache (self.__cache, which is a cStringIO.StringIO instance).  A seek
    # position (self.__pos) is maintained independently of the cache, in order
    # that a single cache may be shared between multiple seek_wrapper objects.
    # Copying using module copy shares the cache in this way.

    def __init__(self, wrapped):
        self.wrapped = wrapped
        self.__have_readline = hasattr(self.wrapped, "readline")
        self.__cache = StringIO()
        self.__pos = 0  # seek position

    def invariant(self):
        # The end of the cache is always at the same place as the end of the
        # wrapped file.
        return self.wrapped.tell() == len(self.__cache.getvalue())

    def __getattr__(self, name):
        wrapped = self.__dict__.get("wrapped")
        if wrapped:
            return getattr(wrapped, name)
        return getattr(self.__class__, name)

    def seek(self, offset, whence=0):
        assert whence in [0,1,2]

        # how much data, if any, do we need to read?
        if whence == 2:  # 2: relative to end of *wrapped* file
            if offset < 0: raise ValueError("negative seek offset")
            # since we don't know yet where the end of that file is, we must
            # read everything
            to_read = None
        else:
            if whence == 0:  # 0: absolute
                if offset < 0: raise ValueError("negative seek offset")
                dest = offset
            else:  # 1: relative to current position
                pos = self.__pos
                if pos < offset:
                    raise ValueError("seek to before start of file")
                dest = pos + offset
            end = len(self.__cache.getvalue())
            to_read = dest - end
            if to_read < 0:
                to_read = 0

        if to_read != 0:
            self.__cache.seek(0, 2)
            if to_read is None:
                assert whence == 2
                self.__cache.write(self.wrapped.read())
                self.__pos = self.__cache.tell() - offset
            else:
                self.__cache.write(self.wrapped.read(to_read))
                # Don't raise an exception even if we've seek()ed past the end
                # of .wrapped, since fseek() doesn't complain in that case.
                # Also like fseek(), pretend we have seek()ed past the end,
                # i.e. not:
                #self.__pos = self.__cache.tell()
                # but rather:
                self.__pos = dest
        else:
            self.__pos = dest

    def tell(self):
        return self.__pos

    def __copy__(self):
        cpy = self.__class__(self.wrapped)
        cpy.__cache = self.__cache
        return cpy

    def get_data(self):
        pos = self.__pos
        try:
            self.seek(0)
            return self.read(-1)
        finally:
            self.__pos = pos

    def read(self, size=-1):
        pos = self.__pos
        end = len(self.__cache.getvalue())
        available = end - pos

        # enough data already cached?
        if size <= available and size != -1:
            self.__cache.seek(pos)
            self.__pos = pos+size
            return self.__cache.read(size)

        # no, so read sufficient data from wrapped file and cache it
        self.__cache.seek(0, 2)
        if size == -1:
            self.__cache.write(self.wrapped.read())
        else:
            to_read = size - available
            assert to_read > 0
            self.__cache.write(self.wrapped.read(to_read))
        self.__cache.seek(pos)

        data = self.__cache.read(size)
        self.__pos = self.__cache.tell()
        assert self.__pos == pos + len(data)
        return data

    def readline(self, size=-1):
        if not self.__have_readline:
            raise NotImplementedError("no readline method on wrapped object")

        # line we're about to read might not be complete in the cache, so
        # read another line first
        pos = self.__pos
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.readline())
        self.__cache.seek(pos)

        data = self.__cache.readline()
        if size != -1:
            r = data[:size]
            self.__pos = pos+size
        else:
            r = data
            self.__pos = pos+len(data)
        return r

    def readlines(self, sizehint=-1):
        pos = self.__pos
        self.__cache.seek(0, 2)
        self.__cache.write(self.wrapped.read())
        self.__cache.seek(pos)
        data = self.__cache.readlines(sizehint)
        self.__pos = self.__cache.tell()
        return data

    def __iter__(self): return self
    def next(self):
        line = self.readline()
        if line == "": raise StopIteration
        return line

    xreadlines = __iter__

    def __repr__(self):
        return ("<%s at %s whose wrapped object = %r>" %
                (self.__class__.__name__, hex(id(self)), self.wrapped))

class response_seek_wrapper(seek_wrapper):

    """
    Supports copying response objects and setting response body data.

    """

    def __init__(self, wrapped):
        seek_wrapper.__init__(self, wrapped)
        self._headers = self.wrapped.info()

    def __copy__(self):
        cpy = seek_wrapper.__copy__(self)
        # copy headers from delegate
        cpy._headers = copy.copy(self.info())
        return cpy

    def info(self):
        return self._headers

    def set_data(self, data):
        self.seek(0)
        self.read()
        self.close()
        cache = self._seek_wrapper__cache = StringIO()
        cache.write(data)
        self.seek(0)

class eoffile:
    # file-like object that always claims to be at end-of-file...
    def read(self, size=-1): return ""
    def readline(self, size=-1): return ""
    def __iter__(self): return self
    def next(self): return ""
    def close(self): pass

class eofresponse(eoffile):
    def __init__(self, url, headers, code, msg):
        self._url = url
        self._headers = headers
        self.code = code
        self.msg = msg
    def geturl(self): return self._url
    def info(self): return self._headers

class closeable_response:
    """Avoids unnecessarily clobbering urllib.addinfourl methods on .close().

    Only supports responses returned by mechanize.HTTPHandler.

    After .close(), the following methods are supported:

    .read()
    .readline()
    .info()
    .geturl()
    .__iter__()
    .next()
    .close()

    and the following attributes are supported:

    .code
    .msg

    Also supports pickling (but the stdlib currently does something to prevent
    it: http://python.org/sf/1144636).

    """
    # presence of this attr indicates is useable after .close()
    closeable_response = None

    def __init__(self, fp, headers, url, code, msg):
        self._set_fp(fp)
        self._headers = headers
        self._url = url
        self.code = code
        self.msg = msg

    def _set_fp(self, fp):
        self.fp = fp
        self.read = self.fp.read
        self.readline = self.fp.readline
        if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
        if hasattr(self.fp, "fileno"):
            self.fileno = self.fp.fileno
        else:
            self.fileno = lambda: None
        self.__iter__ = self.fp.__iter__
        self.next = self.fp.next

    def __repr__(self):
        return '<%s at %s whose fp = %r>' % (
            self.__class__.__name__, hex(id(self)), self.fp)

    def info(self):
        return self._headers

    def geturl(self):
        return self._url

    def close(self):
        wrapped = self.fp
        wrapped.close()
        new_wrapped = eofresponse(
            self._url, self._headers, self.code, self.msg)
        self._set_fp(new_wrapped)

    def __getstate__(self):
        # There are three obvious options here:
        # 1. truncate
        # 2. read to end
        # 3. close socket, pickle state including read position, then open
        #    again on unpickle and use Range header
        # XXXX um, 4. refuse to pickle unless .close()d.  This is better,
        #  actually ("errors should never pass silently").  Pickling doesn't
        #  work anyway ATM, because of http://python.org/sf/1144636 so fix
        #  this later

        # 2 breaks pickle protocol, because one expects the original object
        # to be left unscathed by pickling.  3 is too complicated and
        # surprising (and too much work ;-) to happen in a sane __getstate__.
        # So we do 1.

        state = self.__dict__.copy()
        new_wrapped = eofresponse(
            self._url, self._headers, self.code, self.msg)
        state["wrapped"] = new_wrapped
        return state

def make_response(data, headers, url, code, msg):
    """Convenient factory for objects implementing response interface.

    data: string containing response body data
    headers: sequence of (name, value) pairs
    url: URL of response
    code: integer response code (e.g. 200)
    msg: string response code message (e.g. "OK")

    """
    hdr_text = []
    for name_value in headers:
        hdr_text.append("%s: %s" % name_value)
    mime_headers = mimetools.Message(StringIO("\n".join(hdr_text)))
    r = closeable_response(StringIO(data), mime_headers, url, code, msg)
    return response_seek_wrapper(r)

# Horrible, but needed, at least until fork urllib2.  Even then, may want
# to preseve urllib2 compatibility.
def upgrade_response(response):
    """Return a copy of response that supports mechanize response interface.

    Accepts responses from both mechanize and urllib2 handlers.
    """
    if isinstance(response, urllib2.HTTPError):
        class httperror_seek_wrapper(response_seek_wrapper, response.__class__):
            # this only derives from HTTPError in order to be a subclass --
            # the HTTPError behaviour comes from delegation

            def __init__(self, wrapped):
                assert isinstance(wrapped, closeable_response), wrapped
                response_seek_wrapper.__init__(self, wrapped)
                # be compatible with undocumented HTTPError attributes :-(
                self.hdrs = wrapped._headers
                self.filename = wrapped._url

            # we don't want the HTTPError implementation of these

            def geturl(self):
                return self.wrapped.geturl()

            def close(self):
                self.wrapped.close()
        wrapper_class = httperror_seek_wrapper
    else:
        wrapper_class = response_seek_wrapper

    if hasattr(response, "closeable_response"):
        if not hasattr(response, "seek"):
            response = wrapper_class(response)
        return copy.copy(response)

    # a urllib2 handler constructed the response, i.e. the response is an
    # urllib.addinfourl or a urllib2.HTTPError, instead of a
    # _Util.closeable_response as returned by e.g. mechanize.HTTPHandler
    try:
        code = response.code
    except AttributeError:
        code = None
    try:
        msg = response.msg
    except AttributeError:
        msg = None

    # may have already-.read() data from .seek() cache
    data = None
    get_data = getattr(response, "get_data", None)
    if get_data:
        data = get_data()

    response = closeable_response(
        response.fp, response.info(), response.geturl(), code, msg)
    response = wrapper_class(response)
    if data:
        response.set_data(data)
    return response
<|endoftext|>


<commit_before># *-* coding:utf-8 *-*
"""
Una app para cargar contenidos multimedia

"""
import mimetypes
import datetime
import os
try:
    from Pillow import Image
except ImportError:
    try:
        from PIL import Image
    except ImportError:
        import Image

from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib.contenttypes import generic
from django.utils.encoding import force_unicode
from mediacontent.utils import convert_to_rgb, crop_aspect, resize

def_sizes = {
    'thumb': (75,75),
    'gallery': (250, 250),
    'normal': (600,600)
    }

CONTENT_MEDIA_SIZE = getattr(settings, 'CONTENT_MEDIA_SIZE', def_sizes)
CONTENT_MEDIA_PATH = getattr(settings, 'CONTENT_MEDIA_PATH', 'mediacontent')

class MediaContentManager(models.Manager):
    def get_for_model(self, model):
        ct = ContentType.objects.get_for_model(model)
        qs = self.get_query_set().filter(content_type=ct)

        if isinstance(model, models.Model):
            qs = qs.filter(object_pk=force_unicode(model._get_pk_val()))
        return qs

class MediaContent(models.Model):
    def _get_ct(self):
        mcls = ContentType.objects.get(pk=self.content_type.pk)

        # esto fixea el problema de multise y _default_manager
        if self.content_object:
            return self.content_object
        try:
            cls = mcls.get_object_for_this_type(pk=self.object_pk)
        except mcls.model_class().DoesNotExist:
            cls = mcls.model_class()
        return cls

    def get_content_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.object_pk, filename)

    def get_thumb_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/thumbnail_%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.mimetype, filename)

    def get_gallery_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/gallery_%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.mimetype, filename)

    def content_path(self, filename):
        return self.get_content_path(filename)

    def thumb_path(self, filename):
        return self.get_thumb_path(filename)

    def gallery_path(self, filename):
        return self.get_gallery_path(filename)

    def get_sizes(self):
        return CONTENT_MEDIA_SIZE

    content_type = models.ForeignKey(ContentType, verbose_name=_('content type'), related_name="content_type_set_for_%(class)s")
    object_pk = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey('content_type', 'object_pk')

    mimetype = models.CharField(max_length=100, blank=True)
    title = models.CharField(max_length=100)
    description = models.TextField(blank=True)

    content = models.FileField(upload_to=content_path, max_length=300)
    thumbnail = models.ImageField(upload_to=thumb_path, blank=True, max_length=300)
    gallery = models.ImageField(upload_to=gallery_path, blank=True, max_length=300)

    #hay que actualizar la DB y generar todas las fechas por defecto
    pub_date = models.DateTimeField(blank=True)

    objects = MediaContentManager()

    class Meta:
        ordering = ('pub_date','id')

    def __init__(self, *args, **kwargs):
        super(MediaContent, self).__init__(*args, **kwargs)
        self.__original_image = self.content

    def __unicode__(self):
        return unicode(self.title)

    def save(self, *args, **kwargs):

        changed_image = self.content != self.__original_image

        if not self.id and not self.pub_date:
            self.pub_date = datetime.datetime.today()

        crop_original = kwargs.get('crop_original', False)

        super(MediaContent, self).save(*args, **kwargs)
        self.mimetype = mimetypes.guess_type(self.content.path)[0]

        if self.mimetype:
            content_type = self.mimetype.replace('/', '_')
        else:
            # assume everything else is text/plain
            content_type = 'text_plain'

        i = self.content.name.rindex('/')
        thumbnail = u'%sthumbnail_%s' % (unicode(self.content.name[:i+1]), unicode(self.content.name[i+1:]))
        gallery = u'%sgallery_%s' % (unicode(self.content.name[:i+1]), unicode(self.content.name[i+1:]))
        orig = self.content.name

        if (not self.thumbnail or not self.gallery or changed_image) and content_type.split('_')[0]=='image':
            img_path = self.content.path

            if content_type == 'image_svg+xml':
                try:
                    from nebula.mediacontent import svg_to_png
                    svg_to_png.convert(img_path, svg_to_png.new_name(img_path))
                    img_path = svg_to_png.new_name(img_path)
                    self.content.name = self.content.name[:-3] + self.content.name[-3:].replace('svg', 'png')
                except:
                    pass

            image = Image.open(img_path)
            image = convert_to_rgb(image)
            image = crop_aspect(image, ratio=1.0)

            # hace el thumb
            image_thumb = resize(image.copy(), size=self.get_sizes()['thumb'])
            image_thumb.save(os.path.join(settings.MEDIA_ROOT, thumbnail))
            self.thumbnail = thumbnail

            # guarda la imagen para gallery
            image_gallery = resize(image.copy(), size=self.get_sizes()['gallery'])
            image_gallery.save(os.path.join(settings.MEDIA_ROOT, gallery))
            self.gallery = gallery

            # guarda la imagen al tamaño máximo
            if crop_original:
                image_normal = resize(image.copy(), size=self.get_sizes()['normal'])
                image_normal.save(os.path.join(settings.MEDIA_ROOT, orig))

        elif (not self.thumbnail or not self.gallery or changed_image) and content_type == 'application_pdf':
            print 'carga un pdf'

            # Crea una imagen de la primer pagina de un PDF
            from subprocess import call

            cmd = "gs -q -dQUIET -dPARANOIDSAFER -dBATCH -dNOPAUSE -dNOPROMPT \
                    -dMaxBitmap=500000000 -dLastPage=1 -dAlignToPixels=0 -dGridFitTT=0 \
                    -sDEVICE=jpeg -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -r150 \
                    -sOutputFile=%(fileout)s %(filein)s"

            filein = os.path.join(settings.MEDIA_ROOT, self.content.name)
            filejpg = self.content.name[:-3] + self.content.name[-3:].replace('pdf', 'jpg')
            fileout = os.path.join(settings.MEDIA_ROOT, filejpg)

            if not os.access(filein, os.R_OK):
                raise 'not access %s' % filein

            files = {
                'filein': filein.replace(' ', '\ '),
                'fileout': fileout.replace(' ', '\ '),
            }

            # devuelve 0 si esta OK
            if not call(cmd % files, shell=True):

                i = filejpg.rindex('/')

                thumbnail = u'%sthumbnail_%s' % (unicode(filejpg[:i+1]), unicode(filejpg[i+1:]))
                gallery = u'%sgallery_%s' % (unicode(filejpg[:i+1]), unicode(filejpg[i+1:]))

                image = Image.open(fileout)
                image = convert_to_rgb(image)
                #image = crop_aspect(image, ratio=1.0)

                # hace el thumb
                image_thumb = resize(image.copy(), size=None, max_width=self.get_sizes()['gallery'][0])
                image_thumb.save(os.path.join(settings.MEDIA_ROOT, thumbnail))
                self.thumbnail = thumbnail

                # guarda la imagen para gallery
                #image_gallery = image.copy()
                image_gallery = resize(image.copy(), size=None, max_width=self.get_sizes()['normal'][0])
                image.save(os.path.join(settings.MEDIA_ROOT, gallery))
                self.gallery = gallery

                # borra la original porque es un PDF
                try:
                    os.remove(fileout)
                except (OSError, ValueError):
                    pass

        super(MediaContent, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        try:
            os.remove(self.content.path)
        except (OSError, ValueError):
            pass
        try:
            os.remove(self.gallery.path)
        except (OSError, ValueError):
            pass
        try:
            os.remove(self.thumbnail.path)
        except (OSError, ValueError):
            pass
        return super(MediaContent, self).delete(*args, **kwargs)

    def get_header(self):
        return self.mimetype.split('/')[0]

    def get_file_name(self):
        return self.content.name.split('/')[-1]
<commit_msg>fix get_queryst for compatibility<commit_after># *-* coding:utf-8 *-*
"""
Una app para cargar contenidos multimedia

"""
import mimetypes
import datetime
import os
try:
    from Pillow import Image
except ImportError:
    try:
        from PIL import Image
    except ImportError:
        import Image

from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib.contenttypes import generic
from django.utils.encoding import force_unicode
from mediacontent.utils import convert_to_rgb, crop_aspect, resize

def_sizes = {
    'thumb': (75,75),
    'gallery': (250, 250),
    'normal': (600,600)
    }

CONTENT_MEDIA_SIZE = getattr(settings, 'CONTENT_MEDIA_SIZE', def_sizes)
CONTENT_MEDIA_PATH = getattr(settings, 'CONTENT_MEDIA_PATH', 'mediacontent')

class MediaContentManager(models.Manager):
    def get_for_model(self, model):
        ct = ContentType.objects.get_for_model(model)
        qs = self.get_queryset().filter(content_type=ct)

        if isinstance(model, models.Model):
            qs = qs.filter(object_pk=force_unicode(model._get_pk_val()))
        return qs

class MediaContent(models.Model):
    def _get_ct(self):
        mcls = ContentType.objects.get(pk=self.content_type.pk)

        # esto fixea el problema de multise y _default_manager
        if self.content_object:
            return self.content_object
        try:
            cls = mcls.get_object_for_this_type(pk=self.object_pk)
        except mcls.model_class().DoesNotExist:
            cls = mcls.model_class()
        return cls

    def get_content_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.object_pk, filename)

    def get_thumb_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/thumbnail_%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.mimetype, filename)

    def get_gallery_path(self, filename):
        cls = self._get_ct()
        return u'%s/%s/%s/gallery_%s' % (CONTENT_MEDIA_PATH, cls._meta.module_name, self.mimetype, filename)

    def content_path(self, filename):
        return self.get_content_path(filename)

    def thumb_path(self, filename):
        return self.get_thumb_path(filename)

    def gallery_path(self, filename):
        return self.get_gallery_path(filename)

    def get_sizes(self):
        return CONTENT_MEDIA_SIZE

    content_type = models.ForeignKey(ContentType, verbose_name=_('content type'), related_name="content_type_set_for_%(class)s")
    object_pk = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey('content_type', 'object_pk')

    mimetype = models.CharField(max_length=100, blank=True)
    title = models.CharField(max_length=100)
    description = models.TextField(blank=True)

    content = models.FileField(upload_to=content_path, max_length=300)
    thumbnail = models.ImageField(upload_to=thumb_path, blank=True, max_length=300)
    gallery = models.ImageField(upload_to=gallery_path, blank=True, max_length=300)

    #hay que actualizar la DB y generar todas las fechas por defecto
    pub_date = models.DateTimeField(blank=True)

    objects = MediaContentManager()

    class Meta:
        ordering = ('pub_date','id')

    def __init__(self, *args, **kwargs):
        super(MediaContent, self).__init__(*args, **kwargs)
        self.__original_image = self.content

    def __unicode__(self):
        return unicode(self.title)

    def save(self, *args, **kwargs):

        changed_image = self.content != self.__original_image

        if not self.id and not self.pub_date:
            self.pub_date = datetime.datetime.today()

        crop_original = kwargs.get('crop_original', False)

        super(MediaContent, self).save(*args, **kwargs)
        self.mimetype = mimetypes.guess_type(self.content.path)[0]

        if self.mimetype:
            content_type = self.mimetype.replace('/', '_')
        else:
            # assume everything else is text/plain
            content_type = 'text_plain'

        i = self.content.name.rindex('/')
        thumbnail = u'%sthumbnail_%s' % (unicode(self.content.name[:i+1]), unicode(self.content.name[i+1:]))
        gallery = u'%sgallery_%s' % (unicode(self.content.name[:i+1]), unicode(self.content.name[i+1:]))
        orig = self.content.name

        if (not self.thumbnail or not self.gallery or changed_image) and content_type.split('_')[0]=='image':
            img_path = self.content.path

            if content_type == 'image_svg+xml':
                try:
                    from nebula.mediacontent import svg_to_png
                    svg_to_png.convert(img_path, svg_to_png.new_name(img_path))
                    img_path = svg_to_png.new_name(img_path)
                    self.content.name = self.content.name[:-3] + self.content.name[-3:].replace('svg', 'png')
                except:
                    pass

            image = Image.open(img_path)
            image = convert_to_rgb(image)
            image = crop_aspect(image, ratio=1.0)

            # hace el thumb
            image_thumb = resize(image.copy(), size=self.get_sizes()['thumb'])
            image_thumb.save(os.path.join(settings.MEDIA_ROOT, thumbnail))
            self.thumbnail = thumbnail

            # guarda la imagen para gallery
            image_gallery = resize(image.copy(), size=self.get_sizes()['gallery'])
            image_gallery.save(os.path.join(settings.MEDIA_ROOT, gallery))
            self.gallery = gallery

            # guarda la imagen al tamaño máximo
            if crop_original:
                image_normal = resize(image.copy(), size=self.get_sizes()['normal'])
                image_normal.save(os.path.join(settings.MEDIA_ROOT, orig))

        elif (not self.thumbnail or not self.gallery or changed_image) and content_type == 'application_pdf':
            print 'carga un pdf'

            # Crea una imagen de la primer pagina de un PDF
            from subprocess import call

            cmd = "gs -q -dQUIET -dPARANOIDSAFER -dBATCH -dNOPAUSE -dNOPROMPT \
                    -dMaxBitmap=500000000 -dLastPage=1 -dAlignToPixels=0 -dGridFitTT=0 \
                    -sDEVICE=jpeg -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -r150 \
                    -sOutputFile=%(fileout)s %(filein)s"

            filein = os.path.join(settings.MEDIA_ROOT, self.content.name)
            filejpg = self.content.name[:-3] + self.content.name[-3:].replace('pdf', 'jpg')
            fileout = os.path.join(settings.MEDIA_ROOT, filejpg)

            if not os.access(filein, os.R_OK):
                raise 'not access %s' % filein

            files = {
                'filein': filein.replace(' ', '\ '),
                'fileout': fileout.replace(' ', '\ '),
            }

            # devuelve 0 si esta OK
            if not call(cmd % files, shell=True):

                i = filejpg.rindex('/')

                thumbnail = u'%sthumbnail_%s' % (unicode(filejpg[:i+1]), unicode(filejpg[i+1:]))
                gallery = u'%sgallery_%s' % (unicode(filejpg[:i+1]), unicode(filejpg[i+1:]))

                image = Image.open(fileout)
                image = convert_to_rgb(image)
                #image = crop_aspect(image, ratio=1.0)

                # hace el thumb
                image_thumb = resize(image.copy(), size=None, max_width=self.get_sizes()['gallery'][0])
                image_thumb.save(os.path.join(settings.MEDIA_ROOT, thumbnail))
                self.thumbnail = thumbnail

                # guarda la imagen para gallery
                #image_gallery = image.copy()
                image_gallery = resize(image.copy(), size=None, max_width=self.get_sizes()['normal'][0])
                image.save(os.path.join(settings.MEDIA_ROOT, gallery))
                self.gallery = gallery

                # borra la original porque es un PDF
                try:
                    os.remove(fileout)
                except (OSError, ValueError):
                    pass

        super(MediaContent, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        try:
            os.remove(self.content.path)
        except (OSError, ValueError):
            pass
        try:
            os.remove(self.gallery.path)
        except (OSError, ValueError):
            pass
        try:
            os.remove(self.thumbnail.path)
        except (OSError, ValueError):
            pass
        return super(MediaContent, self).delete(*args, **kwargs)

    def get_header(self):
        return self.mimetype.split('/')[0]

    def get_file_name(self):
        return self.content.name.split('/')[-1]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from autobahn.asyncio.websocket \
	import WebSocketServerProtocol, WebSocketServerFactory

import asyncio
import json
import random
import time
import os

PROCESSORS_SERVER_IP = '0.0.0.0'
PROCESSORS_SERVER_PORT = 12121

CLIENTS_SERVER_IP = '0.0.0.0'
CLIENTS_SERVER_PORT = 21212

MAX_PONG_WAIT_TIME = 3 # Seconds
PING_INTERVAL = 5 # Seconds

MAX_FAILURES = 3

IP_JOB_COUNT_LIMIT = 200
IP_DURATION_LIMIT = 10 # Seconds

SSL_CERT_FILE = '/etc/letsencrypt/live/pooljs.ir/cert.pem'
SSL_KEY_FILE = '/etc/letsencrypt/live/pooljs.ir/privkey.pem'

class Job:

	def __init__(self,code,websocket,args,identity):
		self.code = code
		self.websocket = websocket
		self.args = args
		self.identity = identity
		self.fails = 0

class IpLimit:

	def __init__(self,duration_limit,count_limit):
		self.duration_limit = duration_limit
		self.count_limit = count_limit
		self.expiry_time = None
		self.count = 0

ip_limit = {}
job_id_queue = asyncio.Queue()
job_id_counter = 0
jobs = {}
processor_websockets = set()
client_websockets = set()
processor_exists = asyncio.Condition()

def now():
	return int(time.time())

class ProcessorProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = []
		self.last_ping_time = None
		self.last_pong_time = None

	async def onOpen(self):
		await processor_exists.acquire()
		processor_websockets.add(self)
		processor_exists.notify_all()
		processor_exists.release()

	async def onMessage(self, payload, isBinary):
		self.last_pong_time = now()
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		try:
			jobs[job_id].websocket.result_available(job_id,msg["result"],False)
			jobs[job_id].websocket.job_ids.remove(job_id)
			del jobs[job_id]
		except KeyError:
			pass
		if job_id in self.job_ids:
			self.job_ids.remove(job_id)

	async def onClose(self, wasClean, code, reason):
		if self in processor_websockets:
			processor_websockets.remove(self)
		for job_id in self.job_ids:
			try:
				jobs[job_id].fails += 1
				if jobs[job_id].fails > MAX_FAILURES:
					jobs[job_id].websocket.result_available(job_id,None,True)
					del jobs[job_id]
				else:
					await job_id_queue.put(job_id)
			except KeyError:
				pass
		del self.job_ids[:]

class ClientProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = []
		self.buff = []
		self.buff_size = 1
		self.ip = None
		self.ip_limit = None

	def onConnect(self, request):
		self.ip = request.peer.split(':')[1]
		if self.ip not in ip_limit:
			ip_limit[self.ip] = IpLimit(IP_DURATION_LIMIT,IP_JOB_COUNT_LIMIT)
		self.ip_limit = ip_limit[self.ip]

	def onOpen(self):
		client_websockets.add(self)

	def result_available(self,job_id,result,error):
		if job_id in jobs:
			if error:
				try:
					message = { "type": "result",
								"results": [[None,jobs[job_id].identity]],
								"error": True }
					self.sendMessage(json.dumps(message).encode('utf-8'),False)
				except:
					pass # None of our business!
			else:
				self.buff.append([result,jobs[job_id].identity])
				if len(self.buff) >= self.buff_size:
					self.flush()

	def flush(self):
		try:
			message = { "type": "result",
						"results": self.buff,
						"error": False }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # None of our business!

	def info(self):
		try:
			message = { "type": "info",
						"processorsCount": len(processor_websockets),
						"jobsCount": len(jobs) }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	def limit(self):
		try:
			remaining = self.ip_limit.duration_limit - now() + self.ip_limit.expiry_time
			message = { "type": "limit",
						"remaining": remaining }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	async def new_job(self,code,args,identity):
		global job_id_counter
		if self.ip_limit.count < self.ip_limit.count_limit:
			jobs[job_id_counter] = Job(code,self,args,identity)
			await job_id_queue.put(job_id_counter)
			self.job_ids.append(job_id_counter)
			job_id_counter += 1
			self.ip_limit.count += 1
			return False
		else:
			self.ip_limit.expiry_time = now()
			self.limit()
			return True

	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))

		if msg["type"] == "flush":
			self.flush()

		elif msg["type"] == "info":
			self.info()

		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()

		else:
			if self.ip_limit.expiry_time:
				if now() - self.ip_limit.expiry_time > self.ip_limit.duration_limit:
					self.ip_limit.expiry_time = None
					self.ip_limit.count = 0
				else:
					self.limit()
					return

			if msg["type"] == "run":
				await self.new_job(msg["code"],msg["args"],msg["id"])

			elif msg["type"] == "for":
				for i in range(msg["start"],msg["end"]):
					if await self.new_job(msg["code"],[i] + msg["extraArgs"],msg["id"]):
						break

			elif msg["type"] == "forEach":
				for args in msg["argsList"]:
					if await self.new_job(msg["code"],args + msg["extraArgs"],msg["id"]):
						break

	async def onClose(self, wasClean, code, reason):
		if self in client_websockets:
			client_websockets.remove(self)

		for j in self.job_ids:
			try:
				del jobs[j]
			except KeyError:
				pass
		del self.job_ids[:]

async def balancer():
	while True:
		job_id = await job_id_queue.get()

		await processor_exists.acquire()
		await processor_exists.wait_for(lambda:len(processor_websockets) > 0)
		processor_exists.release()

		websocket = random.sample(processor_websockets, 1)[0]
		if job_id in jobs:
			try:
				message = { "id": job_id,
							"code": jobs[job_id].code,
							"args": jobs[job_id].args }
				websocket.sendMessage(json.dumps(message).encode('utf-8'),False)
				websocket.last_ping_time = now()
				websocket.job_ids.append(job_id)
			except:
				job_id_queue.put(job_id) # Revive the job

async def watcher():
	while True:
		must_close = []
		for ws in processor_websockets:
			if ws.last_ping_time:
				if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
					elapsed = now() - ws.last_ping_time
					if elapsed > MAX_PONG_WAIT_TIME:
						must_close.append(ws)
		for ws in must_close:
			ws.sendClose()
		await asyncio.sleep(PING_INTERVAL)

if __name__ == '__main__':
	ssl_ctx = None
	if os.path.isfile(SSL_CERT_FILE) and os.path.isfile(SSL_KEY_FILE):
		import ssl
		ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
		ssl_ctx.load_cert_chain(certfile = SSL_CERT_FILE, keyfile = SSL_KEY_FILE)

	loop = asyncio.get_event_loop()

	clientFactory = WebSocketServerFactory()
	clientFactory.protocol = ClientProtocol
	clientCoro = loop.create_server(clientFactory, CLIENTS_SERVER_IP, CLIENTS_SERVER_PORT, ssl=ssl_ctx)

	processorFactory = WebSocketServerFactory()
	processorFactory.protocol = ProcessorProtocol
	processorCoro = loop.create_server(processorFactory, PROCESSORS_SERVER_IP, PROCESSORS_SERVER_PORT, ssl=ssl_ctx)

	all_tasks = asyncio.gather(clientCoro,processorCoro,balancer(),watcher())

	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<commit_msg>adding some comments<commit_after>#!/usr/bin/env python3

from autobahn.asyncio.websocket \
	import WebSocketServerProtocol, WebSocketServerFactory

import asyncio
import json
import random
import time
import os

PROCESSORS_SERVER_IP = '0.0.0.0'
PROCESSORS_SERVER_PORT = 12121

CLIENTS_SERVER_IP = '0.0.0.0'
CLIENTS_SERVER_PORT = 21212

MAX_PONG_WAIT_TIME = 3 # Seconds
PING_INTERVAL = 5 # Seconds

MAX_FAILURES = 3

IP_JOB_COUNT_LIMIT = 200
IP_DURATION_LIMIT = 10 # Seconds

SSL_CERT_FILE = '/etc/letsencrypt/live/pooljs.ir/cert.pem'
SSL_KEY_FILE = '/etc/letsencrypt/live/pooljs.ir/privkey.pem'

class Job:

	def __init__(self,code,websocket,args,identity):
		self.code = code
		self.websocket = websocket # Job owner websocket
		self.args = args
		self.identity = identity # Every Job has an identity in the client side
		self.fails = 0 # Jobs will be deleted from the list when having multiple failures (MAX_FAILURES)

class IpLimit:

	def __init__(self,duration_limit,count_limit):
		self.duration_limit = duration_limit
		self.count_limit = count_limit
		self.expiry_time = None
		self.count = 0 # Count of Jobs ran by this IP since the last reset

ip_limit = {} # Mapping IP as strings to IpLimits
job_id_queue = asyncio.Queue() # Queue of requested Job ids
job_id_counter = 0 # A counter for generating Job ids
jobs = {} # Mapping Job ids as integers to Jobs
processor_websockets = set()
client_websockets = set()
processor_exists = asyncio.Condition()

# Get current time in seconds
def now():
	return int(time.time())

class ProcessorProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = []
		self.last_ping_time = None
		self.last_pong_time = None

	async def onOpen(self):
		# Notify the balancer a new Processor has been added
		await processor_exists.acquire()
		processor_websockets.add(self)
		processor_exists.notify_all()
		processor_exists.release()

	async def onMessage(self, payload, isBinary):
		self.last_pong_time = now()
		msg = json.loads(payload.decode('utf8'))
		job_id = msg["id"]
		try:
			jobs[job_id].websocket.result_available(job_id,msg["result"],False)
			jobs[job_id].websocket.job_ids.remove(job_id)
			del jobs[job_id]
		except KeyError:
			pass
		if job_id in self.job_ids:
			self.job_ids.remove(job_id)

	async def onClose(self, wasClean, code, reason):
		if self in processor_websockets:
			processor_websockets.remove(self)
		for job_id in self.job_ids:
			try:
				jobs[job_id].fails += 1
				# Delete the Job from the list when it has multiple failures
				if jobs[job_id].fails > MAX_FAILURES:
					# Send and error to the Client
					jobs[job_id].websocket.result_available(job_id,None,True)
					del jobs[job_id]
				else:
					await job_id_queue.put(job_id)
			except KeyError:
				pass
		del self.job_ids[:]

class ClientProtocol(WebSocketServerProtocol):

	def __init__(self, *args, **kwargs):
		super().__init__(*args, **kwargs)
		self.job_ids = [] # For saving the Job ids requested by this Client
		self.buff = [] # For buffering the Job results
		self.buff_size = 1
		self.ip = None
		self.ip_limit = None

	def onConnect(self, request):
		self.ip = request.peer.split(':')[1]
		if self.ip not in ip_limit:
			ip_limit[self.ip] = IpLimit(IP_DURATION_LIMIT,IP_JOB_COUNT_LIMIT)
		# Each IP has a unique instance of IpLimit
		self.ip_limit = ip_limit[self.ip]

	def onOpen(self):
		client_websockets.add(self)

	def result_available(self,job_id,result,error):
		if job_id in jobs:
			if error:
				try:
					message = { "type": "result",
								"results": [[None,jobs[job_id].identity]],
								"error": True }
					self.sendMessage(json.dumps(message).encode('utf-8'),False)
				except:
					pass # None of our business!
			else:
				self.buff.append([result,jobs[job_id].identity])
				if len(self.buff) >= self.buff_size:
					self.flush()

	def flush(self):
		try:
			message = { "type": "result",
						"results": self.buff,
						"error": False }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
			del self.buff[:]
		except:
			pass # None of our business!

	def info(self):
		try:
			message = { "type": "info",
						"processorsCount": len(processor_websockets),
						"jobsCount": len(jobs) }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	def limit(self):
		try:
			remaining = self.ip_limit.duration_limit - now() + self.ip_limit.expiry_time
			message = { "type": "limit",
						"remaining": remaining }
			self.sendMessage(json.dumps(message).encode('utf-8'),False)
		except:
			pass # None of our business!

	async def new_job(self,code,args,identity):
		global job_id_counter
		if self.ip_limit.count < self.ip_limit.count_limit:
			jobs[job_id_counter] = Job(code,self,args,identity)
			await job_id_queue.put(job_id_counter)
			self.job_ids.append(job_id_counter)
			job_id_counter += 1
			self.ip_limit.count += 1
			return False
		else:
			self.ip_limit.expiry_time = now()
			self.limit()
			return True

	async def onMessage(self, payload, isBinary):
		msg = json.loads(payload.decode('utf8'))

		if msg["type"] == "flush":
			self.flush()

		elif msg["type"] == "info":
			self.info()

		elif msg["type"] == "set":
			if msg["property"] == "bufferSize":
				self.buff_size = msg["value"]
				self.flush()

		else:
			if self.ip_limit.expiry_time:
				if now() - self.ip_limit.expiry_time > self.ip_limit.duration_limit:
					self.ip_limit.expiry_time = None
					self.ip_limit.count = 0
				else:
					self.limit()
					return

			if msg["type"] == "run":
				await self.new_job(msg["code"],msg["args"],msg["id"])

			elif msg["type"] == "for":
				for i in range(msg["start"],msg["end"]):
					if await self.new_job(msg["code"],[i] + msg["extraArgs"],msg["id"]):
						break

			elif msg["type"] == "forEach":
				for args in msg["argsList"]:
					if await self.new_job(msg["code"],args + msg["extraArgs"],msg["id"]):
						break

	async def onClose(self, wasClean, code, reason):
		if self in client_websockets:
			client_websockets.remove(self)

		for j in self.job_ids:
			try:
				del jobs[j]
			except KeyError:
				pass
		del self.job_ids[:]

# Balance the Jobs between Processors
async def balancer():
	while True:
		job_id = await job_id_queue.get()

		# Wait for at least one Processor
		await processor_exists.acquire()
		await processor_exists.wait_for(lambda:len(processor_websockets) > 0)
		processor_exists.release()

		websocket = random.sample(processor_websockets, 1)[0]
		if job_id in jobs:
			try:
				message = { "id": job_id,
							"code": jobs[job_id].code,
							"args": jobs[job_id].args }
				websocket.sendMessage(json.dumps(message).encode('utf-8'),False)
				websocket.last_ping_time = now()
				websocket.job_ids.append(job_id)
			except:
				job_id_queue.put(job_id) # Revive the job

# Close not-responding sockets and revive Jobs
async def watcher():
	while True:
		must_close = []
		for ws in processor_websockets:
			if ws.last_ping_time:
				if not ws.last_pong_time or ws.last_pong_time < ws.last_ping_time:
					elapsed = now() - ws.last_ping_time
					if elapsed > MAX_PONG_WAIT_TIME:
						must_close.append(ws)
		for ws in must_close:
			ws.sendClose()
		await asyncio.sleep(PING_INTERVAL)

if __name__ == '__main__':
	ssl_ctx = None
	if os.path.isfile(SSL_CERT_FILE) and os.path.isfile(SSL_KEY_FILE):
		import ssl
		ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
		ssl_ctx.load_cert_chain(certfile = SSL_CERT_FILE, keyfile = SSL_KEY_FILE)

	loop = asyncio.get_event_loop()

	clientFactory = WebSocketServerFactory()
	clientFactory.protocol = ClientProtocol
	clientCoro = loop.create_server(clientFactory, CLIENTS_SERVER_IP, CLIENTS_SERVER_PORT, ssl=ssl_ctx)

	processorFactory = WebSocketServerFactory()
	processorFactory.protocol = ProcessorProtocol
	processorCoro = loop.create_server(processorFactory, PROCESSORS_SERVER_IP, PROCESSORS_SERVER_PORT, ssl=ssl_ctx)

	all_tasks = asyncio.gather(clientCoro,processorCoro,balancer(),watcher())

	try:
		server = loop.run_until_complete(all_tasks)
	except KeyboardInterrupt:
		# In order to close properly when CTRL-C has been pressed
		all_tasks.cancel()
		loop.run_forever()
		all_tasks.exception()
	finally:
		loop.close()
<|endoftext|>


<commit_before>

REF = 'GRCh38.p4'
FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.p4_genomic.fna'
HISATREF = "/home/ubuntu/refs/hisat_index/GRCh38.p4"

# DATASETS = "SRR1295542".split()
THREADS = 10

SAMPLES = 'SRR959265'.split()

rule all:
	input: expand("{sample}.transferred.logs", sample=SAMPLES), expand("{sample}.transferred", sample=SAMPLES), expand("{sample}.hisat.novel.splicesite.txt", sample=SAMPLES)

	# input: "dSRR959265.GRCh38.p4.hisat.sorted.bam"

# SRA -> PILEUP -> RAW COUNTS OFF NCBI GFF3

rule transfer_logs_s3:
	output: touch("{sample}.transferred.logs")
	input:  "{sample}.hisat1.log", "{sample}.hisat2.log"
	message: "transferring {input}'s logs to S3"
	shell: "s3cmd put {sample}.hisat1.log s3://ncbi-hackathon-aug/rnamapping/; s3cmd put {sample}.hisat2.log s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_bam_s3:
	output: touch("{sample}.transferred")
	input: "{sample}.GRCh38.p4.hisat.sorted.bam"
	message: "transferring {input} to S3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_splices_s3:
	output: touch("{sample}.transferred.splices")
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "transferring hisat splice sites {input} to s3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule sort_bam:
	output: "{sample}.GRCh38.p4.hisat.sorted.bam"
	input: "{sample}.GRCh38.p4.hisat.bam"
	message: "sorting {intput} to {output}"
	shell: "samtools sort {input} {output}.sorted"

rule sam_to_bam:
	output: "{sample}.GRCh38.p4.hisat.bam"
	input: "{sample}.GRCh38.p4.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "samtools view -bS {input} > {output}"

rule hisat_alignment_two:
	output: "{sample}.GRCh38.p4.hisat.sam", "{sample}.hisat2.log"
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "running second pass alignment"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat.sam --novel-splicesite-infile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat2.log"

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat1.log", "{sample}.GRCh38.p4.hisat_firstpass_tmp.sam"
	message: "hisat aligning reads {sample}.fastq to GRCh38.p4 with {THREADS} threads to produce {output}"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat_firstpass_tmp.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat1.log"

# rule star_alignment:
# 	output: "{sample}.GRCh38.p4.star.sam"
# 	input: "{sample}.fastq"
# 	message: "star aligning reads {sample}.fastq to GRCh38.p4 to produce {output}"
# 	shell: <star code>

<commit_msg>logs transferring as well<commit_after>

REF = 'GRCh38.p4'
FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.p4_genomic.fna'
HISATREF = "/home/ubuntu/refs/hisat_index/GRCh38.p4"

# DATASETS = "SRR1295542".split()
THREADS = 10

SAMPLES = 'SRR959265'.split()

rule all:
	input: expand("{sample}.transferred.logs", sample=SAMPLES), expand("{sample}.transferred", sample=SAMPLES), expand("{sample}.hisat.novel.splicesite.txt", sample=SAMPLES)

	# input: "dSRR959265.GRCh38.p4.hisat.sorted.bam"

# SRA -> PILEUP -> RAW COUNTS OFF NCBI GFF3

rule transfer_logs_s3:
	output: touch("{sample}.transferred.logs")
	input:  "{sample}.hisat1.log", "{sample}.hisat2.log"
	message: "transferring {input}'s logs to S3"
	shell: "s3cmd put {sample}.hisat1.log s3://ncbi-hackathon-aug/rnamapping/; s3cmd put {sample}.hisat2.log s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_bam_s3:
	output: touch("{sample}.transferred")
	input: "{sample}.GRCh38.p4.hisat.sorted.bam"
	message: "transferring {input} to S3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_splices_s3:
	output: touch("{sample}.transferred.splices")
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "transferring hisat splice sites {input} to s3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule sort_bam:
	output: "{sample}.GRCh38.p4.hisat.sorted.bam"
	input: "{sample}.GRCh38.p4.hisat.bam"
	message: "sorting {intput} to {output}"
	shell: "samtools sort {input} {output}.sorted"

rule sam_to_bam:
	output: "{sample}.GRCh38.p4.hisat.bam"
	input: "{sample}.GRCh38.p4.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "samtools view -bS {input} > {output}"

rule hisat_alignment_two:
	output: "{sample}.GRCh38.p4.hisat.sam", "{sample}.hisat2.log"
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "running second pass alignment"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat.sam --novel-splicesite-infile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat2.log"

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat1.log", "{sample}.GRCh38.p4.hisat_firstpass_tmp.sam"
	message: "hisat aligning reads from {sample} to GRCh38.p4 with {THREADS} threads to produce new splicesites"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat_firstpass_tmp.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat1.log"

# rule star_alignment:
# 	output: "{sample}.GRCh38.p4.star.sam"
# 	input: "{sample}.fastq"
# 	message: "star aligning reads {sample}.fastq to GRCh38.p4 to produce {output}"
# 	shell: <star code>

<|endoftext|>


<commit_before>

REF = 'GRCh38.p4'
FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.p4_genomic.fna'
HISATREF = "/home/ubuntu/refs/hisat_index/GRCh38.p4"

# DATASETS = "SRR1295542".split()
THREADS = 10

SAMPLES = 'SRR2089122'.split()

rule all:
	input: expand("{sample}.hisat1.log", sample=SAMPLES), expand("{sample}.transferred.logs", sample=SAMPLES), expand("{sample}.transferred", sample=SAMPLES), expand("{sample}.transferred.splices", sample=SAMPLES)

	# input: "dSRR959265.GRCh38.p4.hisat.sorted.bam"

# SRA -> PILEUP -> RAW COUNTS OFF NCBI GFF3

rule transfer_logs_s3:
	output: touch("{sample}.transferred.logs")
	input:  "{sample}.hisat.1.log", "{sample}.hisat.2.log"
	message: "transferring {input}'s logs to S3"
	shell: "s3cmd put {sample}.hisat.1.log s3://ncbi-hackathon-aug/rnamapping/; s3cmd put {sample}.hisat.2.log s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_bam_s3:
	output: touch("{sample}.transferred")
	input: "{sample}.GRCh38.p4.hisat.sorted.bam"
	message: "transferring {input} to S3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_splices_s3:
	output: touch("{sample}.transferred.splices")
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "transferring hisat splice sites {input} to s3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule sort_bam:
	output: "{sample}.GRCh38.p4.hisat.sorted.bam"
	input: "{sample}.GRCh38.p4.hisat.bam"
	message: "sorting {intput} to {output}"
	shell: "samtools sort {input} {output}.sorted"

rule sam_to_bam:
	output: "{sample}.GRCh38.p4.hisat.bam"
	input: "{sample}.GRCh38.p4.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "samtools view -bS {input} > {output}"

rule hisat_alignment_two:
	output: "{sample}.GRCh38.p4.hisat.sam", "{sample}.hisat.2.log"
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "running second pass alignment"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat.sam --novel-splicesite-infile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat.2.log"

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat.1.log"
	message: "hisat aligning reads from {sample} to GRCh38.p4 with {THREADS} to produce splicesites"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S tmp.{sample}.GRCh38.p4.firstpass.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat.1.log"

# rule hisat_alignment_one:
# 	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat1.log", "{sample}.GRCh38.p4.hisat_firstpass_tmp.sam"
# 	message: "hisat aligning reads from {sample} to GRCh38.p4 with {THREADS} threads to produce new splicesites"
# 	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat_firstpass_tmp.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat1.log"

<commit_msg>logs transferring as well<commit_after>

REF = 'GRCh38.p4'
FASTAREF='/home/ubuntu/russ/ncbi/GCF_000001405.30_GRCh38.p4_genomic.fna'
HISATREF = "/home/ubuntu/refs/hisat_index/GRCh38.p4"

# DATASETS = "SRR1295542".split()
THREADS = 10

SAMPLES = 'SRR2089122'.split()

rule all:
	input: expand("{sample}.hisat.1.log", sample=SAMPLES), expand("{sample}.transferred.logs", sample=SAMPLES), expand("{sample}.transferred", sample=SAMPLES), expand("{sample}.transferred.splices", sample=SAMPLES)

	# input: "dSRR959265.GRCh38.p4.hisat.sorted.bam"

# SRA -> PILEUP -> RAW COUNTS OFF NCBI GFF3

rule transfer_logs_s3:
	output: touch("{sample}.transferred.logs")
	input:  "{sample}.hisat.1.log", "{sample}.hisat.2.log"
	message: "transferring {input}'s logs to S3"
	shell: "s3cmd put {sample}.hisat.1.log s3://ncbi-hackathon-aug/rnamapping/; s3cmd put {sample}.hisat.2.log s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_bam_s3:
	output: touch("{sample}.transferred")
	input: "{sample}.GRCh38.p4.hisat.sorted.bam"
	message: "transferring {input} to S3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule transfer_splices_s3:
	output: touch("{sample}.transferred.splices")
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "transferring hisat splice sites {input} to s3"
	shell: "s3cmd put {input} s3://ncbi-hackathon-aug/rnamapping/"

rule sort_bam:
	output: "{sample}.GRCh38.p4.hisat.sorted.bam"
	input: "{sample}.GRCh38.p4.hisat.bam"
	message: "sorting {intput} to {output}"
	shell: "samtools sort {input} {output}.sorted"

rule sam_to_bam:
	output: "{sample}.GRCh38.p4.hisat.bam"
	input: "{sample}.GRCh38.p4.hisat.sam"
	message: "converting sam to bam: {input} to {output}"
	shell: "samtools view -bS {input} > {output}"

rule hisat_alignment_two:
	output: "{sample}.GRCh38.p4.hisat.sam", "{sample}.hisat.2.log"
	input: "{sample}.hisat.novel.splicesite.txt"
	message: "running second pass alignment"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat.sam --novel-splicesite-infile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat.2.log"

rule hisat_alignment_one:
	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat.1.log"
	message: "hisat aligning reads from {sample} to GRCh38.p4 with {THREADS} to produce splicesites"
	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S tmp.{sample}.GRCh38.p4.firstpass.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat.1.log"

# rule hisat_alignment_one:
# 	output: "{sample}.hisat.novel.splicesite.txt", "{sample}.hisat1.log", "{sample}.GRCh38.p4.hisat_firstpass_tmp.sam"
# 	message: "hisat aligning reads from {sample} to GRCh38.p4 with {THREADS} threads to produce new splicesites"
# 	shell: "hisat -D 15 -R 2 -N 0 -L 22 -i S,1,1.15 -x {HISATREF} -p {THREADS} --sra-acc {sample} --mm -t -S {sample}.GRCh38.p4.hisat_firstpass_tmp.sam --novel-splicesite-outfile {sample}.hisat.novel.splicesite.txt 2> {sample}.hisat1.log"

<|endoftext|>


<commit_before># coding=utf-8

"""This module, massive_simulation_runner.py, simply execute many simulations in parallel."""

from c_processes.c_process import CProcess
from c_processes.c_process_manager import ProcessManager

PATH_TO_TRAINER = '/home/databoi/c_code/m0_net_resistance_testing'

class MassiveSimulationRunner(object):
	"""Runs many simulations in parallel."""

	def __init__(self):
		self.best_weights = None
		self.best_score   = None

		self.delta = .001

		self.base_training_flags = [.0001, -.0007, .1, -.4]
		#masari_training_process = CProcess('/home/databoi/c_code/m0_net_resistance_testing')

	def one_training_session(self):
		"""Runs a single training session."""
		# A single training session will train each weight 3000 times with varying values.

		number_of_weights = len(self.base_training_flags)
		#for w in range(number_of_weights):
		self._train_one_weight(0)

	def _train_one_weight(self, weight_index):
		"""Trains a single weight."""
		# Make 3000 processes.
		processes_to_run = []
		for p in range(3000):

			# Calculate the weight delta.
			if p < 1000:
				weight_delta = self.delta * p * -1
			elif p < 2000:
				weight_delta = self.delta * p
			else:
				weight_delta = self.delta * p * 2

			weights_to_use = []
			for i, w in enumerate(self.base_training_flags):
				if i == weight_index:
					weights_to_use.append(w + weight_delta)
				else:
					weights_to_use.append(w)

			for i in range(len(weights_to_use)):
				weights_to_use[i] = str(weights_to_use[i])

			processes_to_run.append(CProcess(PATH_TO_TRAINER, weights_to_use))

		simulation_runner = ProcessManager(processes_to_run)
		results = simulation_runner.run_all_c_processes()

		print('GOT THE FOLLOWING RESULTS')
		for r in results:
			#print(r)
			net_score = float(r[1])
			print(str(r[0].flags) + ' --- ' + str(net_score))

simulation_runner = MassiveSimulationRunner()
simulation_runner.one_training_session()

'''

		data_instances = []
		c_processes = []
		for f in files:
			c_processes.append(CProcess(self._simulation_data_fetcher.executable_file_path, [f]))
		simulation_runner = ProcessManager(c_processes)
		results = simulation_runner.run_all_c_processes()
		for r in results:
			data_instances.append(DataInstance(r))
		return data_instances

'''<commit_msg>finance updating 205<commit_after># coding=utf-8

"""This module, massive_simulation_runner.py, simply execute many simulations in parallel."""

from c_processes.c_process import CProcess
from c_processes.c_process_manager import ProcessManager

PATH_TO_TRAINER = '/home/databoi/c_code/m0_net_resistance_testing'

class MassiveSimulationRunner(object):
	"""Runs many simulations in parallel."""

	def __init__(self):
		self.best_weights = None
		self.best_score   = 1000

		self.delta = .001

		self.base_training_flags = [.0001, -.0007, .1, -.4]
		#masari_training_process = CProcess('/home/databoi/c_code/m0_net_resistance_testing')

	def one_training_session(self):
		"""Runs a single training session."""
		# A single training session will train each weight 3000 times with varying values.

		number_of_weights = len(self.base_training_flags)
		#for w in range(number_of_weights):
		self._train_one_weight(0)

	def _train_one_weight(self, weight_index):
		"""Trains a single weight."""
		# Make 3000 processes.
		processes_to_run = []
		for p in range(3000):

			# Calculate the weight delta.
			if p < 1000:
				weight_delta = self.delta * p * -1
			elif p < 2000:
				weight_delta = self.delta * p
			else:
				weight_delta = self.delta * p * 2

			weights_to_use = []
			for i, w in enumerate(self.base_training_flags):
				if i == weight_index:
					weights_to_use.append(w + weight_delta)
				else:
					weights_to_use.append(w)

			for i in range(len(weights_to_use)):
				weights_to_use[i] = str(weights_to_use[i])

			processes_to_run.append(CProcess(PATH_TO_TRAINER, weights_to_use))

		simulation_runner = ProcessManager(processes_to_run)
		results = simulation_runner.run_all_c_processes()

		print('GOT THE FOLLOWING RESULTS')
		for r in results:
			#print(r)
			net_score = float(r[1])
			flags = r[0].flags
			print(str(flags) + ' --- ' + str(net_score))

			if net_score > self.best_score:
				self.best_score = net_score

				new_weights = []
				for w in flags:
					new_weights.append(float(w))

				self.base_training_flags = new_weights

simulation_runner = MassiveSimulationRunner()
simulation_runner.one_training_session()

'''

		data_instances = []
		c_processes = []
		for f in files:
			c_processes.append(CProcess(self._simulation_data_fetcher.executable_file_path, [f]))
		simulation_runner = ProcessManager(c_processes)
		results = simulation_runner.run_all_c_processes()
		for r in results:
			data_instances.append(DataInstance(r))
		return data_instances

'''<|endoftext|>