<commit_before>import single_robot_behavior
import behavior
from enum import Enum
import main
import evaluation
import constants
import role_assignment
import robocup
import planning_priority
import time

class Capture(single_robot_behavior.SingleRobotBehavior):

    # tunable config values
    CourseApproachErrorThresh = 0.8
    CourseApproachDist = 0.4
    CourseApproachAvoidBall = 0.10
    DelayTime = .5

    # Default dribbler speed, can be overriden by self.dribbler_power
    DribbleSpeed = 100
    FineApproachSpeed = 0.1

    InFrontOfBallCosOfAngleThreshold = 0.95

    class State(Enum):
        course_approach = 1
        fine_approach = 2
        delay = 3

    ## Capture Constructor
    # faceBall - If false, any turning functions are turned off,
    # useful for using capture to reflect/bounce moving ballls.
    def __init__(self, faceBall=True):
        super().__init__(continuous=False)

        self.add_state(Capture.State.course_approach,
                       behavior.Behavior.State.running)
        self.add_state(Capture.State.fine_approach,
                       behavior.Behavior.State.running)
        self.add_state(Capture.State.delay,
                       behavior.Behavior.State.running)

        self.add_transition(behavior.Behavior.State.start,
                            Capture.State.course_approach, lambda: True,
                            'immediately')

        self.add_transition(
            Capture.State.course_approach,
            Capture.State.fine_approach, lambda: (self.bot_in_front_of_ball(
            ) or self.bot_near_ball(Capture.CourseApproachDist)) and main.ball(
            ).valid, 'dist to ball < threshold')

        self.add_transition(
            Capture.State.fine_approach,
            Capture.State.delay, lambda: self.bot_near_ball(
                constants.Robot.Radius + constants.Ball.Radius), 'has ball')

        self.add_transition(
            Capture.State.delay,
            behavior.Behavior.State.completed,
            lambda: time.time() - self.start_time > Capture.DelayTime,
            'delay before finish')

        self.add_transition(
            Capture.State.fine_approach, Capture.State.course_approach, lambda:
            not (self.bot_in_front_of_ball() or self.bot_near_ball(
                Capture.CourseApproachDist)) and (not self.bot_near_ball(
                    Capture.CourseApproachDist * 1.5) or not main.ball().pos),
            'ball went into goal')

        self.dribbler_power = Capture.DribbleSpeed

        self.lastApproachTarget = None
        self.faceBall = faceBall

    def bot_to_ball(self):
        return main.ball().pos - self.robot.pos

    def bot_near_ball(self, distance):
        return (self.bot_to_ball().mag() < distance)

    def bot_in_front_of_ball(self):
        ball2bot = self.bot_to_ball() * -1
        return (ball2bot.normalized().dot(main.ball().vel) > Capture.InFrontOfBallCosOfAngleThreshold) and \
                ((ball2bot).mag() < (evaluation.ball.predict_stop(main.ball().pos, main.ball().vel) - main.ball().pos).mag())

    # normalized vector pointing from the ball to the point the robot should get to in course_aproach

    def find_intercept_point(self):
        return find_robot_intercept_point(self.robot)

    def execute_running(self):
        self.robot.set_planning_priority(planning_priority.CAPTURE)

        if (self.faceBall):
            self.robot.face(main.ball().pos)

    def on_enter_course_approach(self):
        self.lastApproachTarget == None

    def execute_course_approach(self):
        pos = self.find_intercept_point()

        if (self.lastApproachTarget != None and
            (pos - self.lastApproachTarget).mag() < 0.1):
            pos = self.lastApproachTarget

        self.lastApproachTarget = pos

        # don't hit the ball on accident
        if pos.dist_to(main.ball(
        ).pos) < Capture.CourseApproachAvoidBall + constants.Robot.Radius:
            self.robot.disable_avoid_ball()
        else:
            self.robot.set_avoid_ball_radius(Capture.CourseApproachAvoidBall)

        self.robot.move_to(pos)
        main.system_state().draw_circle(self.lastApproachTarget,
                                        constants.Ball.Radius,
                                        constants.Colors.White, "Capture")

    def on_exit_course_approach(self):
        self.lastApproachTarget is None

    def execute_fine_approach(self):
        self.robot.disable_avoid_ball()
        self.robot.set_dribble_speed(self.dribbler_power)

        # TODO(ashaw596): explain this math a bit
        bot2ball = (main.ball().pos - self.robot.pos).normalized()
        multiplier = 1.5
        aproach = self.bot_to_ball(
        ) * multiplier + bot2ball * Capture.FineApproachSpeed / 4 + main.ball(
        ).vel
        if (aproach.mag() > 1):
            aproach = aproach.normalized() * 1
        self.robot.set_world_vel(aproach)

    def on_enter_delay(self):
        self.start_time = time.time()

    def execute_delay(self):
        self.robot.disable_avoid_ball()
        self.robot.set_dribble_speed(self.dribbler_power)

    def role_requirements(self):
        reqs = super().role_requirements()
        reqs.require_kicking = True
        # try to be near the ball
        if main.ball().valid:
            reqs.cost_func = lambda r: reqs.position_cost_multiplier * find_robot_intercept_point(
                r).dist_to(r.pos)

        return reqs

def find_robot_intercept_point(robot):
    if robot is None:
        return main.ball().pos
    approach_vec = approach_vector(robot)
    # sample every 5 cm in the -approach_vector direction from the ball
    pos = None
    for i in range(50):
        dist = i * 0.05
        pos = main.ball().pos + approach_vec * dist
        # how long will it take the ball to get there
        ball_time = evaluation.ball.rev_predict(main.ball().vel, dist)
        robotDist = (pos - robot.pos).mag() * 0.6
        bot_time = robocup.get_trapezoidal_time(robotDist, robotDist, 2.2, 1,
                                                robot.vel.mag(), 0)

        if bot_time < ball_time:
            break

    return pos

def approach_vector(robot):
    if main.ball().vel.mag() > 0.25 \
       and robot.pos.dist_to(main.ball().pos) > 0.2:
        # ball's moving, get on the side it's moving towards
        return main.ball().vel.normalized()
    else:
        return (robot.pos - main.ball().pos).normalized()
<commit_msg>add delay to fine approach transition and bot near ball checks<commit_after>import single_robot_behavior
import behavior
from enum import Enum
import main
import evaluation
import constants
import role_assignment
import robocup
import planning_priority
import time

class Capture(single_robot_behavior.SingleRobotBehavior):

    # tunable config values
    CourseApproachErrorThresh = 0.8
    CourseApproachDist = 0.4
    CourseApproachAvoidBall = 0.10
    DelayTime = .5

    # Default dribbler speed, can be overriden by self.dribbler_power
    DribbleSpeed = 100
    FineApproachSpeed = 0.1

    InFrontOfBallCosOfAngleThreshold = 0.95

    class State(Enum):
        course_approach = 1
        fine_approach = 2
        delay = 3

    ## Capture Constructor
    # faceBall - If false, any turning functions are turned off,
    # useful for using capture to reflect/bounce moving ballls.
    def __init__(self, faceBall=True):
        super().__init__(continuous=False)

        self.add_state(Capture.State.course_approach,
                       behavior.Behavior.State.running)
        self.add_state(Capture.State.fine_approach,
                       behavior.Behavior.State.running)
        self.add_state(Capture.State.delay,
                       behavior.Behavior.State.running)

        self.add_transition(behavior.Behavior.State.start,
                            Capture.State.course_approach, lambda: True,
                            'immediately')

        self.add_transition(
            Capture.State.course_approach,
            Capture.State.fine_approach, lambda: (self.bot_in_front_of_ball(
            ) or self.bot_near_ball(Capture.CourseApproachDist)) and main.ball(
            ).valid, 'dist to ball < threshold')

        self.add_transition(
            Capture.State.fine_approach,
            Capture.State.delay, lambda: self.bot_near_ball(
                constants.Robot.Radius + constants.Ball.Radius), 'has ball')

        self.add_transition(
            Capture.State.delay,
            behavior.Behavior.State.completed,
            lambda: time.time() - self.start_time > Capture.DelayTime and self.bot_near_ball(
                constants.Robot.Radius + constants.Ball.Radius),
            'delay before finish')

        self.add_transition(
            Capture.State.delay,
            Capture.State.fine_approach,
            lambda: not self.bot_near_ball(
                constants.Robot.Radius + constants.Ball.Radius),
            'lost ball during delay')

        self.add_transition(
            Capture.State.fine_approach, Capture.State.course_approach, lambda:
            not (self.bot_in_front_of_ball() or self.bot_near_ball(
                Capture.CourseApproachDist)) and (not self.bot_near_ball(
                    Capture.CourseApproachDist * 1.5) or not main.ball().pos),
            'ball went into goal')

        self.dribbler_power = Capture.DribbleSpeed

        self.lastApproachTarget = None
        self.faceBall = faceBall

    def bot_to_ball(self):
        return main.ball().pos - self.robot.pos

    def bot_near_ball(self, distance):
        return (self.bot_to_ball().mag() < distance)

    def bot_in_front_of_ball(self):
        ball2bot = self.bot_to_ball() * -1
        return (ball2bot.normalized().dot(main.ball().vel) > Capture.InFrontOfBallCosOfAngleThreshold) and \
                ((ball2bot).mag() < (evaluation.ball.predict_stop(main.ball().pos, main.ball().vel) - main.ball().pos).mag())

    # normalized vector pointing from the ball to the point the robot should get to in course_aproach

    def find_intercept_point(self):
        return find_robot_intercept_point(self.robot)

    def execute_running(self):
        self.robot.set_planning_priority(planning_priority.CAPTURE)

        if (self.faceBall):
            self.robot.face(main.ball().pos)

    def on_enter_course_approach(self):
        self.lastApproachTarget == None

    def execute_course_approach(self):
        pos = self.find_intercept_point()

        if (self.lastApproachTarget != None and
            (pos - self.lastApproachTarget).mag() < 0.1):
            pos = self.lastApproachTarget

        self.lastApproachTarget = pos

        # don't hit the ball on accident
        if pos.dist_to(main.ball(
        ).pos) < Capture.CourseApproachAvoidBall + constants.Robot.Radius:
            self.robot.disable_avoid_ball()
        else:
            self.robot.set_avoid_ball_radius(Capture.CourseApproachAvoidBall)

        self.robot.move_to(pos)
        main.system_state().draw_circle(self.lastApproachTarget,
                                        constants.Ball.Radius,
                                        constants.Colors.White, "Capture")

    def on_exit_course_approach(self):
        self.lastApproachTarget is None

    def execute_fine_approach(self):
        self.robot.disable_avoid_ball()
        self.robot.set_dribble_speed(self.dribbler_power)

        # TODO(ashaw596): explain this math a bit
        bot2ball = (main.ball().pos - self.robot.pos).normalized()
        multiplier = 1.5
        aproach = self.bot_to_ball(
        ) * multiplier + bot2ball * Capture.FineApproachSpeed / 4 + main.ball(
        ).vel
        if (aproach.mag() > 1):
            aproach = aproach.normalized() * 1
        self.robot.set_world_vel(aproach)

    def on_enter_delay(self):
        self.start_time = time.time()

    def execute_delay(self):
        self.robot.disable_avoid_ball()
        self.robot.set_dribble_speed(self.dribbler_power)

    def role_requirements(self):
        reqs = super().role_requirements()
        reqs.require_kicking = True
        # try to be near the ball
        if main.ball().valid:
            reqs.cost_func = lambda r: reqs.position_cost_multiplier * find_robot_intercept_point(
                r).dist_to(r.pos)

        return reqs

def find_robot_intercept_point(robot):
    if robot is None:
        return main.ball().pos
    approach_vec = approach_vector(robot)
    # sample every 5 cm in the -approach_vector direction from the ball
    pos = None
    for i in range(50):
        dist = i * 0.05
        pos = main.ball().pos + approach_vec * dist
        # how long will it take the ball to get there
        ball_time = evaluation.ball.rev_predict(main.ball().vel, dist)
        robotDist = (pos - robot.pos).mag() * 0.6
        bot_time = robocup.get_trapezoidal_time(robotDist, robotDist, 2.2, 1,
                                                robot.vel.mag(), 0)

        if bot_time < ball_time:
            break

    return pos

def approach_vector(robot):
    if main.ball().vel.mag() > 0.25 \
       and robot.pos.dist_to(main.ball().pos) > 0.2:
        # ball's moving, get on the side it's moving towards
        return main.ball().vel.normalized()
    else:
        return (robot.pos - main.ball().pos).normalized()
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
general.py: general models for text and images

'''

from som.api.google.storage.client import ClientBase
from som.api.google.storage.models import *
from som.api.google.storage.utils import *
from som.logman import bot
from som.utils import read_json
import six

######################################################################################
# Specs and base data structures for each radiology model
######################################################################################

def entity(uid,collection,metadata=None):
    '''entity returns an entity object
    parent is a collection
    '''
    fields = [{'key':'uid','required':True,'value':uid},
              {'key':'metadata','required':False,'value':metadata}]

    if type(collection) not in six.string_types:
        collection = collection.get_name()
    model = {'fields':fields,
             'key':['Collection',collection,'Entity', uid]}

    return model

def collection(uid,metadata=None):
    '''entity returns an entity object
    parent is an owner
    '''
    fields = [{'key':'uid','required':True,'value':uid},
              {'key':'metadata','required':False,'value':metadata}]

    model = {'fields':fields,
             'key':['Collection', uid]}
    return model

def image(uid,entity,download,url,storage,metadata=None):
    '''image returns an image object. entity is the parent
    '''
    fields = [{'key':'uid','required':True,'value':uid},
              {'key':'metadata','required':False,'value':metadata},
              {'key':'download','required':True,'value':download,
               'key':'url','required':True,'value': url,
               'key':'storage','required':True,'value': storage}]

    collection = entity.collection.get_name()
    entity = entity.get_name()

    model = {'fields':fields,
             'exclude_from_indexes': ['storage','download','url'],
             'key':['Collection', collection, 'Entity',entity,'Image', uid]}

    return model

def text(uid,entity,content,metadata=None):
    '''text returns a text object. entity is the parent
    '''
    fields = [{'key':'uid','required':True,'value':uid},
              {'key':'metadata','required':False,'value':metadata},
              {'key':'content','required':True,'value':content}]

    collection = entity.collection.get_name()
    entity = entity.get_name()

    model =  {'fields':fields,
              'exclude_from_indexes': ['storage','download','url'],
              'key':['Collection', collection, 'Entity',entity,'Text', uid]}
    return model

######################################################################################
# Core Models, extending base model
######################################################################################

class Collection(ModelBase):

    def __init__(self,client,uid,create=True,fields=None):
        self.model = collection(uid=uid)
        super(Collection, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

class Entity(ModelBase):

    def __init__(self,client,collection,uid,create=True,fields=None):
        self.collection = collection
        self.model = entity(uid=uid, collection=collection)
        super(Entity, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

    def collection(self,client):
        '''collection will return the collection associated with
        the entity
        '''
        return client.get(collection.key)

    def images(self,client):
        '''images will return images associated with the entity
        '''
        key = self.get_keypath()
        return client.get(client.key(*key, "Image"))

    def text(self,client):
        '''text will return text associated with the entity
        '''
        key = self.get_keypath()
        return client.get(client.key(*key, "Text"))

class Image(ModelBase):

    def __init__(self,client,uid,entity,create=True,url=None,
                 download=None,storage=None,fields=None):
        self.entity = entity
        self.model = image(uid=uid,entity=entity,url=url,
                           storage=storage,download=download)
        super(Image, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

    def __repr__(self):
        return str(self.this)

class Text(ModelBase):

    def __init__(self,client,uid,entity,content,create=True,fields=None):
        self.entity = entity
        self.model = text(uid=uid,entity=entity,content=content)
        super(Text, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

    def __repr__(self):
        return str(self.this)

######################################################################################
# Radiology Client to interact with Models
######################################################################################

class Client(ClientBase):

    def __init__(self,bucket_name,**kwargs):
        self.bucket_name = bucket_name
        self.batch = BatchManager()
        super(Client, self).__init__(**kwargs)

    def __str__(self):
        return "storage.google.%s" %self.bucket_name

    def __repr__(self):
        return "storage.google.%s" %self.bucket_name

    ###################################################################
    ## Create and get #################################################
    ###################################################################

    def get_collection(self,fields):
        return Collection(client=self.datastore,**fields)

    def get_entity(self,fields):
        return Entity(client=self.datastore,**fields)

    def create_image(self,fields):
        return Image(client=self.datastore,**fields)

    def create_text(self,fields):
        return Text(client=self.datastore,**fields)

    def get_storage_path(self,file_path,entity,return_folder=False):
        folder = '/'.join(entity.get_keypath())
        bucket_path = "%s/%s" %(folder,os.path.basename(file_path))
        if return_folder:
            return os.path.dirname(bucket_path)
        return bucket_path

    ###################################################################
    ## Upload #########################################################
    ###################################################################

    def upload_text(self,text,entity,batch=True):
        '''upload_text will add a text object to the batch manager'''

        uid = self.get_storage_path(text,entity)

        with open(text,'r') as filey:
            content = filey.read()

        fields = {'uid':uid,
                  'entity': entity,
                  'content':content,
                  'create': not batch }

        new_text = self.create_text(fields) # Create if not doing batch
        bot.logger.debug('TEXT: %s',new_text)

        if batch:
            self.batch.add(new_text)
        return new_text

    def upload_image(self,image,entity,batch=True):
        '''upload_images will add an image object to the batch manager
        '''
        bucket_folder = self.get_storage_path(image,entity,return_folder=True)

        image_obj = self.upload_object(file_path=image,
                                       bucket_folder=bucket_folder)

        url = "https://storage.googleapis.com/%s/%s" %(self.bucket['name'],
                                                       image_obj['name'])

        fields = {'uid':image_obj['id'],
                  'entity':entity,
                  'url':url,
                  'storage':image_obj,
                  'download':image_obj['mediaLink'],
                  'create': not batch }

        new_image = self.create_image(fields)
        bot.logger.debug('IMAGE: %s',new_image)

        if batch:
            self.batch.add(new_image)

        return new_image

    def upload_dataset(self,uid,collection,images=None,texts=None,metadata=None,batch=True):
        '''upload takes a list of images, texts, and optional metadata
        and uploads to datastore (metadata) and storage (images)
        :param uid: should be the unique id for the entity, with one or more images/texts
        :param collection: should be the collection to add the entity to
        :param batch: add entities in batches (recommended, default True)
        '''

        fields = {'uid': uid }
        fields['collection'] = collection

        # Add entity
        entity = self.get_entity(fields)

        if metadata is not None:
            entity.update(client=self.datastore,
                          fields=metadata)

        if texts is not None:
            for text in texts:
                self.upload_text(text=text,
                                 entity=entity,
                                 batch=batch)

        if images is not None:
            for img in images:
                self.upload_image(image=img,
                                  entity=entity,
                                  batch=batch)

        # Run a transaction for put (insert) images and text, and clears queue
        self.batch.insert(client=self.datastore)
<commit_msg>	modified:   som/api/google/storage/general.py<commit_after>#!/usr/bin/env python

'''
general.py: general models for text and images

'''

from som.api.google.storage.client import ClientBase
from som.api.google.storage.models import *
from som.api.google.storage.utils import *
from som.logman import bot
from som.utils import read_json
import six

######################################################################################
# Specs and base data structures for each radiology model
######################################################################################

def entity(uid,collection):
    '''entity returns an entity object
    parent is a collection
    '''
    fields = [{'key':'uid','required':True,'value':uid}]

    if type(collection) not in six.string_types:
        collection = collection.get_name()
    model = {'fields':fields,
             'key':['Collection',collection,'Entity', uid]}

    return model

def collection(uid):
    '''entity returns an entity object
    parent is an owner
    '''
    fields = [{'key':'uid','required':True,'value':uid}]

    model = {'fields':fields,
             'key':['Collection', uid]}
    return model

def image(uid,entity,url):
    '''image returns an image object. entity is the parent
    '''
    fields = [{'key':'uid','required':True,'value': uid},
              {'key':'url','required':True,'value': url}]

    collection = entity.collection.get_name()
    entity = entity.get_name()

    model = {'fields':fields,
             'exclude_from_indexes': ['url'],
             'key':['Collection', collection, 'Entity',entity,'Image', uid]}

    return model

def text(uid,entity,content):
    '''text returns a text object. entity is the parent
    '''
    fields = [{'key':'uid','required':True,'value':uid},
              {'key':'content','required':True,'value':content}]

    collection = entity.collection.get_name()
    entity = entity.get_name()

    model =  {'fields':fields,
              'key':['Collection', collection, 'Entity',entity,'Text', uid]}
    return model

######################################################################################
# Core Models, extending base model
######################################################################################

class Collection(ModelBase):

    def __init__(self,client,uid,create=True,fields=None):
        self.model = collection(uid=uid)
        super(Collection, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

class Entity(ModelBase):

    def __init__(self,client,collection,uid,create=True,fields=None):
        self.collection = collection
        self.model = entity(uid,collection)
        super(Entity, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

    def collection(self,client):
        '''collection will return the collection associated with
        the entity
        '''
        return client.get(collection.key)

    def images(self,client):
        '''images will return images associated with the entity
        '''
        key = self.get_keypath()
        return client.get(client.key(*key, "Image"))

    def text(self,client):
        '''text will return text associated with the entity
        '''
        key = self.get_keypath()
        return client.get(client.key(*key, "Text"))

class Image(ModelBase):

    def __init__(self,client,uid,entity,url,create=True,fields=None):
        self.entity = entity
        self.model = image(uid=uid,entity=entity,url=url)
        super(Image, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=metadata)

    def __repr__(self):
        return str(self.this)

class Text(ModelBase):

    def __init__(self,client,uid,entity,content,create=True,fields=None):
        self.entity = entity
        self.model = text(uid=uid,entity=entity,content=content)
        super(Text, self).__init__(client)
        if create:
            self.this = self.update_or_create(client,fields=fields)
        else:
            self.this = self.setup(client,fields=fields)

    def __repr__(self):
        return str(self.this)

######################################################################################
# Radiology Client to interact with Models
######################################################################################

class Client(ClientBase):

    def __init__(self,bucket_name,**kwargs):
        self.bucket_name = bucket_name
        self.batch = BatchManager()
        super(Client, self).__init__(**kwargs)

    def __str__(self):
        return "storage.google.%s" %self.bucket_name

    def __repr__(self):
        return "storage.google.%s" %self.bucket_name

    ###################################################################
    ## Create and get #################################################
    ###################################################################

    def get_collection(self,uid,create=True,fields=None):
        return Collection(client=self.datastore,
                          uid=uid,
                          create=create,
                          fields=fields)

    def get_entity(self,collection,uid,create=True,fields=None):
        return Entity(client=self.datastore,collection=collection,
                      uid=uid,
                      create=create,
                      fields=fields)

    def create_image(self,uid,entity,url,create=True,fields=None):
        return Image(client=self.datastore,
                     uid=uid,
                     entity=entity,
                     url=url,
                     create=create,
                     fields=fields)

    def create_text(self,uid,entity,content,create=True,fields=None):
        return Text(client=self.datastore,
                    uid=uid,
                    entity=entity,
                    content=content,
                    create=create,
                    fields=fields)

    def get_storage_path(self,file_path,entity,return_folder=False):
        folder = '/'.join(entity.get_keypath())
        bucket_path = "%s/%s" %(folder,os.path.basename(file_path))
        if return_folder:
            return os.path.dirname(bucket_path)
        return bucket_path

    ###################################################################
    ## Upload #########################################################
    ###################################################################

    def upload_text(self,text,entity,batch=True):
        '''upload_text will add a text object to the batch manager'''

        uid = self.get_storage_path(text,entity)

        with open(text,'r') as filey:
            content = filey.read()

        new_text = self.create_text(uid=uid,entity=entity,
                                    content=content,create=not batch)

        bot.logger.debug('TEXT: %s',new_text)

        if batch:
            self.batch.add(new_text)
        return new_text

    def upload_image(self,image,entity,batch=True):
        '''upload_images will add an image object to the batch manager
        '''
        bucket_folder = self.get_storage_path(image,entity,return_folder=True)

        image_obj = self.upload_object(file_path=image,
                                       bucket_folder=bucket_folder)

        url = "https://storage.googleapis.com/%s/%s" %(self.bucket['name'],
                                                       image_obj['name'])

        fields = {'storage':image_obj,
                  'download':image_obj['mediaLink']}

        new_image = self.create_image(uid=image_obj['id'],
                                      entity=entity,
                                      url=url,
                                      fields=fields,
                                      create=not batch)

        bot.logger.debug('IMAGE: %s',new_image)

        if batch:
            self.batch.add(new_image)

        return new_image

    def upload_dataset(self,uid,collection,images=None,texts=None,metadata=None,batch=True):
        '''upload takes a list of images, texts, and optional metadata
        and uploads to datastore (metadata) and storage (images)
        :param uid: should be the unique id for the entity, with one or more images/texts
        :param collection: should be the collection to add the entity to
        :param batch: add entities in batches (recommended, default True)
        '''

        fields = {'uid': uid }
        fields['collection'] = collection

        # Add entity
        entity = self.get_entity(fields)

        if metadata is not None:
            entity.update(client=self.datastore,
                          fields=metadata)

        if texts is not None:
            for text in texts:
                self.upload_text(text=text,
                                 entity=entity,
                                 batch=batch)

        if images is not None:
            for img in images:
                self.upload_image(image=img,
                                  entity=entity,
                                  batch=batch)

        # Run a transaction for put (insert) images and text, and clears queue
        self.batch.insert(client=self.datastore)
<|endoftext|>


<commit_before><commit_msg>[FIX] P3: hex() returns text, don't re-decode it<commit_after><|endoftext|>


<commit_before><commit_msg>[#48] additional validation for email<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
def copy_plugins_to(plugin_list, to_placeholder, to_language = None):
    """
    Copies a list of plugins to a placeholder to a language.
    """
    ptree = []
    plugins_ziplist = []
    for old_plugin in plugin_list:
        if to_language:
            plugin_language = to_language
        else:
            plugin_language = old_plugin.language
        # do the simple copying
        new_plugin = old_plugin.copy_plugin(to_placeholder, plugin_language, ptree) # note that ptree comes back modified
        plugins_ziplist.append((new_plugin, old_plugin))
    # this magic is needed for advanced plugins like Text Plugins that can have
    # nested plugins and need to update their content based on the new plugins.
    for new_plugin, old_plugin in plugins_ziplist:
        new_instance = new_plugin.get_plugin_instance()[0]
        if new_instance:
            new_instance.post_copy(old_plugin, plugins_ziplist)
    # returns information about originals and copies
    return plugins_ziplist
<commit_msg>remove whitespace<commit_after># -*- coding: utf-8 -*-
def copy_plugins_to(plugin_list, to_placeholder, to_language = None):
    """
    Copies a list of plugins to a placeholder to a language.
    """
    ptree = []
    plugins_ziplist = []
    for old_plugin in plugin_list:
        if to_language:
            plugin_language = to_language
        else:
            plugin_language = old_plugin.language
        # do the simple copying
        new_plugin = old_plugin.copy_plugin(to_placeholder, plugin_language, ptree) # note that ptree comes back modified
        plugins_ziplist.append((new_plugin, old_plugin))
    # this magic is needed for advanced plugins like Text Plugins that can have
    # nested plugins and need to update their content based on the new plugins.
    for new_plugin, old_plugin in plugins_ziplist:
        new_instance = new_plugin.get_plugin_instance()[0]
        if new_instance:
            new_instance.post_copy(old_plugin, plugins_ziplist)
    # returns information about originals and copies
    return plugins_ziplist
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-<commit_msg>Initial RESTful server. This is just a placeholder, and for debugging purposes.<commit_after># -*- coding: utf-8 -*-
from flask import Flask
from flask.ext import restful

app = Flask(__name__)
api = restful.Api(app)

# If we're run as a script, start a debug server
if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
======================
ELM Classifiers Comparison
======================
A comparison of a several ELMClassifiers with different types of hidden
layer activations.

ELMClassifier is a classifier based on the Extreme Learning Machine,
a single layer feedforward network with random hidden layer components
and least squares fitting of the hidden->output weights by default [1][2]

The point of this example is to illustrate the nature of decision boundaries
with different hidden layer activation types and regressors.

This should be taken with a grain of salt, as the intuition conveyed by
these examples does not necessarily carry over to real datasets.

In particular in high dimensional spaces data can more easily be separated
linearly and the simplicity of classifiers such as naive Bayes and linear SVMs
might lead to better generalization.

The plots show training points in solid colors and testing points
semi-transparent. The lower right shows the classification accuracy on the test
set.

References
__________
.. [1] http://www.extreme-learning-machines.org
.. [2] G.-B. Huang, Q.-Y. Zhu and C.-K. Siew, "Extreme Learning Machine:
          Theory and Applications", Neurocomputing, vol. 70, pp. 489-501,
          2006.

===============================================================================
Basis Functions:
  gaussian rbf  : exp(-gamma * (||x-c||/r)^2)
  tanh          : np.tanh(a)
  sinsq         : np.power(np.sin(a), 2.0)
  tribas        : np.clip(1.0 - np.fabs(a), 0.0, 1.0)
  hardlim       : np.array(a > 0.0, dtype=float)

     where x   : input pattern
           a   : dot_product(x, c) + b
           c,r : randomly generated components

Label Legend:
  ELM(10,tanh)      :10 tanh units
  ELM(10,tanh,LR)   :10 tanh units, LogisticRegression
  ELM(10,sinsq)     :10 sin*sin units
  ELM(10,tribas)    :10 tribas units
  ELM(10,hardlim)   :10 hardlim units
  ELM(20,rbf(0.1))  :20 rbf units gamma=0.1

"""
print __doc__

# Code source: Gael Varoqueux
#              Andreas Mueller
# Modified for Documentation merge by Jaques Grobler
# Modified for Extreme Learning Machine Classifiers by David Lambert
# License: BSD

import numpy as np
import pylab as pl

from matplotlib.colors import ListedColormap
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import LogisticRegression

from elm import ELMClassifier
from random_hidden_layer import RBFRandomHiddenLayer
from random_hidden_layer import SimpleRandomHiddenLayer

def get_data_bounds(X):
    h = .02  # step size in the mesh

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    return (x_min, x_max, y_min, y_max, xx, yy)

def plot_data(ax, X_train, y_train, X_test, y_test, xx, yy):
    cm = ListedColormap(['#FF0000', '#0000FF'])
    # Plot the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm, alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())

def plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z):
    cm = pl.cm.RdBu
    cm_bright = ListedColormap(['#FF0000', '#0000FF'])

    ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)

    # Plot also the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)

    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())

    ax.set_title(name)
    ax.text(xx.max() - 0.3, yy.min() + 0.3, ('%.2f' % score).lstrip('0'),
            size=13, horizontalalignment='right')

def make_datasets():
    return [make_moons(n_samples=200, noise=0.3, random_state=0),
            make_circles(n_samples=200, noise=0.2, factor=0.5, random_state=1),
            make_linearly_separable()]

def make_classifiers():

    names = ["ELM(10,tanh)", "ELM(10,tanh,LR)", "ELM(10,sinsq)",
             "ELM(10,tribas)", "ELM(hardlim)", "ELM(20,rbf(0.1))"]

    nh = 10

    # pass user defined transfer func
    sinsq = (lambda x: np.power(np.sin(x), 2.0))
    srhl_sinsq = SimpleRandomHiddenLayer(n_hidden=nh,
                                         activation_func=sinsq,
                                         random_state=0)

    # use internal transfer funcs
    srhl_tanh = SimpleRandomHiddenLayer(n_hidden=nh,
                                        activation_func='tanh',
                                        random_state=0)

    srhl_tribas = SimpleRandomHiddenLayer(n_hidden=nh,
                                          activation_func='tribas',
                                          random_state=0)

    srhl_hardlim = SimpleRandomHiddenLayer(n_hidden=nh,
                                           activation_func='hardlim',
                                           random_state=0)

    # use gaussian RBF
    srhl_rbf = RBFRandomHiddenLayer(n_hidden=nh*2, gamma=0.1, random_state=0)

    log_reg = LogisticRegression()

    classifiers = [ELMClassifier(srhl_tanh),
                   ELMClassifier(srhl_tanh, regressor=log_reg),
                   ELMClassifier(srhl_sinsq),
                   ELMClassifier(srhl_tribas),
                   ELMClassifier(srhl_hardlim),
                   ELMClassifier(srhl_rbf)]

    return names, classifiers

def make_linearly_separable():
    X, y = make_classification(n_samples=200, n_features=2, n_redundant=0,
                               n_informative=2, random_state=1,
                               n_clusters_per_class=1)
    rng = np.random.RandomState(2)
    X += 2 * rng.uniform(size=X.shape)
    return (X, y)

###############################################################################

datasets = make_datasets()
names, classifiers = make_classifiers()

i = 1
figure = pl.figure(figsize=(18, 9))

# iterate over datasets
for ds in datasets:
    # preprocess dataset, split into training and test part
    X, y = ds
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4,
                                                        random_state=0)

    x_min, x_max, y_min, y_max, xx, yy = get_data_bounds(X)

    # plot dataset first
    ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
    plot_data(ax, X_train, y_train, X_test, y_test, xx, yy)

    i += 1

    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)

        # Plot the decision boundary. For that, we will asign a color to each
        # point in the mesh [x_min, m_max]x[y_min, y_max].
        Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)

        plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z)

        i += 1

figure.subplots_adjust(left=.02, right=.98)
pl.show()
<commit_msg>modified plot_elm_comparison to use RandomLayer instead of RandomHiddenLayer<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
======================
ELM Classifiers Comparison
======================
A comparison of a several ELMClassifiers with different types of hidden
layer activations.

ELMClassifier is a classifier based on the Extreme Learning Machine,
a single layer feedforward network with random hidden layer components
and least squares fitting of the hidden->output weights by default [1][2]

The point of this example is to illustrate the nature of decision boundaries
with different hidden layer activation types and regressors.

This should be taken with a grain of salt, as the intuition conveyed by
these examples does not necessarily carry over to real datasets.

In particular in high dimensional spaces data can more easily be separated
linearly and the simplicity of classifiers such as naive Bayes and linear SVMs
might lead to better generalization.

The plots show training points in solid colors and testing points
semi-transparent. The lower right shows the classification accuracy on the test
set.

References
__________
.. [1] http://www.extreme-learning-machines.org
.. [2] G.-B. Huang, Q.-Y. Zhu and C.-K. Siew, "Extreme Learning Machine:
          Theory and Applications", Neurocomputing, vol. 70, pp. 489-501,
          2006.

===============================================================================
Basis Functions:
  gaussian rbf  : exp(-gamma * (||x-c||/r)^2)
  tanh          : np.tanh(a)
  sinsq         : np.power(np.sin(a), 2.0)
  tribas        : np.clip(1.0 - np.fabs(a), 0.0, 1.0)
  hardlim       : np.array(a > 0.0, dtype=float)

     where x   : input pattern
           a   : dot_product(x, c) + b
           c,r : randomly generated components

Label Legend:
  ELM(10,tanh)      :10 tanh units
  ELM(10,tanh,LR)   :10 tanh units, LogisticRegression
  ELM(10,sinsq)     :10 sin*sin units
  ELM(10,tribas)    :10 tribas units
  ELM(10,hardlim)   :10 hardlim units
  ELM(20,rbf(0.1))  :20 rbf units gamma=0.1

"""
print __doc__

# Code source: Gael Varoqueux
#              Andreas Mueller
# Modified for Documentation merge by Jaques Grobler
# Modified for Extreme Learning Machine Classifiers by David Lambert
# License: BSD

import numpy as np
import pylab as pl

from matplotlib.colors import ListedColormap
from sklearn.datasets import make_moons, make_circles, make_classification
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import LogisticRegression

from elm import ELMClassifier
from random_layer import RBFRandomLayer, SimpleRandomLayer

def get_data_bounds(X):
    h = .02  # step size in the mesh

    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5

    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))

    return (x_min, x_max, y_min, y_max, xx, yy)

def plot_data(ax, X_train, y_train, X_test, y_test, xx, yy):
    cm = ListedColormap(['#FF0000', '#0000FF'])
    # Plot the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm, alpha=0.6)
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())

def plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z):
    cm = pl.cm.RdBu
    cm_bright = ListedColormap(['#FF0000', '#0000FF'])

    ax.contourf(xx, yy, Z, cmap=cm, alpha=.8)

    # Plot also the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright)
    # and testing points
    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6)

    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())

    ax.set_title(name)
    ax.text(xx.max() - 0.3, yy.min() + 0.3, ('%.2f' % score).lstrip('0'),
            size=13, horizontalalignment='right')

def make_datasets():
    return [make_moons(n_samples=200, noise=0.3, random_state=0),
            make_circles(n_samples=200, noise=0.2, factor=0.5, random_state=1),
            make_linearly_separable()]

def make_classifiers():

    names = ["ELM(10,tanh)", "ELM(10,tanh,LR)", "ELM(10,sinsq)",
             "ELM(10,tribas)", "ELM(hardlim)", "ELM(20,rbf(0.1))"]

    nh = 10

    # pass user defined transfer func
    sinsq = (lambda x: np.power(np.sin(x), 2.0))
    srhl_sinsq = SimpleRandomLayer(n_hidden=nh,
                                   activation_func=sinsq,
                                   random_state=0)

    # use internal transfer funcs
    srhl_tanh = SimpleRandomLayer(n_hidden=nh,
                                  activation_func='tanh',
                                  random_state=0)

    srhl_tribas = SimpleRandomLayer(n_hidden=nh,
                                    activation_func='tribas',
                                    random_state=0)

    srhl_hardlim = SimpleRandomLayer(n_hidden=nh,
                                     activation_func='hardlim',
                                     random_state=0)

    # use gaussian RBF
    srhl_rbf = RBFRandomLayer(n_hidden=nh*2, gamma=0.1, random_state=0)

    log_reg = LogisticRegression()

    classifiers = [ELMClassifier(srhl_tanh),
                   ELMClassifier(srhl_tanh, regressor=log_reg),
                   ELMClassifier(srhl_sinsq),
                   ELMClassifier(srhl_tribas),
                   ELMClassifier(srhl_hardlim),
                   ELMClassifier(srhl_rbf)]

    return names, classifiers

def make_linearly_separable():
    X, y = make_classification(n_samples=200, n_features=2, n_redundant=0,
                               n_informative=2, random_state=1,
                               n_clusters_per_class=1)
    rng = np.random.RandomState(2)
    X += 2 * rng.uniform(size=X.shape)
    return (X, y)

###############################################################################

datasets = make_datasets()
names, classifiers = make_classifiers()

i = 1
figure = pl.figure(figsize=(18, 9))

# iterate over datasets
for ds in datasets:
    # preprocess dataset, split into training and test part
    X, y = ds
    X = StandardScaler().fit_transform(X)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.4,
                                                        random_state=0)

    x_min, x_max, y_min, y_max, xx, yy = get_data_bounds(X)

    # plot dataset first
    ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
    plot_data(ax, X_train, y_train, X_test, y_test, xx, yy)

    i += 1

    # iterate over classifiers
    for name, clf in zip(names, classifiers):
        ax = pl.subplot(len(datasets), len(classifiers) + 1, i)
        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)

        # Plot the decision boundary. For that, we will asign a color to each
        # point in the mesh [x_min, m_max]x[y_min, y_max].
        Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)

        plot_contour(ax, X_train, y_train, X_test, y_test, xx, yy, Z)

        i += 1

figure.subplots_adjust(left=.02, right=.98)
pl.show()
<|endoftext|>


<commit_before>"""
Django settings for code_for_good project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import os.path

PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
#SECRET_KEY = 'd^dj6yos5#i)$**wqac)eln&hb8vj-4skh5ci(b11sc(jwt7gj'
SECRET_KEY = 'ys-(a4d!tpulfv+%@j!o#ib*hv5hur9ppiqfrzf(sd3x!9%v1$'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'scraper.apps.ScraperConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'code_for_good.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(PROJECT_ROOT, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'code_for_good.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>REST API is now somewhat secure<commit_after>"""
Django settings for code_for_good project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import os.path

PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
#SECRET_KEY = 'd^dj6yos5#i)$**wqac)eln&hb8vj-4skh5ci(b11sc(jwt7gj'
SECRET_KEY = 'ys-(a4d!tpulfv+%@j!o#ib*hv5hur9ppiqfrzf(sd3x!9%v1$'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'scraper.apps.ScraperConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'rest_framework',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'code_for_good.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(PROJECT_ROOT, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'code_for_good.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticatedOrReadOnly',
    )
}<|endoftext|>


<commit_before>__version__ = "1.0.15"

__authors__ = [
    'Juan Miguel Taboada Godoy <juanmi@juanmitaboada.com>',
    'Juan Soler Ruiz <soleronline@gmail.com>',
]
<commit_msg>New version<commit_after>__version__ = "1.0.16"

__authors__ = [
    'Juan Miguel Taboada Godoy <juanmi@juanmitaboada.com>',
    'Juan Soler Ruiz <soleronline@gmail.com>',
]
<|endoftext|>


<commit_before>###
# Copyright (c) 2004, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

import time
TIME = time # For later use.
from datetime import datetime

import supybot.conf as conf
import supybot.utils as utils
from supybot.commands import *
import supybot.callbacks as callbacks
from supybot.i18n import PluginInternationalization, internationalizeDocstring
_ = PluginInternationalization('Time')

try:
    from dateutil import parser
    def parse(s):
        todo = []
        s = s.replace('noon', '12:00')
        s = s.replace('midnight', '00:00')
        if 'tomorrow' in s:
            todo.append(lambda i: i + 86400)
            s = s.replace('tomorrow', '')
        if 'next week' in s:
            todo.append(lambda i: i + 86400*7)
            s = s.replace('next week', '')
        i = int(time.mktime(parser.parse(s, fuzzy=True).timetuple()))
        for f in todo:
            i = f(i)
        return i
except ImportError:
    parse = None

class Time(callbacks.Plugin):
    @internationalizeDocstring
    def seconds(self, irc, msg, args):
        """[<years>y] [<weeks>w] [<days>d] [<hours>h] [<minutes>m] [<seconds>s]

        Returns the number of seconds in the number of <years>, <weeks>,
        <days>, <hours>, <minutes>, and <seconds> given.  An example usage is
        "seconds 2h 30m", which would return 9000, which is '3600*2 + 30*60'.
        Useful for scheduling events at a given number of seconds in the
        future.
        """
        if not args:
            raise callbacks.ArgumentError
        seconds = 0
        for arg in args:
            if not arg or arg[-1] not in 'ywdhms':
                raise callbacks.ArgumentError
            (s, kind) = arg[:-1], arg[-1]
            try:
                i = int(s)
            except ValueError:
                irc.errorInvalid('argument', arg, Raise=True)
            if kind == 'y':
                seconds += i*31536000
            elif kind == 'w':
                seconds += i*604800
            elif kind == 'd':
                seconds += i*86400
            elif kind == 'h':
                seconds += i*3600
            elif kind == 'm':
                seconds += i*60
            elif kind == 's':
                seconds += i
        irc.reply(str(seconds))

    @internationalizeDocstring
    def at(self, irc, msg, args, s=None):
        """[<time string>]

        Returns the number of seconds since epoch <time string> is.
        <time string> can be any number of natural formats; just try something
        and see if it will work.
        If the <time string> is not given, defaults to now.
        """
        if not s:
            irc.reply(str(int(time.time())))
            return
        if not parse:
            irc.error(_('This command is not available on this bot, ask the '
                'owner to install the python-dateutil library.'), Raise=True)
        now = int(time.time())
        new = parse(s)
        if new != now:
            irc.reply(str(new))
        else:
            irc.error(_('That\'s right now!'))
    at = wrap(at, [optional('text')])

    @internationalizeDocstring
    def until(self, irc, msg, args, s):
        """<time string>

        Returns the number of seconds until <time string>.
        """
        if not parse:
            irc.error(_('This command is not available on this bot, ask the '
                'owner to install the python-dateutil library.'), Raise=True)
        now = int(time.time())
        new = parse(s)
        if new != now:
            if new - now < 0:
                new += 86400
            irc.reply(str(new-now))
        else:
            irc.error(_('That\'s right now!'))
    until = wrap(until, ['text'])

    @internationalizeDocstring
    def ctime(self, irc, msg, args, seconds):
        """[<seconds since epoch>]

        Returns the ctime for <seconds since epoch>, or the current ctime if
        no <seconds since epoch> is given.
        """
        irc.reply(time.ctime(seconds))
    ctime = wrap(ctime,[additional(('int', _('number of seconds since epoch')),
                                    TIME.time)])

    @internationalizeDocstring
    def time(self, irc, msg, args, channel, format, seconds):
        """[<format>] [<seconds since epoch>]

        Returns the current time in <format> format, or, if <format> is not
        given, uses the configurable format for the current channel.  If no
        <seconds since epoch> time is given, the current time is used.
        """
        if not format:
            if channel:
                format = self.registryValue('format', channel)
            else:
                format = self.registryValue('format')
        irc.reply(time.strftime(format, time.localtime(seconds)))
    time = wrap(time, [optional('channel'), optional('nonInt'),
                       additional('float', TIME.time)])

    @internationalizeDocstring
    def elapsed(self, irc, msg, args, seconds):
        """<seconds>

        Returns a pretty string that is the amount of time represented by
        <seconds>.
        """
        irc.reply(utils.timeElapsed(seconds))
    elapsed = wrap(elapsed, ['int'])

    @internationalizeDocstring
    def tztime(self, irc, msg, args, timezone):
        """<region>/<city>

        Takes a city and its region, and returns the locale time. This
        command uses the IANA Time Zone Database."""
        try:
            import pytz
        except ImportError:
            irc.error(_('Python-tz is required by the command, but is not '
                        'installed on this computer.'))
            return
        try:
            timezone = pytz.timezone(timezone)
        except pytz.UnknownTimeZoneError:
            irc.error(_('Unknown timezone'))
            return
        irc.reply(str(datetime.now(timezone)))
    tztime = wrap(tztime, ['text'])

Class = Time

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:
<commit_msg>Time.time: Work around a Python bug<commit_after>###
# Copyright (c) 2004, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

import time
TIME = time # For later use.
from datetime import datetime

import supybot.conf as conf
import supybot.utils as utils
from supybot.commands import *
import supybot.callbacks as callbacks
from supybot.i18n import PluginInternationalization, internationalizeDocstring
_ = PluginInternationalization('Time')

try:
    from dateutil import parser
    def parse(s):
        todo = []
        s = s.replace('noon', '12:00')
        s = s.replace('midnight', '00:00')
        if 'tomorrow' in s:
            todo.append(lambda i: i + 86400)
            s = s.replace('tomorrow', '')
        if 'next week' in s:
            todo.append(lambda i: i + 86400*7)
            s = s.replace('next week', '')
        i = int(time.mktime(parser.parse(s, fuzzy=True).timetuple()))
        for f in todo:
            i = f(i)
        return i
except ImportError:
    parse = None

try:
    from dateutil.tz import tzlocal
except ImportError:
    tzlocal = None

class Time(callbacks.Plugin):
    @internationalizeDocstring
    def seconds(self, irc, msg, args):
        """[<years>y] [<weeks>w] [<days>d] [<hours>h] [<minutes>m] [<seconds>s]

        Returns the number of seconds in the number of <years>, <weeks>,
        <days>, <hours>, <minutes>, and <seconds> given.  An example usage is
        "seconds 2h 30m", which would return 9000, which is '3600*2 + 30*60'.
        Useful for scheduling events at a given number of seconds in the
        future.
        """
        if not args:
            raise callbacks.ArgumentError
        seconds = 0
        for arg in args:
            if not arg or arg[-1] not in 'ywdhms':
                raise callbacks.ArgumentError
            (s, kind) = arg[:-1], arg[-1]
            try:
                i = int(s)
            except ValueError:
                irc.errorInvalid('argument', arg, Raise=True)
            if kind == 'y':
                seconds += i*31536000
            elif kind == 'w':
                seconds += i*604800
            elif kind == 'd':
                seconds += i*86400
            elif kind == 'h':
                seconds += i*3600
            elif kind == 'm':
                seconds += i*60
            elif kind == 's':
                seconds += i
        irc.reply(str(seconds))

    @internationalizeDocstring
    def at(self, irc, msg, args, s=None):
        """[<time string>]

        Returns the number of seconds since epoch <time string> is.
        <time string> can be any number of natural formats; just try something
        and see if it will work.
        If the <time string> is not given, defaults to now.
        """
        if not s:
            irc.reply(str(int(time.time())))
            return
        if not parse:
            irc.error(_('This command is not available on this bot, ask the '
                'owner to install the python-dateutil library.'), Raise=True)
        now = int(time.time())
        new = parse(s)
        if new != now:
            irc.reply(str(new))
        else:
            irc.error(_('That\'s right now!'))
    at = wrap(at, [optional('text')])

    @internationalizeDocstring
    def until(self, irc, msg, args, s):
        """<time string>

        Returns the number of seconds until <time string>.
        """
        if not parse:
            irc.error(_('This command is not available on this bot, ask the '
                'owner to install the python-dateutil library.'), Raise=True)
        now = int(time.time())
        new = parse(s)
        if new != now:
            if new - now < 0:
                new += 86400
            irc.reply(str(new-now))
        else:
            irc.error(_('That\'s right now!'))
    until = wrap(until, ['text'])

    @internationalizeDocstring
    def ctime(self, irc, msg, args, seconds):
        """[<seconds since epoch>]

        Returns the ctime for <seconds since epoch>, or the current ctime if
        no <seconds since epoch> is given.
        """
        irc.reply(time.ctime(seconds))
    ctime = wrap(ctime,[additional(('int', _('number of seconds since epoch')),
                                    TIME.time)])

    @internationalizeDocstring
    def time(self, irc, msg, args, channel, format, seconds):
        """[<format>] [<seconds since epoch>]

        Returns the current time in <format> format, or, if <format> is not
        given, uses the configurable format for the current channel.  If no
        <seconds since epoch> time is given, the current time is used.
        """
        if not format:
            if channel:
                format = self.registryValue('format', channel)
            else:
                format = self.registryValue('format')
        if tzlocal:
            irc.reply(datetime.fromtimestamp(seconds, tzlocal()).strftime(format))
        else:
            # NOTE: This has erroneous behavior on some older Python versions,
            # including at least up to 2.7.5 and 3.2.3. Install dateutil if you
            # can't upgrade Python.
            irc.reply(time.strftime(format, time.localtime(seconds)))
    time = wrap(time, [optional('channel'), optional('nonInt'),
                       additional('float', TIME.time)])

    @internationalizeDocstring
    def elapsed(self, irc, msg, args, seconds):
        """<seconds>

        Returns a pretty string that is the amount of time represented by
        <seconds>.
        """
        irc.reply(utils.timeElapsed(seconds))
    elapsed = wrap(elapsed, ['int'])

    @internationalizeDocstring
    def tztime(self, irc, msg, args, timezone):
        """<region>/<city>

        Takes a city and its region, and returns the locale time. This
        command uses the IANA Time Zone Database."""
        try:
            import pytz
        except ImportError:
            irc.error(_('Python-tz is required by the command, but is not '
                        'installed on this computer.'))
            return
        try:
            timezone = pytz.timezone(timezone)
        except pytz.UnknownTimeZoneError:
            irc.error(_('Unknown timezone'))
            return
        irc.reply(str(datetime.now(timezone)))
    tztime = wrap(tztime, ['text'])

Class = Time

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:
<|endoftext|>


<commit_before>#! /usr/bin/python

import logging
import os
import StringIO
import struct
import sys
import urllib2
import xml.etree.ElementTree as etree
import zipfile

def file_hash(path):

    """
    Hash a file.

    The algorithm and original code comes from this page:
    http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes

    Takes:
        path - path to the file

    Returns:
        hex string of hash

    Raises:
        Exception - file too small: < 128 KiB
    """

    def chunk_hash(f, hash_value):

        fmt = "q"  # long long
        bufsize = struct.calcsize(fmt)

        for _ in range(64 * 1024 / bufsize):
            buf = f.read(bufsize)
            hash_value += struct.unpack(fmt, buf)[0]
            hash_value &= 0xFFFFFFFFFFFFFFFF  # to remain as 64 bit number

        return hash_value

    filesize = os.path.getsize(path)
    if filesize < 2 * 64 * 1024:
        raise Exception("file too small: < 128 KiB")

    hash_value = filesize
    with open(path, "rb") as f:
        hash_value = chunk_hash(f, hash_value)
        f.seek(max(0, filesize - 64 * 1024), 0)
        hash_value = chunk_hash(f, hash_value)

    hex_string = "{:016x}".format(hash_value)
    logging.info("hash: {}".format(hex_string))
    return hex_string

def movie_hash(file_list):

    """
    Hash a multi-CD movie by the first file, ignore the rest.

    Takes:
        file_list - list of video files belonging to the movie

    Returns:
        hex string of hash
    """

    movie_hash = file_hash(file_list[0])
    return movie_hash

class SubtitleNotFound(Exception):
    pass

def extract_subtitle_urls(xml_file_object):

    """
    Extract search result URLs from simplexml.

    Takes:
        xml - file-like object of simplexml search results

    Returns:
        list of search result URLs (same order as in xml)
    """

    tree = etree.parse(xml_file_object)

    # FIXME use absolute xpath: /search/results/subtitle/download
    #
    # findall with an absolute xpath is broken in xml.etree.Elementree 1.3.0
    # I couldn't find the issue on bugs.python.org, but here is the code
    # issuing the warning: /usr/lib/python2.7/xml/etree/ElementTree.py:745
    elements = tree.findall("./results/subtitle/download")

    if len(elements) == 0:
        raise SubtitleNotFound()

    url_list = map(lambda elem: elem.text, elements)
    logging.debug("search_result_urls: {}".format(url_list))
    return url_list

class Subtitle(object):

    """Subtitle struct."""

    def __init__(self, name, content):

        self.name = name
        self.content = content

    def __repr__(self):

        return "{}({!r})".format(self.__class__, self.__dict__)

    def __str__(self):

        return "{}({!r})".format(self.__class__, self.name)

# NOTE zipfile.ZipFile requires a seekable file handle.
#      File-like objects returned by urlopen are not seekable.
#      So we must take the whole archive content.
#
# NOTE I tried returning the open file-like objects (plus the name).
#      That way we wouldn't have to keep both compressed and uncompressed
#      subtitles in memory. The zipfile package prevents us doing this,
#      because it shares seek positions between open files inside the archive.
#      For details see:
#      http://stackoverflow.com/questions/5624669
#      http://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.open
#
def extract_subtitles(
    zip_content,
    extensions=set(
        [".srt", ".sub", ".smi", ".txt", ".ssa", ".ass", ".mpl"])
    ):

    """
    Extract all subtitle files from subtitle zip archive.

    Ignore anything else in the archive (e.g. .nfo files).
    Store everything in memory, write nothing to the file system.

    Source of recognized extension list:
        http://trac.opensubtitles.org/projects/opensubtitles
            /wiki/DevReadFirst#Subtitlefilesextensions

    Takes:
        zip_content - zip archive content as a string
        extensions - iterable of recognized subtitle extensions

    Returns:
        list of subtitles as Subtitle objects (order unspecified)

    Raises:
        Exception - subtitle not found in archive
    """

    with zipfile.ZipFile(StringIO.StringIO(zip_content)) as z:

        logging.debug("files_in_archive: {}".format(z.namelist()))

        subtitle_list = list()
        for name in z.namelist():
            ext = os.path.splitext(name)[1]
            if ext in extensions:
                with z.open(name) as s:
                    subtitle_list.append(
                        Subtitle(name=name, content=s.read()))

        if len(subtitle_list) == 0:
            raise Exception("subtitle not found in archive")

    return subtitle_list

class UserAgent(object):

    """Communicate with subtitle servers."""

    def __init__(self, server, opener=urllib2.build_opener()):

        self.server = server
        self.opener = opener

    def __repr__(self):

        return "{}({!r})".format(self.__class__, self.__dict__)

    def __str__(self):

        return "{}({!r})".format(self.__class__, self.server)

    # maybe FIXME encode urls
    #
    # Depending on where the inputs come from you may need to
    # construct the urls more carefully and care for url encoding.

    def search_page_url(
        self, movie_hash, language, cd_count=1, _fmt="simplexml"):

        """
        Construct search page URL.

        Takes:
            movie_hash - hash of movie (hex string)
                cf. movie_hash()
            language - subtitle language according to ISO 639 (string)
            cd_count - how many video files make up the movie?

        Returns:
            search page URL
        """

        url = ("http://"
            + self.server
            + "/en"
            + "/search"
            + "/sublanguageid-{}".format(language)
            + "/moviehash-{}".format(movie_hash)
            + "/subsumcd-{}".format(cd_count)
            + "/{}".format(_fmt)
            )
        logging.debug("search_page_url: {}".format(url))
        return url

    def get_subtitle(self, movie, language):

        """
        Get subtitles of movie.

        Query for subtitles of movie.
        Download first search result.
        Extract archive.
        Return subtitle objects ordered by subtitle name.

        Takes:
            movie - list of video files belonging to movie ("natural order")
            language - preferred language (ISO 639 code string)

        Returns:
            list of Subtitle objects (sorted by subtitle name)
        """

        def best(lst):
            return lst[0]

        subtitle_archive_url = best(
            extract_subtitle_urls(
            self.opener.open(
            self.search_page_url(
                language=language,
                movie_hash=movie_hash(movie),
                cd_count=len(movie),
                ))))

        zip_fo = self.opener.open(subtitle_archive_url)
        zip_content = zip_fo.read()
        zip_fo.close()

        subtitle_list = extract_subtitles(zip_content=zip_content)

        if len(movie) != len(subtitle_list):
            raise Exception(
                "unexpected number of subtitle files in archive: {}".format(
                    len(subtitle_list)))

        subtitle_list.sort(key=lambda obj: obj.name)

        return subtitle_list
<commit_msg>Add FIXMEs.<commit_after>#! /usr/bin/python

import logging
import os
import StringIO
import struct
import sys
import urllib2
import xml.etree.ElementTree as etree
import zipfile

def file_hash(path):

    """
    Hash a file.

    The algorithm and original code comes from this page:
    http://trac.opensubtitles.org/projects/opensubtitles/wiki/HashSourceCodes

    Takes:
        path - path to the file

    Returns:
        hex string of hash

    Raises:
        Exception - file too small: < 128 KiB
    """

    def chunk_hash(f, hash_value):

        fmt = "q"  # long long
        bufsize = struct.calcsize(fmt)

        for _ in range(64 * 1024 / bufsize):
            buf = f.read(bufsize)
            hash_value += struct.unpack(fmt, buf)[0]
            hash_value &= 0xFFFFFFFFFFFFFFFF  # to remain as 64 bit number

        return hash_value

    filesize = os.path.getsize(path)
    if filesize < 2 * 64 * 1024:
        raise Exception("file too small: < 128 KiB")

    hash_value = filesize
    with open(path, "rb") as f:
        hash_value = chunk_hash(f, hash_value)
        f.seek(max(0, filesize - 64 * 1024), 0)
        hash_value = chunk_hash(f, hash_value)

    hex_string = "{:016x}".format(hash_value)
    logging.info("hash: {}".format(hex_string))
    return hex_string

def movie_hash(file_list):

    """
    Hash a multi-CD movie by the first file, ignore the rest.

    Takes:
        file_list - list of video files belonging to the movie

    Returns:
        hex string of hash
    """

    movie_hash = file_hash(file_list[0])
    return movie_hash

class SubtitleNotFound(Exception):
    pass

def extract_subtitle_urls(xml_file_object):

    """
    Extract search result URLs from simplexml.

    Takes:
        xml - file-like object of simplexml search results

    Returns:
        list of search result URLs (same order as in xml)
    """

    tree = etree.parse(xml_file_object)

    # FIXME use absolute xpath: /search/results/subtitle/download
    #
    # findall with an absolute xpath is broken in xml.etree.Elementree 1.3.0
    # I couldn't find the issue on bugs.python.org, but here is the code
    # issuing the warning: /usr/lib/python2.7/xml/etree/ElementTree.py:745
    elements = tree.findall("./results/subtitle/download")

    if len(elements) == 0:
        raise SubtitleNotFound()

    url_list = map(lambda elem: elem.text, elements)
    logging.debug("search_result_urls: {}".format(url_list))
    return url_list

class Subtitle(object):

    """Subtitle struct."""

    def __init__(self, name, content):

        self.name = name
        self.content = content

    def __repr__(self):

        return "{}({!r})".format(self.__class__, self.__dict__)

    def __str__(self):

        return "{}({!r})".format(self.__class__, self.name)

# NOTE zipfile.ZipFile requires a seekable file handle.
#      File-like objects returned by urlopen are not seekable.
#      So we must take the whole archive content.
#
# NOTE I tried returning the open file-like objects (plus the name).
#      That way we wouldn't have to keep both compressed and uncompressed
#      subtitles in memory. The zipfile package prevents us doing this,
#      because it shares seek positions between open files inside the archive.
#      For details see:
#      http://stackoverflow.com/questions/5624669
#      http://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.open
#
def extract_subtitles(
    zip_content,
    extensions=set(
        [".srt", ".sub", ".smi", ".txt", ".ssa", ".ass", ".mpl"])
    ):

    """
    Extract all subtitle files from subtitle zip archive.

    Ignore anything else in the archive (e.g. .nfo files).
    Store everything in memory, write nothing to the file system.

    Source of recognized extension list:
        http://trac.opensubtitles.org/projects/opensubtitles
            /wiki/DevReadFirst#Subtitlefilesextensions

    Takes:
        zip_content - zip archive content as a string
        extensions - iterable of recognized subtitle extensions

    Returns:
        list of subtitles as Subtitle objects (order unspecified)

    Raises:
        Exception - subtitle not found in archive
    """

    # FIXME stringio vs bytearrayio
    with zipfile.ZipFile(StringIO.StringIO(zip_content)) as z:

        logging.debug("files_in_archive: {}".format(z.namelist()))

        subtitle_list = list()
        for name in z.namelist():
            ext = os.path.splitext(name)[1]
            if ext in extensions:
                with z.open(name) as s:
                    subtitle_list.append(
                        Subtitle(name=name, content=s.read()))

        if len(subtitle_list) == 0:
            raise Exception("subtitle not found in archive")

    return subtitle_list

class UserAgent(object):

    """Communicate with subtitle servers."""

    def __init__(self, server, opener=urllib2.build_opener()):

        self.server = server
        self.opener = opener

    def __repr__(self):

        return "{}({!r})".format(self.__class__, self.__dict__)

    def __str__(self):

        return "{}({!r})".format(self.__class__, self.server)

    # maybe FIXME encode urls
    #
    # Depending on where the inputs come from you may need to
    # construct the urls more carefully and care for url encoding.

    # FIXME Which variant of ISO 639 is accepted?
    #
    # So far I have used the 3-letter codes like 'eng', 'hun'...

    def search_page_url(
        self, movie_hash, language, cd_count=1, _fmt="simplexml"):

        """
        Construct search page URL.

        Takes:
            movie_hash - hash of movie (hex string)
                cf. movie_hash()
            language - preferred language (ISO 639 code string)
            cd_count - how many video files make up the movie?

        Returns:
            search page URL
        """

        url = ("http://"
            + self.server
            + "/en"
            + "/search"
            + "/sublanguageid-{}".format(language)
            + "/moviehash-{}".format(movie_hash)
            + "/subsumcd-{}".format(cd_count)
            + "/{}".format(_fmt)
            )
        logging.debug("search_page_url: {}".format(url))
        return url

    def get_subtitle(self, movie, language):

        """
        Get subtitles of movie.

        Query for subtitles of movie.
        Download first search result.
        Extract archive.
        Return subtitle objects ordered by subtitle name.

        Takes:
            movie - list of video files belonging to movie ("natural order")
            language - preferred language (ISO 639 code string)

        Returns:
            list of Subtitle objects (sorted by subtitle name)
        """

        def best(lst):
            return lst[0]

        subtitle_archive_url = best(
            extract_subtitle_urls(
            self.opener.open(
            self.search_page_url(
                language=language,
                movie_hash=movie_hash(movie),
                cd_count=len(movie),
                ))))

        zip_fo = self.opener.open(subtitle_archive_url)
        zip_content = zip_fo.read()
        zip_fo.close()

        subtitle_list = extract_subtitles(zip_content=zip_content)

        if len(movie) != len(subtitle_list):
            raise Exception(
                "unexpected number of subtitle files in archive: {}".format(
                    len(subtitle_list)))

        subtitle_list.sort(key=lambda obj: obj.name)

        return subtitle_list
<|endoftext|>


<commit_before>import re
from cookielib import Cookie
from util import hook, http, timesince

def format_comment(data):
    if not data:
        return None

    if data['over_18']:
        data['nsfw_tag'] = '- \x034NSFW\x0f'
    else:
        data['nsfw_tag'] = ''

    data['time_ago'] = timesince.timesince(data['created_utc'])

    return (
        "\x02{title:s}\x02 (\x0307{ups:d}\x0f|\x0312{downs:d}\x0f) "
        "- {time_ago:s} ago by {author:s} to /r/{subreddit:s} - {num_comments:d} comments{nsfw_tag:s}"
    ).format(**data)

    return data

def info_by_reddit_id(type, id):
    url = 'http://www.reddit.com/by_id/%s_%s.json' % (type, id)

    data = http.get_json(url)

    if type == 't3':
        children = data['data']['children']

        if len(children) == 0:
            return None

        return format_comment(children[0]['data'])

    return data

@hook.regex(r'reddit.com/r/([A-Za-z0-9]+)/comments/([a-z0-9A-Z]+)')
def link_reddit_comments(match):
    return info_by_reddit_id('t3', match.group(2))
<commit_msg>add subreddit linking too because why not<commit_after>import re
from cookielib import Cookie
from util import hook, http, timesince

def format_comment(data):
    if not data:
        return None

    if data['over_18']:
        data['nsfw_tag'] = '- \x034NSFW\x0f'
    else:
        data['nsfw_tag'] = ''

    data['time_ago'] = timesince.timesince(data['created_utc'])

    return (
        "\x02{title:s}\x02 (\x0307{ups:d}\x0f|\x0312{downs:d}\x0f) "
        "- {time_ago:s} ago by {author:s} to /r/{subreddit:s} - {num_comments:d} comments{nsfw_tag:s}"
    ).format(**data)

    return data

def format_subreddit(data):
    if not data:
        return None

    if data['over18']:
        data['nsfw_tag'] = '- \x034NSFW\x0f'
    else:
        data['nsfw_tag'] = ''

    if data['quarantine']:
        data['quarantine_tag'] = ' - QUARANTINED '
    else:
        data['quarantine_tag'] = ''

    data['time_ago'] = timesince.timesince(data['created_utc'])

    return (
        "\x02/r/{display_name} - {description:s}\x02 - {subscribers:d} subscribers "
        "- created {time_ago:s} ago {nsfw_tag:s}{quarantine_tag:s}"
    ).format(**data)

    return data

def info_by_reddit_id(type, id):
    url = 'http://www.reddit.com/by_id/%s_%s.json' % (type, id)

    data = http.get_json(url)

    if type == 't3':
        children = data['data']['children']

        if len(children) == 0:
            return None

        return format_comment(children[0]['data'])

    return data

def info_by_reddit_subreddit(subreddit):
    url = 'http://www.reddit.com/r/%s/about.json' % (subreddit)

    data = http.get_json(url)

    return format_subreddit(data['data'])

@hook.regex(r'reddit.com/r/([^/]+)/comments/([^/]+)')
def link_reddit_comments(match):
    return info_by_reddit_id('t3', match.group(2))

@hook.regex(r'reddit.com/r/([^/]+)$')
def link_reddit_subreddit(match):
    return info_by_reddit_subreddit(match.group(1))
<|endoftext|>


<commit_before>import os

from pmxbot.dictlib import ConfigDict

config = ConfigDict(
	web_base = '',
	host = '::0',
	port = os.environ.get('PORT', 8080),
)
<commit_msg>Accept the web port from the environment<commit_after>import os

from pmxbot.dictlib import ConfigDict

config = ConfigDict(
	web_base = '',
	host = '::0',
	port = int(os.environ.get('PORT', 8080)),
)
<|endoftext|>


<commit_before><commit_msg>Support Python 3 without 2to3<commit_after><|endoftext|>


<commit_before><commit_msg>SPECPROD optional for desi_preproc<commit_after><|endoftext|>


<commit_before><commit_msg>typo not being able to addChannel<commit_after><|endoftext|>