<commit_before>class DenseMatrix(list):
    def __init__(self, l):
        def a(x):
            if hasattr(x, '__iter__'):
                return DenseMatrix([a(y) for y in x])
            return x
        if hasattr(l, '__iter__'):
            self.extend(a(x) for x in l)
        else:
            self.append(l)

    def inplaceArrayOperation(self, other, f):
        assert(len(self)==len(other))
        for i in range(len(self)):
            self[i] = f(self[i], other[i])
        return self

    def arrayOperation(self, other, f):
        assert(len(self)==len(other))
        return DenseMatrix(f(self[i], other[i]) for i in range(len(self)))

    def inplaceScalarOperation(self, scalar, f):
        for i in range(len(self)):
            self[i] = f(self[i], scalar)
        return self

    def scalarOperation(self, scalar, f):
        return DenseMatrix(f(self[i], scalar) for i in range(len(self)))

    def __iadd__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a+b)

    def __add__(self, other):
        return self.arrayOperation(other, lambda a, b: a+b)

    def __isub__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a-b)

    def __sub__(self, other):
        return self.arrayOperation(other, lambda a, b: a-b)

    def __imul__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a*b)

    def __mul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __rmul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __itruediv__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a/b)

    def __truediv__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def __div__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def addColumn(self, element, columnID=0):
        """
            Add a whole column made of the given element at the columnID position.
        """
        for l in self:
            l[columnID:columnID] = [element]

    def removeColumn(self, columnID=0):
        """
            Remove the whole column.
        """
        for l in self:
            del l[columnID]

    def argmin(self, inf=0, sup=None):
        sup = sup if sup is not None else len(self)
        return (lambda array: min(zip(array, range(len(array))))[1])(self[inf:sup]) + inf

    def copy(self):
        return DenseMatrix(self)

class SparseLine(dict):
    def __init__(self, l=[]):
        if isinstance(l, dict):
            for k, elt in l.items():
                self[k] = elt
        else:
            for (i, x) in enumerate(l):
                if x != 0:
                    self[i] = x
        self.__nbitem__ = len(l)

    def __getitem__(self, i):
        if i<0:
            i=len(self)+i
        return self.get(i, 0)

    def __len__(self):
        return self.__nbitem__

    def __eq__(self, other):
        if isinstance(other, (dict, SparseLine)):
            return self.items() == other.items()
        else:
            return len(self) == len(other) and all(self[i] == other[i] for i in range(len(other)))

    def inplaceArrayOperation(self, other, f):
        for k in self.keys()|other.keys():
            elt = f(self[k], other[k])
            if elt:
                self[k] = elt
            else:
                self.pop(k, None)
        return self

    def arrayOperation(self, other, f):
        s = SparseLine()
        for k in self.keys()|other.keys():
            elt = f(self[k], other[k])
            if elt:
                s[k] = elt
        return s

    def inplaceScalarOperation(self, scalar, f):
        l = list(self.items())
        for k, elt in l:
            elt = f(self[k], scalar)
            if elt:
                self[k] = elt
            else:
                self.pop(k)
        return self

    def scalarOperation(self, scalar, f):
        s = SparseLine()
        for k in self.keys():
            elt = f(self[k], scalar)
            if elt:
                s[k] = elt
        return s

    def __iadd__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a+b)

    def __add__(self, other):
        return self.arrayOperation(other, lambda a, b: a+b)

    def __isub__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a-b)

    def __sub__(self, other):
        return self.arrayOperation(other, lambda a, b: a-b)

    def __imul__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a*b)

    def __mul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __rmul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __itruediv__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a/b)

    def __truediv__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def __div__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def addColumn(self, element, columnID=0):
        for k in sorted(self.keys(), key = lambda x:-x):
            if k < columnID:
                break
            self[k+1] = self[k]
            self.pop(k)
        self[columnID] = element

    def removeColumn(self, columnID=0):
        for k in sorted(self.keys()):
            if k < columnID:
                continue
            if k > columnID:
                self[k-1] = self[k]
            self.pop(k)

    def argmin(self, inf=0, sup=None):
        sup = sup if sup is not None else len(self)
        if sup == -1:
            sup = len(self)-1
        if inf >= sup:
            raise Exception('inf is greater than sup, no argmin')
        m, minIndex = None, None
        keys = sorted(k for k in self.keys() if k >= inf and k < sup)
        if len(keys) == 0: # all elts are 0
            return 0
        elif keys[0] > inf:
            m, minIndex = 0, inf
        elif keys[-1] < sup-1:
            m, minIndex = 0, sup-1
        oldK = keys[0]-1
        for k in keys:
            if k-oldK > 1 and m > 0:
                m = 0
                minIndex = k
            else:
                if m is None or self[k] < m:
                    m = self[k]
                    minIndex = k
            oldK = k
        return minIndex if m is not None else inf # in case the min is 0

    def copy(self):
        return SparseLine(self)

class SparseMatrix(list):
    def __init__(self, l):
        self.extend(SparseLine(elt) for elt in l)

    def addColumn(self, element, columnID=0):
        """
            Add a whole column made of the given element at the columnID position.
        """
        for l in self:
            l.addColumn(element, columnID) #[columnID:columnID] = [element]

    def removeColumn(self, columnID=0):
        """
            Remove the whole column.
        """
        for l in self:
            l.removeColumn(columnID)

class Array(SparseMatrix):
    pass
<commit_msg>Another bug.<commit_after>class DenseMatrix(list):
    def __init__(self, l):
        def a(x):
            if hasattr(x, '__iter__'):
                return DenseMatrix([a(y) for y in x])
            return x
        if hasattr(l, '__iter__'):
            self.extend(a(x) for x in l)
        else:
            self.append(l)

    def inplaceArrayOperation(self, other, f):
        assert(len(self)==len(other))
        for i in range(len(self)):
            self[i] = f(self[i], other[i])
        return self

    def arrayOperation(self, other, f):
        assert(len(self)==len(other))
        return DenseMatrix(f(self[i], other[i]) for i in range(len(self)))

    def inplaceScalarOperation(self, scalar, f):
        for i in range(len(self)):
            self[i] = f(self[i], scalar)
        return self

    def scalarOperation(self, scalar, f):
        return DenseMatrix(f(self[i], scalar) for i in range(len(self)))

    def __iadd__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a+b)

    def __add__(self, other):
        return self.arrayOperation(other, lambda a, b: a+b)

    def __isub__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a-b)

    def __sub__(self, other):
        return self.arrayOperation(other, lambda a, b: a-b)

    def __imul__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a*b)

    def __mul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __rmul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __itruediv__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a/b)

    def __truediv__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def __div__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def addColumn(self, element, columnID=0):
        """
            Add a whole column made of the given element at the columnID position.
        """
        for l in self:
            l[columnID:columnID] = [element]

    def removeColumn(self, columnID=0):
        """
            Remove the whole column.
        """
        for l in self:
            del l[columnID]

    def argmin(self, inf=0, sup=None):
        sup = sup if sup is not None else len(self)
        return (lambda array: min(zip(array, range(len(array))))[1])(self[inf:sup]) + inf

    def copy(self):
        return DenseMatrix(self)

class SparseLine(dict):
    def __init__(self, l=[]):
        if isinstance(l, dict):
            for k, elt in l.items():
                self[k] = elt
        else:
            for (i, x) in enumerate(l):
                if x != 0:
                    self[i] = x
        self.__nbitem__ = len(l)

    def __getitem__(self, i):
        if i<0:
            i=len(self)+i
        return self.get(i, 0)

    def __len__(self):
        return self.__nbitem__

    def __eq__(self, other):
        if isinstance(other, (dict, SparseLine)):
            return self.items() == other.items()
        else:
            return len(self) == len(other) and all(self[i] == other[i] for i in range(len(other)))

    def inplaceArrayOperation(self, other, f):
        for k in self.keys()|other.keys():
            elt = f(self[k], other[k])
            if elt:
                self[k] = elt
            else:
                self.pop(k, None)
        return self

    def arrayOperation(self, other, f):
        s = SparseLine()
        for k in self.keys()|other.keys():
            elt = f(self[k], other[k])
            if elt:
                s[k] = elt
        return s

    def inplaceScalarOperation(self, scalar, f):
        l = list(self.items())
        for k, elt in l:
            elt = f(self[k], scalar)
            if elt:
                self[k] = elt
            else:
                self.pop(k)
        return self

    def scalarOperation(self, scalar, f):
        s = SparseLine()
        for k in self.keys():
            elt = f(self[k], scalar)
            if elt:
                s[k] = elt
        return s

    def __iadd__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a+b)

    def __add__(self, other):
        return self.arrayOperation(other, lambda a, b: a+b)

    def __isub__(self, other):
        return self.inplaceArrayOperation(other, lambda a, b: a-b)

    def __sub__(self, other):
        return self.arrayOperation(other, lambda a, b: a-b)

    def __imul__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a*b)

    def __mul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __rmul__(self, other):
        return self.scalarOperation(other, lambda a, b: a*b)

    def __itruediv__(self, other):
        return self.inplaceScalarOperation(other, lambda a, b: a/b)

    def __truediv__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def __div__(self, other):
        return self.scalarOperation(other, lambda a, b: a/b)

    def addColumn(self, element, columnID=0):
        for k in sorted(self.keys(), key = lambda x:-x):
            if k < columnID:
                break
            self[k+1] = self[k]
            self.pop(k)
        self[columnID] = element
        self.__nbitem__ += 1

    def removeColumn(self, columnID=0):
        for k in sorted(self.keys()):
            if k < columnID:
                continue
            if k > columnID:
                self[k-1] = self[k]
            self.pop(k)
        self.__nbitem__ -= 1

    def argmin(self, inf=0, sup=None):
        sup = sup if sup is not None else len(self)
        if sup == -1:
            sup = len(self)-1
        if inf >= sup:
            raise Exception('inf is greater than sup, no argmin')
        m, minIndex = None, None
        keys = sorted(k for k in self.keys() if k >= inf and k < sup)
        if len(keys) == 0: # all elts are 0
            return 0
        elif keys[0] > inf:
            m, minIndex = 0, inf
        elif keys[-1] < sup-1:
            m, minIndex = 0, sup-1
        oldK = keys[0]-1
        for k in keys:
            if k-oldK > 1 and m > 0:
                m = 0
                minIndex = k
            else:
                if m is None or self[k] < m:
                    m = self[k]
                    minIndex = k
            oldK = k
        return minIndex if m is not None else inf # in case the min is 0

    def copy(self):
        return SparseLine(self)

class SparseMatrix(list):
    def __init__(self, l):
        self.extend(SparseLine(elt) for elt in l)

    def addColumn(self, element, columnID=0):
        """
            Add a whole column made of the given element at the columnID position.
        """
        for l in self:
            l.addColumn(element, columnID) #[columnID:columnID] = [element]

    def removeColumn(self, columnID=0):
        """
            Remove the whole column.
        """
        for l in self:
            l.removeColumn(columnID)

class Array(SparseMatrix):
    pass
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
import json

import celery
from emails import send_email
from pytz import timezone
import requests
import stripe

from salesforce import SalesforceConnection
from config import STRIPE_KEYS
from config import ACCOUNTING_MAIL_RECIPIENT
from config import TIMEZONE

zone = timezone(TIMEZONE)

stripe.api_key = STRIPE_KEYS['secret_key']

class Log(object):
    """
    This encapulates sending to the console/stdout and email all in one.

    """
    def __init__(self):
        self.log = list()

    def it(self, string):
        """
        Add something to the log.
        """
        print(string)
        self.log.append(string)

    def send(self):
        """
        Send the assembled log out as an email.
        """
        body = '\n'.join(self.log)
        recipient = ACCOUNTING_MAIL_RECIPIENT
        subject = 'Batch run'
        send_email(body=body, recipient=recipient, subject=subject)

def amount_to_charge(entry):
    """
    Determine the amount to charge. This depends on whether the payer agreed
    to pay fees or not. If they did then we add that to the amount charged.
    Stripe charges 2.9% + $0.30.

    Stripe wants the amount to charge in cents. So we multiply by 100 and
    return that.
    """
    amount = int(entry['Amount'])
    if entry['Stripe_Agreed_to_pay_fees__c']:
        fees = amount * .029 + .30
    else:
        fees = 0
    total = amount + fees
    total_in_cents = total * 100

    return int(total_in_cents)

def process_charges(query, log):

    sf = SalesforceConnection()

    response = sf.query(query)
    # TODO: check response code

    log.it('Found {} opportunities available to process.'.format(
        len(response)))

    for item in response:
        amount = amount_to_charge(item)
        try:
            log.it("---- Charging ${} to {} ({})".format(amount / 100,
                item['Stripe_Customer_ID__c'],
                item['Name']))
            charge = stripe.Charge.create(
                    customer=item['Stripe_Customer_ID__c'],
                    amount=amount,
                    currency='usd',
                    description=item['Description'],
                    )
        except stripe.error.CardError as e:
            # look for decline code:
            decline_code = ''
            try:
                decline_code = e.json_body['error']['decline_code']
            except:
                print('Unable to extract decline code')
            log.it("The card has been declined: {} ({})".format(e,
                decline_code))
            continue
        except stripe.error.InvalidRequestError as e:
            log.it("Problem: {}".format(e))
            continue
        # print ('Charge: {}'.format(charge))
        # TODO: check for success
        # TODO: catch other errors

        # print ("Charge id: {}".format(charge.id))
        update = {
                'Stripe_Transaction_Id__c': charge.id,
                'Stripe_Card__c': charge.source.id,
                'StageName': 'Closed Won',
                }
        path = item['attributes']['url']
        url = '{}{}'.format(sf.instance_url, path)
        resp = requests.patch(url, headers=sf.headers, data=json.dumps(update))
        # TODO: check 'errors' and 'success' too
        if resp.status_code == 204:
            log.it("ok")
        else:
            log.it("problem")
            raise Exception('problem')

@celery.task()
def charge_cards():

    log = Log()

    log.it('---Starting batch job...')

    three_days_ago = (datetime.now(tz=zone) - timedelta(
        days=3)).strftime('%Y-%m-%d')
    today = datetime.now(tz=zone).strftime('%Y-%m-%d')

    # regular (non Circle) pledges:
    log.it('---Processing regular charges...')

    query = """
        SELECT Amount, Name, Stripe_Customer_Id__c, Description,
            Stripe_Agreed_to_pay_fees__c
        FROM Opportunity
        WHERE CloseDate <= {}
        AND CloseDate >= {}
        AND StageName = 'Pledged'
        AND Stripe_Customer_Id__c != ''
        AND Type != 'Giving Circle'
        """.format(today, three_days_ago)

    process_charges(query, log)

    #
    # Circle transactions are different from the others. The Close Dates for a
    # given Circle donation are all identical. That's so that the gift can be
    # recognized all at once on the donor wall. So we use another field to
    # determine when the card is actually charged: Expected_Giving_Date__c.
    # So we process charges separately for Circles.
    #

    log.it('---Processing Circle charges...')

    query = """
        SELECT Amount, Name, Stripe_Customer_Id__c, Description,
            Stripe_Agreed_to_pay_fees__c
        FROM Opportunity
        WHERE Expected_Giving_Date__c <= {}
        AND Expected_Giving_Date__c >= {}
        AND StageName = 'Pledged'
        AND Stripe_Customer_Id__c != ''
        AND Type = 'Giving Circle'
        """.format(today, three_days_ago)

    process_charges(query, log)
    log.send()

if __name__ == '__main__':
    charge_cards()
<commit_msg>check stripe.Charge object for success<commit_after>from datetime import datetime, timedelta
import json

import celery
from emails import send_email
from pytz import timezone
import requests
import stripe

from salesforce import SalesforceConnection
from config import STRIPE_KEYS
from config import ACCOUNTING_MAIL_RECIPIENT
from config import TIMEZONE

zone = timezone(TIMEZONE)

stripe.api_key = STRIPE_KEYS['secret_key']

class Log(object):
    """
    This encapulates sending to the console/stdout and email all in one.

    """
    def __init__(self):
        self.log = list()

    def it(self, string):
        """
        Add something to the log.
        """
        print(string)
        self.log.append(string)

    def send(self):
        """
        Send the assembled log out as an email.
        """
        body = '\n'.join(self.log)
        recipient = ACCOUNTING_MAIL_RECIPIENT
        subject = 'Batch run'
        send_email(body=body, recipient=recipient, subject=subject)

def amount_to_charge(entry):
    """
    Determine the amount to charge. This depends on whether the payer agreed
    to pay fees or not. If they did then we add that to the amount charged.
    Stripe charges 2.9% + $0.30.

    Stripe wants the amount to charge in cents. So we multiply by 100 and
    return that.
    """
    amount = int(entry['Amount'])
    if entry['Stripe_Agreed_to_pay_fees__c']:
        fees = amount * .029 + .30
    else:
        fees = 0
    total = amount + fees
    total_in_cents = total * 100

    return int(total_in_cents)

def process_charges(query, log):

    sf = SalesforceConnection()

    response = sf.query(query)
    # TODO: check response code

    log.it('Found {} opportunities available to process.'.format(
        len(response)))

    for item in response:
        amount = amount_to_charge(item)
        try:
            log.it("---- Charging ${} to {} ({})".format(amount / 100,
                item['Stripe_Customer_ID__c'],
                item['Name']))
            charge = stripe.Charge.create(
                    customer=item['Stripe_Customer_ID__c'],
                    amount=amount,
                    currency='usd',
                    description=item['Description'],
                    )
        except stripe.error.CardError as e:
            # look for decline code:
            decline_code = ''
            try:
                decline_code = e.json_body['error']['decline_code']
            except:
                print('Unable to extract decline code')
            log.it("The card has been declined: {} ({})".format(e,
                decline_code))
            continue
        except stripe.error.InvalidRequestError as e:
            log.it("Problem: {}".format(e))
            continue
        except Exception as e:
            log.it("Problem: {}".format(e))
        if charge.status != 'succeeded':
            log.it("Charge failed. Check Stripe logs.")
            continue
        update = {
                'Stripe_Transaction_Id__c': charge.id,
                'Stripe_Card__c': charge.source.id,
                'StageName': 'Closed Won',
                }
        path = item['attributes']['url']
        url = '{}{}'.format(sf.instance_url, path)
        resp = requests.patch(url, headers=sf.headers, data=json.dumps(update))
        # TODO: check 'errors' and 'success' too
        if resp.status_code == 204:
            log.it("ok")
        else:
            log.it("problem")
            raise Exception('problem')

@celery.task()
def charge_cards():

    log = Log()

    log.it('---Starting batch job...')

    three_days_ago = (datetime.now(tz=zone) - timedelta(
        days=3)).strftime('%Y-%m-%d')
    today = datetime.now(tz=zone).strftime('%Y-%m-%d')

    # regular (non Circle) pledges:
    log.it('---Processing regular charges...')

    query = """
        SELECT Amount, Name, Stripe_Customer_Id__c, Description,
            Stripe_Agreed_to_pay_fees__c
        FROM Opportunity
        WHERE CloseDate <= {}
        AND CloseDate >= {}
        AND StageName = 'Pledged'
        AND Stripe_Customer_Id__c != ''
        AND Type != 'Giving Circle'
        """.format(today, three_days_ago)

    process_charges(query, log)

    #
    # Circle transactions are different from the others. The Close Dates for a
    # given Circle donation are all identical. That's so that the gift can be
    # recognized all at once on the donor wall. So we use another field to
    # determine when the card is actually charged: Expected_Giving_Date__c.
    # So we process charges separately for Circles.
    #

    log.it('---Processing Circle charges...')

    query = """
        SELECT Amount, Name, Stripe_Customer_Id__c, Description,
            Stripe_Agreed_to_pay_fees__c
        FROM Opportunity
        WHERE Expected_Giving_Date__c <= {}
        AND Expected_Giving_Date__c >= {}
        AND StageName = 'Pledged'
        AND Stripe_Customer_Id__c != ''
        AND Type = 'Giving Circle'
        """.format(today, three_days_ago)

    process_charges(query, log)
    log.send()

if __name__ == '__main__':
    charge_cards()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os
import getopt
import logging
from threading import Thread
from Queue import Queue, Empty
from collections import deque

from helpers import colored_print

from dialog_exceptions import UnsufficientInputError

from sentence import Sentence
import sentence

from speaker_identification import SpeakerIdentifier
from parsing.parser import Parser
from interpretation.resolution import Resolver
from interpretation.content_analysis import ContentAnalyser
from verbalization.verbalization import Verbalizer

VERSION = "0.1"

class Dialog(Thread):
    def __init__(self):
        Thread.__init__(self)

        self.go_on = True
        self._logger = logging.getLogger('dialog')

        self._speaker = SpeakerIdentifier()
        self._parser = Parser()
        self._resolver = Resolver()
        self._content_analyser = ContentAnalyser()
        self._verbalizer = Verbalizer()

        self._nl_input_queue = Queue()
        self._sentence_output_queue = Queue()

        #true when the Dialog module is within an interaction with a speaker
        self.in_interaction = False

        #the current set of sentences that dialog is dealing with
        self.sentences = deque()

        #the current sentence being worked on. Specifically, if active_sentence
        #is set, the parser will _complete_ this sentence instead of creating a
        #new one.
        self.active_sentence = None

        #true when, during an interaction, more infos is needed.
        self.waiting_for_more_info = False

        #contains the last result of a failed resolution (including the object
        #that could not be resolved. Cf doc of UnsufficientInputError for details.
        self._last_output = None

        #the object currently discussed. Used to resolve anaphors (like 'this
        # one').
        self.current_object = None

        #the ID of the speaker we are talking with. used to resolve references
        #like 'me', 'you', etc.
        self.current_speaker = None

        #the set of the statements generated by the analysis of the last sentence.
        self.last_stmts_set = []

    def run(self):
        while self.go_on:

            try:
                input = self._nl_input_queue.get(block = False).strip()
                self._logger.info(colored_print("###################################", 'green'))
                self._logger.info(colored_print("#             NL INPUT            #", 'green'))
                self._logger.info(colored_print("###################################\n", 'green'))
                self._logger.info(colored_print(input + "\n", 'blue'))
                self.in_interaction = True
                self.waiting_for_more_info = False

                try:
                    self._process(input)
                except UnsufficientInputError as uie:
                    self._logger.info(colored_print("##########################################", 'green'))
                    self._logger.info(colored_print("#  Missing content! Going back to human  #", 'green'))
                    self._logger.info(colored_print("##########################################", 'green'))
                    self._last_output = uie.value

                    #Waiting for more info to solve content
                    self.waiting_for_more_info = True

            except Empty:
                pass

            try:
                output = self._sentence_output_queue.get(block = False)
                self._logger.debug(colored_print("> Got output to verbalize: ", 'bold'))

                sys.stdout.write(colored_print( \
                            self._verbalizer.verbalize(output), \
                            'red') + "\n")
            except Empty:
                pass

    def stop(self):
        while(not self._nl_input_queue.empty()):
            pass
        self.go_on = False

    def input(self, input, speaker = None):
        if speaker:
            self.current_speaker = speaker
        else:
            self.current_speaker = self._speaker.get_current_speaker_id()

        if self.waiting_for_more_info and self._last_output:
            self._logger.info(colored_print("##########################################", 'green'))
            self._logger.info(colored_print("#   New content provided by human!       #", 'green'))
            self._logger.info(colored_print("##########################################", 'green'))
            self._logger.info(colored_print(input + "\n", 'blue'))

            self._last_output['object_with_more_info'] = sentence.nom_gr_remerge(self._parser.parse(input, None),
                                                                self._last_output['status'],
                                                                self._last_output['object'])
            #No more info needed
            self.waiting_for_more_info = False

        self._nl_input_queue.put(input)

    def test(self, speaker, input, answer = None):
        """This method eases the testing of dialog by returning only when a
        sentence is completely processed.
        The optional 'answer' argument is used to answer a request for more
        details from the resolution code.
        """
        self.in_interaction = True
        self.input(input, speaker)
        while(self.in_interaction):
            if answer and self.waiting_for_more_info:
                logging.debug(colored_print("> Automatically answering: ", 'bold'))
                logging.debug(colored_print(answer, 'red'))
                self.input(answer, speaker)

                answer = None
            elif self.waiting_for_more_info:
                return None
            pass

        return self.last_stmts_set

    def _process(self, nl_input):
        #Parsing
        self._logger.info(colored_print("###################################", 'green'))
        self._logger.info(colored_print("#             PARSING             #", 'green'))
        self._logger.info(colored_print("###################################", 'green'))

        uie_sentence = self._last_output['sentence'] if self._last_output else None

        self.sentences.appendleft(self._parser.parse(nl_input, uie_sentence)[0])

        for s in range(len(self.sentences)): #sentences is a deque. Cannot do a simple [:] to iterate over a copy
            self.active_sentence = self.sentences.popleft()

            #Resolution
            self._logger.info(colored_print("###################################", 'green'))
            self._logger.info(colored_print("#       RESOLVING SENTENCE        #", 'green'))
            self._logger.info(colored_print("###################################", 'green'))

            uie_object = self._last_output['object'] if self._last_output else None
            uie_object_with_more_info = self._last_output['object_with_more_info'] if uie_object else None
            self._last_output = None # No Needed no more
            self.active_sentence = self._resolver.references_resolution(self.active_sentence,
                                                                        self.current_speaker,
                                                                        self.current_object)
            self.active_sentence = self._resolver.noun_phrases_resolution(self.active_sentence,
                                                                          self.current_speaker,
                                                                          uie_object,
                                                                          uie_object_with_more_info)
            self.active_sentence = self._resolver.verbal_phrases_resolution(self.active_sentence)

            self._logger.debug(colored_print("###################################", 'green'))
            self._logger.debug("Sentence after resolution:\n" + str(self.active_sentence))

            #Content analysis
            self._logger.info(colored_print("###################################", 'green'))
            self._logger.info(colored_print("#        CONTENT ANALYSIS         #", 'green'))
            self._logger.info(colored_print("###################################", 'green'))
            self.last_stmts_set = self._content_analyser.analyse(self.active_sentence,
                                                                    self.current_speaker)
            self._sentence_output_queue.put(self._content_analyser.analyse_output())

        #Finalizing the processing
        self._logger.info(colored_print("\n###################################\n", 'green'))
        self.active_sentence = None
        self._logger.info(colored_print("NL sentence \"" + nl_input + "\" processed!", 'green'))
        self.in_interaction = False

def usage():
    print """The LAAS-CNRS dialog module.

Usage:
dialog.py [OPTIONS]
  -h, --help			Displays this message and exits
  -t, --test			Runs unit-tests
  -d, --debug			Sets verbosity to debug
  -s, --silent			The module won't output anything

This module reads on stdin user input in natural language, parse it, call
resolution routines when ambiguous concepts are used, and finally generate RDF
statements that are an interpretation of the input.

It includes as well a verbalization module that conversely turns RDF statements
into a sentence in natural language.

dialog fits into the so-called CHRIS architecture @ LAAS. More details on the
LAAS internal Wiki:
https://intranet.laas.fr/intranet/robots/wiki/ChrisArchitecture
"""

def unit_tests():
    print("Please run the dialog_test.py Python script.")
    print()
    print("> ./dialog_test.py")

def main():

    debug_level = logging.INFO

    try:
        optlist, args = getopt.getopt(sys.argv[1:], 'htds', ['help', 'test', 'debug', 'silent'])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    for o, a in optlist:
        if o in ("-h", "--help"):
            usage()
            sys.exit(0)
        elif o in ("-t", "--test"):
            unit_tests()
            sys.exit(0)
        elif o in ("-d", "--debug"):
            debug_level = logging.DEBUG
        elif o in ("-s", "--silent"):
            debug_level = logging.CRITICAL
        else:
            print "Unhandled option " + o
            usage()
            sys.exit(2)

    logging.basicConfig(level=debug_level, format="%(message)s")
    logging.info("**** DIALOG module ****")
    logging.info("v." + VERSION + "\n")

    dialog = Dialog()

    dialog.start()

    running = True
    while running:
        try:
            data = sys.stdin.readline()
            dialog.input(data)

        except KeyboardInterrupt:
            logging.info("Leaving now.")
            running = False
            dialog.stop()

    dialog.join()
    sys.exit()

if __name__ == '__main__':
    main()
<commit_msg>Anaphora resolution on process. Added the static variable 'dialog_history' that holds an history of the conversation for all instances of class dialog.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os
import getopt
import logging
from threading import Thread
from Queue import Queue, Empty
from collections import deque

from helpers import colored_print

from dialog_exceptions import UnsufficientInputError

from sentence import Sentence
import sentence

from speaker_identification import SpeakerIdentifier
from parsing.parser import Parser
from interpretation.resolution import Resolver
from interpretation.content_analysis import ContentAnalyser
from verbalization.verbalization import Verbalizer

VERSION = "0.1"

class Dialog(Thread):

    #This holds the history of conversation of all the instances of Dialog.
    dialog_history = []

    def __init__(self):
        Thread.__init__(self)

        self.go_on = True
        self._logger = logging.getLogger('dialog')

        self._speaker = SpeakerIdentifier()
        self._parser = Parser()
        self._resolver = Resolver()
        self._content_analyser = ContentAnalyser()
        self._verbalizer = Verbalizer()

        self._nl_input_queue = Queue()
        self._sentence_output_queue = Queue()

        #true when the Dialog module is within an interaction with a speaker
        self.in_interaction = False

        #the current set of sentences that dialog is dealing with
        self.sentences = deque()

        #the current sentence being worked on. Specifically, if active_sentence
        #is set, the parser will _complete_ this sentence instead of creating a
        #new one.
        self.active_sentence = None

        #true when, during an interaction, more infos is needed.
        self.waiting_for_more_info = False

        #contains the last result of a failed resolution (including the object
        #that could not be resolved. Cf doc of UnsufficientInputError for details.
        self._last_output = None

        #the object currently discussed. Used to resolve anaphors (like 'this
        # one').
        self.current_object = None

        #the ID of the speaker we are talking with. used to resolve references
        #like 'me', 'you', etc.
        self.current_speaker = None

        #the set of the statements generated by the analysis of the last sentence.
        self.last_stmts_set = []

    def run(self):
        while self.go_on:

            try:
                input = self._nl_input_queue.get(block = False).strip()
                self._logger.info(colored_print("###################################", 'green'))
                self._logger.info(colored_print("#             NL INPUT            #", 'green'))
                self._logger.info(colored_print("###################################\n", 'green'))
                self._logger.info(colored_print(input + "\n", 'blue'))
                self.in_interaction = True
                self.waiting_for_more_info = False

                try:
                    self._process(input)
                except UnsufficientInputError as uie:
                    self._logger.info(colored_print("##########################################", 'green'))
                    self._logger.info(colored_print("#  Missing content! Going back to human  #", 'green'))
                    self._logger.info(colored_print("##########################################", 'green'))
                    self._last_output = uie.value

                    #Waiting for more info to solve content
                    self.waiting_for_more_info = True

            except Empty:
                pass

            try:
                output = self._sentence_output_queue.get(block = False)
                self._logger.debug(colored_print("> Got output to verbalize: ", 'bold'))

                sys.stdout.write(colored_print( \
                            self._verbalizer.verbalize(output), \
                            'red') + "\n")

                # Store output
                Dialog.dialog_history.extend(output)

            except Empty:
                pass

    def stop(self):
        while(not self._nl_input_queue.empty()):
            pass
        self.go_on = False

    def input(self, input, speaker = None):
        if speaker:
            self.current_speaker = speaker
        else:
            self.current_speaker = self._speaker.get_current_speaker_id()

        if self.waiting_for_more_info and self._last_output:
            self._logger.info(colored_print("##########################################", 'green'))
            self._logger.info(colored_print("#   New content provided by human!       #", 'green'))
            self._logger.info(colored_print("##########################################", 'green'))
            self._logger.info(colored_print(input + "\n", 'blue'))

            self._last_output['object_with_more_info'] = sentence.nom_gr_remerge(self._parser.parse(input, None),
                                                                self._last_output['status'],
                                                                self._last_output['object'])
            #No more info needed
            self.waiting_for_more_info = False

        self._nl_input_queue.put(input)

    def test(self, speaker, input, answer = None):
        """This method eases the testing of dialog by returning only when a
        sentence is completely processed.
        The optional 'answer' argument is used to answer a request for more
        details from the resolution code.
        """
        self.in_interaction = True
        self.input(input, speaker)
        while(self.in_interaction):
            if answer and self.waiting_for_more_info:
                logging.debug(colored_print("> Automatically answering: ", 'bold'))
                logging.debug(colored_print(answer, 'red'))
                self.input(answer, speaker)

                answer = None
            elif self.waiting_for_more_info:
                return None
            pass

        return self.last_stmts_set

    def _process(self, nl_input):
        #Parsing
        self._logger.info(colored_print("###################################", 'green'))
        self._logger.info(colored_print("#             PARSING             #", 'green'))
        self._logger.info(colored_print("###################################", 'green'))

        uie_sentence = self._last_output['sentence'] if self._last_output else None

        self.sentences.appendleft(self._parser.parse(nl_input, uie_sentence)[0])

        for s in range(len(self.sentences)): #sentences is a deque. Cannot do a simple [:] to iterate over a copy
            self.active_sentence = self.sentences.popleft()

            #Resolution
            self._logger.info(colored_print("###################################", 'green'))
            self._logger.info(colored_print("#       RESOLVING SENTENCE        #", 'green'))
            self._logger.info(colored_print("###################################", 'green'))

            uie_object = self._last_output['object'] if self._last_output else None
            uie_object_with_more_info = self._last_output['object_with_more_info'] if uie_object else None
            self._last_output = None # No Needed no more
            self.active_sentence = self._resolver.references_resolution(self.active_sentence,
                                                                        self.current_speaker,
                                                                        self.current_object)
            self.active_sentence = self._resolver.noun_phrases_resolution(self.active_sentence,
                                                                          self.current_speaker,
                                                                          uie_object,
                                                                          uie_object_with_more_info)
            self.active_sentence = self._resolver.verbal_phrases_resolution(self.active_sentence)

            self._logger.debug(colored_print("###################################", 'green'))
            self._logger.debug("Sentence after resolution:\n" + str(self.active_sentence))

            #Content analysis
            self._logger.info(colored_print("###################################", 'green'))
            self._logger.info(colored_print("#        CONTENT ANALYSIS         #", 'green'))
            self._logger.info(colored_print("###################################", 'green'))
            self.last_stmts_set = self._content_analyser.analyse(self.active_sentence,
                                                                    self.current_speaker)
            self._sentence_output_queue.put(self._content_analyser.analyse_output())

            #Dialog History
            self._logger.debug(colored_print(" \t Sentence saved in history", 'green'))
            Dialog.dialog_history.append(self.active_sentence)

        #Finalizing the processing
        self._logger.info(colored_print("\n###################################\n", 'green'))
        self.active_sentence = None
        self._logger.info(colored_print("NL sentence \"" + nl_input + "\" processed!", 'green'))
        self.in_interaction = False

def usage():
    print """The LAAS-CNRS dialog module.

Usage:
dialog.py [OPTIONS]
  -h, --help			Displays this message and exits
  -t, --test			Runs unit-tests
  -d, --debug			Sets verbosity to debug
  -s, --silent			The module won't output anything

This module reads on stdin user input in natural language, parse it, call
resolution routines when ambiguous concepts are used, and finally generate RDF
statements that are an interpretation of the input.

It includes as well a verbalization module that conversely turns RDF statements
into a sentence in natural language.

dialog fits into the so-called CHRIS architecture @ LAAS. More details on the
LAAS internal Wiki:
https://intranet.laas.fr/intranet/robots/wiki/ChrisArchitecture
"""

def unit_tests():
    print("Please run the dialog_test.py Python script.")
    print()
    print("> ./dialog_test.py")

def main():

    debug_level = logging.INFO

    try:
        optlist, args = getopt.getopt(sys.argv[1:], 'htds', ['help', 'test', 'debug', 'silent'])
    except getopt.GetoptError, err:
        # print help information and exit:
        print str(err) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)

    for o, a in optlist:
        if o in ("-h", "--help"):
            usage()
            sys.exit(0)
        elif o in ("-t", "--test"):
            unit_tests()
            sys.exit(0)
        elif o in ("-d", "--debug"):
            debug_level = logging.DEBUG
        elif o in ("-s", "--silent"):
            debug_level = logging.CRITICAL
        else:
            print "Unhandled option " + o
            usage()
            sys.exit(2)

    logging.basicConfig(level=debug_level, format="%(message)s")
    logging.info("**** DIALOG module ****")
    logging.info("v." + VERSION + "\n")

    dialog = Dialog()

    dialog.start()

    running = True
    while running:
        try:
            data = sys.stdin.readline()
            dialog.input(data)

        except KeyboardInterrupt:
            logging.info("Leaving now.")
            running = False
            dialog.stop()

    dialog.join()
    sys.exit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""The Django url to handler mappings for mDS.

:Authors: Sana dev team
:Version: 2.0
"""

from django.conf import settings
from django.conf.urls.defaults import patterns, url, include
from django.views.generic.simple import redirect_to

from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns(
    '',
    url(r'^$', 'sana.core.views.home', name="home"),
    url(r'^core/', include('sana.core.urls', namespace='core')),
    url(r'^mds/', include('sana.mds.urls', namespace='mds')),
    url(r'^mrs/', include('sana.mrs.urls', namespace='mrs')),
    # ADMIN
    (r'^admin/', include(admin.site.urls)),
)

"""The mappings Django uses to send requests to the appropriate handlers."""

if settings.DEBUG:
    urlpatterns += patterns(
        'django.contrib.staticfiles.views',
        url(r'^static/(?P<path>.*)$', 'serve'),
        )
    urlpatterns += patterns('',
            url(r'^media/(?P<path>.*)$', 'django.views.static.serve',  {'document_root': settings.MEDIA_ROOT }),
        )

if 'v1' in settings.APICOMPAT_INCLUDE:
    from piston.resource import Resource
    from sana.mrs.handlers import *

    v1auth_resource = Resource(AuthHandler)
    v1notification_resource = Resource(NotificationHandler)
    v1smtp_resource = Resource(SMTPHandler)
    v1patient_resource = Resource(PatientHandler)
    v1savedprocedure_resource = Resource(SavedProcedureHandler)
    v1event_resource = Resource(EventHandler)
    v1requestlog_resource = Resource(EventHandler)
    v1binary_resource = Resource(BinaryHandler)
    v1binarypacket_resource = Resource(BinaryPacketHandler)
    v1base64packet_resource = Resource(Base64PacketHandler)
    v1requestlog_resource = Resource(RequestLogHandler)

    v1patterns = patterns(
        '',

        url(r'^notifications/$',
            v1notification_resource,
            name="sana-list-notifications"),

        url(r'^notifications/submit/$',
            v1notification_resource,
            name="sana-api-notification-submit"),

        url(r'^notifications/submit/email/$',
            v1smtp_resource,
            name="sana-api-email-notification-submit"),

         url(r'^json/patient/list/$',
            v1patient_resource,
             name="sana-json-patient-list"),

         url(r'^json/patient/(?P<id>[0-9-]+)/$',
            v1patient_resource,
            name="sana-json-patient-get"),

        url(r'^json/validate/credentials/$',
            v1auth_resource,
            name = "sana-json-validate-credentials"),

        url(r'^procedure/submit/$',
            redirect_to,
            name="sana-html-procedure-submit"),

        url(r'^json/procedure/submit/$',
            redirect_to,
            name="sana-json-procedure-submit"),

        url(r'^json/binary/submit/$',
            v1binary_resource,
            name="sana-json-binary-submit"),

        url(r'^json/binarychunk/submit/$',
            v1binarypacket_resource,
            name="sana-json-binarychunk-submit"),

        url(r'^json/textchunk/submit/$',
            v1base64packet_resource,
            name="sana-json-binarychunk-hack-submit"),

        url(r'^json/eventlog/submit/$',
            v1event_resource,
            name="sana-json-eventlog-submit"),

        # LOGGING
        url(r'^log-detail/$',
            'sana.api.v1.util.log_json_detail',
            name="log-json-detail-noarg"),

        url(r'^log-detail/(?P<log_id>\d+)$',
            'sana.api.v1.util.log_json_detail',
            name="log-json-detail"),

        url(r'^log/$',
            v1requestlog_resource,
            name="log-view"),

        url(r'^log/web/$',
            v1requestlog_resource,
            name="log-web-view"),

        url(r'^log/list/$',
            v1requestlog_resource,
            name="requestlog-list"),

        url(r'^log/(?P<uuid>[^/]+)/$',
            v1requestlog_resource,
            name="requestlog"),
    )
    # Add v1 compat urls
    urlpatterns += v1patterns

    <commit_msg>Removed unused urls<commit_after>"""The Django url to handler mappings for mDS.

:Authors: Sana dev team
:Version: 2.0
"""

from django.conf import settings
from django.conf.urls.defaults import patterns, url, include
from django.views.generic.simple import redirect_to

from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns(
    '',
    url(r'^$', 'sana.core.views.home', name="home"),
    url(r'^core/', include('sana.core.urls', namespace='core')),
    # ADMIN
    (r'^admin/', include(admin.site.urls)),
)

"""The mappings Django uses to send requests to the appropriate handlers."""

if settings.DEBUG:
    urlpatterns += patterns(
        'django.contrib.staticfiles.views',
        url(r'^static/(?P<path>.*)$', 'serve'),
        )
    urlpatterns += patterns('',
            url(r'^media/(?P<path>.*)$', 'django.views.static.serve',  {'document_root': settings.MEDIA_ROOT }),
        )

if 'v1' in settings.APICOMPAT_INCLUDE:
    from piston.resource import Resource
    from sana.mrs.handlers import *

    v1auth_resource = Resource(AuthHandler)
    v1notification_resource = Resource(NotificationHandler)
    v1smtp_resource = Resource(SMTPHandler)
    v1patient_resource = Resource(PatientHandler)
    v1savedprocedure_resource = Resource(SavedProcedureHandler)
    v1event_resource = Resource(EventHandler)
    v1requestlog_resource = Resource(EventHandler)
    v1binary_resource = Resource(BinaryHandler)
    v1binarypacket_resource = Resource(BinaryPacketHandler)
    v1base64packet_resource = Resource(Base64PacketHandler)
    v1requestlog_resource = Resource(RequestLogHandler)

    v1patterns = patterns(
        '',

        url(r'^notifications/$',
            v1notification_resource,
            name="sana-list-notifications"),

        url(r'^notifications/submit/$',
            v1notification_resource,
            name="sana-api-notification-submit"),

        url(r'^notifications/submit/email/$',
            v1smtp_resource,
            name="sana-api-email-notification-submit"),

         url(r'^json/patient/list/$',
            v1patient_resource,
             name="sana-json-patient-list"),

         url(r'^json/patient/(?P<id>[0-9-]+)/$',
            v1patient_resource,
            name="sana-json-patient-get"),

        url(r'^json/validate/credentials/$',
            v1auth_resource,
            name = "sana-json-validate-credentials"),

        url(r'^procedure/submit/$',
            v1savedprocedure_resource,
            name="sana-html-procedure-submit"),

        url(r'^json/procedure/submit/$',
            v1savedprocedure_resource,
            name="sana-json-procedure-submit"),

        url(r'^json/binary/submit/$',
            v1binary_resource,
            name="sana-json-binary-submit"),

        url(r'^json/binarychunk/submit/$',
            v1binarypacket_resource,
            name="sana-json-binarychunk-submit"),

        url(r'^json/textchunk/submit/$',
            v1base64packet_resource,
            name="sana-json-binarychunk-hack-submit"),

        url(r'^json/eventlog/submit/$',
            v1event_resource,
            name="sana-json-eventlog-submit"),

        # LOGGING
        url(r'^log-detail/$',
            v1requestlog_resource,
            #'sana.api.v1.util.log_json_detail',
            name="log-json-detail-noarg"),

        url(r'^log-detail/(?P<uuid>\d+)$',
            v1requestlog_resource,
            name="log-json-detail"),

        url(r'^log/$',
            v1requestlog_resource,
            name="log-view"),

        url(r'^log/web/$',
            v1requestlog_resource,
            name="log-web-view"),

        url(r'^log/list/$',
            v1requestlog_resource,
            name="requestlog-list"),

        url(r'^log/(?P<uuid>[^/]+)/$',
            v1requestlog_resource,
            name="requestlog"),
    )
    # Add v1 compat urls
    urlpatterns += v1patterns

    <|endoftext|>


<commit_before><commit_msg>Added a warning and auto-exit on main thread exit if the Bus is not shut down. This will help avoid issues like #750.<commit_after><|endoftext|>


<commit_before>"""
Created on 11 Jul 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import time

# --------------------------------------------------------------------------------------------------------------------

class IntervalTimer(object):
    """
    generate indices at pre-set intervals
    """

    def __init__(self, interval):
        self.__interval = interval
        self.__next_yield = time.time() + self.__interval

    # ----------------------------------------------------------------------------------------------------------------

    def range(self, stop):
        for i in range(stop):
            try:
                self.__sleep_until_next_yield()
            except KeyboardInterrupt:
                return stop

            yield i

    def true(self):
        try:
            self.__sleep_until_next_yield()
        except KeyboardInterrupt:
            return False

        return True

    # ----------------------------------------------------------------------------------------------------------------

    def __sleep_until_next_yield(self):
        if self.__interval == 0:
            return

        sleep_time = (self.__next_yield - time.time()) % self.__interval        # prevents negative intervals

        if sleep_time == 0:
            sleep_time = self.__interval        # prevents zero intervals

        time.sleep(sleep_time)

        self.__next_yield += self.__interval

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def interval(self):
        return self.__interval

    @property
    def time_to_next_yield(self):
        return self.__next_yield - time.time()

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "IntervalTimer:{interval:%s, time_to_next_yield:%0.3f}" % (self.interval, self.time_to_next_yield)
<commit_msg>Fixed a bug in IntervalTimer.<commit_after>"""
Created on 11 Jul 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)
"""

import time

# --------------------------------------------------------------------------------------------------------------------

class IntervalTimer(object):
    """
    generate indices at pre-set intervals
    """

    def __init__(self, interval):
        self.__interval = interval
        self.__next_yield = time.time() + self.__interval

    # ----------------------------------------------------------------------------------------------------------------

    def range(self, stop):
        for i in range(stop):
            try:
                self.__sleep_until_next_yield()
            except KeyboardInterrupt:
                return stop

            yield i

    def true(self):
        try:
            self.__sleep_until_next_yield()
        except KeyboardInterrupt:
            return False

        return True

    # ----------------------------------------------------------------------------------------------------------------

    def __sleep_until_next_yield(self):
        if self.__interval == 0:
            return

        sleep_time = (self.__next_yield - time.time()) % self.__interval        # prevents negative intervals

        if sleep_time == 0:
            sleep_time = self.__interval        # prevents zero intervals

        time.sleep(sleep_time)

        self.__next_yield += self.__interval

    # ----------------------------------------------------------------------------------------------------------------

    @property
    def interval(self):
        return self.__interval

    @property
    def time_to_next_yield(self):
        return abs(self.__next_yield - time.time())         # time to next must always be positive

    # ----------------------------------------------------------------------------------------------------------------

    def __str__(self, *args, **kwargs):
        return "IntervalTimer:{interval:%s, time_to_next_yield:%0.3f}" % (self.interval, self.time_to_next_yield)
<|endoftext|>


<commit_before>import flask
app = flask.Flask(__name__, static_folder='package_data/static')

@app.route('/srw')
def srw_root():
    return app.send_static_file('html/srw.html')

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000, debug=1)
<commit_msg>added srw_run()<commit_after>import flask
app = flask.Flask(__name__, static_folder='package_data/static')

@app.route('/srw')
def srw_root():
    return app.send_static_file('html/srw.html')

@app.route('/srw/run', methods=('GET', 'POST'))
def srw_run():
    print("in srw_run()")
    print(flask.request.json)
    return 'OK'

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000, debug=1)
<|endoftext|>


<commit_before>import itertools

'''
An `accumulator` adds some push-based stream semantics to Python iterators,
which are pull-based (they do stuff when you ask for values, not when data is
available). To do this we define an accumulator as a function from iterators to
iterators, where the output iterator will make exactly one request to the input
iterator before yielding. This allows us to run multiple accumulators on one
input stream without having to buffer the whole list in memory, which is the
`batch` function below: all of the accumulators are run one step, then the cycle
continues again. It also lets us filter the calls to one accumulator based on a
predicate function while still ensuring that it does not get ahead of the batch,
this is the `prefilter` function. Finally `premap` takes an accumulator and
returns another accumulator which transforms the input stream by mapping it
before acting on it.
'''

def batch(*accs):
    def out(iterator):
        copies = itertools.tee(iterator, len(accs) + 1)
        result_streams = tuple(a(c) for (a, c) in itertools.izip(accs, copies))
        for i in copies[-1]:
            yield tuple(stream.next() for stream in result_streams)
    return out

def prefilter(acc, predicate):
    '''Filter the input to an accumulator based on whether it satisfies the
    predicate.'''
    def out(iterator):
        (i1, i2) = itertools.tee(iterator, 2)
        output = acc(itertools.ifilter(predicate, i2))
        last = None
        for i in i1:
            # There's a way to write this without duplicating the check but this
            # just looks so much cleaner.
            if predicate(i):
                last = output.next()
            yield last
    return out
<commit_msg>docstring for batch<commit_after>import itertools

'''
An `accumulator` adds some push-based stream semantics to Python iterators,
which are pull-based (they do stuff when you ask for values, not when data is
available). To do this we define an accumulator as a function from iterators to
iterators, where the output iterator will make exactly one request to the input
iterator before yielding. This allows us to run multiple accumulators on one
input stream without having to buffer the whole list in memory, which is the
`batch` function below: all of the accumulators are run one step, then the cycle
continues again. It also lets us filter the calls to one accumulator based on a
predicate function while still ensuring that it does not get ahead of the batch,
this is the `prefilter` function. Finally `premap` takes an accumulator and
returns another accumulator which transforms the input stream by mapping it
before acting on it.
'''

def batch(*accs):
    '''Batch together all of the accumulators `*accs` so that they all act on a
    single iterator.'''
    def out(iterator):
        copies = itertools.tee(iterator, len(accs) + 1)
        result_streams = tuple(a(c) for (a, c) in itertools.izip(accs, copies))
        for i in copies[-1]:
            yield tuple(stream.next() for stream in result_streams)
    return out

def prefilter(acc, predicate):
    '''Filter the input to an accumulator based on whether it satisfies the
    predicate.'''
    def out(iterator):
        (i1, i2) = itertools.tee(iterator, 2)
        output = acc(itertools.ifilter(predicate, i2))
        last = None
        for i in i1:
            # There's a way to write this without duplicating the check but this
            # just looks so much cleaner.
            if predicate(i):
                last = output.next()
            yield last
    return out
<|endoftext|>


<commit_before>from collections import namedtuple, defaultdict, deque
from random import Random
import constraint_maxteams
import constraint_distribute
import constraint_balance_zones
import constraint_balance_matches
import constraint_once_only
import constraint_roundwise
import constraint_compound

ScheduleConfiguration = namedtuple('ScheduleConfiguration',
                                   ['zones', 'teams', 'weight_zones',
                                    'round_length', 'imbalance_action',
                                    'match_count'])

INFINITY = float('inf')

class Scheduler(object):
    def __init__(self, configuration):
        self.configuration = configuration
        self.matches = []
        self._constraint = None
        self._rng = Random(3)
        self._init_constraint()
        self._fill_order = deque(self.configuration.teams)
        self._rng.shuffle(self._fill_order)

    def _init_constraint(self):
        subconstraints = []
        if self.configuration.round_length is not None:
            subconstraints.append(constraint_roundwise.RoundwiseConstraint(sub = constraint_once_only.OnceOnlyConstraint(),
                                                      length = self.configuration.round_length))
        subconstraints.append(constraint_balance_matches.BalanceMatchCountConstraint())
        subconstraints.append(constraint_maxteams.MaximiseTeamsConstraint(2))
        subconstraints.append(constraint_distribute.DistributeMatchesConstraint(1, 2))
        if self.configuration.weight_zones:
            subconstraints.append(constraint_balance_zones.BalanceZoneCountConstraint())
        self._constraint = constraint_compound.CompoundConstraint(subconstraints)

    def add_match(self, match):
        self.matches.append(match)
        self._constraint.update(match)

    def _random_match(self):
        return self._rng.sample(self.configuration.teams +
                               ['-']*self.configuration.zones,
                               self.configuration.zones)

    def _random_match_pool(self, pool_size = 1400):
        return [self._random_match() for i in xrange(pool_size)]

    def _best_match_from_pool(self, pool):
        return min(pool, key = self._constraint.evaluate)

    def _is_viable(self, match):
        return self._constraint.evaluate(match) < INFINITY

    def _imbalance_surrogate(self, match):
        assert self.configuration.imbalance_action != 'empty'
        for i, team in enumerate(match):
            if team == '-':
                while True:
                    team = self._fill_order.popleft()
                    self._fill_order.append(team)
                    if team in match:
                        continue
                    match[i] = '-{0}'.format(team)
                    break

    def pick_match(self):
        pool = self._random_match_pool()
        next_match = self._best_match_from_pool(pool)
        if not self._is_viable(next_match):
            raise ValueError("no viable matches")
        if self.configuration.imbalance_action == 'surrogate':
            self._imbalance_surrogate(next_match)
        self.add_match(next_match)

    def pick_all_matches(self):
        while len(self.matches) < self.configuration.match_count:
            self.pick_match()

    def reset(self):
        if self.matches:
            self._constraint.reset()
            self.matches = []

    def compute_full_schedule(self):
        self.reset()
        self.pick_all_matches()

    def compute_partial_schedule(self, past_matches):
        self.reset()
        for match in past_matches:
            self.add_match(match)
        self.pick_all_matches()

def full_schedule(configuration):
    scheduler = Scheduler(configuration)
    scheduler.compute_full_schedule()
    return scheduler.matches

def partial_schedule(configuration, past_matches):
    scheduler = Scheduler(configuration)
    scheduler.compute_partial_schedule(past_matches)
    return scheduler.matches

def match_statistics(matches):
    from collections import Counter, defaultdict
    match_counter = Counter()
    opponents = defaultdict(set)
    prev_match = []
    collisions = set()
    for match in matches:
        for team in match:
            if team in prev_match:
                collisions.add(team)
            match_counter[team] += 1
            for other_team in match:
                if team != other_team:
                    opponents[team].add(other_team)
    return match_counter, opponents, collisions

def schedule_check(matches):
    print '{0} matches total'.format(len(matches))
    match_counter, opponents, collisions = match_statistics(matches)
    all_teams = set(match_counter.iterkeys())
    for team, matches in match_counter.iteritems():
        print '{0}: {1} matches, missed opponents: {2}'.format(team, matches, ', '.join(all_teams - opponents[team] - {team}))
        if team in collisions:
            print '\t{0} DO have a match collision'.format(team)

if __name__ == '__main__':
    config = ScheduleConfiguration(zones = 4,
                                   teams = ['Team {0}'.format(x) for x in xrange(1, 11)],
                                   weight_zones = True,
                                   round_length = None,
                                   imbalance_action = 'empty',
                                   match_count = 25)
    schedule = full_schedule(config)
    schedule_check(schedule)
    import json
    print json.dumps(schedule)

<commit_msg>Remove INFINITY<commit_after>from collections import namedtuple, defaultdict, deque
from random import Random
import constraint_maxteams
import constraint_distribute
import constraint_balance_zones
import constraint_balance_matches
import constraint_once_only
import constraint_roundwise
import constraint_compound

ScheduleConfiguration = namedtuple('ScheduleConfiguration',
                                   ['zones', 'teams', 'weight_zones',
                                    'round_length', 'imbalance_action',
                                    'match_count'])

class Scheduler(object):
    def __init__(self, configuration):
        self.configuration = configuration
        self.matches = []
        self._constraint = None
        self._rng = Random(3)
        self._init_constraint()
        self._fill_order = deque(self.configuration.teams)
        self._rng.shuffle(self._fill_order)

    def _init_constraint(self):
        subconstraints = []
        if self.configuration.round_length is not None:
            subconstraints.append(constraint_roundwise.RoundwiseConstraint(sub = constraint_once_only.OnceOnlyConstraint(),
                                                      length = self.configuration.round_length))
        subconstraints.append(constraint_balance_matches.BalanceMatchCountConstraint())
        subconstraints.append(constraint_maxteams.MaximiseTeamsConstraint(2))
        subconstraints.append(constraint_distribute.DistributeMatchesConstraint(1, 2))
        if self.configuration.weight_zones:
            subconstraints.append(constraint_balance_zones.BalanceZoneCountConstraint())
        self._constraint = constraint_compound.CompoundConstraint(subconstraints)

    def add_match(self, match):
        self.matches.append(match)
        self._constraint.update(match)

    def _random_match(self):
        return self._rng.sample(self.configuration.teams +
                               ['-']*self.configuration.zones,
                               self.configuration.zones)

    def _random_match_pool(self, pool_size = 1400):
        return [self._random_match() for i in xrange(pool_size)]

    def _best_match_from_pool(self, pool):
        return min(pool, key = self._constraint.evaluate)

    def _is_viable(self, match):
        return self._constraint.evaluate(match) < float('inf')

    def _imbalance_surrogate(self, match):
        assert self.configuration.imbalance_action != 'empty'
        for i, team in enumerate(match):
            if team == '-':
                while True:
                    team = self._fill_order.popleft()
                    self._fill_order.append(team)
                    if team in match:
                        continue
                    match[i] = '-{0}'.format(team)
                    break

    def pick_match(self):
        pool = self._random_match_pool()
        next_match = self._best_match_from_pool(pool)
        if not self._is_viable(next_match):
            raise ValueError("no viable matches")
        if self.configuration.imbalance_action == 'surrogate':
            self._imbalance_surrogate(next_match)
        self.add_match(next_match)

    def pick_all_matches(self):
        while len(self.matches) < self.configuration.match_count:
            self.pick_match()

    def reset(self):
        if self.matches:
            self._constraint.reset()
            self.matches = []

    def compute_full_schedule(self):
        self.reset()
        self.pick_all_matches()

    def compute_partial_schedule(self, past_matches):
        self.reset()
        for match in past_matches:
            self.add_match(match)
        self.pick_all_matches()

def full_schedule(configuration):
    scheduler = Scheduler(configuration)
    scheduler.compute_full_schedule()
    return scheduler.matches

def partial_schedule(configuration, past_matches):
    scheduler = Scheduler(configuration)
    scheduler.compute_partial_schedule(past_matches)
    return scheduler.matches

def match_statistics(matches):
    from collections import Counter, defaultdict
    match_counter = Counter()
    opponents = defaultdict(set)
    prev_match = []
    collisions = set()
    for match in matches:
        for team in match:
            if team in prev_match:
                collisions.add(team)
            match_counter[team] += 1
            for other_team in match:
                if team != other_team:
                    opponents[team].add(other_team)
    return match_counter, opponents, collisions

def schedule_check(matches):
    print '{0} matches total'.format(len(matches))
    match_counter, opponents, collisions = match_statistics(matches)
    all_teams = set(match_counter.iterkeys())
    for team, matches in match_counter.iteritems():
        print '{0}: {1} matches, missed opponents: {2}'.format(team, matches, ', '.join(all_teams - opponents[team] - {team}))
        if team in collisions:
            print '\t{0} DO have a match collision'.format(team)

if __name__ == '__main__':
    config = ScheduleConfiguration(zones = 4,
                                   teams = ['Team {0}'.format(x) for x in xrange(1, 11)],
                                   weight_zones = True,
                                   round_length = None,
                                   imbalance_action = 'empty',
                                   match_count = 25)
    schedule = full_schedule(config)
    schedule_check(schedule)
    import json
    print json.dumps(schedule)

<|endoftext|>


<commit_before>"""
Sync Media to S3
================

Django command that scans all files in your settings.MEDIA_ROOT folder and
uploads them to S3 with the same directory structure.

This command can optionally do the following but it is off by default:
* gzip compress any CSS and Javascript files it finds and adds the appropriate
  'Content-Encoding' header.
* set a far future 'Expires' header for optimal caching.

Note: This script requires the Python boto library and valid Amazon Web
Services API keys.

Required settings.py variables:
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_BUCKET_NAME = ''

Command options are:
  -p PREFIX, --prefix=PREFIX
                        The prefix to prepend to the path on S3.
  --gzip                Enables gzipping CSS and Javascript files.
  --expires             Enables setting a far future expires header.
  --force               Skip the file mtime check to force upload of all
                        files.

TODO:
* Make FILTER_LIST an optional argument

"""
import datetime
import email
import mimetypes
import optparse
import os
import sys
import time

from django.core.management.base import BaseCommand, CommandError

# Make sure boto is available
try:
    import boto
    import boto.exception
except ImportError:
    raise ImportError, "The boto Python library is not installed."

class Command(BaseCommand):

    # Extra variables to avoid passing these around
    AWS_ACCESS_KEY_ID = ''
    AWS_SECRET_ACCESS_KEY = ''
    AWS_BUCKET_NAME = ''
    DIRECTORY = ''
    FILTER_LIST = ['.DS_Store',]
    GZIP_CONTENT_TYPES = (
        'text/css',
        'application/javascript',
        'application/x-javascript'
    )

    upload_count = 0
    skip_count = 0

    option_list = BaseCommand.option_list + (
        optparse.make_option('-p', '--prefix',
            dest='prefix', default='',
            help="The prefix to prepend to the path on S3."),
        optparse.make_option('--gzip',
            action='store_true', dest='gzip', default=False,
            help="Enables gzipping CSS and Javascript files."),
        optparse.make_option('--expires',
            action='store_true', dest='expires', default=False,
            help="Enables setting a far future expires header."),
        optparse.make_option('--force',
            action='store_true', dest='force', default=False,
            help="Skip the file mtime check to force upload of all files.")
    )

    help = 'Syncs the complete MEDIA_ROOT structure and files to S3 into the given bucket name.'
    args = 'bucket_name'

    can_import_settings = True

    def handle(self, *args, **options):
        from django.conf import settings

        # Check for AWS keys in settings
        if not hasattr(settings, 'AWS_ACCESS_KEY_ID') or \
           not hasattr(settings, 'AWS_SECRET_ACCESS_KEY'):
           raise CommandError('Missing AWS keys from settings file.  Please' +
                     'supply both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.')
        else:
            self.AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
            self.AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY

        if not hasattr(settings, 'AWS_BUCKET_NAME'):
            raise CommandError('Missing bucket name from settings file. Please' +
                ' add the AWS_BUCKET_NAME to your settings file.')
        else:
            if not settings.AWS_BUCKET_NAME:
                raise CommandError('AWS_BUCKET_NAME cannot be empty.')
        self.AWS_BUCKET_NAME = settings.AWS_BUCKET_NAME

        if not hasattr(settings, 'MEDIA_ROOT'):
            raise CommandError('MEDIA_ROOT must be set in your settings.')
        else:
            if not settings.MEDIA_ROOT:
                raise CommandError('MEDIA_ROOT must be set in your settings.')
        self.DIRECTORY = settings.MEDIA_ROOT

        self.verbosity = int(options.get('verbosity'))
        self.prefix = options.get('prefix')
        self.do_gzip = options.get('gzip')
        self.do_expires = options.get('expires')
        self.do_force = options.get('force')

        # Now call the syncing method to walk the MEDIA_ROOT directory and
        # upload all files found.
        self.sync_s3()

        print
        print "%d files uploaded." % (self.upload_count)
        print "%d files skipped." % (self.skip_count)

    def sync_s3(self):
        """
        Walks the media directory and syncs files to S3
        """
        bucket, key = self.open_s3()
        os.path.walk(self.DIRECTORY, self.upload_s3,
            (bucket, key, self.AWS_BUCKET_NAME, self.DIRECTORY))

    def compress_string(self, s):
        """Gzip a given string."""
        import cStringIO, gzip
        zbuf = cStringIO.StringIO()
        zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
        zfile.write(s)
        zfile.close()
        return zbuf.getvalue()

    def open_s3(self):
        """
        Opens connection to S3 returning bucket and key
        """
        conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
        try:
            bucket = conn.get_bucket(self.AWS_BUCKET_NAME)
        except boto.exception.S3ResponseError:
            bucket = conn.create_bucket(self.AWS_BUCKET_NAME)
        return bucket, boto.s3.key.Key(bucket)

    def upload_s3(self, arg, dirname, names):
        """
        This is the callback to os.path.walk and where much of the work happens
        """
        bucket, key, bucket_name, root_dir = arg # expand arg tuple

        if root_dir == dirname:
            return # We're in the root media folder

        for file in names:
            headers = {}

            if file in self.FILTER_LIST:
                continue # Skip files we don't want to sync

            filename = os.path.join(dirname, file)
            if os.path.isdir(filename):
                continue # Don't try to upload directories

            file_key = filename[len(root_dir)+1:]
            if self.prefix:
                file_key = '%s/%s' % (self.prefix, file_key)

            # Check if file on S3 is older than local file, if so, upload
            if not self.do_force:
                s3_key = bucket.get_key(file_key)
                if s3_key:
                    s3_datetime = datetime.datetime(*time.strptime(
                        s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
                    local_datetime = datetime.datetime.utcfromtimestamp(
                        os.stat(filename).st_mtime)
                    if local_datetime < s3_datetime:
                        self.skip_count += 1
                        if self.verbosity > 1:
                            print "File %s hasn't been modified since last " \
                                "being uploaded" % (file_key)
                        continue

            # File is newer, let's process and upload
            if self.verbosity > 0:
                print "Uploading %s..." % (file_key)

            content_type = mimetypes.guess_type(filename)[0]
            if content_type:
                headers['Content-Type'] = content_type
            file_obj = open(filename, 'rb')
            file_size = os.fstat(file_obj.fileno()).st_size
            filedata = file_obj.read()
            if self.do_gzip:
                # Gzipping only if file is large enough (>1K is recommended)
                # and only if file is a common text type (not a binary file)
                if file_size > 1024 and content_type in self.GZIP_CONTENT_TYPES:
                    filedata = self.compress_string(filedata)
                    headers['Content-Encoding'] = 'gzip'
                    if self.verbosity > 1:
                        print "\tgzipped: %dk to %dk" % \
                            (file_size/1024, len(filedata)/1024)
            if self.do_expires:
                headers['Expires'] = '%s GMT' % (email.Utils.formatdate(
                    time.mktime((datetime.datetime.now() +
                    datetime.timedelta(days=365*2)).timetuple())))
                if self.verbosity > 1:
                    print "\texpires: %s" % (headers['Expires'])

            try:
                key.name = file_key
                key.set_contents_from_string(filedata, headers, replace=True)
                key.make_public()
            except boto.s3.connection.S3CreateError, e:
                print "Failed: %s" % e
            except Exception, e:
                print e
                raise
            else:
                self.upload_count += 1

            file_obj.close()

# Backwards compatibility for Django r9110
if not [opt for opt in Command.option_list if opt.dest=='verbosity']:
    Command.option_list += (
        optparse.make_option('-v', '--verbosity',
            dest='verbosity', default=1, action='count',
            help="Verbose mode. Multiple -v options increase the verbosity."),
    )
<commit_msg>Adding HTTP/1.1 Cache-Control with max-age header when using --expires.<commit_after>"""
Sync Media to S3
================

Django command that scans all files in your settings.MEDIA_ROOT folder and
uploads them to S3 with the same directory structure.

This command can optionally do the following but it is off by default:
* gzip compress any CSS and Javascript files it finds and adds the appropriate
  'Content-Encoding' header.
* set a far future 'Expires' header for optimal caching.

Note: This script requires the Python boto library and valid Amazon Web
Services API keys.

Required settings.py variables:
AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
AWS_BUCKET_NAME = ''

Command options are:
  -p PREFIX, --prefix=PREFIX
                        The prefix to prepend to the path on S3.
  --gzip                Enables gzipping CSS and Javascript files.
  --expires             Enables setting a far future expires header.
  --force               Skip the file mtime check to force upload of all
                        files.

TODO:
* Make FILTER_LIST an optional argument

"""
import datetime
import email
import mimetypes
import optparse
import os
import sys
import time

from django.core.management.base import BaseCommand, CommandError

# Make sure boto is available
try:
    import boto
    import boto.exception
except ImportError:
    raise ImportError, "The boto Python library is not installed."

class Command(BaseCommand):

    # Extra variables to avoid passing these around
    AWS_ACCESS_KEY_ID = ''
    AWS_SECRET_ACCESS_KEY = ''
    AWS_BUCKET_NAME = ''
    DIRECTORY = ''
    FILTER_LIST = ['.DS_Store',]
    GZIP_CONTENT_TYPES = (
        'text/css',
        'application/javascript',
        'application/x-javascript'
    )

    upload_count = 0
    skip_count = 0

    option_list = BaseCommand.option_list + (
        optparse.make_option('-p', '--prefix',
            dest='prefix', default='',
            help="The prefix to prepend to the path on S3."),
        optparse.make_option('--gzip',
            action='store_true', dest='gzip', default=False,
            help="Enables gzipping CSS and Javascript files."),
        optparse.make_option('--expires',
            action='store_true', dest='expires', default=False,
            help="Enables setting a far future expires header."),
        optparse.make_option('--force',
            action='store_true', dest='force', default=False,
            help="Skip the file mtime check to force upload of all files.")
    )

    help = 'Syncs the complete MEDIA_ROOT structure and files to S3 into the given bucket name.'
    args = 'bucket_name'

    can_import_settings = True

    def handle(self, *args, **options):
        from django.conf import settings

        # Check for AWS keys in settings
        if not hasattr(settings, 'AWS_ACCESS_KEY_ID') or \
           not hasattr(settings, 'AWS_SECRET_ACCESS_KEY'):
           raise CommandError('Missing AWS keys from settings file.  Please' +
                     'supply both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.')
        else:
            self.AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
            self.AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY

        if not hasattr(settings, 'AWS_BUCKET_NAME'):
            raise CommandError('Missing bucket name from settings file. Please' +
                ' add the AWS_BUCKET_NAME to your settings file.')
        else:
            if not settings.AWS_BUCKET_NAME:
                raise CommandError('AWS_BUCKET_NAME cannot be empty.')
        self.AWS_BUCKET_NAME = settings.AWS_BUCKET_NAME

        if not hasattr(settings, 'MEDIA_ROOT'):
            raise CommandError('MEDIA_ROOT must be set in your settings.')
        else:
            if not settings.MEDIA_ROOT:
                raise CommandError('MEDIA_ROOT must be set in your settings.')
        self.DIRECTORY = settings.MEDIA_ROOT

        self.verbosity = int(options.get('verbosity'))
        self.prefix = options.get('prefix')
        self.do_gzip = options.get('gzip')
        self.do_expires = options.get('expires')
        self.do_force = options.get('force')

        # Now call the syncing method to walk the MEDIA_ROOT directory and
        # upload all files found.
        self.sync_s3()

        print
        print "%d files uploaded." % (self.upload_count)
        print "%d files skipped." % (self.skip_count)

    def sync_s3(self):
        """
        Walks the media directory and syncs files to S3
        """
        bucket, key = self.open_s3()
        os.path.walk(self.DIRECTORY, self.upload_s3,
            (bucket, key, self.AWS_BUCKET_NAME, self.DIRECTORY))

    def compress_string(self, s):
        """Gzip a given string."""
        import cStringIO, gzip
        zbuf = cStringIO.StringIO()
        zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
        zfile.write(s)
        zfile.close()
        return zbuf.getvalue()

    def open_s3(self):
        """
        Opens connection to S3 returning bucket and key
        """
        conn = boto.connect_s3(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY)
        try:
            bucket = conn.get_bucket(self.AWS_BUCKET_NAME)
        except boto.exception.S3ResponseError:
            bucket = conn.create_bucket(self.AWS_BUCKET_NAME)
        return bucket, boto.s3.key.Key(bucket)

    def upload_s3(self, arg, dirname, names):
        """
        This is the callback to os.path.walk and where much of the work happens
        """
        bucket, key, bucket_name, root_dir = arg # expand arg tuple

        if root_dir == dirname:
            return # We're in the root media folder

        for file in names:
            headers = {}

            if file in self.FILTER_LIST:
                continue # Skip files we don't want to sync

            filename = os.path.join(dirname, file)
            if os.path.isdir(filename):
                continue # Don't try to upload directories

            file_key = filename[len(root_dir)+1:]
            if self.prefix:
                file_key = '%s/%s' % (self.prefix, file_key)

            # Check if file on S3 is older than local file, if so, upload
            if not self.do_force:
                s3_key = bucket.get_key(file_key)
                if s3_key:
                    s3_datetime = datetime.datetime(*time.strptime(
                        s3_key.last_modified, '%a, %d %b %Y %H:%M:%S %Z')[0:6])
                    local_datetime = datetime.datetime.utcfromtimestamp(
                        os.stat(filename).st_mtime)
                    if local_datetime < s3_datetime:
                        self.skip_count += 1
                        if self.verbosity > 1:
                            print "File %s hasn't been modified since last " \
                                "being uploaded" % (file_key)
                        continue

            # File is newer, let's process and upload
            if self.verbosity > 0:
                print "Uploading %s..." % (file_key)

            content_type = mimetypes.guess_type(filename)[0]
            if content_type:
                headers['Content-Type'] = content_type
            file_obj = open(filename, 'rb')
            file_size = os.fstat(file_obj.fileno()).st_size
            filedata = file_obj.read()
            if self.do_gzip:
                # Gzipping only if file is large enough (>1K is recommended)
                # and only if file is a common text type (not a binary file)
                if file_size > 1024 and content_type in self.GZIP_CONTENT_TYPES:
                    filedata = self.compress_string(filedata)
                    headers['Content-Encoding'] = 'gzip'
                    if self.verbosity > 1:
                        print "\tgzipped: %dk to %dk" % \
                            (file_size/1024, len(filedata)/1024)
            if self.do_expires:
                # HTTP/1.0
                headers['Expires'] = '%s GMT' % (email.Utils.formatdate(
                    time.mktime((datetime.datetime.now() +
                    datetime.timedelta(days=365*2)).timetuple())))
                # HTTP/1.1
                headers['Cache-Control'] = 'max-age %d' % (3600 * 24 * 365 * 2)
                if self.verbosity > 1:
                    print "\texpires: %s" % (headers['Expires'])
                    print "\tcache-control: %s" % (headers['Cache-Control'])

            try:
                key.name = file_key
                key.set_contents_from_string(filedata, headers, replace=True)
                key.make_public()
            except boto.s3.connection.S3CreateError, e:
                print "Failed: %s" % e
            except Exception, e:
                print e
                raise
            else:
                self.upload_count += 1

            file_obj.close()

# Backwards compatibility for Django r9110
if not [opt for opt in Command.option_list if opt.dest=='verbosity']:
    Command.option_list += (
        optparse.make_option('-v', '--verbosity',
            dest='verbosity', default=1, action='count',
            help="Verbose mode. Multiple -v options increase the verbosity."),
    )
<|endoftext|>