<commit_before>#!/usr/bin/env python
"""This is a script for managing MassPay each week.

Most of our payouts are handled by Balanced, but they're limited to people in
the U.S. We need to payout to people outside the U.S. (#126), and while we work
on a long-term solution, we are using PayPal. However, we've grown past the
point that PayPal's Instant Transfer feature is workable. This script is for
interfacing with PayPal's MassPay feature.

This script provides for:

  1. Computing an input CSV by hitting the Gratipay database directly.
  2. Computing two output CSVs (one to upload to PayPal, the second to use for POSTing
      the exchanges back to Gratipay)
  3. POSTing the exchanges back to Gratipay via the HTTP API.

The idea is that you run steps 1 and 2, then run through the MassPay UI on the
PayPal website using the appropriate CSV from step 2, then run step 3.

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import csv
import datetime
import getpass
import os
import sys
from decimal import Decimal as D, ROUND_HALF_UP

import requests
from gratipay import wireup
from gratipay.billing.exchanges import get_ready_payout_routes_by_network
from httplib import IncompleteRead

os.chdir('../masspay')
ts = datetime.datetime.now().strftime('%Y-%m-%d')
INPUT_CSV = '{}.input.csv'.format(ts)
PAYPAL_CSV = '{}.output.paypal.csv'.format(ts)
GRATIPAY_CSV = '{}.output.gratipay.csv'.format(ts)
REPORT_CSV = '{}.report.paypal.csv'.format(ts)

def round_(d):
    return d.quantize(D('0.01'), rounding=ROUND_HALF_UP)

def print_rule(w=80):
    print("-" * w)

class Payee(object):
    username = None
    email = None
    gross = None
    gross_perc = None
    fee = None
    net = None
    additional_note = ""

    def __init__(self, rec):
        self.username, self.email, fee_cap, amount = rec
        self.gross = D(amount)
        self.fee = D(0)
        self.fee_cap = D(fee_cap)
        self.net = self.gross

    def assess_fee(self):

        # In order to avoid slowly leaking escrow, we need to be careful about
        # how we compute the fee. It's complicated, but it goes something like
        # this:
        #
        #   1. We want to pass PayPal's fees through to each payee.
        #
        #   2. With MassPay there is no option to have the receiver pay the fee,
        #       as there is with Instant Transfer.
        #
        #   3. We have to subtract the fee before uploading the spreadsheet
        #       to PayPal.
        #
        #   4. If we upload 15.24, PayPal upcharges to 15.54.
        #
        #   6. If we upload 15.25, PayPal upcharges to 15.56.
        #
        #   7. They only accept whole cents. We can't upload 15.245.
        #
        #   8. What if we want to hit 15.55?
        #
        #   9. We can't.
        #
        #  10. Our solution is to leave a penny behind in Gratipay for
        #       affected payees.
        #
        #  11. BUT ... if we upload 1.25, PayPal upcharges to 1.28. Think about
        #       it.
        #
        # See also: https://github.com/gratipay/gratipay.com/issues/1673
        #           https://github.com/gratipay/gratipay.com/issues/2029
        #           https://github.com/gratipay/gratipay.com/issues/2198
        #           https://github.com/gratipay/gratipay.com/pull/2209
        #           https://github.com/gratipay/gratipay.com/issues/2296

        target = net = self.gross
        while 1:
            net -= D('0.01')
            fee = round_(net * D('0.02'))
            fee = min(fee, self.fee_cap)
            gross = net + fee
            if gross <= target:
                break
        self.gross = gross
        self.net = net
        self.fee = fee

        remainder = target - gross
        if remainder > 0:
            n = "{:.2} remaining due to PayPal rounding limitation.".format(remainder)
            self.additional_note = n

        return fee

def compute_input_csv():
    db = wireup.db(wireup.env())
    routes = get_ready_payout_routes_by_network(db, 'paypal')
    writer = csv.writer(open(INPUT_CSV, 'w+'))
    print_rule(88)
    headers = "username", "email", "fee cap", "amount"
    print("{:<24}{:<32} {:^7} {:^7}".format(*headers))
    print_rule(88)
    total_gross = 0
    for route in routes:
        amount = route.participant.balance
        if amount < 0.50:
            # Minimum payout of 50 cents. I think that otherwise PayPal upcharges to a penny.
            # See https://github.com/gratipay/gratipay.com/issues/1958.
            continue
        total_gross += amount
        print("{:<24}{:<32} {:>7} {:>7}".format( route.participant.username
                                                           , route.address
                                                           , route.fee_cap
                                                           , amount
                                                            ))
        row = (route.participant.username, route.address, route.fee_cap, amount)
        writer.writerow(row)
    print(" "*80, "-"*7)
    print("{:>88}".format(total_gross))

def compute_output_csvs():
    payees = [Payee(rec) for rec in csv.reader(open(INPUT_CSV))]
    payees.sort(key=lambda o: o.gross, reverse=True)

    total_fees = sum([payee.assess_fee() for payee in payees])  # side-effective!
    total_net = sum([p.net for p in payees])
    total_gross = sum([p.gross for p in payees])
    assert total_fees + total_net == total_gross

    paypal_csv = csv.writer(open(PAYPAL_CSV, 'w+'))
    gratipay_csv = csv.writer(open(GRATIPAY_CSV, 'w+'))
    print_rule()
    print("{:<24}{:<32} {:^7} {:^7} {:^7}".format("username", "email", "gross", "fee", "net"))
    print_rule()
    for payee in payees:
        paypal_csv.writerow((payee.email, payee.net, "usd"))
        gratipay_csv.writerow(( payee.username
                            , payee.email
                            , payee.gross
                            , payee.fee
                            , payee.net
                            , payee.additional_note
                             ))
        print("{username:<24}{email:<32} {gross:>7} {fee:>7} {net:>7}".format(**payee.__dict__))

    print(" "*56, "-"*23)
    print("{:>64} {:>7} {:>7}".format(total_gross, total_fees, total_net))

def load_statuses():
    _status_map = { 'Completed': 'succeeded'
                  , 'Unclaimed': 'pending'
                  , 'Denied': 'failed'
                   } # PayPal -> Gratipay
    statuses = {}
    fp = open(REPORT_CSV)
    for line in fp:
        if line.startswith('Transaction ID,Recipient'):
            break
    for rec in csv.reader(fp):
        statuses[rec[1]] = _status_map[rec[5]]
    return statuses

def post_back_to_gratipay():

    try:
        gratipay_api_key = os.environ['GRATIPAY_API_KEY']
    except KeyError:
        gratipay_api_key = getpass.getpass("Gratipay API key: ")

    try:
        gratipay_base_url = os.environ['GRATIPAY_BASE_URL']
    except KeyError:
        gratipay_base_url = 'https://gratipay.com'

    statuses = load_statuses()

    nposts = 0
    for username, email, gross, fee, net, additional_note in csv.reader(open(GRATIPAY_CSV)):
        url = '{}/~{}/history/record-an-exchange'.format(gratipay_base_url, username)
        note = 'PayPal MassPay to {}.'.format(email)
        if additional_note:
            note += " " + additional_note
        print(note)
        status = statuses[email]

        data = {'amount': '-' + net, 'fee': fee, 'note': note, 'status': status}
        try:
            response = requests.post(url, auth=(gratipay_api_key, ''), data=data)
        except IncompleteRead:
            print('IncompleteRead, proceeding (but double-check!)')
        else:
            if response.status_code == 200:
                nposts += 1
            else:
                if response.status_code == 404:
                    print('Got 404, is your API key good? {}'.format(gratipay_api_key))
                else:
                    print('... resulted in a {} response:'.format(response.status_code))
                    print(response.text)
                raise SystemExit
        print("POSTed MassPay back to Gratipay for {} users.".format(nposts))

def run_report():
    """Print a report to help Determine how much escrow we should store in PayPal.
    """
    totals = []
    max_masspay = max_weekly_growth = D(0)
    for filename in os.listdir('.'):
        if not filename.endswith('.input.csv'):
            continue

        datestamp = filename.split('.')[0]

        totals.append(D(0))
        for rec in csv.reader(open(filename)):
            amount = rec[-1]
            totals[-1] += D(amount)

        max_masspay = max(max_masspay, totals[-1])
        if len(totals) == 1:
            print("{} {:8}".format(datestamp, totals[-1]))
        else:
            weekly_growth = totals[-1] / totals[-2]
            max_weekly_growth = max(max_weekly_growth, weekly_growth)
            print("{} {:8} {:4.1f}".format(datestamp, totals[-1], weekly_growth))

    print()
    print("Max Withdrawal:    ${:9,.2f}".format(max_masspay))
    print("Max Weekly Growth:  {:8.1f}".format(max_weekly_growth))
    print("5x Current:        ${:9,.2f}".format(5 * totals[-1]))

def main():
    if not sys.argv[1:]:
        print("Looking for files for {} ...".format(ts))
        for filename in (INPUT_CSV, PAYPAL_CSV, GRATIPAY_CSV):
            print("  [{}] {}".format('x' if os.path.exists(filename) else ' ', filename))
        print("Rerun with one of these options:")
        print("  -i - hits db to generate input CSV (needs envvars via heroku + honcho)")
        print("  -o - computes output CSVs (doesn't need anything but input CSV)")
        print("  -p - posts back to Gratipay (prompts for API key)")
    elif '-i' in sys.argv:
        compute_input_csv()
    elif '-o' in sys.argv:
        compute_output_csvs()
    elif '-p' in sys.argv:
        post_back_to_gratipay()

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print()
<commit_msg>Fix MassPay output alignment<commit_after>#!/usr/bin/env python
"""This is a script for managing MassPay each week.

Most of our payouts are handled by Balanced, but they're limited to people in
the U.S. We need to payout to people outside the U.S. (#126), and while we work
on a long-term solution, we are using PayPal. However, we've grown past the
point that PayPal's Instant Transfer feature is workable. This script is for
interfacing with PayPal's MassPay feature.

This script provides for:

  1. Computing an input CSV by hitting the Gratipay database directly.
  2. Computing two output CSVs (one to upload to PayPal, the second to use for POSTing
      the exchanges back to Gratipay)
  3. POSTing the exchanges back to Gratipay via the HTTP API.

The idea is that you run steps 1 and 2, then run through the MassPay UI on the
PayPal website using the appropriate CSV from step 2, then run step 3.

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import csv
import datetime
import getpass
import os
import sys
from decimal import Decimal as D, ROUND_HALF_UP

import requests
from gratipay import wireup
from gratipay.billing.exchanges import get_ready_payout_routes_by_network
from httplib import IncompleteRead

os.chdir('../masspay')
ts = datetime.datetime.now().strftime('%Y-%m-%d')
INPUT_CSV = '{}.input.csv'.format(ts)
PAYPAL_CSV = '{}.output.paypal.csv'.format(ts)
GRATIPAY_CSV = '{}.output.gratipay.csv'.format(ts)
REPORT_CSV = '{}.report.paypal.csv'.format(ts)

def round_(d):
    return d.quantize(D('0.01'), rounding=ROUND_HALF_UP)

def print_rule(w=80):
    print("-" * w)

class Payee(object):
    username = None
    email = None
    gross = None
    gross_perc = None
    fee = None
    net = None
    additional_note = ""

    def __init__(self, rec):
        self.username, self.email, fee_cap, amount = rec
        self.gross = D(amount)
        self.fee = D(0)
        self.fee_cap = D(fee_cap)
        self.net = self.gross

    def assess_fee(self):

        # In order to avoid slowly leaking escrow, we need to be careful about
        # how we compute the fee. It's complicated, but it goes something like
        # this:
        #
        #   1. We want to pass PayPal's fees through to each payee.
        #
        #   2. With MassPay there is no option to have the receiver pay the fee,
        #       as there is with Instant Transfer.
        #
        #   3. We have to subtract the fee before uploading the spreadsheet
        #       to PayPal.
        #
        #   4. If we upload 15.24, PayPal upcharges to 15.54.
        #
        #   6. If we upload 15.25, PayPal upcharges to 15.56.
        #
        #   7. They only accept whole cents. We can't upload 15.245.
        #
        #   8. What if we want to hit 15.55?
        #
        #   9. We can't.
        #
        #  10. Our solution is to leave a penny behind in Gratipay for
        #       affected payees.
        #
        #  11. BUT ... if we upload 1.25, PayPal upcharges to 1.28. Think about
        #       it.
        #
        # See also: https://github.com/gratipay/gratipay.com/issues/1673
        #           https://github.com/gratipay/gratipay.com/issues/2029
        #           https://github.com/gratipay/gratipay.com/issues/2198
        #           https://github.com/gratipay/gratipay.com/pull/2209
        #           https://github.com/gratipay/gratipay.com/issues/2296

        target = net = self.gross
        while 1:
            net -= D('0.01')
            fee = round_(net * D('0.02'))
            fee = min(fee, self.fee_cap)
            gross = net + fee
            if gross <= target:
                break
        self.gross = gross
        self.net = net
        self.fee = fee

        remainder = target - gross
        if remainder > 0:
            n = "{:.2} remaining due to PayPal rounding limitation.".format(remainder)
            self.additional_note = n

        return fee

def compute_input_csv():
    db = wireup.db(wireup.env())
    routes = get_ready_payout_routes_by_network(db, 'paypal')
    writer = csv.writer(open(INPUT_CSV, 'w+'))
    print_rule(88)
    headers = "username", "email", "fee cap", "amount"
    print("{:<24}{:<32} {:^7} {:^7}".format(*headers))
    print_rule(88)
    total_gross = 0
    for route in routes:
        amount = route.participant.balance
        if amount < 0.50:
            # Minimum payout of 50 cents. I think that otherwise PayPal upcharges to a penny.
            # See https://github.com/gratipay/gratipay.com/issues/1958.
            continue
        total_gross += amount
        print("{:<24}{:<32} {:>7} {:>7}".format( route.participant.username
                                                           , route.address
                                                           , route.fee_cap
                                                           , amount
                                                            ))
        row = (route.participant.username, route.address, route.fee_cap, amount)
        writer.writerow(row)
    print(" "*64, "-"*7)
    print("{:>72}".format(total_gross))

def compute_output_csvs():
    payees = [Payee(rec) for rec in csv.reader(open(INPUT_CSV))]
    payees.sort(key=lambda o: o.gross, reverse=True)

    total_fees = sum([payee.assess_fee() for payee in payees])  # side-effective!
    total_net = sum([p.net for p in payees])
    total_gross = sum([p.gross for p in payees])
    assert total_fees + total_net == total_gross

    paypal_csv = csv.writer(open(PAYPAL_CSV, 'w+'))
    gratipay_csv = csv.writer(open(GRATIPAY_CSV, 'w+'))
    print_rule()
    print("{:<24}{:<32} {:^7} {:^7} {:^7}".format("username", "email", "gross", "fee", "net"))
    print_rule()
    for payee in payees:
        paypal_csv.writerow((payee.email, payee.net, "usd"))
        gratipay_csv.writerow(( payee.username
                            , payee.email
                            , payee.gross
                            , payee.fee
                            , payee.net
                            , payee.additional_note
                             ))
        print("{username:<24}{email:<32} {gross:>7} {fee:>7} {net:>7}".format(**payee.__dict__))

    print(" "*56, "-"*23)
    print("{:>64} {:>7} {:>7}".format(total_gross, total_fees, total_net))

def load_statuses():
    _status_map = { 'Completed': 'succeeded'
                  , 'Unclaimed': 'pending'
                  , 'Denied': 'failed'
                   } # PayPal -> Gratipay
    statuses = {}
    fp = open(REPORT_CSV)
    for line in fp:
        if line.startswith('Transaction ID,Recipient'):
            break
    for rec in csv.reader(fp):
        statuses[rec[1]] = _status_map[rec[5]]
    return statuses

def post_back_to_gratipay():

    try:
        gratipay_api_key = os.environ['GRATIPAY_API_KEY']
    except KeyError:
        gratipay_api_key = getpass.getpass("Gratipay API key: ")

    try:
        gratipay_base_url = os.environ['GRATIPAY_BASE_URL']
    except KeyError:
        gratipay_base_url = 'https://gratipay.com'

    statuses = load_statuses()

    nposts = 0
    for username, email, gross, fee, net, additional_note in csv.reader(open(GRATIPAY_CSV)):
        url = '{}/~{}/history/record-an-exchange'.format(gratipay_base_url, username)
        note = 'PayPal MassPay to {}.'.format(email)
        if additional_note:
            note += " " + additional_note
        print(note)
        status = statuses[email]

        data = {'amount': '-' + net, 'fee': fee, 'note': note, 'status': status}
        try:
            response = requests.post(url, auth=(gratipay_api_key, ''), data=data)
        except IncompleteRead:
            print('IncompleteRead, proceeding (but double-check!)')
        else:
            if response.status_code == 200:
                nposts += 1
            else:
                if response.status_code == 404:
                    print('Got 404, is your API key good? {}'.format(gratipay_api_key))
                else:
                    print('... resulted in a {} response:'.format(response.status_code))
                    print(response.text)
                raise SystemExit
        print("POSTed MassPay back to Gratipay for {} users.".format(nposts))

def run_report():
    """Print a report to help Determine how much escrow we should store in PayPal.
    """
    totals = []
    max_masspay = max_weekly_growth = D(0)
    for filename in os.listdir('.'):
        if not filename.endswith('.input.csv'):
            continue

        datestamp = filename.split('.')[0]

        totals.append(D(0))
        for rec in csv.reader(open(filename)):
            amount = rec[-1]
            totals[-1] += D(amount)

        max_masspay = max(max_masspay, totals[-1])
        if len(totals) == 1:
            print("{} {:8}".format(datestamp, totals[-1]))
        else:
            weekly_growth = totals[-1] / totals[-2]
            max_weekly_growth = max(max_weekly_growth, weekly_growth)
            print("{} {:8} {:4.1f}".format(datestamp, totals[-1], weekly_growth))

    print()
    print("Max Withdrawal:    ${:9,.2f}".format(max_masspay))
    print("Max Weekly Growth:  {:8.1f}".format(max_weekly_growth))
    print("5x Current:        ${:9,.2f}".format(5 * totals[-1]))

def main():
    if not sys.argv[1:]:
        print("Looking for files for {} ...".format(ts))
        for filename in (INPUT_CSV, PAYPAL_CSV, GRATIPAY_CSV):
            print("  [{}] {}".format('x' if os.path.exists(filename) else ' ', filename))
        print("Rerun with one of these options:")
        print("  -i - hits db to generate input CSV (needs envvars via heroku + honcho)")
        print("  -o - computes output CSVs (doesn't need anything but input CSV)")
        print("  -p - posts back to Gratipay (prompts for API key)")
    elif '-i' in sys.argv:
        compute_input_csv()
    elif '-o' in sys.argv:
        compute_output_csvs()
    elif '-p' in sys.argv:
        post_back_to_gratipay()

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Anton Prokhorov'
SITENAME = 'Write Python Blog'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'Asia/Krasnoyarsk'

DEFAULT_LANG = 'ru'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

THEME = 'themes/custom'

STATIC_PATHS = ['images']

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True
<commit_msg>Rename blog<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals

AUTHOR = 'Anton Prokhorov'
SITENAME = 'Import Blog'
SITEURL = ''

PATH = 'content'

TIMEZONE = 'Asia/Krasnoyarsk'

DEFAULT_LANG = 'ru'

# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None

# Blogroll
LINKS = (('Pelican', 'http://getpelican.com/'),
         ('Python.org', 'http://python.org/'),
         ('Jinja2', 'http://jinja.pocoo.org/'),
         ('You can modify those links in your config file', '#'),)

# Social widget
SOCIAL = (('You can add links in your config file', '#'),
          ('Another social link', '#'),)

DEFAULT_PAGINATION = 10

THEME = 'themes/custom'

STATIC_PATHS = ['images']

# Uncomment following line if you want document-relative URLs when developing
# RELATIVE_URLS = True
<|endoftext|>


<commit_before>from __future__ import print_function
import serial
import time

# TODO: EVERYTHING

inp = raw_input("Enter space separated: port testname\n")

input_list = inp.split(" ")
params = [i.strip() for i in input_list]
port = params[0]
testname = params[1]

ser = serial.Serial(port, 9600)
print(ser.name)

logfile = open("logs/" + testname, "w")
ser.flush()

while True:
    x = ser.readline()
    print(x, end="")
    logfile.write(x)
    # time.sleep(0.25)
    logfile.flush()
<commit_msg>small logging updates<commit_after>from __future__ import print_function
import serial
import time

# TODO: EVERYTHING

inp = raw_input("Enter space separated: port testname\n")

input_list = inp.split(" ")
params = [i.strip() for i in input_list]
port = params[0]
testname = params[1]

ser = serial.Serial(port, 9600)
print(ser.name)

logfile = open("logs/" + testname, "w")
ser.flush()

while True:
    x = ser.readline()
    print(x, end="")
    logfile.write(x)
    # time.sleep(0.25)
    logfile.flush()
    time.sleep(0.25)
<|endoftext|>


<commit_before>
import logging
import pickle

logging.basicConfig(format='%(asctime)s : %(levelname)s :: %(message)s', level=logging.DEBUG)

logging.debug('started.')
input_file = './most_common.pickle'
data = None
with open(input_file, 'rb') as input_fp:
    data = pickle.load(input_fp)

logging.debug('read preprocessed data from  %s.' % input_file)
<commit_msg>cleanup; no functional changes.<commit_after>import logging
import pickle

logging.basicConfig(format='%(asctime)s : %(levelname)s :: %(message)s', level=logging.DEBUG)

logging.debug('started.')
input_file = './most_common.pickle'
with open(input_file, 'rb') as input_fp:
    data = pickle.load(input_fp)

logging.debug('read preprocessed data from  %s.' % input_file)
<|endoftext|>


<commit_before>import logging
import pickle
import collections
logging.basicConfig(format='%(asctime)s : %(levelname)s :: %(message)s', level=logging.DEBUG)

logging.debug('started.')
input_file = './most_common.pickle'
with open(input_file, 'rb') as input_fp:
    data = pickle.load(input_fp)

logging.debug('read preprocessed data from  %s.' % input_file)

for key in data.keys():
    logging.debug('input data has key %s' % key)

file_names = list()
if 'file_names' in data.keys():
    file_names = data['file_names']

most = collections.Counter()
if 'counts_from_corpus' in data.keys():
    most = data['counts_from_corpus']

per_file_most = list()
if 'counts_from_documents' in data.keys():
    per_file_most = data['counts_from_documents']

t0 = most.most_common(10)

if True:
    for index, item in enumerate(t0):
        logging.debug('%s: %s :: %d' % (index + 1, item[0], item[1]))
    # logging.debug('%d unique words/tokens' % len(counts))
    most_words = set([each[0] for each in t0])
    logging.debug(most_words)
    for item in per_file_most:
        t1 = item.most_common(10)
        current = set([each[0] for each in t1])
        logging.debug(current.difference(t0))
<commit_msg>feature: renamed some variables; separated the most popular tokens from the corpus from the document words.<commit_after>import logging
import pickle
import collections
logging.basicConfig(format='%(asctime)s : %(levelname)s :: %(message)s', level=logging.DEBUG)

logging.debug('started.')
input_file = './most_common.pickle'
with open(input_file, 'rb') as input_fp:
    data = pickle.load(input_fp)

logging.debug('read preprocessed data from  %s.' % input_file)

for key in data.keys():
    logging.debug('input data has key %s' % key)

file_names = list()
if 'file_names' in data.keys():
    file_names = data['file_names']

counts_from_corpus = collections.Counter()
if 'counts_from_corpus' in data.keys():
    counts_from_corpus = data['counts_from_corpus']

counts_from_documents = list()
if 'counts_from_documents' in data.keys():
    counts_from_documents = data['counts_from_documents']

t0 = counts_from_corpus.most_common(20)

if True:
    for index, item in enumerate(t0):
        logging.debug('%s: %s :: %d' % (index + 1, item[0], item[1]))
    most_words = set([each[0] for each in t0])
    logging.debug(most_words)
    for item in counts_from_documents:
        t1 = item.most_common(10)
        current = set([each[0] for each in t1])
        logging.debug(current.difference(t0))
<|endoftext|>


<commit_before>from datetime import timedelta
import hashlib
from itertools import chain
import uuid
import os
import subprocess as sp
from bb_binary import DataSource, FrameContainer, \
    parse_image_fname, parse_video_fname, get_timezone
from pipeline.objects import PipelineResult
import numpy as np

class VideoReader:
    def __init__(self, video_path,
                 ffmpeg_stderr_fd=None,
                 format='guess_on_ext',
                 ffmpeg_bin='ffmpeg',
                 ffprobe_bin='ffprobe'):
        if format == 'guess_on_ext':
            format = self.guess_format_on_extension(video_path)

        vidread_command = [
            ffmpeg_bin,
            '-i', video_path,
            '-f', 'image2pipe',
            '-pix_fmt', 'gray',
            '-vsync', '0',
            '-vcodec', 'rawvideo', '-'
        ]

        if format is not None:
            vidread_command.insert(1, '-vcodec')
            vidread_command.insert(2, format)

        resolution_command = [
            ffprobe_bin,
            '-v', 'error',
            '-of', 'flat=s=_',
            '-select_streams', 'v:0',
            '-show_entries', 'stream=height,width',
            video_path
        ]

        pipe = sp.Popen(resolution_command, stdout=sp.PIPE, stderr=sp.PIPE)
        infos = pipe.stdout.readlines()
        self.w, self.h = [int(s.decode('utf-8').strip().split('=')[1]) for s in infos]

        self.video_pipe = sp.Popen(vidread_command,
                                   stdout=sp.PIPE,
                                   stderr=ffmpeg_stderr_fd)
        self.frames = 0

    @staticmethod
    def guess_format_on_extension(video_path):
        _, ext = os.path.splitext(video_path)
        if ext == '.mkv':
            format = None
        elif ext == '.avi':
            format = 'hevc'
        else:
            raise Exception("Unknown extension {}.".format(ext))
        return format

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        raw_image = self.video_pipe.stdout.read(self.h * self.w * 1)

        if len(raw_image) != self.h * self.w * 1:
            assert(len(raw_image) == 0)
            raise StopIteration()

        self.frames += 1

        image = np.fromstring(raw_image, dtype='uint8')
        image = image.reshape((self.h, self.w))
        self.video_pipe.stdout.flush()
        return image

def raw_frames_generator(path_video, format='guess_on_ext', stderr_fd=None):
    "yields the frames of the video."
    return VideoReader(path_video, stderr_fd)

def video_generator(path_video, ts_format='2016', path_filelists=None,
                    log_callback=None, stderr_fd=sp.DEVNULL):
    timestamps = get_timestamps(path_video, ts_format, path_filelists)
    fname_video = os.path.basename(path_video)
    data_source = DataSource.new_message(filename=fname_video)
    for i, frame in enumerate(VideoReader(path_video, stderr_fd)):
        if log_callback is not None:
            log_callback(i)
        img = frame
        yield data_source, img, timestamps[i]

class Sink:
    def add_frame(self, data_source, frame):
        raise NotImplemented()

    def finish(self):
        raise NotImplemented()

def unique_id():
    hasher = hashlib.sha1()
    hasher.update(uuid.uuid4().bytes)
    hash = int.from_bytes(hasher.digest(), byteorder='big')
    # strip to 64 bits
    hash = hash >> (hash.bit_length() - 64)
    return hash

class BBBinaryRepoSink(Sink):
    def __init__(self, repo, camId):
        self.repo = repo
        self.frames = []
        self.data_sources_fname = []
        self.data_sources = []
        self.camId = camId

    def add_frame(self, data_source, results, timestamp):
        detections = results[PipelineResult]
        fname = data_source.filename
        if fname not in self.data_sources_fname:
            self.data_sources.append(data_source)
            self.data_sources_fname.append(fname)
        data_source_idx = self.data_sources_fname.index(fname)
        self.frames.append((data_source_idx, detections, timestamp))

    def _get_container(self):
        TIMESTAMP_IDX = 2
        self.frames.sort(key=lambda x: x[TIMESTAMP_IDX])
        start_ts = self.frames[0][TIMESTAMP_IDX]
        end_ts = self.frames[-1][TIMESTAMP_IDX]
        fc = FrameContainer.new_message(fromTimestamp=start_ts,
                                        toTimestamp=end_ts,
                                        camId=self.camId,
                                        id=unique_id())
        dataSources = fc.init('dataSources', len(self.data_sources))
        for i, dsource in enumerate(self.data_sources):
            dataSources[i] = dsource
            dataSources[i].idx = int(i)

        frames = fc.init('frames', len(self.frames))
        for i, (data_source_idx, detection, timestamp) in enumerate(self.frames):
            frame = frames[i]
            frame.dataSourceIdx = data_source_idx
            frame.frameIdx = int(i)
            frame.timestamp = timestamp
            detections_builder = frame.detectionsUnion.init(
                'detectionsDP', len(detection.positions))
            for j, db in enumerate(detections_builder):
                db.idx = j
                db.xpos = int(detection.positions[j, 0])
                db.ypos = int(detection.positions[j, 1])
                db.xposHive = int(detection.hive_positions[j, 0])
                db.yposHive = int(detection.hive_positions[j, 1])
                db.zRotation = float(detection.orientations[j, 0])
                db.yRotation = float(detection.orientations[j, 1])
                db.xRotation = float(detection.orientations[j, 2])
                db.localizerSaliency = float(detection.saliencies[j, 0])
                db.radius = float(detection.radii[j])
                descriptor = db.init('descriptor', len(detection.descriptors[j]))
                for k, part in enumerate(detection.descriptors[j]):
                    descriptor[k] = int(part)
                decodedId = db.init('decodedId', len(detection.ids[j]))
                for k, bit in enumerate(detection.ids[j]):
                    decodedId[k] = int(round(255*bit))

        return fc

    def finish(self):
        self.repo.add(self._get_container())

def get_seperate_timestamps(path_video, ts_format, path_filelists):
    def get_flist_name(dt_utc):
        fmt = '%Y%m%d'
        dt = dt_utc.astimezone(get_timezone())
        if ts_format == '2014':
            return dt.strftime(fmt) + '.txt'
        elif ts_format == '2015':
            return os.path.join(dt.strftime(fmt), 'images.txt')
        else:
            assert(False)

    def find_file(name, path):
        for root, dirs, files in os.walk(path):
            if name in [os.path.join(os.path.basename(root), f) for f in files]:
                return os.path.join(path, name)
        return None

    def next_day(dt):
        return dt + timedelta(days=1)

    fname_video = os.path.basename(path_video)
    cam, from_dt, to_dt = parse_video_fname(fname_video)
    # due to a bug in the data aquistion software we have to check in the
    # filename list of the next day as well
    timestamps = [from_dt, to_dt, next_day(from_dt), next_day(to_dt)]
    txt_files = set([get_flist_name(dt) for dt in timestamps])
    txt_paths = [find_file(f, path_filelists) for f in txt_files]
    txt_paths = [path for path in txt_paths if path is not None]
    assert len(txt_paths) > 0

    image_fnames = list(chain.from_iterable([open(path, 'r').readlines() for path in txt_paths]))
    first_fname = fname_video.split('_TO_')[0] + '.jpeg\n'
    second_fname = fname_video.split('_TO_')[1].split('.mkv')[0] + '.jpeg\n'
    image_fnames.sort()

    fnames = image_fnames[image_fnames.index(first_fname):image_fnames.index(second_fname) + 1]
    return [parse_image_fname(fn, format='beesbook')[1].timestamp() for fn in fnames]

def get_timestamps(path_video, ts_format, path_filelists):
    if ts_format == '2016':
        assert path_filelists is None
        txt_path = path_video.replace('mkv', 'txt')
        fnames = open(txt_path, 'r').readlines()
        return [parse_image_fname(fn, format='iso')[1].timestamp() for fn in fnames]
    elif ts_format in ('2014', '2015'):
        assert path_filelists is not None
        return get_seperate_timestamps(path_video, ts_format, path_filelists)
    else:
        assert False, 'Unknown timestamp format'
<commit_msg>Add explanation of the different timestamp formats<commit_after>from datetime import timedelta
import hashlib
from itertools import chain
import uuid
import os
import subprocess as sp
from bb_binary import DataSource, FrameContainer, \
    parse_image_fname, parse_video_fname, get_timezone
from pipeline.objects import PipelineResult
import numpy as np

class VideoReader:
    def __init__(self, video_path,
                 ffmpeg_stderr_fd=None,
                 format='guess_on_ext',
                 ffmpeg_bin='ffmpeg',
                 ffprobe_bin='ffprobe'):
        if format == 'guess_on_ext':
            format = self.guess_format_on_extension(video_path)

        vidread_command = [
            ffmpeg_bin,
            '-i', video_path,
            '-f', 'image2pipe',
            '-pix_fmt', 'gray',
            '-vsync', '0',
            '-vcodec', 'rawvideo', '-'
        ]

        if format is not None:
            vidread_command.insert(1, '-vcodec')
            vidread_command.insert(2, format)

        resolution_command = [
            ffprobe_bin,
            '-v', 'error',
            '-of', 'flat=s=_',
            '-select_streams', 'v:0',
            '-show_entries', 'stream=height,width',
            video_path
        ]

        pipe = sp.Popen(resolution_command, stdout=sp.PIPE, stderr=sp.PIPE)
        infos = pipe.stdout.readlines()
        self.w, self.h = [int(s.decode('utf-8').strip().split('=')[1]) for s in infos]

        self.video_pipe = sp.Popen(vidread_command,
                                   stdout=sp.PIPE,
                                   stderr=ffmpeg_stderr_fd)
        self.frames = 0

    @staticmethod
    def guess_format_on_extension(video_path):
        _, ext = os.path.splitext(video_path)
        if ext == '.mkv':
            format = None
        elif ext == '.avi':
            format = 'hevc'
        else:
            raise Exception("Unknown extension {}.".format(ext))
        return format

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        raw_image = self.video_pipe.stdout.read(self.h * self.w * 1)

        if len(raw_image) != self.h * self.w * 1:
            assert(len(raw_image) == 0)
            raise StopIteration()

        self.frames += 1

        image = np.fromstring(raw_image, dtype='uint8')
        image = image.reshape((self.h, self.w))
        self.video_pipe.stdout.flush()
        return image

def raw_frames_generator(path_video, format='guess_on_ext', stderr_fd=None):
    "yields the frames of the video."
    return VideoReader(path_video, stderr_fd)

def video_generator(path_video, ts_format='2016', path_filelists=None,
                    log_callback=None, stderr_fd=sp.DEVNULL):
    timestamps = get_timestamps(path_video, ts_format, path_filelists)
    fname_video = os.path.basename(path_video)
    data_source = DataSource.new_message(filename=fname_video)
    for i, frame in enumerate(VideoReader(path_video, stderr_fd)):
        if log_callback is not None:
            log_callback(i)
        img = frame
        yield data_source, img, timestamps[i]

class Sink:
    def add_frame(self, data_source, frame):
        raise NotImplemented()

    def finish(self):
        raise NotImplemented()

def unique_id():
    hasher = hashlib.sha1()
    hasher.update(uuid.uuid4().bytes)
    hash = int.from_bytes(hasher.digest(), byteorder='big')
    # strip to 64 bits
    hash = hash >> (hash.bit_length() - 64)
    return hash

class BBBinaryRepoSink(Sink):
    def __init__(self, repo, camId):
        self.repo = repo
        self.frames = []
        self.data_sources_fname = []
        self.data_sources = []
        self.camId = camId

    def add_frame(self, data_source, results, timestamp):
        detections = results[PipelineResult]
        fname = data_source.filename
        if fname not in self.data_sources_fname:
            self.data_sources.append(data_source)
            self.data_sources_fname.append(fname)
        data_source_idx = self.data_sources_fname.index(fname)
        self.frames.append((data_source_idx, detections, timestamp))

    def _get_container(self):
        TIMESTAMP_IDX = 2
        self.frames.sort(key=lambda x: x[TIMESTAMP_IDX])
        start_ts = self.frames[0][TIMESTAMP_IDX]
        end_ts = self.frames[-1][TIMESTAMP_IDX]
        fc = FrameContainer.new_message(fromTimestamp=start_ts,
                                        toTimestamp=end_ts,
                                        camId=self.camId,
                                        id=unique_id())
        dataSources = fc.init('dataSources', len(self.data_sources))
        for i, dsource in enumerate(self.data_sources):
            dataSources[i] = dsource
            dataSources[i].idx = int(i)

        frames = fc.init('frames', len(self.frames))
        for i, (data_source_idx, detection, timestamp) in enumerate(self.frames):
            frame = frames[i]
            frame.dataSourceIdx = data_source_idx
            frame.frameIdx = int(i)
            frame.timestamp = timestamp
            detections_builder = frame.detectionsUnion.init(
                'detectionsDP', len(detection.positions))
            for j, db in enumerate(detections_builder):
                db.idx = j
                db.xpos = int(detection.positions[j, 0])
                db.ypos = int(detection.positions[j, 1])
                db.xposHive = int(detection.hive_positions[j, 0])
                db.yposHive = int(detection.hive_positions[j, 1])
                db.zRotation = float(detection.orientations[j, 0])
                db.yRotation = float(detection.orientations[j, 1])
                db.xRotation = float(detection.orientations[j, 2])
                db.localizerSaliency = float(detection.saliencies[j, 0])
                db.radius = float(detection.radii[j])
                descriptor = db.init('descriptor', len(detection.descriptors[j]))
                for k, part in enumerate(detection.descriptors[j]):
                    descriptor[k] = int(part)
                decodedId = db.init('decodedId', len(detection.ids[j]))
                for k, bit in enumerate(detection.ids[j]):
                    decodedId[k] = int(round(255*bit))

        return fc

    def finish(self):
        self.repo.add(self._get_container())

def get_seperate_timestamps(path_video, ts_format, path_filelists):
    def get_flist_name(dt_utc):
        fmt = '%Y%m%d'
        dt = dt_utc.astimezone(get_timezone())
        if ts_format == '2014':
            return dt.strftime(fmt) + '.txt'
        elif ts_format == '2015':
            return os.path.join(dt.strftime(fmt), 'images.txt')
        else:
            assert(False)

    def find_file(name, path):
        for root, dirs, files in os.walk(path):
            if name in [os.path.join(os.path.basename(root), f) for f in files]:
                return os.path.join(path, name)
        return None

    def next_day(dt):
        return dt + timedelta(days=1)

    fname_video = os.path.basename(path_video)
    cam, from_dt, to_dt = parse_video_fname(fname_video)
    # due to a bug in the data aquistion software we have to check in the
    # filename list of the next day as well
    timestamps = [from_dt, to_dt, next_day(from_dt), next_day(to_dt)]
    txt_files = set([get_flist_name(dt) for dt in timestamps])
    txt_paths = [find_file(f, path_filelists) for f in txt_files]
    txt_paths = [path for path in txt_paths if path is not None]
    assert len(txt_paths) > 0

    image_fnames = list(chain.from_iterable([open(path, 'r').readlines() for path in txt_paths]))
    first_fname = fname_video.split('_TO_')[0] + '.jpeg\n'
    second_fname = fname_video.split('_TO_')[1].split('.mkv')[0] + '.jpeg\n'
    image_fnames.sort()

    fnames = image_fnames[image_fnames.index(first_fname):image_fnames.index(second_fname) + 1]
    return [parse_image_fname(fn, format='beesbook')[1].timestamp() for fn in fnames]

def get_timestamps(path_video, ts_format, path_filelists):
    # In season 2014 and 2015, the timestamps for each video were stored in a seperate location
    # from the videos themselves. From 2016 onwards a text file with the timestamps of all frames
    # is stored next to the videos with the same filename (apart from the extension).
    if ts_format == '2016':
        assert path_filelists is None
        txt_path = path_video.replace('mkv', 'txt')
        fnames = open(txt_path, 'r').readlines()
        return [parse_image_fname(fn, format='iso')[1].timestamp() for fn in fnames]
    elif ts_format in ('2014', '2015'):
        assert path_filelists is not None
        return get_seperate_timestamps(path_video, ts_format, path_filelists)
    else:
        assert False, 'Unknown timestamp format'
<|endoftext|>


<commit_before>""":mod:`plastic.app` --- Application factories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""
import sys
import dis

from werkzeug.routing import Map, Rule, RequestRedirect
from werkzeug.wrappers import Request, Response
from werkzeug.exceptions import HTTPException, NotFound

class BaseApp(object):
    """The application base class."""

    #: (:class:`collections.Mapping`) The :class:`dict` of endpoints to
    #: view functions.
    endpoints = {}

    #: (:class:`collections.Sequence`) The list of routing rules.
    rules = []

    @classmethod
    def clone(cls, __module__=None, __name__=None, **values):
        """Subclasses the application class.  It is a just shorthand of
        normal subclassing.  If your application becomes bigger stop
        using this and normaly subclass the application class
        using :keyword:`class` keyword instead.

        """
        try:
            getframe = sys._getframe
        except AttributeError:
            pass
        else:
            frame = getframe(1)
            if __module__ is None:
                __module__ = frame.f_globals.get('__name__')
            if __name__ is None:
                code = frame.f_code.co_code[frame.f_lasti:]
                code = code[3 if ord(code[0]) >= dis.HAVE_ARGUMENT else 1:]
                stores = (dis.opmap['STORE_NAME'],
                          dis.opmap['STORE_GLOBAL'],
                          dis.opmap['STORE_FAST'])
                opcode = ord(code[0])
                if opcode in stores:
                    name_pos = ord(code[1])
                    if opcode == dis.opmap['STORE_FAST']:
                        index = frame.f_code.co_varnames
                    else:
                        index = frame.f_code.co_names
                    try:
                        __name__ = index[name_pos]
                    except IndexError:
                        pass
        if __name__ is None:
            __name__ = 'App'
        subclass = type(__name__, (cls,), values)
        subclass.__module__ = __module__
        return subclass

    @classmethod
    def add_rule(cls, rule, function):
        """Adds a new ``rule`` and maps it to the ``function``.
        It is a lower-level primitive method.  In most cases you
        don't need it and just use :meth:`route()` decorator or
        other higher-level methods.

        :param rule: the rule to add
        :type rule: :class:`werkzeug.routing.Rule`
        :param function: the function to map to the ``rule``
        :type function: :class:`collections.Callable`

        """
        if not isinstance(rule, Rule):
            raise TypeError('rule must be a werkzeug.routing.Rule instance')
        elif not callable(function):
            raise TypeError('function must be callable')
        if getattr(cls, 'initialized_app_class', None) is not cls:
            cls.endpoints = cls.endpoints.copy()
            cls.rules = cls.rules[:]
            cls.initialized_app_class = cls
        cls.rules.append(rule)
        cls.endpoints[rule.endpoint] = function

    @classmethod
    def route(cls, *rule_args, **rule_kwargs):
        """The function decorator which maps the path to the decorated
        view function.

        It takes the same arguments to the constructor of
        :class:`werkzeug.routing.Rule` class.  In most case simply give
        a path string to route::

            @App.route('/people/<name>')
            def person(request, name):
                return 'Hi, ' + name

        """
        rule = Rule(*rule_args, **rule_kwargs)
        def decorate(function):
            if rule.endpoint is None:
                try:
                    rule.endpoint = function.__name__
                except AttributeError:
                    raise TypeError(
                        'function has to be an ordinary function (not just '
                        'callable) or endpoint has to be specified'
                    )
            cls.add_rule(rule, function)
            return function
        return decorate

    def __init__(self):
        self.endpoints = self.endpoints.copy()
        self.routing_map = Map(self.rules, strict_slashes=True)

    def __call__(self, environ, start_response):
        return self.wsgi_function(environ, start_response)

    def wsgi_function(self, environ, start_response):
        """The actual WSGI function.  Replace it when the application
        should be wrapped by middlewares e.g.::

            app.wsgi_function = Middleware(app.wsgi_function)

        """
        bound = self.routing_map.bind_to_environ(environ)
        try:
            try:
                endpoint, values = bound.match()
            except RequestRedirect as result:
                pass
            else:
                try:
                    view_func = self.endpoints[endpoint]
                except KeyError:
                    result = NotFound()
                else:
                    request = Request(environ)  # FIXME: custom request cls
                    request.endpoint = endpoint
                    result = view_func(request, **values)
                    if isinstance(result, str):
                        result = Response(result)
        except HTTPException as result:
            pass
        response = Response.force_type(result, environ)
        return response(environ, start_response)

<commit_msg>Rename wsgi_function to wsgi_app which is shorter.<commit_after>""":mod:`plastic.app` --- Application factories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""
import sys
import dis

from werkzeug.routing import Map, Rule, RequestRedirect
from werkzeug.wrappers import Request, Response
from werkzeug.exceptions import HTTPException, NotFound

class BaseApp(object):
    """The application base class."""

    #: (:class:`collections.Mapping`) The :class:`dict` of endpoints to
    #: view functions.
    endpoints = {}

    #: (:class:`collections.Sequence`) The list of routing rules.
    rules = []

    @classmethod
    def clone(cls, __module__=None, __name__=None, **values):
        """Subclasses the application class.  It is a just shorthand of
        normal subclassing.  If your application becomes bigger stop
        using this and normaly subclass the application class
        using :keyword:`class` keyword instead.

        """
        try:
            getframe = sys._getframe
        except AttributeError:
            pass
        else:
            frame = getframe(1)
            if __module__ is None:
                __module__ = frame.f_globals.get('__name__')
            if __name__ is None:
                code = frame.f_code.co_code[frame.f_lasti:]
                code = code[3 if ord(code[0]) >= dis.HAVE_ARGUMENT else 1:]
                stores = (dis.opmap['STORE_NAME'],
                          dis.opmap['STORE_GLOBAL'],
                          dis.opmap['STORE_FAST'])
                opcode = ord(code[0])
                if opcode in stores:
                    name_pos = ord(code[1])
                    if opcode == dis.opmap['STORE_FAST']:
                        index = frame.f_code.co_varnames
                    else:
                        index = frame.f_code.co_names
                    try:
                        __name__ = index[name_pos]
                    except IndexError:
                        pass
        if __name__ is None:
            __name__ = 'App'
        subclass = type(__name__, (cls,), values)
        subclass.__module__ = __module__
        return subclass

    @classmethod
    def add_rule(cls, rule, function):
        """Adds a new ``rule`` and maps it to the ``function``.
        It is a lower-level primitive method.  In most cases you
        don't need it and just use :meth:`route()` decorator or
        other higher-level methods.

        :param rule: the rule to add
        :type rule: :class:`werkzeug.routing.Rule`
        :param function: the function to map to the ``rule``
        :type function: :class:`collections.Callable`

        """
        if not isinstance(rule, Rule):
            raise TypeError('rule must be a werkzeug.routing.Rule instance')
        elif not callable(function):
            raise TypeError('function must be callable')
        if getattr(cls, 'initialized_app_class', None) is not cls:
            cls.endpoints = cls.endpoints.copy()
            cls.rules = cls.rules[:]
            cls.initialized_app_class = cls
        cls.rules.append(rule)
        cls.endpoints[rule.endpoint] = function

    @classmethod
    def route(cls, *rule_args, **rule_kwargs):
        """The function decorator which maps the path to the decorated
        view function.

        It takes the same arguments to the constructor of
        :class:`werkzeug.routing.Rule` class.  In most case simply give
        a path string to route::

            @App.route('/people/<name>')
            def person(request, name):
                return 'Hi, ' + name

        """
        rule = Rule(*rule_args, **rule_kwargs)
        def decorate(function):
            if rule.endpoint is None:
                try:
                    rule.endpoint = function.__name__
                except AttributeError:
                    raise TypeError(
                        'function has to be an ordinary function (not just '
                        'callable) or endpoint has to be specified'
                    )
            cls.add_rule(rule, function)
            return function
        return decorate

    def __init__(self):
        self.endpoints = self.endpoints.copy()
        self.routing_map = Map(self.rules, strict_slashes=True)

    def __call__(self, environ, start_response):
        return self.wsgi_app(environ, start_response)

    def wsgi_app(self, environ, start_response):
        """The actual WSGI function.  Replace it when the application
        should be wrapped by middlewares::

            app.wsgi_app = Middleware(app.wsgi_app)

        """
        bound = self.routing_map.bind_to_environ(environ)
        try:
            try:
                endpoint, values = bound.match()
            except RequestRedirect as result:
                pass
            else:
                try:
                    view_func = self.endpoints[endpoint]
                except KeyError:
                    result = NotFound()
                else:
                    request = Request(environ)  # FIXME: custom request cls
                    request.endpoint = endpoint
                    result = view_func(request, **values)
                    if isinstance(result, str):
                        result = Response(result)
        except HTTPException as result:
            pass
        response = Response.force_type(result, environ)
        return response(environ, start_response)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Rahul Handay <rahulha@saltstack.com>`
'''

# Import Python libs
from __future__ import absolute_import

# Import Salt Testing Libs
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import (
    MagicMock,
    patch,
    NO_MOCK,
    NO_MOCK_REASON
)

ensure_in_syspath('../../')

# Import Salt Libs
from salt.modules import systemd

# Globals
systemd.__salt__ = {}
systemd.__context__ = {}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class SystemdTestCase(TestCase):
    '''
        Test case for salt.modules.systemd
    '''
    def test_systemctl_reload(self):
        '''
            Test to Reloads systemctl
        '''
        mock = MagicMock(side_effect=[1, 0])
        with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
            self.assertFalse(systemd.systemctl_reload())

            self.assertTrue(systemd.systemctl_reload())

    def test_get_enabled(self):
        '''
            Test to return a list of all enabled services
        '''
        def sysv(name):
            if name in ['d', 'e']:
                return True
            return False

        def sysve(name):
            if name in ['e']:
                return True
            return False

        mock = MagicMock(return_value={"a": "enabled", "b": "enabled",
                                       "c": "disabled"})
        lmock = MagicMock(return_value={"d": "disabled",
                                        "a": "disabled",
                                        "b": "disabled",
                                        "e": "disabled"})
        with patch.object(systemd, "_sysv_is_disabled", sysve):
            with patch.object(systemd, "_service_is_sysv", sysv):
                with patch.object(systemd, '_get_all_unit_files', mock):
                    with patch.object(systemd, '_get_all_units', lmock):
                        self.assertListEqual(
                            systemd.get_enabled(), ["a", "b", "d"])

    def test_get_disabled(self):
        '''
            Test to return a list of all disabled services
        '''
        mock = MagicMock(return_value={"a": "enabled", "b": "enabled",
                                       "c": "disabled"})
        with patch.object(systemd, '_get_all_unit_files', mock):
            mock = MagicMock(return_value={})
            with patch.object(systemd, '_get_all_legacy_init_scripts', mock):
                self.assertListEqual(systemd.get_disabled(), ["c"])

    def test_get_all(self):
        '''
            Test to return a list of all available services
        '''
        mock = MagicMock(return_value={"a": "enabled", "b": "enabled",
                                       "c": "disabled"})
        with patch.object(systemd, '_get_all_units', mock):
            mock = MagicMock(return_value={"a1": "enabled", "b1": "disabled",
                                           "c1": "enabled"})
            with patch.object(systemd, '_get_all_unit_files', mock):
                mock = MagicMock(return_value={})
                with patch.object(systemd,
                                  '_get_all_legacy_init_scripts', mock):
                    self.assertListEqual(systemd.get_all(),
                                         ['a', 'a1', 'b', 'b1', 'c', 'c1'])

    def test_available(self):
        '''
            Test to check that the given service is available
        '''
        mock = MagicMock(side_effect=["a", "@", "c"])
        with patch.object(systemd, '_canonical_template_unit_name', mock):
            mock = MagicMock(side_effect=[{"a": "z", "b": "z"},
                                          {"@": "z", "b": "z"},
                                          {"a": "z", "b": "z"}])
            with patch.object(systemd, 'get_all', mock):
                self.assertTrue(systemd.available("sshd"))

                self.assertTrue(systemd.available("sshd"))

                self.assertFalse(systemd.available("sshd"))

    def test_missing(self):
        '''
            Test to the inverse of service.available.
        '''
        mock = MagicMock(return_value=True)
        with patch.object(systemd, 'available', mock):
            self.assertFalse(systemd.missing("sshd"))

    def test_unmask(self):
        '''
            Test to unmask the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.unmask("sshd"))

    def test_start(self):
        '''
            Test to start the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.start("sshd"))

    def test_stop(self):
        '''
            Test to stop the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.stop("sshd"))

    def test_restart(self):
        '''
            Test to restart the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.restart("sshd"))

    def test_reload_(self):
        '''
            Test to Reload the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.reload_("sshd"))

    def test_force_reload(self):
        '''
            Test to force-reload the specified service with systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.force_reload("sshd"))

    def test_status(self):
        '''
            Test to return the status for a service via systemd
        '''
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    self.assertTrue(systemd.status("sshd"))

    def test_enable(self):
        '''
            Test to enable the named service to start when the system boots
        '''
        exe = MagicMock(return_value='foo')
        tmock = MagicMock(return_value=True)
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    with patch.object(systemd, "_service_is_sysv", mock):
                        self.assertTrue(systemd.enable("sshd"))
                    with patch.object(systemd, "_get_service_exec", exe):
                        with patch.object(systemd, "_service_is_sysv", tmock):
                            self.assertTrue(systemd.enable("sshd"))

    def test_disable(self):
        '''
            Test to disable the named service to not
            start when the system boots
        '''
        exe = MagicMock(return_value='foo')
        tmock = MagicMock(return_value=True)
        mock = MagicMock(return_value=False)
        with patch.object(systemd, '_untracked_custom_unit_found', mock):
            with patch.object(systemd, '_unit_file_changed', mock):
                with patch.dict(systemd.__salt__, {'cmd.retcode': mock}):
                    with patch.object(systemd, "_service_is_sysv", mock):
                        self.assertTrue(systemd.disable("sshd"))
                    with patch.object(systemd, "_get_service_exec", exe):
                        with patch.object(systemd, "_service_is_sysv", tmock):
                            self.assertTrue(systemd.disable("sshd"))

    def test_enabled(self):
        '''
            Test to return if the named service is enabled to start on boot
        '''
        mock = MagicMock(return_value=True)
        with patch.object(systemd, '_enabled', mock):
            self.assertTrue(systemd.enabled("sshd"))

    def test_disabled(self):
        '''
            Test to Return if the named service is disabled to start on boot
        '''
        mock = MagicMock(return_value=True)
        with patch.object(systemd, '_enabled', mock):
            self.assertFalse(systemd.disabled("sshd"))

    def test_show(self):
        '''
            Test to show properties of one or more units/jobs or the manager
        '''
        mock = MagicMock(return_value="a = b , c = d")
        with patch.dict(systemd.__salt__, {'cmd.run': mock}):
            self.assertDictEqual(systemd.show("sshd"), {'a ': ' b , c = d'})

    def test_execs(self):
        '''
            Test to return a list of all files specified as ``ExecStart``
            for all services
        '''
        mock = MagicMock(return_value=["a", "b"])
        with patch.object(systemd, 'get_all', mock):
            mock = MagicMock(return_value={"ExecStart": {"path": "c"}})
            with patch.object(systemd, 'show', mock):
                self.assertDictEqual(systemd.execs(), {'a': 'c', 'b': 'c'})

if __name__ == '__main__':
    from integration import run_tests
    run_tests(SystemdTestCase, needs_daemon=False)
<commit_msg>Update systemd tests<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Rahul Handay <rahulha@saltstack.com>`
'''

# Import Python libs
from __future__ import absolute_import
import os

# Import Salt Testing Libs
from salt.exceptions import CommandExecutionError
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import (
    MagicMock,
    patch,
    NO_MOCK,
    NO_MOCK_REASON
)

ensure_in_syspath('../../')

# Import Salt Libs
from salt.modules import systemd

# Globals
systemd.__salt__ = {}
systemd.__context__ = {}

_SYSTEMCTL_STATUS = {
    'sshd.service': '''\
* sshd.service - OpenSSH Daemon
   Loaded: loaded (/usr/lib/systemd/system/sshd.service; disabled; vendor preset: disabled)
   Active: inactive (dead)''',

    'foo.service': '''\
* foo.service
   Loaded: not-found (Reason: No such file or directory)
   Active: inactive (dead)'''
}

_LIST_UNIT_FILES = '''\
service1.service                           enabled
service2.service                           disabled
service3.service                           static
timer1.timer                               enabled
timer2.timer                               disabled
timer3.timer                               static'''

@skipIf(NO_MOCK, NO_MOCK_REASON)
class SystemdTestCase(TestCase):
    '''
        Test case for salt.modules.systemd
    '''
    def test_systemctl_reload(self):
        '''
            Test to Reloads systemctl
        '''
        mock = MagicMock(side_effect=[
            {'stdout': 'Who knows why?',
             'stderr': '',
             'retcode': 1,
             'pid': 12345},
            {'stdout': '',
             'stderr': '',
             'retcode': 0,
             'pid': 54321},
        ])
        with patch.dict(systemd.__salt__, {'cmd.run_all': mock}):
            self.assertRaisesRegexp(
                CommandExecutionError,
                'Problem performing systemctl daemon-reload: Who knows why?',
                systemd.systemctl_reload
            )
            self.assertTrue(systemd.systemctl_reload())

    def test_get_enabled(self):
        '''
        Test to return a list of all enabled services
        '''
        cmd_mock = MagicMock(return_value=_LIST_UNIT_FILES)
        listdir_mock = MagicMock(return_value=['foo', 'bar', 'baz', 'README'])
        sd_mock = MagicMock(
            return_value=set(
                [x.replace('.service', '') for x in _SYSTEMCTL_STATUS]
            )
        )
        access_mock = MagicMock(
            side_effect=lambda x, y: x != os.path.join(
                systemd.INITSCRIPT_PATH,
                'README'
            )
        )
        sysv_enabled_mock = MagicMock(side_effect=lambda x: x == 'baz')

        with patch.dict(systemd.__salt__, {'cmd.run': cmd_mock}):
            with patch.object(os, 'listdir', listdir_mock):
                with patch.object(systemd, '_get_systemd_services', sd_mock):
                    with patch.object(os, 'access', side_effect=access_mock):
                        with patch.object(systemd, '_sysv_enabled',
                                          sysv_enabled_mock):
                            self.assertListEqual(
                                systemd.get_enabled(),
                                ['baz', 'service1', 'timer1.timer']
                            )

    def test_get_disabled(self):
        '''
        Test to return a list of all disabled services
        '''
        cmd_mock = MagicMock(return_value=_LIST_UNIT_FILES)
        # 'foo' should collide with the systemd services (as returned by
        # sd_mock) and thus not be returned by _get_sysv_services(). It doesn't
        # matter that it's not part of the _LIST_UNIT_FILES output, we just
        # want to ensure that 'foo' isn't identified as a disabled initscript
        # even though below we are mocking it to show as not enabled (since
        # only 'baz' will be considered an enabled sysv service).
        listdir_mock = MagicMock(return_value=['foo', 'bar', 'baz', 'README'])
        sd_mock = MagicMock(
            return_value=set(
                [x.replace('.service', '') for x in _SYSTEMCTL_STATUS]
            )
        )
        access_mock = MagicMock(
            side_effect=lambda x, y: x != os.path.join(
                systemd.INITSCRIPT_PATH,
                'README'
            )
        )
        sysv_enabled_mock = MagicMock(side_effect=lambda x: x == 'baz')

        with patch.dict(systemd.__salt__, {'cmd.run': cmd_mock}):
            with patch.object(os, 'listdir', listdir_mock):
                with patch.object(systemd, '_get_systemd_services', sd_mock):
                    with patch.object(os, 'access', side_effect=access_mock):
                        with patch.object(systemd, '_sysv_enabled',
                                          sysv_enabled_mock):
                            self.assertListEqual(
                                systemd.get_disabled(),
                                ['bar', 'service2', 'timer2.timer']
                            )

    def test_get_all(self):
        '''
        Test to return a list of all available services
        '''
        listdir_mock = MagicMock(side_effect=[
            ['foo.service', 'multi-user.target.wants', 'mytimer.timer'],
            ['foo.service', 'multi-user.target.wants', 'bar.service'],
            ['mysql', 'nginx', 'README']
        ])
        access_mock = MagicMock(
            side_effect=lambda x, y: x != os.path.join(
                systemd.INITSCRIPT_PATH,
                'README'
            )
        )
        with patch.object(os, 'listdir', listdir_mock):
            with patch.object(os, 'access', side_effect=access_mock):
                self.assertListEqual(
                    systemd.get_all(),
                    ['bar', 'foo', 'mysql', 'mytimer.timer', 'nginx']
                )

    def test_available(self):
        '''
        Test to check that the given service is available
        '''
        mock = MagicMock(side_effect=lambda x: _SYSTEMCTL_STATUS[x])
        with patch.object(systemd, '_systemctl_status', mock):
            self.assertTrue(systemd.available('sshd.service'))
            self.assertFalse(systemd.available('foo.service'))

    def test_missing(self):
        '''
            Test to the inverse of service.available.
        '''
        mock = MagicMock(side_effect=lambda x: _SYSTEMCTL_STATUS[x])
        with patch.object(systemd, '_systemctl_status', mock):
            self.assertFalse(systemd.missing('sshd.service'))
            self.assertTrue(systemd.missing('foo.service'))

    def test_show(self):
        '''
            Test to show properties of one or more units/jobs or the manager
        '''
        mock = MagicMock(return_value="a = b , c = d")
        with patch.dict(systemd.__salt__, {'cmd.run': mock}):
            self.assertDictEqual(systemd.show("sshd"), {'a ': ' b , c = d'})

    def test_execs(self):
        '''
            Test to return a list of all files specified as ``ExecStart``
            for all services
        '''
        mock = MagicMock(return_value=["a", "b"])
        with patch.object(systemd, 'get_all', mock):
            mock = MagicMock(return_value={"ExecStart": {"path": "c"}})
            with patch.object(systemd, 'show', mock):
                self.assertDictEqual(systemd.execs(), {'a': 'c', 'b': 'c'})

if __name__ == '__main__':
    from integration import run_tests
    run_tests(SystemdTestCase, needs_daemon=False)
<|endoftext|>


<commit_before>__author__ = "Jurismarches <informatique@jurismarches.com>"
__version__ = "2.2.5"
<commit_msg>update to 2.2.6<commit_after>__author__ = "Jurismarches <informatique@jurismarches.com>"
__version__ = "2.2.6"
<|endoftext|>


<commit_before>#! /usr/bin/python3

"""
Allow simultaneous lock and transfer.
"""

import struct
import decimal
D = decimal.Decimal

from . import (config, util, exceptions, bitcoin, util)

FORMAT_1 = '>QQ?'
LENGTH_1 = 8 + 8 + 1
FORMAT_2 = '>QQ??If42p'
LENGTH_2 = 8 + 8 + 1 + 1 + 4 + 4 + 42
ID = 20

def validate (db, source, destination, asset, quantity, divisible, callable_, call_date, call_price, description, block_index):
    problems = []
    fee = 0

    if asset in (config.BTC, config.XCP):
        problems.append('cannot issue {} or {}'.format(config.BTC, config.XCP))

    if call_date is None: call_date = 0
    if call_price is None: call_price = 0.0

    if isinstance(call_price, int): call_price = float(call_price)
    #^ helps especially with calls from JS-based clients, where parseFloat(15) returns 15 (not 15.0), which json takes as an int

    if not isinstance(quantity, int):
        problems.append('quantity must be in satoshis')
        return problems, fee
    if call_date and not isinstance(call_date, int):
        problems.append('call_date must be epoch integer')
        return problems, fee
    if call_price and not isinstance(call_price, float):
        problems.append('call_price must be a float')
        return problems, fee

    if quantity < 0: problems.append('negative quantity')
    if call_price < 0: problems.append('negative call price')
    if call_date < 0: problems.append('negative call date')

    # Callable, or not.
    if not callable_:
        if block_index >= 51950 or config.TESTNET: # Protocol change.
            call_date = 0
            call_price = 0.0
        elif block_index >= 51800:                 # Protocol change.
            if call_date:
                problems.append('call date for non‐callable asset')
            if call_price:
                problems.append('call price for non‐callable asset')

    # Valid re-issuance?
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM issuances \
                      WHERE (status = ? AND asset = ?)
                      ORDER BY tx_index ASC''', ('valid', asset))
    issuances = cursor.fetchall()
    cursor.close()
    if issuances:
        reissuance = True
        last_issuance = issuances[-1]
        if call_date is None: call_date = 0
        if call_price is None: call_price = 0.0

        if last_issuance['issuer'] != source:
            problems.append('issued by another address')
        if bool(last_issuance['divisible']) != bool(divisible):
            problems.append('cannot change divisibility')
        if bool(last_issuance['callable']) != bool(callable_):
            problems.append('cannot change callability')
        if last_issuance['call_date'] > call_date and (call_date != 0 or (block_index < 51950 and not config.TESTNET)):
            problems.append('cannot advance call date')
        if last_issuance['call_price'] > call_price:
            problems.append('cannot reduce call price')
        if last_issuance['locked'] and quantity:
            problems.append('locked asset and non‐zero quantity')
    else:
        reissuance = False
        if description.lower() == 'lock':
            problems.append('cannot lock a non‐existent asset')
        if destination:
            problems.append('cannot transfer a non‐existent asset')

    # Check for existence of fee funds.
    if quantity or (block_index >= 95000 or config.TESTNET):   # Protocol change.
        if not reissuance or (block_index < 51800 or config.TESTNET):  # Pay fee only upon first issuance. (Protocol change.)
            cursor = db.cursor()
            cursor.execute('''SELECT * FROM balances \
                              WHERE (address = ? AND asset = ?)''', (source, config.XCP))
            balances = cursor.fetchall()
            cursor.close()
            if block_index >= 50700 or config.TESTNET:     # Protocol change.
                fee = int(1 * config.UNIT)
            elif block_index >= 50300 or config.TESTNET:   # Protocol change.
                fee = 5 * config.UNIT
            elif block_index > 50000 or config.TESTNET:    # Protocol change.
                fee = 5
            if fee and (not balances or balances[0]['quantity'] < fee):
                problems.append('insufficient funds')

    # For SQLite3
    call_date = min(call_date, config.MAX_INT)
    total = sum([issuance['quantity'] for issuance in issuances])
    assert isinstance(quantity, int)
    if total + quantity > config.MAX_INT:
        problems.append('total quantity overflow')

    if destination and quantity:
        problems.append('cannot issue and transfer simultaneously')

    return call_date, call_price, problems, fee

def compose (db, source, transfer_destination, asset, quantity, divisible, callable_, call_date, call_price, description):
    call_date, call_price, problems, fee = validate(db, source, transfer_destination, asset, quantity, divisible, callable_, call_date, call_price, description, util.last_block(db)['block_index'])
    if problems: raise exceptions.IssuanceError(problems)

    asset_id = util.asset_id(asset)
    data = config.PREFIX + struct.pack(config.TXTYPE_FORMAT, ID)
    data += struct.pack(FORMAT_2, asset_id, quantity, 1 if divisible else 0, 1 if callable_ else 0,
        call_date or 0, call_price or 0.0, description.encode('utf-8'))
    if len(data) > 80:
        raise exceptions.IssuanceError('Description is greater than 52 bytes.')
    if transfer_destination:
        destination_outputs = [(transfer_destination, None)]
    else:
        destination_outputs = []
    return (source, destination_outputs, data)

def parse (db, tx, message):
    issuance_parse_cursor = db.cursor()

    # Unpack message.
    try:
        if (tx['block_index'] > 50100 or config.TESTNET) and len(message) == LENGTH_2: # Protocol change.
            asset_id, quantity, divisible, callable_, call_date, call_price, description = struct.unpack(FORMAT_2, message)
            call_price = round(call_price, 6) # TODO: arbitrary
            try:
                description = description.decode('utf-8')
            except UnicodeDecodeError:
                description = ''
        else:
            asset_id, quantity, divisible = struct.unpack(FORMAT_1, message)
            callable_, call_date, call_price, description = False, 0, 0.0, ''
        try:
            asset = util.asset_name(asset_id)
        except:
            asset = None
            status = 'invalid: bad asset name'
        status = 'valid'
    except (AssertionError, struct.error) as e:
        asset, quantity, divisible, callable_, call_date, call_price, description = None, None, None, None, None, None, None
        status = 'invalid: could not unpack'

    fee = 0
    if status == 'valid':
        call_date, call_price, problems, fee = validate(db, tx['source'], tx['destination'], asset, quantity, divisible, callable_, call_date, call_price, description, block_index=tx['block_index'])
        if problems: status = 'invalid: ' + '; '.join(problems)
        if 'total quantity overflow' in problems:
            quantity = 0

    if tx['destination']:
        issuer = tx['destination']
        transfer = True
        quantity = 0
    else:
        issuer = tx['source']
        transfer = False

    # Debit fee.
    if status == 'valid':
        util.debit(db, tx['block_index'], tx['source'], config.XCP, fee)

    # Lock?
    lock = False
    if status == 'valid':
        if description and description.lower() == 'lock':
            lock = True
            cursor = db.cursor()
            issuances = list(cursor.execute('''SELECT * FROM issuances \
                                               WHERE (status = ? AND asset = ?)
                                               ORDER BY tx_index ASC''', ('valid', asset)))
            cursor.close()
            description = issuances[-1]['description']  # Use last description. (Assume previous issuance exists because tx is valid.)
            timestamp, value_int, fee_fraction_int = None, None, None

    # Add parsed transaction to message-type–specific table.
    bindings= {
        'tx_index': tx['tx_index'],
        'tx_hash': tx['tx_hash'],
        'block_index': tx['block_index'],
        'asset': asset,
        'quantity': quantity,
        'divisible': divisible,
        'source': tx['source'],
        'issuer': issuer,
        'transfer': transfer,
        'callable': callable_,
        'call_date': call_date,
        'call_price': call_price,
        'description': description,
        'fee_paid': fee,
        'locked': lock,
        'status': status,
    }
    sql='insert into issuances values(:tx_index, :tx_hash, :block_index, :asset, :quantity, :divisible, :source, :issuer, :transfer, :callable, :call_date, :call_price, :description, :fee_paid, :locked, :status)'
    issuance_parse_cursor.execute(sql, bindings)

    # Credit.
    if status == 'valid' and quantity:
        util.credit(db, tx['block_index'], tx['source'], asset, quantity)

    issuance_parse_cursor.close()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>Increase issuance fee<commit_after>#! /usr/bin/python3

"""
Allow simultaneous lock and transfer.
"""

import struct
import decimal
D = decimal.Decimal

from . import (config, util, exceptions, bitcoin, util)

FORMAT_1 = '>QQ?'
LENGTH_1 = 8 + 8 + 1
FORMAT_2 = '>QQ??If42p'
LENGTH_2 = 8 + 8 + 1 + 1 + 4 + 4 + 42
ID = 20

def validate (db, source, destination, asset, quantity, divisible, callable_, call_date, call_price, description, block_index):
    problems = []
    fee = 0

    if asset in (config.BTC, config.XCP):
        problems.append('cannot issue {} or {}'.format(config.BTC, config.XCP))

    if call_date is None: call_date = 0
    if call_price is None: call_price = 0.0

    if isinstance(call_price, int): call_price = float(call_price)
    #^ helps especially with calls from JS-based clients, where parseFloat(15) returns 15 (not 15.0), which json takes as an int

    if not isinstance(quantity, int):
        problems.append('quantity must be in satoshis')
        return problems, fee
    if call_date and not isinstance(call_date, int):
        problems.append('call_date must be epoch integer')
        return problems, fee
    if call_price and not isinstance(call_price, float):
        problems.append('call_price must be a float')
        return problems, fee

    if quantity < 0: problems.append('negative quantity')
    if call_price < 0: problems.append('negative call price')
    if call_date < 0: problems.append('negative call date')

    # Callable, or not.
    if not callable_:
        if block_index >= 51950 or config.TESTNET: # Protocol change.
            call_date = 0
            call_price = 0.0
        elif block_index >= 51800:                 # Protocol change.
            if call_date:
                problems.append('call date for non‐callable asset')
            if call_price:
                problems.append('call price for non‐callable asset')

    # Valid re-issuance?
    cursor = db.cursor()
    cursor.execute('''SELECT * FROM issuances \
                      WHERE (status = ? AND asset = ?)
                      ORDER BY tx_index ASC''', ('valid', asset))
    issuances = cursor.fetchall()
    cursor.close()
    if issuances:
        reissuance = True
        last_issuance = issuances[-1]
        if call_date is None: call_date = 0
        if call_price is None: call_price = 0.0

        if last_issuance['issuer'] != source:
            problems.append('issued by another address')
        if bool(last_issuance['divisible']) != bool(divisible):
            problems.append('cannot change divisibility')
        if bool(last_issuance['callable']) != bool(callable_):
            problems.append('cannot change callability')
        if last_issuance['call_date'] > call_date and (call_date != 0 or (block_index < 51950 and not config.TESTNET)):
            problems.append('cannot advance call date')
        if last_issuance['call_price'] > call_price:
            problems.append('cannot reduce call price')
        if last_issuance['locked'] and quantity:
            problems.append('locked asset and non‐zero quantity')
    else:
        reissuance = False
        if description.lower() == 'lock':
            problems.append('cannot lock a non‐existent asset')
        if destination:
            problems.append('cannot transfer a non‐existent asset')

    # Check for existence of fee funds.
    if quantity or (block_index >= 95000 or config.TESTNET):   # Protocol change.
        if not reissuance or (block_index < 51800 or config.TESTNET):  # Pay fee only upon first issuance. (Protocol change.)
            cursor = db.cursor()
            cursor.execute('''SELECT * FROM balances \
                              WHERE (address = ? AND asset = ?)''', (source, config.XCP))
            balances = cursor.fetchall()
            cursor.close()
            if block_index >= 50700 or config.TESTNET:     # Protocol change.
                fee = int(100 * config.UNIT)
            elif block_index >= 50300 or config.TESTNET:   # Protocol change.
                fee = 5 * config.UNIT
            elif block_index > 50000 or config.TESTNET:    # Protocol change.
                fee = 5
            if fee and (not balances or balances[0]['quantity'] < fee):
                problems.append('insufficient funds')

    # For SQLite3
    call_date = min(call_date, config.MAX_INT)
    total = sum([issuance['quantity'] for issuance in issuances])
    assert isinstance(quantity, int)
    if total + quantity > config.MAX_INT:
        problems.append('total quantity overflow')

    if destination and quantity:
        problems.append('cannot issue and transfer simultaneously')

    return call_date, call_price, problems, fee

def compose (db, source, transfer_destination, asset, quantity, divisible, callable_, call_date, call_price, description):
    call_date, call_price, problems, fee = validate(db, source, transfer_destination, asset, quantity, divisible, callable_, call_date, call_price, description, util.last_block(db)['block_index'])
    if problems: raise exceptions.IssuanceError(problems)

    asset_id = util.asset_id(asset)
    data = config.PREFIX + struct.pack(config.TXTYPE_FORMAT, ID)
    data += struct.pack(FORMAT_2, asset_id, quantity, 1 if divisible else 0, 1 if callable_ else 0,
        call_date or 0, call_price or 0.0, description.encode('utf-8'))
    if len(data) > 80:
        raise exceptions.IssuanceError('Description is greater than 52 bytes.')
    if transfer_destination:
        destination_outputs = [(transfer_destination, None)]
    else:
        destination_outputs = []
    return (source, destination_outputs, data)

def parse (db, tx, message):
    issuance_parse_cursor = db.cursor()

    # Unpack message.
    try:
        if (tx['block_index'] > 50100 or config.TESTNET) and len(message) == LENGTH_2: # Protocol change.
            asset_id, quantity, divisible, callable_, call_date, call_price, description = struct.unpack(FORMAT_2, message)
            call_price = round(call_price, 6) # TODO: arbitrary
            try:
                description = description.decode('utf-8')
            except UnicodeDecodeError:
                description = ''
        else:
            asset_id, quantity, divisible = struct.unpack(FORMAT_1, message)
            callable_, call_date, call_price, description = False, 0, 0.0, ''
        try:
            asset = util.asset_name(asset_id)
        except:
            asset = None
            status = 'invalid: bad asset name'
        status = 'valid'
    except (AssertionError, struct.error) as e:
        asset, quantity, divisible, callable_, call_date, call_price, description = None, None, None, None, None, None, None
        status = 'invalid: could not unpack'

    fee = 0
    if status == 'valid':
        call_date, call_price, problems, fee = validate(db, tx['source'], tx['destination'], asset, quantity, divisible, callable_, call_date, call_price, description, block_index=tx['block_index'])
        if problems: status = 'invalid: ' + '; '.join(problems)
        if 'total quantity overflow' in problems:
            quantity = 0

    if tx['destination']:
        issuer = tx['destination']
        transfer = True
        quantity = 0
    else:
        issuer = tx['source']
        transfer = False

    # Debit fee.
    if status == 'valid':
        util.debit(db, tx['block_index'], tx['source'], config.XCP, fee)

    # Lock?
    lock = False
    if status == 'valid':
        if description and description.lower() == 'lock':
            lock = True
            cursor = db.cursor()
            issuances = list(cursor.execute('''SELECT * FROM issuances \
                                               WHERE (status = ? AND asset = ?)
                                               ORDER BY tx_index ASC''', ('valid', asset)))
            cursor.close()
            description = issuances[-1]['description']  # Use last description. (Assume previous issuance exists because tx is valid.)
            timestamp, value_int, fee_fraction_int = None, None, None

    # Add parsed transaction to message-type–specific table.
    bindings= {
        'tx_index': tx['tx_index'],
        'tx_hash': tx['tx_hash'],
        'block_index': tx['block_index'],
        'asset': asset,
        'quantity': quantity,
        'divisible': divisible,
        'source': tx['source'],
        'issuer': issuer,
        'transfer': transfer,
        'callable': callable_,
        'call_date': call_date,
        'call_price': call_price,
        'description': description,
        'fee_paid': fee,
        'locked': lock,
        'status': status,
    }
    sql='insert into issuances values(:tx_index, :tx_hash, :block_index, :asset, :quantity, :divisible, :source, :issuer, :transfer, :callable, :call_date, :call_price, :description, :fee_paid, :locked, :status)'
    issuance_parse_cursor.execute(sql, bindings)

    # Credit.
    if status == 'valid' and quantity:
        util.credit(db, tx['block_index'], tx['source'], asset, quantity)

    issuance_parse_cursor.close()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>
# Exports
from .executor import LuigiSwfExecutor
from .tasks import SwfHeartbeatCancel

__version__ = '0.7'
<commit_msg>version bump to 0.8<commit_after>
# Exports
from .executor import LuigiSwfExecutor
from .tasks import SwfHeartbeatCancel

__version__ = '0.8'
<|endoftext|>