<commit_before>from datetime import datetime
from enum import Enum
from uuid import UUID
from flask import request

# This is somewhat :(
#
# So we've already batched the database fetches for a bunch of sqlalchemy
# objects together. These objects probably have joins/joinedloads, though,
# which may also need to do data fetches (and/or have joins themselves that
# have data fetches.) Without the function you're reading about now, those
# would all happen serially.
#
# So here's what we'll do. We'll write a function that operates on a list of
# already-converted dictionaries (from a Crumbler.) We can
# find every key/value pair within that dict that has its own data fetching to
# do [1], batch all of that data fetching together and crumble those objects.
# Now we can recursively run ourselves on that resulting list, and replace
# the original value with the final result from this greedy fetch.
#
# Another way of describing this is that we do a depth first search and crumble
# everything we come across, transforming the original structure in place.
#
# [1] The implmentation does this for every value that has its own Crumble object
# even if get_extra_attrs_from_db is a no-op
#
# Based on the way serialization is written, this cannot change what we'd
# output... we're basically allowed to run object_to_dict on objects whenever
# we want, as long as we make sure to run serialize on them afterwards. In this
# case, the caller of this function will run serialize on this entire tree
# after we're done.
#
# Philosophically, this function is written so that running it can only make
# things better: there may be corner cases where it fails to batch something
# that should be batched, but if it fails that won't break anything
#
# @param (data): a list of objects that have already had serializer run on them

def greedily_try_to_batch_data_fetches(data, extended_registry):
    # we only care about lists of dicts
    if not isinstance(data, list) or not isinstance(data[0], dict):
        return data

    for k in data[0].keys():
        if isinstance(data[0][k], _PASSTHROUGH):
            # cheap check to see if value is boring before calling
            # get_crumbler
            continue

        crumbler = get_crumbler(data[0][k], extended_registry)
        if not crumbler:
            # this isn't a key that might do data fetching
            continue

        objs_to_crumble = [o[k] for o in data]

        # let's double check that all of these objects are the same type
        # (missing objects/None values are ok)
        if len(set(type(o) for o in objs_to_crumble if o)) != 1:
            continue

        # ok, batch the data fetch for all of these child keys and run
        # serializer
        attrs = crumbler.get_extra_attrs_from_db(objs_to_crumble)
        replacements = [crumbler(o, attrs=attrs.get(o))
                        for o in objs_to_crumble]

        # we've serialized the child data fetch. But it might also have
        # children that want to fetch data, so let's recursively call greed!
        if isinstance(replacements[0], dict):
            replacements = greedily_try_to_batch_data_fetches(replacements,
                                                              extended_registry)

        # replace the originals with our fetched&crumbled objects
        replacement_dict = dict(zip(objs_to_crumble, replacements))
        for o in data:
            if o[k] in replacement_dict:
                o[k] = replacement_dict[o[k]]
    return data

# Types for which serialization is a no-op.
_PASSTHROUGH = (basestring, bool, int, long, type(None), float)

def serialize(data, extended_registry=None):
    """
    Converts a data structure of dicts, lists, SQLAlchemy objects, and other
    random python objects into something that can be passed to JSON.dumps. This
    is not a guarantee...if data contains an object that we don't know how to
    handle, we'll just leave it in.

    extended_registry: additional Crumblers to use. e.g. one API might
    want to use a special crumbler for Jobs that also adds Build
    information and passes { Job: JobWithBuildCrumbler }

    Its safe (but CPU-expensive) to rerun serialize on data multiple times
    """
    if extended_registry is None:
        extended_registry = {}

    if isinstance(data, _PASSTHROUGH):
        return data

    if isinstance(data, dict):
        for k, v in data.iteritems():
            if not isinstance(v, _PASSTHROUGH) or not isinstance(k, _PASSTHROUGH):
                # Gotta do it the hard way.
                return dict(zip(serialize(data.keys(), extended_registry),
                                serialize(data.values(), extended_registry)))
        # All keys and values were passthrough, so the dict is already
        # serialized.
        return data

    if isinstance(data, (list, tuple, set, frozenset)):
        if not data:
            return []

        # if every item in the list is the same, we want to batch fetch any
        # necessary data from the db before serializing them all
        if len(set(type(g) for g in data)) == 1:
            # Make sure it is a list.
            if not isinstance(data, list):
                data = list(data)

            # If we have a list of passthrough, we're done.
            if isinstance(data[0], _PASSTHROUGH):
                return data

            crumbler = get_crumbler(data[0], extended_registry)

            if crumbler:
                attrs = crumbler.get_extra_attrs_from_db(data)
                data = [crumbler(o, attrs=attrs.get(o)) for o in data]

                disable_greedy_batching = int(request.args.get('__nobatch__', 0))
                if not disable_greedy_batching:
                    greedily_try_to_batch_data_fetches(data, extended_registry)

        return [serialize(j, extended_registry) for j in data]

    # if we're here, we have a single object that we probably need to convert
    # using a crumbler
    crumbler = get_crumbler(data, extended_registry)

    if crumbler is None:
        return data

    attrs = crumbler.get_extra_attrs_from_db([data])
    data = crumbler(data, attrs=attrs.get(data))

    return serialize(data, extended_registry)

#
# Crumbler code: the code that converts SQLAlchemy objects into dictionaries.
# Classes for individual SQLAlchemy objects are in models/
#
# We create a registry of crumblers using class decorators, e.g. use this class
# to convert DateTime objects to strings. serialize() just looks up the right
# class to use and calls it on its objects
#

_registry = {}

def register(type):
    def wrapped(cls):
        _registry[type] = cls()
        return cls
    return wrapped

def get_crumbler(item, registry):
    item_type = type(item)

    crumbler = registry.get(item_type, _registry.get(item_type))

    if crumbler is None:
        for cls, _crumbler in _registry.iteritems():
            if issubclass(item_type, cls):
                crumbler = _crumbler
                break

    return crumbler

class Crumbler(object):
    """
    Converts an object (most often a SQLAlchemy object) to a dict/string/int.
    This is shallow: the returned dict may have values that need to be crumbled
    themselves.

    Why "Crumble"? The name is suggestive of what the class does, and you very likely
    went to these docs to find out more.
    """

    def __call__(self, item, attrs):
        return self.crumble(item, attrs)

    def get_extra_attrs_from_db(self, item_list):
        """
        We may need to do additional data fetching to convert an object: for
        example, we want to look up the phabricator callsign when returning
        revision objects. This function can take a list of objects to crumble
        and returns an attrs dict (object => additional fetched data to use in
        crumble)
        """
        return {}

    def crumble(self, item, attrs):
        """
        Does the actual conversion from object to something simpler. attrs
        should come from get_extra_attrs_from_db, e.g.:

        all_attrs = cls.get_extra_attrs_from_db(item_list)
        s = [cls.crumble(item, all_attrs.get(item)) for item in item_list]
        """
        return {}

@register(datetime)
class DateTimeCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return item.isoformat()

@register(Enum)
class EnumCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return {
            'id': item.name,
            'name': unicode(item),
        }

@register(UUID)
class UUIDCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return item.hex
<commit_msg>Fix changes<commit_after>from datetime import datetime
from enum import Enum
from uuid import UUID
from flask import request

# This is somewhat :(
#
# So we've already batched the database fetches for a bunch of sqlalchemy
# objects together. These objects probably have joins/joinedloads, though,
# which may also need to do data fetches (and/or have joins themselves that
# have data fetches.) Without the function you're reading about now, those
# would all happen serially.
#
# So here's what we'll do. We'll write a function that operates on a list of
# already-converted dictionaries (from a Crumbler.) We can
# find every key/value pair within that dict that has its own data fetching to
# do [1], batch all of that data fetching together and crumble those objects.
# Now we can recursively run ourselves on that resulting list, and replace
# the original value with the final result from this greedy fetch.
#
# Another way of describing this is that we do a depth first search and crumble
# everything we come across, transforming the original structure in place.
#
# [1] The implmentation does this for every value that has its own Crumble object
# even if get_extra_attrs_from_db is a no-op
#
# Based on the way serialization is written, this cannot change what we'd
# output... we're basically allowed to run object_to_dict on objects whenever
# we want, as long as we make sure to run serialize on them afterwards. In this
# case, the caller of this function will run serialize on this entire tree
# after we're done.
#
# Philosophically, this function is written so that running it can only make
# things better: there may be corner cases where it fails to batch something
# that should be batched, but if it fails that won't break anything
#
# @param (data): a list of objects that have already had serializer run on them

def greedily_try_to_batch_data_fetches(data, extended_registry):
    # we only care about lists of dicts
    if not isinstance(data, list) or not isinstance(data[0], dict):
        return data

    for k in data[0].keys():
        if isinstance(data[0][k], _PASSTHROUGH):
            # cheap check to see if value is boring before calling
            # get_crumbler
            continue

        crumbler = get_crumbler(data[0][k], extended_registry)
        if not crumbler:
            # this isn't a key that might do data fetching
            continue

        objs_to_crumble = [o[k] for o in data]

        # let's double check that all of these objects are the same type
        # (missing objects/None values are ok)
        if len(set(type(o) for o in objs_to_crumble if o)) != 1:
            continue

        # ok, batch the data fetch for all of these child keys and run
        # serializer
        attrs = crumbler.get_extra_attrs_from_db(objs_to_crumble)
        replacements = [crumbler(o, attrs=attrs.get(o)) if o else None
                        for o in objs_to_crumble]

        # we've serialized the child data fetch. But it might also have
        # children that want to fetch data, so let's recursively call greed!
        if isinstance(replacements[0], dict):
            replacements = greedily_try_to_batch_data_fetches(replacements,
                                                              extended_registry)

        # replace the originals with our fetched&crumbled objects
        replacement_dict = dict(zip(objs_to_crumble, replacements))
        for o in data:
            if o[k] in replacement_dict:
                o[k] = replacement_dict[o[k]]
    return data

# Types for which serialization is a no-op.
_PASSTHROUGH = (basestring, bool, int, long, type(None), float)

def serialize(data, extended_registry=None):
    """
    Converts a data structure of dicts, lists, SQLAlchemy objects, and other
    random python objects into something that can be passed to JSON.dumps. This
    is not a guarantee...if data contains an object that we don't know how to
    handle, we'll just leave it in.

    extended_registry: additional Crumblers to use. e.g. one API might
    want to use a special crumbler for Jobs that also adds Build
    information and passes { Job: JobWithBuildCrumbler }

    Its safe (but CPU-expensive) to rerun serialize on data multiple times
    """
    if extended_registry is None:
        extended_registry = {}

    if isinstance(data, _PASSTHROUGH):
        return data

    if isinstance(data, dict):
        for k, v in data.iteritems():
            if not isinstance(v, _PASSTHROUGH) or not isinstance(k, _PASSTHROUGH):
                # Gotta do it the hard way.
                return dict(zip(serialize(data.keys(), extended_registry),
                                serialize(data.values(), extended_registry)))
        # All keys and values were passthrough, so the dict is already
        # serialized.
        return data

    if isinstance(data, (list, tuple, set, frozenset)):
        if not data:
            return []

        # if every item in the list is the same, we want to batch fetch any
        # necessary data from the db before serializing them all
        if len(set(type(g) for g in data)) == 1:
            # Make sure it is a list.
            if not isinstance(data, list):
                data = list(data)

            # If we have a list of passthrough, we're done.
            if isinstance(data[0], _PASSTHROUGH):
                return data

            crumbler = get_crumbler(data[0], extended_registry)

            if crumbler:
                attrs = crumbler.get_extra_attrs_from_db(data)
                data = [crumbler(o, attrs=attrs.get(o)) for o in data]

                disable_greedy_batching = int(request.args.get('__nobatch__', 0))
                if not disable_greedy_batching:
                    greedily_try_to_batch_data_fetches(data, extended_registry)

        return [serialize(j, extended_registry) for j in data]

    # if we're here, we have a single object that we probably need to convert
    # using a crumbler
    crumbler = get_crumbler(data, extended_registry)

    if crumbler is None:
        return data

    attrs = crumbler.get_extra_attrs_from_db([data])
    data = crumbler(data, attrs=attrs.get(data))

    return serialize(data, extended_registry)

#
# Crumbler code: the code that converts SQLAlchemy objects into dictionaries.
# Classes for individual SQLAlchemy objects are in models/
#
# We create a registry of crumblers using class decorators, e.g. use this class
# to convert DateTime objects to strings. serialize() just looks up the right
# class to use and calls it on its objects
#

_registry = {}

def register(type):
    def wrapped(cls):
        _registry[type] = cls()
        return cls
    return wrapped

def get_crumbler(item, registry):
    item_type = type(item)

    crumbler = registry.get(item_type, _registry.get(item_type))

    if crumbler is None:
        for cls, _crumbler in _registry.iteritems():
            if issubclass(item_type, cls):
                crumbler = _crumbler
                break

    return crumbler

class Crumbler(object):
    """
    Converts an object (most often a SQLAlchemy object) to a dict/string/int.
    This is shallow: the returned dict may have values that need to be crumbled
    themselves.

    Why "Crumble"? The name is suggestive of what the class does, and you very likely
    went to these docs to find out more.
    """

    def __call__(self, item, attrs):
        return self.crumble(item, attrs)

    def get_extra_attrs_from_db(self, item_list):
        """
        We may need to do additional data fetching to convert an object: for
        example, we want to look up the phabricator callsign when returning
        revision objects. This function can take a list of objects to crumble
        and returns an attrs dict (object => additional fetched data to use in
        crumble)
        """
        return {}

    def crumble(self, item, attrs):
        """
        Does the actual conversion from object to something simpler. attrs
        should come from get_extra_attrs_from_db, e.g.:

        all_attrs = cls.get_extra_attrs_from_db(item_list)
        s = [cls.crumble(item, all_attrs.get(item)) for item in item_list]
        """
        return {}

@register(datetime)
class DateTimeCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return item.isoformat()

@register(Enum)
class EnumCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return {
            'id': item.name,
            'name': unicode(item),
        }

@register(UUID)
class UUIDCrumbler(Crumbler):
    def crumble(self, item, attrs):
        return item.hex
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Usage: seq2ffv1.py source_parent_directory output_directory
The script will look through all subdirectories beneath the source_parent_directory for a DPX or image sequence.
The script will then:
Create folder structure for each image sequence in your designated output_directory
Create framemd5 values of the source sequence
Transcode to a single FFV1 in Matroska file
Create framemd5 values for the FFV1 in Matroska file
Verify losslessness (There will most likely be a warning about pixel aspect ratio - https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html)
Generate CSV log that will be saved to your desktop
'''
import subprocess
import os
import argparse
import datetime
import time
import itertools
import ififuncs
from ififuncs import diff_textfiles
from ififuncs import get_mediainfo
from ififuncs import create_csv
from ififuncs import append_csv
from ififuncs import get_image_sequence_files
from ififuncs import get_ffmpeg_friendly_name

def read_lines(infile):
    '''
    Returns line number and text from an textfile.
    '''
    for lineno, line in enumerate(infile):
        yield lineno, line

def make_framemd5(directory, log_filename_alteration, args):
    '''
    Apparently this makes framemd5s. But it clearly does a lot more.
    '''
    sequence_length, basename, ffmpeg_friendly_name, start_number, images, container, output_dirname = make_folder_structure(directory, args)
    output = output_dirname + '/metadata/%ssource.framemd5' % (basename)
    logfile = output_dirname + '/logs/%s%s.log' % (basename, log_filename_alteration)
    logfile = "\'" + logfile + "\'"
    env_dict = ififuncs.set_environment(logfile)
    image_seq_without_container = ffmpeg_friendly_name
    start_number_length = len(start_number)
    number_regex = "%0" + str(start_number_length) + 'd.'
    if len(images[0].split("_")[-1].split(".")) > 2:
        image_seq_without_container = ffmpeg_friendly_name[:-1] + ffmpeg_friendly_name[-1].replace('_', '.')
        ffmpeg_friendly_name = image_seq_without_container
    ffmpeg_friendly_name += number_regex + '%s' % container
    framemd5 = [
        'ffmpeg', '-start_number',
        start_number, '-report',
        '-f', 'image2',
        '-framerate', '24',
        '-i', ffmpeg_friendly_name,
        '-f', 'framemd5', output
    ]
    print framemd5
    subprocess.call(framemd5, env=env_dict)
    info = [
        output_dirname,
        output,
        image_seq_without_container,
        start_number,
        container,
        ffmpeg_friendly_name,
        number_regex,
        sequence_length
    ]
    return info

def remove_bad_files(root_dir):
    '''
    removes unwanted files. unless something is altered, just use the ififuncs
    function instead.
    '''
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, _, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)

def setup():
    '''
    Sets up a lot of the variables and filepaths.
    '''
    csv_report_filename = os.path.join(
        os.path.expanduser("~/Desktop/"),
        'dpx_transcode_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
    )
    parser = argparse.ArgumentParser(description='Transcode all DPX or TIFF'
                                    ' image sequence in the subfolders of your'
                                    ' source directory to FFV1 Version 3'
                                    ' in a Matroska Container.'
                                    'CSV report is generated on your desktop.'
                                    ' Written by Kieran O\'Leary.')
    parser.add_argument('source_directory', help='Input directory')
    parser.add_argument('destination', help='Destination directory')
    args = parser.parse_args()
    create_csv(csv_report_filename, (
        'Sequence Name', 'Lossless?',
        'Start time', 'Finish Time',
        'Transcode Start Time', 'Transcode Finish Time',
        'Transcode Time', 'Frame Count',
        'Encode FPS', 'Sequence Size',
        'FFV1 Size', 'Pixel Format',
        'Sequence Type', 'Width',
        'Height', 'Compression Ratio'
        ))
    return args, csv_report_filename

def make_folder_structure(directory, args):
    images = get_image_sequence_files(directory)
    if images == 'none':
        return 'none'
    sequence_length = len(images)
    ffmpeg_friendly_name, container, start_number = get_ffmpeg_friendly_name(images)
    output_parent_directory = args.destination
    if start_number == '864000':
        output_dirname = os.path.join(
            output_parent_directory,
            os.path.basename(directory) + time.strftime("%Y_%m_%dT%H_%M_%S")
        )
        basename = os.path.basename(directory)
    else:
        output_dirname = os.path.join(
            output_parent_directory,
            ffmpeg_friendly_name + time.strftime("%Y_%m_%dT%H_%M_%S")
        )
        basename = ffmpeg_friendly_name
    logs_dir = output_dirname + '/logs'
    objects_dir = output_dirname + '/objects'
    metadata_dir = output_dirname + '/metadata'
    try:
        os.makedirs(output_dirname)
        os.makedirs(logs_dir)
        os.makedirs(objects_dir)
        os.makedirs(metadata_dir)
    except: OSError
    return sequence_length, basename, ffmpeg_friendly_name, start_number, images, container, output_dirname

def make_ffv1(
        start_number,
        dpx_filename,
        output_dirname,
        output_filename,
        image_seq_without_container,
        parent_basename,
        env_dict
    ):
    '''
    This launches the image sequence to FFV1/Matroska process
    as well as framemd5 losslessness verification.
    '''
    ffprobe_cmd = [
        'ffprobe',
        '-start_number', start_number,
        '-i', os.path.abspath(dpx_filename),
        '-v', 'error',
        '-select_streams', 'v:0',
        '-show_entries',
        'stream=pix_fmt',
        '-of', 'default=noprint_wrappers=1:nokey=1'
    ]
    pix_fmt = subprocess.check_output(ffprobe_cmd).rstrip()
    ffv12dpx = [
        'ffmpeg', '-report',
        '-f', 'image2',
        '-framerate', '24',
        '-start_number', start_number,
        '-i', os.path.abspath(dpx_filename),
        '-strict', '-2',
        '-c:v', 'ffv1',
        '-level', '3',
        '-g', '1',
        '-slicecrc', '1',
        '-slices', '16',
        '-pix_fmt', pix_fmt,
        output_dirname +  '/objects/' + output_filename + '.mkv'
    ]
    print ffv12dpx
    transcode_start = datetime.datetime.now()
    transcode_start_machine = time.time()
    subprocess.call(ffv12dpx, env=env_dict)
    transcode_finish = datetime.datetime.now()
    transcode_finish_machine = time.time()
    transcode_time = transcode_finish_machine - transcode_start_machine
    manifest_textfile = os.path.join(
        os.path.dirname(output_dirname), parent_basename + '_manifest.md5'
    )
    ffv1_path = output_dirname +  '/objects/'  + output_filename + '.mkv'
    width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path)
    height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path)
    ffv1_md5 = output_dirname +  '/metadata/' + image_seq_without_container + 'ffv1.framemd5'
    ffv1_fmd5_cmd = [
        'ffmpeg',
        '-i', ffv1_path,
        '-pix_fmt', pix_fmt,
        '-f', 'framemd5',
        ffv1_md5
    ]
    ffv1_fmd5_logfile = os.path.join(
        output_dirname, 'logs/%s_ffv1_framemd5.log' % output_filename
    )
    ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
    ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
    subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
    finish = datetime.datetime.now()
    return (
        ffv1_path, ffv1_md5,
        transcode_time, pix_fmt,
        width, height,
        finish, transcode_start,
        transcode_finish
    )

def run_loop(args, csv_report_filename):
    for root, _, filenames in os.walk(args.source_directory):
        source_directory = root
        total_size = 0
        start = datetime.datetime.now()
        info = make_framemd5(source_directory, 'dpx_framemd5', args)
        if info == 'none':
            continue
        for files in filenames:
            total_size += os.path.getsize(os.path.join(root, files))
        output_dirname = info[0]
        source_textfile = info[1]
        image_seq_without_container = info[2]
        start_number = info[3]
        container = info[4]
        dpx_filename = info[5]
        sequence_length = info[7]
        output_filename = image_seq_without_container[:-1]
        parent_basename = os.path.basename(output_dirname)
        logfile = output_dirname + '/logs/%s_ffv1_transcode.log' % output_filename
        logfile = "\'" + logfile + "\'"
        env_dict = ififuncs.set_environment(logfile)
        ffv1_path, ffv1_md5, transcode_time, pix_fmt, width, height, finish, transcode_start, transcode_finish = make_ffv1(
            start_number,
            dpx_filename,
            output_dirname,
            output_filename,
            image_seq_without_container,
            parent_basename,
            env_dict
        )
        ffv1_size = os.path.getsize(ffv1_path)
        comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
        judgement = diff_textfiles(source_textfile, ffv1_md5)
        fps = float(sequence_length) / float(transcode_time)
        checksum_mismatches = []
        with open(source_textfile) as source_md5_object:
            with open(ffv1_md5) as ffv1_md5_object:
                for (lineno1, line1), (lineno2, line2) in itertools.izip(
                        read_lines(source_md5_object), read_lines(ffv1_md5_object)
                ):
                    if line1 != line2:
                        if 'sar' in line1:
                            checksum_mismatches = ['sar']
                        else:
                            checksum_mismatches.append(1)
        if len(checksum_mismatches) == 0:
            print 'LOSSLESS'
            append_csv(
                csv_report_filename, (
                    parent_basename, judgement,
                    start, finish,
                    transcode_start, transcode_finish,
                    transcode_time, sequence_length,
                    fps, total_size,
                    ffv1_size, pix_fmt,
                    container, width,
                    height, comp_ratio
                )
            )

        elif len(checksum_mismatches) == 1:
            if checksum_mismatches[0] == 'sar':
                print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
                append_csv(csv_report_filename, (
                    parent_basename, 'LOSSLESS - different PAR',
                    start, finish, transcode_start,
                    transcode_finish, transcode_time,
                    sequence_length, fps,
                    total_size, ffv1_size,
                    pix_fmt, container,
                    width, height,
                    comp_ratio
                )
                          )
        elif len(checksum_mismatches) > 1:
            print 'NOT LOSSLESS'
            append_csv(csv_report_filename, (
                parent_basename, judgement,
                start, finish, transcode_start,
                transcode_finish, transcode_time,
                sequence_length, fps,
                total_size, ffv1_size,
                pix_fmt, container,
                width, height,
                comp_ratio
            ))
def main():
    '''
    Overly long main function that does most of the heavy lifting.
    This needs to be broken up into smaller functions.
    '''
    args, csv_report_filename = setup()
    run_loop(args, csv_report_filename)

if __name__ == '__main__':
    main()
<commit_msg>seq2ffv1 more PEP-08 refactoring<commit_after>#!/usr/bin/env python

'''
Usage: seq2ffv1.py source_parent_directory output_directory
The script will look through all subdirectories beneath the
source_parent_directory for a DPX or image sequence.
The script will then:
Create folder structure for each image sequence in
your designated output_directory.
Create framemd5 values of the source sequence
Transcode to a single FFV1 in Matroska file
Create framemd5 values for the FFV1 in Matroska file
Verify losslessness
(There will most likely be a warning about pixel aspect ratio
- https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html)
Generate CSV log that will be saved to your desktop
'''
import subprocess
import os
import argparse
import datetime
import time
import itertools
import ififuncs
from ififuncs import diff_textfiles
from ififuncs import get_mediainfo
from ififuncs import create_csv
from ififuncs import append_csv
from ififuncs import get_image_sequence_files
from ififuncs import get_ffmpeg_friendly_name

def read_lines(infile):
    '''
    Returns line number and text from an textfile.
    '''
    for lineno, line in enumerate(infile):
        yield lineno, line

def make_framemd5(directory, log_filename_alteration, args):
    '''
    Apparently this makes framemd5s. But it clearly does a lot more.
    '''
    (
    sequence_length,
    basename,
    ffmpeg_friendly_name,
    start_number,
    images,
    container,
    output_dirname) = make_folder_structure(directory, args)
    output = output_dirname + '/metadata/%ssource.framemd5' % (basename)
    logfile = output_dirname + '/logs/%s%s.log' % (
        basename,
        log_filename_alteration
    )
    logfile = "\'" + logfile + "\'"
    env_dict = ififuncs.set_environment(logfile)
    image_seq_without_container = ffmpeg_friendly_name
    start_number_length = len(start_number)
    number_regex = "%0" + str(start_number_length) + 'd.'
    if len(images[0].split("_")[-1].split(".")) > 2:
        image_seq_without_container = ffmpeg_friendly_name[:-1] + ffmpeg_friendly_name[-1].replace('_', '.')
        ffmpeg_friendly_name = image_seq_without_container
    ffmpeg_friendly_name += number_regex + '%s' % container
    framemd5 = [
        'ffmpeg', '-start_number',
        start_number, '-report',
        '-f', 'image2',
        '-framerate', '24',
        '-i', ffmpeg_friendly_name,
        '-f', 'framemd5', output
    ]
    print framemd5
    subprocess.call(framemd5, env=env_dict)
    info = [
        output_dirname,
        output,
        image_seq_without_container,
        start_number,
        container,
        ffmpeg_friendly_name,
        number_regex,
        sequence_length
    ]
    return info

def remove_bad_files(root_dir):
    '''
    removes unwanted files. unless something is altered, just use the ififuncs
    function instead.
    '''
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, _, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)

def setup():
    '''
    Sets up a lot of the variables and filepaths.
    '''
    csv_report_filename = os.path.join(
        os.path.expanduser("~/Desktop/"),
        'dpx_transcode_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
    )
    parser = argparse.ArgumentParser(description='Transcode all DPX or TIFF'
                                    ' image sequence in the subfolders of your'
                                    ' source directory to FFV1 Version 3'
                                    ' in a Matroska Container.'
                                    'CSV report is generated on your desktop.'
                                    ' Written by Kieran O\'Leary.')
    parser.add_argument('source_directory', help='Input directory')
    parser.add_argument('destination', help='Destination directory')
    args = parser.parse_args()
    create_csv(csv_report_filename, (
        'Sequence Name', 'Lossless?',
        'Start time', 'Finish Time',
        'Transcode Start Time', 'Transcode Finish Time',
        'Transcode Time', 'Frame Count',
        'Encode FPS', 'Sequence Size',
        'FFV1 Size', 'Pixel Format',
        'Sequence Type', 'Width',
        'Height', 'Compression Ratio'
        ))
    return args, csv_report_filename

def make_folder_structure(directory, args):
    '''
    Creates output folder structure and sets more variables
    '''
    images = get_image_sequence_files(directory)
    if images == 'none':
        return 'none'
    sequence_length = len(images)
    (ffmpeg_friendly_name,
    container,
    start_number) = get_ffmpeg_friendly_name(images)
    output_parent_directory = args.destination
    if start_number == '864000':
        output_dirname = os.path.join(
            output_parent_directory,
            os.path.basename(directory) + time.strftime("%Y_%m_%dT%H_%M_%S")
        )
        basename = os.path.basename(directory)
    else:
        output_dirname = os.path.join(
            output_parent_directory,
            ffmpeg_friendly_name + time.strftime("%Y_%m_%dT%H_%M_%S")
        )
        basename = ffmpeg_friendly_name
    logs_dir = output_dirname + '/logs'
    objects_dir = output_dirname + '/objects'
    metadata_dir = output_dirname + '/metadata'
    try:
        os.makedirs(output_dirname)
        os.makedirs(logs_dir)
        os.makedirs(objects_dir)
        os.makedirs(metadata_dir)
    except OSError:
        print 'directories already exist'
    return (sequence_length,
            basename,
            ffmpeg_friendly_name,
            start_number,
            images,
            container,
            output_dirname)

def make_ffv1(
        start_number,
        dpx_filename,
        output_dirname,
        output_filename,
        image_seq_without_container,
        env_dict
    ):
    '''
    This launches the image sequence to FFV1/Matroska process
    as well as framemd5 losslessness verification.
    '''
    ffprobe_cmd = [
        'ffprobe',
        '-start_number', start_number,
        '-i', os.path.abspath(dpx_filename),
        '-v', 'error',
        '-select_streams', 'v:0',
        '-show_entries',
        'stream=pix_fmt',
        '-of', 'default=noprint_wrappers=1:nokey=1'
    ]
    pix_fmt = subprocess.check_output(ffprobe_cmd).rstrip()
    ffv12dpx = [
        'ffmpeg', '-report',
        '-f', 'image2',
        '-framerate', '24',
        '-start_number', start_number,
        '-i', os.path.abspath(dpx_filename),
        '-strict', '-2',
        '-c:v', 'ffv1',
        '-level', '3',
        '-g', '1',
        '-slicecrc', '1',
        '-slices', '16',
        '-pix_fmt', pix_fmt,
        output_dirname +  '/objects/' + output_filename + '.mkv'
    ]
    print ffv12dpx
    transcode_start = datetime.datetime.now()
    transcode_start_machine = time.time()
    subprocess.call(ffv12dpx, env=env_dict)
    transcode_finish = datetime.datetime.now()
    transcode_finish_machine = time.time()
    transcode_time = transcode_finish_machine - transcode_start_machine
    ffv1_path = output_dirname +  '/objects/'  + output_filename + '.mkv'
    width = get_mediainfo('duration', '--inform=Video;%Width%', ffv1_path)
    height = get_mediainfo('duration', '--inform=Video;%Height%', ffv1_path)
    ffv1_md5 = os.path.join(
        output_dirname +  '/metadata',
        image_seq_without_container + 'ffv1.framemd5'
    )
    ffv1_fmd5_cmd = [
        'ffmpeg',
        '-i', ffv1_path,
        '-pix_fmt', pix_fmt,
        '-f', 'framemd5',
        ffv1_md5
    ]
    ffv1_fmd5_logfile = os.path.join(
        output_dirname, 'logs/%s_ffv1_framemd5.log' % output_filename
    )
    ffv1_fmd5_logfile = "\'" + ffv1_fmd5_logfile + "\'"
    ffv1_fmd5_env_dict = ififuncs.set_environment(ffv1_fmd5_logfile)
    subprocess.call(ffv1_fmd5_cmd, env=ffv1_fmd5_env_dict)
    finish = datetime.datetime.now()
    return (
        ffv1_path, ffv1_md5,
        transcode_time, pix_fmt,
        width, height,
        finish, transcode_start,
        transcode_finish
    )

def run_loop(args, csv_report_filename):
    '''
    Launches a recursive loop to process all images sequences in your
    subdirectories.
    '''
    for root, _, filenames in os.walk(args.source_directory):
        source_directory = root
        total_size = 0
        start = datetime.datetime.now()
        info = make_framemd5(source_directory, 'dpx_framemd5', args)
        if info == 'none':
            continue
        for files in filenames:
            total_size += os.path.getsize(os.path.join(root, files))
        output_dirname = info[0]
        source_textfile = info[1]
        image_seq_without_container = info[2]
        start_number = info[3]
        container = info[4]
        dpx_filename = info[5]
        sequence_length = info[7]
        output_filename = image_seq_without_container[:-1]
        parent_basename = os.path.basename(output_dirname)
        logfile = os.path.join(
            output_dirname,
            'logs/%s_ffv1_transcode.log' % output_filename
        )
        logfile = "\'" + logfile + "\'"
        env_dict = ififuncs.set_environment(logfile)
        (ffv1_path,
         ffv1_md5,
         transcode_time,
         pix_fmt,
         width,
         height,
         finish,
         transcode_start,
         transcode_finish) = make_ffv1(
            start_number,
            dpx_filename,
            output_dirname,
            output_filename,
            image_seq_without_container,
            env_dict
        )
        ffv1_size = os.path.getsize(ffv1_path)
        comp_ratio = float(total_size) / float(os.path.getsize(ffv1_path))
        judgement = diff_textfiles(source_textfile, ffv1_md5)
        fps = float(sequence_length) / float(transcode_time)
        checksum_mismatches = []
        with open(source_textfile) as source_md5_object:
            with open(ffv1_md5) as ffv1_md5_object:
                for (line1), (line2) in itertools.izip(
                    read_lines(source_md5_object), read_lines(ffv1_md5_object)
                ):
                    if line1 != line2:
                        if 'sar' in line1:
                            checksum_mismatches = ['sar']
                        else:
                            checksum_mismatches.append(1)
        log_results(
            checksum_mismatches,
            csv_report_filename,
            parent_basename,
            judgement,
            start, finish,
            transcode_start, transcode_finish,
            transcode_time, sequence_length,
            fps, total_size,
            ffv1_size, pix_fmt,
            container, width,
            height, comp_ratio)

def log_results(
    checksum_mismatches,
    csv_report_filename,
    parent_basename,
    judgement,
    start, finish,
    transcode_start, transcode_finish,
    transcode_time, sequence_length,
    fps, total_size,
    ffv1_size, pix_fmt,
    container, width,
    height, comp_ratio
):
    if len(checksum_mismatches) == 0:
        print 'LOSSLESS'
        append_csv(
            csv_report_filename, (
                parent_basename, judgement,
                start, finish,
                transcode_start, transcode_finish,
                transcode_time, sequence_length,
                fps, total_size,
                ffv1_size, pix_fmt,
                container, width,
                height, comp_ratio
            )
        )

    elif len(checksum_mismatches) == 1:
        if checksum_mismatches[0] == 'sar':
            print 'Image content is lossless, Pixel Aspect Ratio has been altered - This is mostly likely because your source had no pixel aspect ratio information, and Matroska is specifying 1:1. https://www.ietf.org/mail-archive/web/cellar/current/msg00739.html '
            append_csv(csv_report_filename, (
                parent_basename, 'LOSSLESS - different PAR',
                start, finish, transcode_start,
                transcode_finish, transcode_time,
                sequence_length, fps,
                total_size, ffv1_size,
                pix_fmt, container,
                width, height,
                comp_ratio
            )
                      )
    elif len(checksum_mismatches) > 1:
        print 'NOT LOSSLESS'
        append_csv(csv_report_filename, (
            parent_basename, judgement,
            start, finish, transcode_start,
            transcode_finish, transcode_time,
            sequence_length, fps,
            total_size, ffv1_size,
            pix_fmt, container,
            width, height,
            comp_ratio
        ))

def main():
    '''
    Overly long main function that does most of the heavy lifting.
    This needs to be broken up into smaller functions.
    '''
    args, csv_report_filename = setup()
    run_loop(args, csv_report_filename)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2013 Jordan Halterman <jordan.halterman@gmail.com>
# See LICENSE for details.
from active_redis.core import DataType, Observer, Script
from active_redis.registry import datatype

class SetDefault(Script):
  """
  Sets the value or default value of a dict item.
  """
  keys = ['key']
  args = ['field', 'default']

  script = """
  local key = KEYS[1]
  local field = ARGV[1]

  var exists = redis.call('HEXISTS', key, field)
  if exists then
    return redis.call('HGET', key, field)
  else
    local default = ARGV[2]
    redis.call('HSET', key, field, default)
    return default
  end
  """

class PopItem(Script):
  """
  Pops and returns an item from the dictionary.
  """
  keys = ['key']

  script = """
  local key = KEYS[1]
  local keys = redis.call('HKEYS', key)
  if keys[1] != nil then
    local val = redis.call('HGET', key, keys[1])
    redis.call('HDEL', key, keys[1])
    return val
  end
  return nil
  """

@datatype
class Dict(DataType, Observer):
  """
  A Redis dict data type.
  """
  type = 'dict'
  _scripts = {'set_default': SetDefault}

  def notify(self, subject, index):
    """Updates a dict subject."""
    self.__setitem__(index, subject)

  def clear(self):
    """Clears the dict."""
    self.client.delete(self.key)

  def get(self, key, default=None):
    """Gets a value from the dict."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    return self.observe(default, key)

  def has_key(self, key):
    """Indicates whether the given key exists."""
    return self.client.hexists(self.key, key)

  def items(self):
    """Returns all dict items."""
    return [(key, self.observe(self.client.decode(item), key)) for key, item in self.client.hgetall(self.key).items()]

  def iteritems(self):
    """Returns an iterator over dict items."""
    for key in self.client.hkeys(self.key):
      yield key, self.observe(self.client.decode(self.client.hget(self.key, key)), key)

  def keys(self):
    """Returns all dict keys."""
    return self.client.hkeys(self.key)

  def iterkeys(self):
    """Returns an iterator over dict keys."""
    return iter(self.client.hkeys(self.key))

  def values(self):
    """Returns all dict values."""
    return [self.observe(self.client.decode(self.client.hget(self.key, key)), key) for key in self.client.hkeys(self.key)]

  def itervalues(self):
    """Returns an iterator over dict values."""
    for key in self.client.hkeys(self.key):
      yield self.observe(self.client.decode(self.client.hget(self.key, key)), key)

  def pop(self, key, *args):
    """Pops a value from the dictionary."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    else:
      try:
        return args[0]
      except IndexError:
        raise KeyError("Invalid key %s." % (key,))

  def popitem(self):
    """Pops a random item from the dictionary."""
    item = self._execute_script('popitem', self.key)
    if item is not None:
      return self.client.decode(item)
    else:
      raise KeyError("Dictionary is empty.")

  def setdefault(self, key, default=None):
    """Sets a dict item value or default value."""
    return self._execute_script('setdefault', self.key, key, default)

  def delete(self):
    """Deletes the dictionary."""
    for key, item in self.iteritems():
      if isinstance(item, DataType):
        item.delete()
    self.client.delete(self.key)

  def __len__(self):
    return self.client.hlen(self.key)

  def __iter__(self):
    """Iterates over dict keys."""
    return self.iterkeys()

  def __getitem__(self, key):
    """Gets a dict item."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    else:
      raise KeyError("Key %s not found." % (key,))

  def __setitem__(self, key, item):
    """Sets a dict item."""
    return self.client.hset(self.key, key, self.client.encode(item))

  def __delitem__(self, key):
    """Deletes an item from the dict."""
    return self.client.hdel(self.key, key)

  def __contains__(self, key):
    """Supports using 'in' and 'not in' operators."""
    return self.has_key(key)

  def __repr__(self):
    return repr(dict(self.items()))
<commit_msg>Fix bad Lua script variable in dict.<commit_after># Copyright (c) 2013 Jordan Halterman <jordan.halterman@gmail.com>
# See LICENSE for details.
from active_redis.core import DataType, Observer, Script
from active_redis.registry import datatype

class SetDefault(Script):
  """
  Sets the value or default value of a dict item.
  """
  keys = ['key']
  args = ['field', 'default']

  script = """
  local key = KEYS[1]
  local field = ARGV[1]

  local exists = redis.call('HEXISTS', key, field)
  if exists then
    return redis.call('HGET', key, field)
  else
    local default = ARGV[2]
    redis.call('HSET', key, field, default)
    return default
  end
  """

class PopItem(Script):
  """
  Pops and returns an item from the dictionary.
  """
  keys = ['key']

  script = """
  local key = KEYS[1]
  local keys = redis.call('HKEYS', key)
  if keys[1] != nil then
    local val = redis.call('HGET', key, keys[1])
    redis.call('HDEL', key, keys[1])
    return val
  end
  return nil
  """

@datatype
class Dict(DataType, Observer):
  """
  A Redis dict data type.
  """
  type = 'dict'
  _scripts = {'set_default': SetDefault}

  def notify(self, subject, index):
    """Updates a dict subject."""
    self.__setitem__(index, subject)

  def clear(self):
    """Clears the dict."""
    self.client.delete(self.key)

  def get(self, key, default=None):
    """Gets a value from the dict."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    return self.observe(default, key)

  def has_key(self, key):
    """Indicates whether the given key exists."""
    return self.client.hexists(self.key, key)

  def items(self):
    """Returns all dict items."""
    return [(key, self.observe(self.client.decode(item), key)) for key, item in self.client.hgetall(self.key).items()]

  def iteritems(self):
    """Returns an iterator over dict items."""
    for key in self.client.hkeys(self.key):
      yield key, self.observe(self.client.decode(self.client.hget(self.key, key)), key)

  def keys(self):
    """Returns all dict keys."""
    return self.client.hkeys(self.key)

  def iterkeys(self):
    """Returns an iterator over dict keys."""
    return iter(self.client.hkeys(self.key))

  def values(self):
    """Returns all dict values."""
    return [self.observe(self.client.decode(self.client.hget(self.key, key)), key) for key in self.client.hkeys(self.key)]

  def itervalues(self):
    """Returns an iterator over dict values."""
    for key in self.client.hkeys(self.key):
      yield self.observe(self.client.decode(self.client.hget(self.key, key)), key)

  def pop(self, key, *args):
    """Pops a value from the dictionary."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    else:
      try:
        return args[0]
      except IndexError:
        raise KeyError("Invalid key %s." % (key,))

  def popitem(self):
    """Pops a random item from the dictionary."""
    item = self._execute_script('popitem', self.key)
    if item is not None:
      return self.client.decode(item)
    else:
      raise KeyError("Dictionary is empty.")

  def setdefault(self, key, default=None):
    """Sets a dict item value or default value."""
    return self._execute_script('setdefault', self.key, key, default)

  def delete(self):
    """Deletes the dictionary."""
    for key, item in self.iteritems():
      if isinstance(item, DataType):
        item.delete()
    self.client.delete(self.key)

  def __len__(self):
    return self.client.hlen(self.key)

  def __iter__(self):
    """Iterates over dict keys."""
    return self.iterkeys()

  def __getitem__(self, key):
    """Gets a dict item."""
    item = self.client.hget(self.key, key)
    if item is not None:
      return self.observe(self.client.decode(item), key)
    else:
      raise KeyError("Key %s not found." % (key,))

  def __setitem__(self, key, item):
    """Sets a dict item."""
    return self.client.hset(self.key, key, self.client.encode(item))

  def __delitem__(self, key):
    """Deletes an item from the dict."""
    return self.client.hdel(self.key, key)

  def __contains__(self, key):
    """Supports using 'in' and 'not in' operators."""
    return self.has_key(key)

  def __repr__(self):
    return repr(dict(self.items()))
<|endoftext|>


<commit_before>"""
    Example usage of Memcache class:

    >>> from pony import caching
    >>> cache = caching.Memcache()
    >>> cache.set('k1', 'v1')
    True
    >>> cache.set('k2', 'v2', 10)
    True
    >>> cache.set('k3', 'v3')
    True
    >>> cache.set('k4', 'v4')
    True
    >>> cache.items()
    [('k4', 'v4'), ('k3', 'v3'), ('k2', 'v2'), ('k1', 'v1')]
    >>> cache.get('k2')
    'v2'
    >>> from time import sleep; sleep(10)
    >>> print cache.get('k2')
    None
    >>> cache.items()
    [('k4', 'v4'), ('k3', 'v3'), ('k1', 'v1')]
    >>> cache.add('k5', 'v5')
    True
    >>> cache.add('k5', 'v6')
    False
    >>> cache.items()
    [('k5', 'v5'), ('k4', 'v4'), ('k3', 'v3'), ('k1', 'v1')]
    >>> print cache.incr('x')
    None
    >>> cache.set('x', '100')
    True
    >>> cache.incr('x')
    101
    >>> cache.decr('x', 10)
    91
    >>> cache.get('k3')
    'v3'
    >>> cache.items()
    [('k3', 'v3'), ('x', '91'), ('k5', 'v5'), ('k4', 'v4'), ('k1', 'v1')]
    >>> len(cache)
    5
    >>> cache.replace('k4', 'v4_new')
    True
    >>> cache.replace('notexists', 'value')
    False
    >>> cache.items()
    [('k4', 'v4_new'), ('k3', 'v3'), ('x', '91'), ('k5', 'v5'), ('k1', 'v1')]
"""

from heapq import heappush, heappop, heapify
from threading import Lock
from time import time as gettime
from weakref import ref

from pony.utils import simple_decorator

class Node(object):
    __slots__ = 'prev', 'next', 'key', 'value', 'expire', 'access', '__weakref__'

MONTH = 30*24*60*60
MAX_KEY_LENGTH = 250
MAX_VALUE_LENGTH = 1024*1024  # 1 Megabyte

char_table = "?" * 256
noncontrol_chars = "".join(chr(i) for i in range(33, 256) if i != 127)

def normalize(key, value="", expire=None):
    if isinstance(key, tuple): hash_value, key = key
    elif not isinstance(key, str): raise ValueError('Key must be tuple or string. Got: %s' % key.__class__.__name__)
    if len(key) > MAX_KEY_LENGTH: raise ValueError('Key size too big: %d' % len(key))
    if key.translate(char_table, noncontrol_chars): raise ValueError('Key cannot contains spaces or control characters')

    if not isinstance(value, str): raise ValueError('Value must be string. Got: %s' % value.__class__.__name__)
    if len(value) > MAX_VALUE_LENGTH: raise ValueError('Value size too big: %d' % len(value))
    if expire is not None:
        expire = int(expire)
        if expire == 0: expire = None
        elif expire < 0: raise ValueError('Expiration must not be negative')
        elif expire <= MONTH: expire = int(gettime()) + expire
        elif expire <= MONTH * 100: raise ValueError('Invalid expire value: %d' % expire)
    return key, value, expire

@simple_decorator
def with_lock(func, self, *args, **keyargs):
    self._lock.acquire()
    try: return func(self, *args, **keyargs)
    finally: self._lock.release()

DEFAULT_MAX_DATA_SIZE = 64*1024*1024  # 64 Megabytes

class Memcache(object):
    def __init__(self, max_data_size=DEFAULT_MAX_DATA_SIZE):
        self._lock = Lock()
        self._heap = []
        self._dict = {}
        self._list = list = Node()
        list.prev = list.next = list.expire = list.key = list.value = None
        list.prev = list.next = list
        list.access = int(gettime())
        self._data_size = 0
        if not isinstance(max_data_size, int): raise TypeError, 'Max data size must be int. Got: %s' % type(max_data_size).__name__
        self.max_data_size = max_data_size
        self._stat_get_hits = self._stat_get_misses = self._stat_evictions = 0
        self._stat_cmd_get = self._stat_cmd_set = 0
        self._stat_total_items = 0
        self._start_time = int(gettime())
    def __len__(self):
        return len(self._dict)
    def __iter__(self):
        return iter(self.items())
    @with_lock
    def items(self):
        now = int(gettime())
        result = []
        append = result.append
        list = self._list
        node = list.next
        while node is not list:
            expire = node.expire
            if expire is not None and expire <= now: self._delete_node(node)
            elif node.value is not None: append((node.key, node.value))
            node = node.next
        return result
    def _delete_node(self, node, unlink=True):
        if unlink:
            prev, next = node.prev, node.next
            prev.next = next
            next.prev = prev
        del self._dict[node.key]
        self._data_size -= len(node.key)
        if node.value is not None: self._data_size -= len(node.value)
    def _find_node(self, key):
        node = self._dict.get(key)
        if node is None: return None
        prev, next = node.prev, node.next
        prev.next = next
        next.prev = prev
        expire = node.expire
        if expire is None or expire > int(gettime()): return node
        self._delete_node(node, unlink=False)
        return None
    def _create_node(self, key):
        self._dict[key] = node = Node()
        node.key = key
        node.value = None
        self._data_size += len(node.key)
        self._stat_total_items += 1
        return node
    def _place_on_top(self, node):
        list = self._list
        old_top = list.next
        node.prev, node.next = list, old_top
        list.next = old_top.prev = node
        node.access = int(gettime())
    def _set_node_value(self, node, value, expire):
        if node.value is not None: self._data_size -= len(node.value)
        if value is not None: self._data_size += len(value)
        node.value, node.expire = value, expire
        if expire is not None: heappush(self._heap, (expire, ref(node)))
        self._delete_expired_nodes()
        self._conform_to_limits()
        if len(self._heap) > len(self._dict) * 2: self._pack_heap()
    def _delete_expired_nodes(self):
        now = int(gettime())
        heap = self._heap
        while heap:
            expire, node_ref = heap[0]
            if expire > now: break
            heappop(heap)
            node = node_ref()
            if node is not None:
                expire = node.expire
                if expire is not None and expire <= now: self._delete_node(node)
    def _conform_to_limits(self):
        list = self._list
        while self._data_size > self.max_data_size:
            bottom = list.prev
            self._delete_node(bottom)
            self._stat_evictions += 1
    def _pack_heap(self):
        new_heap = []
        for item in self._heap:
            expire, node_ref = item
            node = node_ref()
            if node is not None and expire == node.expire: new_heap.append(item)
        heapify(new_heap)
        self._heap = new_heap
    def _get(self, key):
        self._stat_cmd_get += 1
        key, _, _ = normalize(key)
        node = self._find_node(key)
        if node is None:
            self._stat_get_misses += 1
            return None
        self._place_on_top(node)
        self._stat_get_hits += 1
        return node.value
    get = with_lock(_get)
    @with_lock
    def get_multi(self, keys, key_prefix=''):
        result = {}
        for key in keys:
            val = self._get(key_prefix + key)
            if val is not None: result[key] = val
        return result
    def _set(self, key, value, time=None):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is None: node = self._create_node(key)
        self._place_on_top(node)
        self._set_node_value(node, value, expire)
        return True
    set = with_lock(_set)
    @with_lock
    def set_multi(self, mapping, time=None, key_prefix=''):
        for key, value in mapping.iteritems():
            self._set(key_prefix + key, value, time)
        return []
    def _add(self, key, value, time=None):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is not None:
            self._place_on_top(node)
            return False
        node = self._create_node(key)
        self._place_on_top(node)
        self._set_node_value(node, value, expire)
        return True
    add = with_lock(_add)
    @with_lock
    def add_multi(self, mapping, time=None, key_prefix=''):
        result = []
        for key, value in mapping.iteritems():
            if not self._add(key_prefix + key, value, time):
                result.append(key)
        return result
    def _replace(self, key, value, time=None):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._set_node_value(node, value, expire)
        return True
    replace = with_lock(_replace)
    @with_lock
    def replace_multi(self, mapping, time=None, key_prefix=''):
        result = []
        for key, value in mapping.iteritems():
            if not self._replace(key_prefix + key, value, time):
                result.append(key)
        return result
    def _delete(self, key, seconds=None):
        self._stat_cmd_set += 1
        key, _, seconds = normalize(key, "", seconds)
        node = self._find_node(key)
        if node is None or node.value is None: return 1
        if seconds is None: self._delete_node(node)
        else:
            self._place_on_top(node)
            self._set_node_value(node, None, seconds)
        return 2
    delete = with_lock(_delete)
    @with_lock
    def delete_multi(self, keys, seconds=None, key_prefix=''):
        for key in keys:
            self._delete(key_prefix + key, seconds)
        return []
    @with_lock
    def incr(self, key, delta=1):
        self._stat_cmd_set += 1
        key, _, _ = normalize(key)
        node = self._find_node(key)
        if node is None: return None
        self._place_on_top(node)
        value = node.value
        if value is None: return None
        try: value = int(value) + delta
        except ValueError: return None
        if value < 0: value = 0
        node.value = str(value)
        return value
    def decr(self, key, delta=1):
        return self.incr(key, -delta)
    @with_lock
    def append(self, key, value):
        key, value, _ = normalize(key, value)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._data_size += len(value)
        node.value += value
        self._delete_expired_nodes()
        self._conform_to_limits()
        return True
    @with_lock
    def prepend(self, key, value):
        key, value, _ = normalize(key, value)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._data_size += len(value)
        node.value = value + node.value
        self._delete_expired_nodes()
        self._conform_to_limits()
        return True
    @with_lock
    def flush_all(self):
        self._dict.clear()
        self._heap = []
        self._list.prev = self._list.next = self._list
        for node in self._dict.itervalues():
            node.prev = node.next = None
    @with_lock
    def get_stats(self):
        now = int(gettime())
        return dict(curr_items=len(self._dict), total_items=self._stat_total_items,
                    bytes=self._data_size, limit_maxbytes=self.max_data_size,
                    get_hits=self._stat_get_hits, get_misses=self._stat_get_misses, evictions=self._stat_evictions,
                    oldest_item_age=int(gettime())-self._list.prev.access,
                    cmd_get=self._stat_cmd_get, cmd_set=self._stat_cmd_set,
                    time=now, uptime=now-self._start_time)
<commit_msg>Backward-compatible default value change: time=None -> time=0<commit_after>"""
    Example usage of Memcache class:

    >>> from pony import caching
    >>> cache = caching.Memcache()
    >>> cache.set('k1', 'v1')
    True
    >>> cache.set('k2', 'v2', 10)
    True
    >>> cache.set('k3', 'v3')
    True
    >>> cache.set('k4', 'v4')
    True
    >>> cache.items()
    [('k4', 'v4'), ('k3', 'v3'), ('k2', 'v2'), ('k1', 'v1')]
    >>> cache.get('k2')
    'v2'
    >>> from time import sleep; sleep(10)
    >>> print cache.get('k2')
    None
    >>> cache.items()
    [('k4', 'v4'), ('k3', 'v3'), ('k1', 'v1')]
    >>> cache.add('k5', 'v5')
    True
    >>> cache.add('k5', 'v6')
    False
    >>> cache.items()
    [('k5', 'v5'), ('k4', 'v4'), ('k3', 'v3'), ('k1', 'v1')]
    >>> print cache.incr('x')
    None
    >>> cache.set('x', '100')
    True
    >>> cache.incr('x')
    101
    >>> cache.decr('x', 10)
    91
    >>> cache.get('k3')
    'v3'
    >>> cache.items()
    [('k3', 'v3'), ('x', '91'), ('k5', 'v5'), ('k4', 'v4'), ('k1', 'v1')]
    >>> len(cache)
    5
    >>> cache.replace('k4', 'v4_new')
    True
    >>> cache.replace('notexists', 'value')
    False
    >>> cache.items()
    [('k4', 'v4_new'), ('k3', 'v3'), ('x', '91'), ('k5', 'v5'), ('k1', 'v1')]
"""

from heapq import heappush, heappop, heapify
from threading import Lock
from time import time as gettime
from weakref import ref

from pony.utils import simple_decorator

class Node(object):
    __slots__ = 'prev', 'next', 'key', 'value', 'expire', 'access', '__weakref__'

MONTH = 30*24*60*60
MAX_KEY_LENGTH = 250
MAX_VALUE_LENGTH = 1024*1024  # 1 Megabyte

char_table = "?" * 256
noncontrol_chars = "".join(chr(i) for i in range(33, 256) if i != 127)

def normalize(key, value="", expire=None):
    if isinstance(key, tuple): hash_value, key = key
    elif not isinstance(key, str): raise ValueError('Key must be tuple or string. Got: %s' % key.__class__.__name__)
    if len(key) > MAX_KEY_LENGTH: raise ValueError('Key size too big: %d' % len(key))
    if key.translate(char_table, noncontrol_chars): raise ValueError('Key cannot contains spaces or control characters')

    if not isinstance(value, str): raise ValueError('Value must be string. Got: %s' % value.__class__.__name__)
    if len(value) > MAX_VALUE_LENGTH: raise ValueError('Value size too big: %d' % len(value))
    if expire is not None:
        expire = int(expire)
        if expire == 0: expire = None
        elif expire < 0: raise ValueError('Expiration must not be negative')
        elif expire <= MONTH: expire = int(gettime()) + expire
        elif expire <= MONTH * 100: raise ValueError('Invalid expire value: %d' % expire)
    return key, value, expire

@simple_decorator
def with_lock(func, self, *args, **keyargs):
    self._lock.acquire()
    try: return func(self, *args, **keyargs)
    finally: self._lock.release()

DEFAULT_MAX_DATA_SIZE = 64*1024*1024  # 64 Megabytes

class Memcache(object):
    def __init__(self, max_data_size=DEFAULT_MAX_DATA_SIZE):
        self._lock = Lock()
        self._heap = []
        self._dict = {}
        self._list = list = Node()
        list.prev = list.next = list.expire = list.key = list.value = None
        list.prev = list.next = list
        list.access = int(gettime())
        self._data_size = 0
        if not isinstance(max_data_size, int): raise TypeError, 'Max data size must be int. Got: %s' % type(max_data_size).__name__
        self.max_data_size = max_data_size
        self._stat_get_hits = self._stat_get_misses = self._stat_evictions = 0
        self._stat_cmd_get = self._stat_cmd_set = 0
        self._stat_total_items = 0
        self._start_time = int(gettime())
    def __len__(self):
        return len(self._dict)
    def __iter__(self):
        return iter(self.items())
    @with_lock
    def items(self):
        now = int(gettime())
        result = []
        append = result.append
        list = self._list
        node = list.next
        while node is not list:
            expire = node.expire
            if expire is not None and expire <= now: self._delete_node(node)
            elif node.value is not None: append((node.key, node.value))
            node = node.next
        return result
    def _delete_node(self, node, unlink=True):
        if unlink:
            prev, next = node.prev, node.next
            prev.next = next
            next.prev = prev
        del self._dict[node.key]
        self._data_size -= len(node.key)
        if node.value is not None: self._data_size -= len(node.value)
    def _find_node(self, key):
        node = self._dict.get(key)
        if node is None: return None
        prev, next = node.prev, node.next
        prev.next = next
        next.prev = prev
        expire = node.expire
        if expire is None or expire > int(gettime()): return node
        self._delete_node(node, unlink=False)
        return None
    def _create_node(self, key):
        self._dict[key] = node = Node()
        node.key = key
        node.value = None
        self._data_size += len(node.key)
        self._stat_total_items += 1
        return node
    def _place_on_top(self, node):
        list = self._list
        old_top = list.next
        node.prev, node.next = list, old_top
        list.next = old_top.prev = node
        node.access = int(gettime())
    def _set_node_value(self, node, value, expire):
        if node.value is not None: self._data_size -= len(node.value)
        if value is not None: self._data_size += len(value)
        node.value, node.expire = value, expire
        if expire is not None: heappush(self._heap, (expire, ref(node)))
        self._delete_expired_nodes()
        self._conform_to_limits()
        if len(self._heap) > len(self._dict) * 2: self._pack_heap()
    def _delete_expired_nodes(self):
        now = int(gettime())
        heap = self._heap
        while heap:
            expire, node_ref = heap[0]
            if expire > now: break
            heappop(heap)
            node = node_ref()
            if node is not None:
                expire = node.expire
                if expire is not None and expire <= now: self._delete_node(node)
    def _conform_to_limits(self):
        list = self._list
        while self._data_size > self.max_data_size:
            bottom = list.prev
            self._delete_node(bottom)
            self._stat_evictions += 1
    def _pack_heap(self):
        new_heap = []
        for item in self._heap:
            expire, node_ref = item
            node = node_ref()
            if node is not None and expire == node.expire: new_heap.append(item)
        heapify(new_heap)
        self._heap = new_heap
    def _get(self, key):
        self._stat_cmd_get += 1
        key, _, _ = normalize(key)
        node = self._find_node(key)
        if node is None:
            self._stat_get_misses += 1
            return None
        self._place_on_top(node)
        self._stat_get_hits += 1
        return node.value
    get = with_lock(_get)
    @with_lock
    def get_multi(self, keys, key_prefix=''):
        result = {}
        for key in keys:
            val = self._get(key_prefix + key)
            if val is not None: result[key] = val
        return result
    def _set(self, key, value, time=0):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is None: node = self._create_node(key)
        self._place_on_top(node)
        self._set_node_value(node, value, expire)
        return True
    set = with_lock(_set)
    @with_lock
    def set_multi(self, mapping, time=0, key_prefix=''):
        for key, value in mapping.iteritems():
            self._set(key_prefix + key, value, time)
        return []
    def _add(self, key, value, time=0):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is not None:
            self._place_on_top(node)
            return False
        node = self._create_node(key)
        self._place_on_top(node)
        self._set_node_value(node, value, expire)
        return True
    add = with_lock(_add)
    @with_lock
    def add_multi(self, mapping, time=0, key_prefix=''):
        result = []
        for key, value in mapping.iteritems():
            if not self._add(key_prefix + key, value, time):
                result.append(key)
        return result
    def _replace(self, key, value, time=0):
        self._stat_cmd_set += 1
        key, value, expire = normalize(key, value, time)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._set_node_value(node, value, expire)
        return True
    replace = with_lock(_replace)
    @with_lock
    def replace_multi(self, mapping, time=0, key_prefix=''):
        result = []
        for key, value in mapping.iteritems():
            if not self._replace(key_prefix + key, value, time):
                result.append(key)
        return result
    def _delete(self, key, seconds=0):
        self._stat_cmd_set += 1
        key, _, seconds = normalize(key, "", seconds)
        node = self._find_node(key)
        if node is None or node.value is None: return 1
        if seconds is None: self._delete_node(node)
        else:
            self._place_on_top(node)
            self._set_node_value(node, None, seconds)
        return 2
    delete = with_lock(_delete)
    @with_lock
    def delete_multi(self, keys, seconds=0, key_prefix=''):
        for key in keys:
            self._delete(key_prefix + key, seconds)
        return []
    @with_lock
    def incr(self, key, delta=1):
        self._stat_cmd_set += 1
        key, _, _ = normalize(key)
        node = self._find_node(key)
        if node is None: return None
        self._place_on_top(node)
        value = node.value
        if value is None: return None
        try: value = int(value) + delta
        except ValueError: return None
        if value < 0: value = 0
        node.value = str(value)
        return value
    def decr(self, key, delta=1):
        return self.incr(key, -delta)
    @with_lock
    def append(self, key, value):
        key, value, _ = normalize(key, value)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._data_size += len(value)
        node.value += value
        self._delete_expired_nodes()
        self._conform_to_limits()
        return True
    @with_lock
    def prepend(self, key, value):
        key, value, _ = normalize(key, value)
        node = self._find_node(key)
        if node is None: return False
        self._place_on_top(node)
        if node.value is None: return False
        self._data_size += len(value)
        node.value = value + node.value
        self._delete_expired_nodes()
        self._conform_to_limits()
        return True
    @with_lock
    def flush_all(self):
        self._dict.clear()
        self._heap = []
        self._list.prev = self._list.next = self._list
        for node in self._dict.itervalues():
            node.prev = node.next = None
    @with_lock
    def get_stats(self):
        now = int(gettime())
        return dict(curr_items=len(self._dict), total_items=self._stat_total_items,
                    bytes=self._data_size, limit_maxbytes=self.max_data_size,
                    get_hits=self._stat_get_hits, get_misses=self._stat_get_misses, evictions=self._stat_evictions,
                    oldest_item_age=int(gettime())-self._list.prev.access,
                    cmd_get=self._stat_cmd_get, cmd_set=self._stat_cmd_set,
                    time=now, uptime=now-self._start_time)
<|endoftext|>


<commit_before>from datetime import datetime, timedelta

from changes.config import db, queue
from changes.constants import Status
from changes.models.build import Build

def cleanup_builds():
    """
    Look for any jobs which haven't checked in (but are listed in a pending state)
    and mark them as finished in an unknown state.
    """
    now = datetime.utcnow()
    cutoff = now - timedelta(minutes=5)

    build_list = Build.query.filter(
        Build.status != Status.finished,
        Build.date_modified < cutoff,
    )

    db.session.query(Build).filter(
        Build.id.in_(b.id for b in build_list),
    ).update({
        Build.date_modified: now,
    })

    for build in build_list:

        queue.delay('sync_build', kwargs={
            'build_id': build.id.hex,
        })
<commit_msg>Use func.now for timestamp update<commit_after>from datetime import datetime, timedelta
from sqlalchemy.sql import func

from changes.config import db, queue
from changes.constants import Status
from changes.models.build import Build

def cleanup_builds():
    """
    Look for any jobs which haven't checked in (but are listed in a pending state)
    and mark them as finished in an unknown state.
    """
    now = datetime.utcnow()
    cutoff = now - timedelta(minutes=5)

    build_list = Build.query.filter(
        Build.status != Status.finished,
        Build.date_modified < cutoff,
    )

    db.session.query(Build).filter(
        Build.id.in_(b.id for b in build_list),
    ).update({
        Build.date_modified: func.now(),
    })

    for build in build_list:

        queue.delay('sync_build', kwargs={
            'build_id': build.id.hex,
        })
<|endoftext|>


<commit_before>from gevent import Greenlet, monkey, core; monkey.patch_all()

import bottle, jsonrpc, sys

class DuplicateKeyError(Exception):
    pass

class CoinBank(object):
    def __init__(self, config):
        self.acct      = config['sericata.acct']
        self.url       = config['sericata.url']
        self.ratio     = float(config['sericata.ratio'])
        self.txfee     = float(config['sericata.txfee'])
        self.interval  = float(config['sericata.interval'])

        self.pending = {}
        self.paid = [(0,0)]
        self.pay_periods = 0
        self.last_pay_time = core.time()

        self._balance_stat = 0
        self._balance = self.balance

        self._public_address_stat = 0
        self._public_address = self.public_address

    @property
    def balance(self):
        svc = None
        while self._balance_stat != self.get_pay_status():
            if svc is None:
                svc = self.get_proxy()
            self._balance_stat = self.get_pay_status()
            self._balance = float(svc.getbalance(self.acct))
        return self._balance

    @property
    def public_address(self):
        svc = None
        while self._public_address_stat != self.get_pay_status():
            if svc is None:
                svc = self.get_proxy()
            self._public_address_stat = self.get_pay_status()
            self._public_address = svc.getaccountaddress(self.acct)
        return self._public_address

    def get_proxy(self):
        return jsonrpc.ServiceProxy(self.url)

    def get_total_pending(self):
        return sum(self.pending.values())

    def get_available(self):
        net = self.balance - self.get_total_pending() - self.txfee
        if net < 0:
            net = 0
        return net

    def get_current_payout(self):
        return min(self.ratio * self.get_available(), 1000)

    def get_pay_status(self):
        '''This method should be greenthread-atomic'''
        return (len(self.paid), sum(self.pending.values()), self.pay_periods)

    def get_public_status(self):

        time = core.time()
        last_time, last_amt = self.paid[-1]
        return {
            'current_funds':     self.get_available(),
            'current_payout':    self.get_current_payout(),
            'last_payout_time':  last_time,
            'last_payout_total': last_amt,
            'total_pay_periods': self.pay_periods,
            'payout_interval':   self.interval,
            'current_address':   self.public_address,
            'current_time':      time,
            'next_payout_time':  self.last_pay_time + self.interval,
            'next_payout_total': self.get_total_pending(),
        }

    def schedule_payment(self, addr):
        svc = self.get_proxy()
        if not svc.validateaddress(addr)['isvalid']:
            raise ValueError
        if addr in self.pending:
            raise DuplicateKeyError

        while 1:
            pay_status = self.get_pay_status()
            amt = self.get_current_payout()
            # Check to make sure nothing happened
            if pay_status == self.get_pay_status():
                break

        self.pending[addr] = amt
        return amt

def with_bank(orig_func):
    '''Function decorator to provide CoinBank instance'''
    def wrapped_func(*arg, **kwarg):
        app = bottle.default_app()
        bank = app.config['sericata.bank']
        return orig_func(bank, *arg, **kwarg)
    return wrapped_func

@with_bank
def make_payments(bank):

    bank.pay_periods += 1
    bank.last_pay_time = core.time()
    amt = bank.get_total_pending()
    Greenlet.spawn_later(bank.interval, globals()['make_payments'])

    if amt == 0:
        return False

    svc = bank.get_proxy()
    to_pay = bank.pending
    bank.pending = {}
    bank.paid += [(core.time(), amt)]

    svc.sendmany(bank.acct, to_pay)

@bottle.get('/')
@with_bank
def make_main_page(bank):
    return """
Donation address is: {}

Current payout is: {}

<form action="/payout" method="post">
  Address: <input name="addr" type="text" />
  <input value="payout" type="Submit">
</form>
""".format(bank.public_address, bank.get_current_payout())

@bottle.get('/stats')
@with_bank
def make_stats_page(bank):
    last_time, last_amt = bank.paid[-1]
    next_time = bank.last_pay_time + bank.interval
    return """
Donation address is: {}

Available funds are: {}

Current payout is: {}

Total pending payouts are: {}

Total pay periods elapsed: {}

Last payout was on {} for {}

Next payout will be in {} seconds
""".format(
    bank.public_address,
    bank.get_available(),
    bank.get_current_payout(),
    bank.get_total_pending(),
    bank.pay_periods,
    last_time,
    last_amt,
    next_time - core.time(),
)

@bottle.post('/payout')
@with_bank
def attempt_payout(bank):
    form = bottle.request.forms
    addr = form.get('addr')
    try:
        amt = bank.schedule_payment(addr)
    except ValueError:
        return bottle.HTTPResponse(status = 400, body = "Bad address: "+addr)
    except DuplicateKeyError:
        return bottle.HTTPResponse(status = 400, body = addr+" already queued")

    return "Scheduled {} to {}".format(amt, addr)

if __name__ == '__main__':
    app = bottle.default_app()
    try:
        conf_file = sys.argv[1]
    except IndexError:
        conf_file = 'sericata.conf'

    config = app.config.load_config(conf_file)

    url = 'http://%s:%s@%s:%s' % (
        config['sericata.user'],
        config['sericata.pass'],
        config['sericata.host'],
        config['sericata.port'],
    )
    config['sericata.url'] = url

    bank = CoinBank(config)
    config['sericata.bank'] = bank

    # GeventServer is more fragile than the default WSGIRefServer: extra
    # arguments must be filtered out of config
    gevent_conf = {'server': 'gevent'}
    for key, val in config.items():
        if '.' not in key and key not in ('catchall','autojson'):
            gevent_conf[key] = val

    Greenlet.spawn_later(bank.interval, make_payments)
    app.run(**gevent_conf)
<commit_msg>Use get_public_status() for stats page<commit_after>from gevent import Greenlet, monkey, core; monkey.patch_all()

import bottle, jsonrpc, sys

class DuplicateKeyError(Exception):
    pass

class CoinBank(object):
    def __init__(self, config):
        self.acct      = config['sericata.acct']
        self.url       = config['sericata.url']
        self.ratio     = float(config['sericata.ratio'])
        self.txfee     = float(config['sericata.txfee'])
        self.interval  = float(config['sericata.interval'])

        self.pending = {}
        self.paid = [(0,0)]
        self.pay_periods = 0
        self.last_pay_time = core.time()

        self._balance_stat = 0
        self._balance = self.balance

        self._public_address_stat = 0
        self._public_address = self.public_address

    @property
    def balance(self):
        svc = None
        while self._balance_stat != self.get_pay_status():
            if svc is None:
                svc = self.get_proxy()
            self._balance_stat = self.get_pay_status()
            self._balance = float(svc.getbalance(self.acct))
        return self._balance

    @property
    def public_address(self):
        svc = None
        while self._public_address_stat != self.get_pay_status():
            if svc is None:
                svc = self.get_proxy()
            self._public_address_stat = self.get_pay_status()
            self._public_address = svc.getaccountaddress(self.acct)
        return self._public_address

    def get_proxy(self):
        return jsonrpc.ServiceProxy(self.url)

    def get_total_pending(self):
        return sum(self.pending.values())

    def get_available(self):
        net = self.balance - self.get_total_pending() - self.txfee
        if net < 0:
            net = 0
        return net

    def get_current_payout(self):
        return min(self.ratio * self.get_available(), 1000)

    def get_pay_status(self):
        '''This method should be greenthread-atomic'''
        return (len(self.paid), sum(self.pending.values()), self.pay_periods)

    def get_public_status(self):

        time = core.time()
        last_time, last_amt = self.paid[-1]
        return {
            'current_funds':     self.get_available(),
            'current_payout':    self.get_current_payout(),
            'last_payout_time':  last_time,
            'last_payout_total': last_amt,
            'total_pay_periods': self.pay_periods,
            'payout_interval':   self.interval,
            'current_address':   self.public_address,
            'current_time':      time,
            'next_payout_time':  self.last_pay_time + self.interval,
            'next_payout_total': self.get_total_pending(),
        }

    def schedule_payment(self, addr):
        svc = self.get_proxy()
        if not svc.validateaddress(addr)['isvalid']:
            raise ValueError
        if addr in self.pending:
            raise DuplicateKeyError

        while 1:
            pay_status = self.get_pay_status()
            amt = self.get_current_payout()
            # Check to make sure nothing happened
            if pay_status == self.get_pay_status():
                break

        self.pending[addr] = amt
        return amt

def with_bank(orig_func):
    '''Function decorator to provide CoinBank instance'''
    def wrapped_func(*arg, **kwarg):
        app = bottle.default_app()
        bank = app.config['sericata.bank']
        return orig_func(bank, *arg, **kwarg)
    return wrapped_func

@with_bank
def make_payments(bank):

    bank.pay_periods += 1
    bank.last_pay_time = core.time()
    amt = bank.get_total_pending()
    Greenlet.spawn_later(bank.interval, globals()['make_payments'])

    if amt == 0:
        return False

    svc = bank.get_proxy()
    to_pay = bank.pending
    bank.pending = {}
    bank.paid += [(core.time(), amt)]

    svc.sendmany(bank.acct, to_pay)

@bottle.get('/')
@with_bank
def make_main_page(bank):
    return """
Donation address is: {}

Current payout is: {}

<form action="/payout" method="post">
  Address: <input name="addr" type="text" />
  <input value="payout" type="Submit">
</form>
""".format(bank.public_address, bank.get_current_payout())

@bottle.get('/stats')
@with_bank
def make_stats_page(bank):
    return """
Donation address is: {current_address}

Available funds are: {current_funds}

Current payout is: {current_payout}

Total pending payouts are: {next_payout_total}

Total pay periods elapsed: {total_pay_periods}

Last payout was on {last_payout_time} for {last_payout_total}

Next payout will be at {next_payout_time} for {next_payout_total}

Current server time is {current_time}
""".format(**bank.get_public_status())

@bottle.post('/payout')
@with_bank
def attempt_payout(bank):
    form = bottle.request.forms
    addr = form.get('addr')
    try:
        amt = bank.schedule_payment(addr)
    except ValueError:
        return bottle.HTTPResponse(status = 400, body = "Bad address: "+addr)
    except DuplicateKeyError:
        return bottle.HTTPResponse(status = 400, body = addr+" already queued")

    return "Scheduled {} to {}".format(amt, addr)

if __name__ == '__main__':
    app = bottle.default_app()
    try:
        conf_file = sys.argv[1]
    except IndexError:
        conf_file = 'sericata.conf'

    config = app.config.load_config(conf_file)

    url = 'http://%s:%s@%s:%s' % (
        config['sericata.user'],
        config['sericata.pass'],
        config['sericata.host'],
        config['sericata.port'],
    )
    config['sericata.url'] = url

    bank = CoinBank(config)
    config['sericata.bank'] = bank

    # GeventServer is more fragile than the default WSGIRefServer: extra
    # arguments must be filtered out of config
    gevent_conf = {'server': 'gevent'}
    for key, val in config.items():
        if '.' not in key and key not in ('catchall','autojson'):
            gevent_conf[key] = val

    Greenlet.spawn_later(bank.interval, make_payments)
    app.run(**gevent_conf)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '0.8.2'
<commit_msg>Updated version.<commit_after># -*- coding: utf-8 -*-

__version__ = '0.9.0'
<|endoftext|>