<commit_before>#!/usr/bin/python
# -*- encoding: utf-8 -*-
"""

"""
from datetime import datetime, date, time
import collections
import itertools
from sqlalchemy.orm import object_mapper
from sqlalchemy.orm.exc import UnmappedInstanceError
from sqlalchemy.orm.query import Query

from .errors import IllegalArgumentError, DictConvertionError
from .wrapper import ModelWrapper

__author__ = 'Martin Martimeo <martin@martimeo.de>'
__date__ = '23.05.13 - 17:41'

__datetypes__ = (datetime, time, date)
__basetypes__ = (str, int, bool, float)

def to_filter(instance,
              filters=None,
              order_by=None):
    """
        Returns a list of filters made by arguments

        :param instance:
        :param filters: List of filters in restless 3-tuple op string format
        :param order_by: List of orders to be appended aswell
    """

    # Get all provided filters
    argument_filters = filters and filters or []

    # Parse order by as filters
    argument_orders = order_by and order_by or []
    for argument_order in argument_orders:
        direction = argument_order['direction']
        if direction not in ["asc", "desc"]:
            raise IllegalArgumentError("Direction unknown")
        argument_filters.append({'name': argument_order['field'], 'op': direction})

    # Create Alchemy Filters
    alchemy_filters = []
    for argument_filter in argument_filters:

        # Resolve right attribute
        if "field" in argument_filter.keys():
            right = getattr(instance, argument_filter["field"])
        elif "val" in argument_filter.keys():
            right = argument_filter["val"]
        elif "value" in argument_filter.keys():  # Because we hate abbr sometimes ...
            right = argument_filter["value"]
        else:
            right = None

        # Operator
        op = argument_filter["op"]

        # Resolve left attribute
        if "name" not in argument_filter:
            raise IllegalArgumentError("Missing fieldname attribute 'name'")

        if "__" in argument_filter["name"] or "." in argument_filter["name"]:
            relation, _, name = argument_filter["name"].replace("__", ".").partition(".")
            left = getattr(instance, relation)
            op = "has"
            argument_filter["name"] = name
            right = to_filter(instance=left.property.mapper.class_, filters=[argument_filter])
        elif argument_filter["name"] == "~":
            left = instance
            op = "attr_is"
        else:
            left = getattr(instance, argument_filter["name"])

        # Operators from flask-restless
        if op in ["is_null"]:
            alchemy_filters.append(left.is_(None))
        elif op in ["is_not_null"]:
            alchemy_filters.append(left.isnot(None))
        elif op in ["is"]:
            alchemy_filters.append(left.is_(right))
        elif op in ["is_not"]:
            alchemy_filters.append(left.isnot(right))
        elif op in ["==", "eq", "equals", "equals_to"]:
            alchemy_filters.append(left == right)
        elif op in ["!=", "ne", "neq", "not_equal_to", "does_not_equal"]:
            alchemy_filters.append(left != right)
        elif op in [">", "gt"]:
            alchemy_filters.append(left > right)
        elif op in ["<", "lt"]:
            alchemy_filters.append(left < right)
        elif op in [">=", "ge", "gte", "geq"]:
            alchemy_filters.append(left >= right)
        elif op in ["<=", "le", "lte", "leq"]:
            alchemy_filters.append(left <= right)
        elif op in ["ilike"]:
            alchemy_filters.append(left.ilike(right))
        elif op in ["not_ilike"]:
            alchemy_filters.append(left.notilike(right))
        elif op in ["like"]:
            alchemy_filters.append(left.like(right))
        elif op in ["not_like"]:
            alchemy_filters.append(left.notlike(right))
        elif op in ["match"]:
            alchemy_filters.append(left.match(right))
        elif op in ["in"]:
            alchemy_filters.append(left.in_(right))
        elif op in ["not_in"]:
            alchemy_filters.append(left.notin_(right))
        elif op in ["has"] and isinstance(right, list):
            alchemy_filters.append(left.has(*right))
        elif op in ["has"]:
            alchemy_filters.append(left.has(right))
        elif op in ["any"]:
            alchemy_filters.append(left.any(right))

        # Additional Operators
        elif op in ["between"]:
            alchemy_filters.append(left.between(*right))
        elif op in ["contains"]:
            alchemy_filters.append(left.contains(right))
        elif op in ["startswith"]:
            alchemy_filters.append(left.startswith(right))
        elif op in ["endswith"]:
            alchemy_filters.append(left.endswith(right))

        # Order By Operators
        elif op in ["asc"]:
            alchemy_filters.append(left.asc())
        elif op in ["desc"]:
            alchemy_filters.append(left.desc())

        # Additional Checks
        elif op in ["attr_is"]:
            alchemy_filters.append(getattr(left, right))
        elif op in ["method_is"]:
            alchemy_filters.append(getattr(left, right)())

        # Test comparator
        elif hasattr(left.comparator, op):
            alchemy_filters.append(getattr(left.comparator, op)(right))

        # Raise Exception
        else:
            raise IllegalArgumentError("Unknown operator")
    return alchemy_filters

def to_deep(include,
            exclude,
            key):
    """
        Extract the include/exclude information for key

        :param include: Columns and Relations that should be included for an instance
        :param exclude: Columns and Relations that should not be included for an instance
        :param key: The key that should be extracted
    """
    rtn = {}

    try:
        rtn['include'] = include.setdefault(key, False)
    except AttributeError:
        rtn['include'] = False

    try:
        rtn['exclude'] = exclude[key]
    except TypeError:
        rtn['exclude'] = None

    return rtn

def to_dict(instance,
            options=collections.defaultdict(bool),
            include=None,
            exclude=None):
    """
        Translates sqlalchemy instance to dictionary

        Inspired by flask-restless.helpers.to_dict

        :param instance:
        :param options: Dictionary of flags
                          * execute_queries: Execute Query Objects
                          * execute_hybrids: Execute Hybrids
        :param include: Columns and Relations that should be included for an instance
        :param exclude: Columns and Relations that should not be included for an instance
    """
    if exclude is not None and include is not None:
        raise ValueError('Cannot specify both include and exclude.')

    # None
    if instance is None:
        return None

    # Int / Float / Str
    if isinstance(instance, __basetypes__):
        return instance

    # Date & Time
    if isinstance(instance, __datetypes__):
        return instance.isoformat()

    # Any Dictionary
    if isinstance(instance, dict) or hasattr(instance, 'items'):
        return {k: to_dict(v, options=options, **to_deep(include, exclude, k)) for k, v in instance.items()}

    # Any List
    if isinstance(instance, list) or hasattr(instance, '__iter__'):
        return [to_dict(x, options=options, include=include, exclude=exclude) for x in instance]

    # Include Columns given
    if isinstance(include, collections.Iterable):
        rtn = {}
        for column in include:
            rtn[column] = to_dict(getattr(instance, column), **to_deep(include, exclude, column))
        return rtn

    # Include all columns if it is a SQLAlchemy instance
    try:
        columns = ModelWrapper.get_columns(object_mapper(instance)).keys()
        relations = ModelWrapper.get_relations(object_mapper(instance)).keys()
        attributes = ModelWrapper.get_attributes(object_mapper(instance)).keys()
        proxies = [p.key for p in ModelWrapper.get_proxies(object_mapper(instance))]
        hybrids = [p.key for p in ModelWrapper.get_hybrids(object_mapper(instance))]
        attributes = itertools.chain(columns, relations, proxies, hybrids, attributes)
    except UnmappedInstanceError:
        raise DictConvertionError("Could not convert argument to plain dict")

    rtn = {}

    # Include AssociationProxy and Hybrids (may be list/dict/col)
    for column in attributes:

        if exclude is not None and column in exclude:
            continue
        if column in rtn:
            continue

        # Prevent unnec. db calls
        if include is False and column not in hybrids and column not in columns:
            continue

        if column not in instance.__dict__ and not options.get('execute_queries', True):
            if column not in hybrids or not options.get('execute_hybrids', True):
                continue

        # Get Attribute
        node = getattr(instance, column)

        # Don't execute queries if stopping deepnes
        if include is False and isinstance(node, Query):
            continue
        # Otherwise query it
        elif isinstance(node, Query) and options['execute_queries']:
            node = node.all()

        # Convert it
        rtn[column] = to_dict(node, **to_deep(include, exclude, column))
    return rtn<commit_msg>Fix has for collections to be any<commit_after>#!/usr/bin/python
# -*- encoding: utf-8 -*-
"""

"""
from datetime import datetime, date, time
import collections
import itertools
from sqlalchemy.orm import object_mapper
from sqlalchemy.orm.exc import UnmappedInstanceError
from sqlalchemy.orm.query import Query

from .errors import IllegalArgumentError, DictConvertionError
from .wrapper import ModelWrapper

__author__ = 'Martin Martimeo <martin@martimeo.de>'
__date__ = '23.05.13 - 17:41'

__datetypes__ = (datetime, time, date)
__basetypes__ = (str, int, bool, float)

def to_filter(instance,
              filters=None,
              order_by=None):
    """
        Returns a list of filters made by arguments

        :param instance:
        :param filters: List of filters in restless 3-tuple op string format
        :param order_by: List of orders to be appended aswell
    """

    # Get all provided filters
    argument_filters = filters and filters or []

    # Parse order by as filters
    argument_orders = order_by and order_by or []
    for argument_order in argument_orders:
        direction = argument_order['direction']
        if direction not in ["asc", "desc"]:
            raise IllegalArgumentError("Direction unknown")
        argument_filters.append({'name': argument_order['field'], 'op': direction})

    # Create Alchemy Filters
    alchemy_filters = []
    for argument_filter in argument_filters:

        # Resolve right attribute
        if "field" in argument_filter.keys():
            right = getattr(instance, argument_filter["field"])
        elif "val" in argument_filter.keys():
            right = argument_filter["val"]
        elif "value" in argument_filter.keys():  # Because we hate abbr sometimes ...
            right = argument_filter["value"]
        else:
            right = None

        # Operator
        op = argument_filter["op"]

        # Resolve left attribute
        if "name" not in argument_filter:
            raise IllegalArgumentError("Missing fieldname attribute 'name'")

        if "__" in argument_filter["name"] or "." in argument_filter["name"]:
            relation, _, name = argument_filter["name"].replace("__", ".").partition(".")
            left = getattr(instance, relation)
            op = "has"
            argument_filter["name"] = name
            right = to_filter(instance=left.property.mapper.class_, filters=[argument_filter])
        elif argument_filter["name"] == "~":
            left = instance
            op = "attr_is"
        else:
            left = getattr(instance, argument_filter["name"])

        # Operators from flask-restless
        if op in ["is_null"]:
            alchemy_filters.append(left.is_(None))
        elif op in ["is_not_null"]:
            alchemy_filters.append(left.isnot(None))
        elif op in ["is"]:
            alchemy_filters.append(left.is_(right))
        elif op in ["is_not"]:
            alchemy_filters.append(left.isnot(right))
        elif op in ["==", "eq", "equals", "equals_to"]:
            alchemy_filters.append(left == right)
        elif op in ["!=", "ne", "neq", "not_equal_to", "does_not_equal"]:
            alchemy_filters.append(left != right)
        elif op in [">", "gt"]:
            alchemy_filters.append(left > right)
        elif op in ["<", "lt"]:
            alchemy_filters.append(left < right)
        elif op in [">=", "ge", "gte", "geq"]:
            alchemy_filters.append(left >= right)
        elif op in ["<=", "le", "lte", "leq"]:
            alchemy_filters.append(left <= right)
        elif op in ["ilike"]:
            alchemy_filters.append(left.ilike(right))
        elif op in ["not_ilike"]:
            alchemy_filters.append(left.notilike(right))
        elif op in ["like"]:
            alchemy_filters.append(left.like(right))
        elif op in ["not_like"]:
            alchemy_filters.append(left.notlike(right))
        elif op in ["match"]:
            alchemy_filters.append(left.match(right))
        elif op in ["in"]:
            alchemy_filters.append(left.in_(right))
        elif op in ["not_in"]:
            alchemy_filters.append(left.notin_(right))
        elif op in ["has"] and isinstance(right, list):
            alchemy_filters.append(left.any(*right))
        elif op in ["has"]:
            alchemy_filters.append(left.has(right))
        elif op in ["any"]:
            alchemy_filters.append(left.any(right))

        # Additional Operators
        elif op in ["between"]:
            alchemy_filters.append(left.between(*right))
        elif op in ["contains"]:
            alchemy_filters.append(left.contains(right))
        elif op in ["startswith"]:
            alchemy_filters.append(left.startswith(right))
        elif op in ["endswith"]:
            alchemy_filters.append(left.endswith(right))

        # Order By Operators
        elif op in ["asc"]:
            alchemy_filters.append(left.asc())
        elif op in ["desc"]:
            alchemy_filters.append(left.desc())

        # Additional Checks
        elif op in ["attr_is"]:
            alchemy_filters.append(getattr(left, right))
        elif op in ["method_is"]:
            alchemy_filters.append(getattr(left, right)())

        # Test comparator
        elif hasattr(left.comparator, op):
            alchemy_filters.append(getattr(left.comparator, op)(right))

        # Raise Exception
        else:
            raise IllegalArgumentError("Unknown operator")
    return alchemy_filters

def to_deep(include,
            exclude,
            key):
    """
        Extract the include/exclude information for key

        :param include: Columns and Relations that should be included for an instance
        :param exclude: Columns and Relations that should not be included for an instance
        :param key: The key that should be extracted
    """
    rtn = {}

    try:
        rtn['include'] = include.setdefault(key, False)
    except AttributeError:
        rtn['include'] = False

    try:
        rtn['exclude'] = exclude[key]
    except TypeError:
        rtn['exclude'] = None

    return rtn

def to_dict(instance,
            options=collections.defaultdict(bool),
            include=None,
            exclude=None):
    """
        Translates sqlalchemy instance to dictionary

        Inspired by flask-restless.helpers.to_dict

        :param instance:
        :param options: Dictionary of flags
                          * execute_queries: Execute Query Objects
                          * execute_hybrids: Execute Hybrids
        :param include: Columns and Relations that should be included for an instance
        :param exclude: Columns and Relations that should not be included for an instance
    """
    if exclude is not None and include is not None:
        raise ValueError('Cannot specify both include and exclude.')

    # None
    if instance is None:
        return None

    # Int / Float / Str
    if isinstance(instance, __basetypes__):
        return instance

    # Date & Time
    if isinstance(instance, __datetypes__):
        return instance.isoformat()

    # Any Dictionary
    if isinstance(instance, dict) or hasattr(instance, 'items'):
        return {k: to_dict(v, options=options, **to_deep(include, exclude, k)) for k, v in instance.items()}

    # Any List
    if isinstance(instance, list) or hasattr(instance, '__iter__'):
        return [to_dict(x, options=options, include=include, exclude=exclude) for x in instance]

    # Include Columns given
    if isinstance(include, collections.Iterable):
        rtn = {}
        for column in include:
            rtn[column] = to_dict(getattr(instance, column), **to_deep(include, exclude, column))
        return rtn

    # Include all columns if it is a SQLAlchemy instance
    try:
        columns = ModelWrapper.get_columns(object_mapper(instance)).keys()
        relations = ModelWrapper.get_relations(object_mapper(instance)).keys()
        attributes = ModelWrapper.get_attributes(object_mapper(instance)).keys()
        proxies = [p.key for p in ModelWrapper.get_proxies(object_mapper(instance))]
        hybrids = [p.key for p in ModelWrapper.get_hybrids(object_mapper(instance))]
        attributes = itertools.chain(columns, relations, proxies, hybrids, attributes)
    except UnmappedInstanceError:
        raise DictConvertionError("Could not convert argument to plain dict")

    rtn = {}

    # Include AssociationProxy and Hybrids (may be list/dict/col)
    for column in attributes:

        if exclude is not None and column in exclude:
            continue
        if column in rtn:
            continue

        # Prevent unnec. db calls
        if include is False and column not in hybrids and column not in columns:
            continue

        if column not in instance.__dict__ and not options.get('execute_queries', True):
            if column not in hybrids or not options.get('execute_hybrids', True):
                continue

        # Get Attribute
        node = getattr(instance, column)

        # Don't execute queries if stopping deepnes
        if include is False and isinstance(node, Query):
            continue
        # Otherwise query it
        elif isinstance(node, Query) and options['execute_queries']:
            node = node.all()

        # Convert it
        rtn[column] = to_dict(node, **to_deep(include, exclude, column))
    return rtn<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys, re, errno, stat, time, math
import hashsplit, git, options, index, client
from helpers import *

optspec = """
bup save [-tc] [-n name] <filenames...>
--
r,remote=  remote repository path
t,tree     output a tree id
c,commit   output a commit id
n,name=    name of backup set to update (if any)
v,verbose  increase log output (can be used more than once)
q,quiet    don't show progress meter
smaller=   only back up files smaller than n bytes
"""
o = options.Options('bup save', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

git.check_repo_or_die()
if not (opt.tree or opt.commit or opt.name):
    log("bup save: use one or more of -t, -c, -n\n")
    o.usage()
if not extra:
    log("bup save: no filenames given.\n")
    o.usage()

opt.progress = (istty and not opt.quiet)

refname = opt.name and 'refs/heads/%s' % opt.name or None
if opt.remote:
    cli = client.Client(opt.remote)
    oldref = refname and cli.read_ref(refname) or None
    w = cli.new_packwriter()
else:
    cli = None
    oldref = refname and git.read_ref(refname) or None
    w = git.PackWriter()

def eatslash(dir):
    if dir.endswith('/'):
        return dir[:-1]
    else:
        return dir

parts = ['']
shalists = [[]]

def _push(part):
    assert(part)
    parts.append(part)
    shalists.append([])

def _pop():
    assert(len(parts) > 1)
    part = parts.pop()
    shalist = shalists.pop()
    tree = w.new_tree(shalist)
    shalists[-1].append(('40000', part, tree))

lastremain = None
def progress_report(n):
    global count, lastremain
    count += n
    pct = count*100.0/total
    now = time.time()
    elapsed = now - tstart
    kps = elapsed and int(count/1024./elapsed)
    kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
    kps = int(kps/kps_frac)*kps_frac
    if count:
        remain = elapsed*1.0/count * (total-count)
    else:
        remain = 0.0
    if (lastremain and (remain > lastremain)
          and ((remain - lastremain)/lastremain < 0.05)):
        remain = lastremain
    else:
        lastremain = remain
    hours = int(remain/60/60)
    mins = int(remain/60 - hours*60)
    secs = int(remain - hours*60*60 - mins*60)
    if elapsed < 30:
        remainstr = ''
        kpsstr = ''
    else:
        kpsstr = '%dk/s' % kps
        if hours:
            remainstr = '%dh%dm' % (hours, mins)
        elif mins:
            remainstr = '%dm%d' % (mins, secs)
        else:
            remainstr = '%ds' % secs
    progress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s     \r'
             % (pct, count/1024, total/1024, fcount, ftotal,
                remainstr, kpsstr))

r = index.Reader(git.repo('bupindex'))

total = ftotal = 0
if opt.progress:
    for (transname,ent) in r.filter(extra):
        if not (ftotal % 10024):
            progress('Reading index: %d\r' % ftotal)
        exists = (ent.flags & index.IX_EXISTS)
        hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
        if exists and not hashvalid:
            total += ent.size
        ftotal += 1
    progress('Reading index: %d, done.\n' % ftotal)
    hashsplit.progress_callback = progress_report

tstart = time.time()
count = fcount = 0
for (transname,ent) in r.filter(extra):
    (dir, file) = os.path.split(ent.name)
    exists = (ent.flags & index.IX_EXISTS)
    hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
    if opt.verbose:
        if not exists:
            status = 'D'
        elif not hashvalid:
            if ent.sha == index.EMPTY_SHA:
                status = 'A'
            else:
                status = 'M'
        else:
            status = ' '
        if opt.verbose >= 2 or stat.S_ISDIR(ent.mode):
            log('%s %-70s\n' % (status, ent.name))

    if opt.progress:
        progress_report(0)
    fcount += 1

    if not exists:
        continue

    assert(dir.startswith('/'))
    dirp = dir.split('/')
    while parts > dirp:
        _pop()
    if dir != '/':
        for part in dirp[len(parts):]:
            _push(part)

    if not file:
        # directory already handled.
        # FIXME: not using the indexed tree sha1's for anything, which is
        # a waste.  That's a potential optimization...
        count += ent.size
        continue

    id = None
    if hashvalid:
        mode = '%o' % ent.mode
        id = ent.sha
        shalists[-1].append((mode, file, id))
    elif opt.smaller and ent.size >= opt.smaller:
        add_error('skipping large file "%s"' % ent.name)
    else:
        if stat.S_ISREG(ent.mode):
            try:
                f = open(ent.name)
            except IOError, e:
                add_error(e)
            except OSError, e:
                add_error(e)
            else:
                (mode, id) = hashsplit.split_to_blob_or_tree(w, [f])
        else:
            if stat.S_ISDIR(ent.mode):
                assert(0)  # handled above
            elif stat.S_ISLNK(ent.mode):
                try:
                    rl = os.readlink(ent.name)
                except OSError, e:
                    add_error(e)
                except IOError, e:
                    add_error(e)
                else:
                    (mode, id) = ('120000', w.new_blob(rl))
            else:
                add_error(Exception('skipping special file "%s"' % ent.name))
            count += ent.size
        if id:
            ent.validate(id)
            ent.repack()
            shalists[-1].append((mode, file, id))

if opt.progress:
    pct = total and count*100.0/total or 100
    progress('Saving: %.2f%% (%d/%dk, %d/%d files), done.    \n'
             % (pct, count/1024, total/1024, fcount, ftotal))

#log('parts out: %r\n' % parts)
#log('stk out: %r\n' % shalists)
while len(parts) > 1:
    _pop()
#log('parts out: %r\n' % parts)
#log('stk out: %r\n' % shalists)
assert(len(shalists) == 1)
tree = w.new_tree(shalists[-1])
if opt.tree:
    print tree.encode('hex')
if opt.commit or opt.name:
    msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
    ref = opt.name and ('refs/heads/%s' % opt.name) or None
    commit = w.new_commit(oldref, tree, msg)
    if opt.commit:
        print commit.encode('hex')

w.close()  # must close before we can update the ref

if opt.name:
    if cli:
        cli.update_ref(refname, commit, oldref)
    else:
        git.update_ref(refname, commit, oldref)

if cli:
    cli.close()

if saved_errors:
    log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
<commit_msg>cmd-save: fix a potential divide by zero error.<commit_after>#!/usr/bin/env python
import sys, re, errno, stat, time, math
import hashsplit, git, options, index, client
from helpers import *

optspec = """
bup save [-tc] [-n name] <filenames...>
--
r,remote=  remote repository path
t,tree     output a tree id
c,commit   output a commit id
n,name=    name of backup set to update (if any)
v,verbose  increase log output (can be used more than once)
q,quiet    don't show progress meter
smaller=   only back up files smaller than n bytes
"""
o = options.Options('bup save', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])

git.check_repo_or_die()
if not (opt.tree or opt.commit or opt.name):
    log("bup save: use one or more of -t, -c, -n\n")
    o.usage()
if not extra:
    log("bup save: no filenames given.\n")
    o.usage()

opt.progress = (istty and not opt.quiet)

refname = opt.name and 'refs/heads/%s' % opt.name or None
if opt.remote:
    cli = client.Client(opt.remote)
    oldref = refname and cli.read_ref(refname) or None
    w = cli.new_packwriter()
else:
    cli = None
    oldref = refname and git.read_ref(refname) or None
    w = git.PackWriter()

def eatslash(dir):
    if dir.endswith('/'):
        return dir[:-1]
    else:
        return dir

parts = ['']
shalists = [[]]

def _push(part):
    assert(part)
    parts.append(part)
    shalists.append([])

def _pop():
    assert(len(parts) > 1)
    part = parts.pop()
    shalist = shalists.pop()
    tree = w.new_tree(shalist)
    shalists[-1].append(('40000', part, tree))

lastremain = None
def progress_report(n):
    global count, lastremain
    count += n
    pct = total and (count*100.0/total) or 0
    now = time.time()
    elapsed = now - tstart
    kps = elapsed and int(count/1024./elapsed)
    kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
    kps = int(kps/kps_frac)*kps_frac
    if count:
        remain = elapsed*1.0/count * (total-count)
    else:
        remain = 0.0
    if (lastremain and (remain > lastremain)
          and ((remain - lastremain)/lastremain < 0.05)):
        remain = lastremain
    else:
        lastremain = remain
    hours = int(remain/60/60)
    mins = int(remain/60 - hours*60)
    secs = int(remain - hours*60*60 - mins*60)
    if elapsed < 30:
        remainstr = ''
        kpsstr = ''
    else:
        kpsstr = '%dk/s' % kps
        if hours:
            remainstr = '%dh%dm' % (hours, mins)
        elif mins:
            remainstr = '%dm%d' % (mins, secs)
        else:
            remainstr = '%ds' % secs
    progress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s     \r'
             % (pct, count/1024, total/1024, fcount, ftotal,
                remainstr, kpsstr))

r = index.Reader(git.repo('bupindex'))

total = ftotal = 0
if opt.progress:
    for (transname,ent) in r.filter(extra):
        if not (ftotal % 10024):
            progress('Reading index: %d\r' % ftotal)
        exists = (ent.flags & index.IX_EXISTS)
        hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
        if exists and not hashvalid:
            total += ent.size
        ftotal += 1
    progress('Reading index: %d, done.\n' % ftotal)
    hashsplit.progress_callback = progress_report

tstart = time.time()
count = fcount = 0
for (transname,ent) in r.filter(extra):
    (dir, file) = os.path.split(ent.name)
    exists = (ent.flags & index.IX_EXISTS)
    hashvalid = (ent.flags & index.IX_HASHVALID) and w.exists(ent.sha)
    if opt.verbose:
        if not exists:
            status = 'D'
        elif not hashvalid:
            if ent.sha == index.EMPTY_SHA:
                status = 'A'
            else:
                status = 'M'
        else:
            status = ' '
        if opt.verbose >= 2 or stat.S_ISDIR(ent.mode):
            log('%s %-70s\n' % (status, ent.name))

    if opt.progress:
        progress_report(0)
    fcount += 1

    if not exists:
        continue

    assert(dir.startswith('/'))
    dirp = dir.split('/')
    while parts > dirp:
        _pop()
    if dir != '/':
        for part in dirp[len(parts):]:
            _push(part)

    if not file:
        # directory already handled.
        # FIXME: not using the indexed tree sha1's for anything, which is
        # a waste.  That's a potential optimization...
        count += ent.size
        continue

    id = None
    if hashvalid:
        mode = '%o' % ent.mode
        id = ent.sha
        shalists[-1].append((mode, file, id))
    elif opt.smaller and ent.size >= opt.smaller:
        add_error('skipping large file "%s"' % ent.name)
    else:
        if stat.S_ISREG(ent.mode):
            try:
                f = open(ent.name)
            except IOError, e:
                add_error(e)
            except OSError, e:
                add_error(e)
            else:
                (mode, id) = hashsplit.split_to_blob_or_tree(w, [f])
        else:
            if stat.S_ISDIR(ent.mode):
                assert(0)  # handled above
            elif stat.S_ISLNK(ent.mode):
                try:
                    rl = os.readlink(ent.name)
                except OSError, e:
                    add_error(e)
                except IOError, e:
                    add_error(e)
                else:
                    (mode, id) = ('120000', w.new_blob(rl))
            else:
                add_error(Exception('skipping special file "%s"' % ent.name))
            count += ent.size
        if id:
            ent.validate(id)
            ent.repack()
            shalists[-1].append((mode, file, id))

if opt.progress:
    pct = total and count*100.0/total or 100
    progress('Saving: %.2f%% (%d/%dk, %d/%d files), done.    \n'
             % (pct, count/1024, total/1024, fcount, ftotal))

#log('parts out: %r\n' % parts)
#log('stk out: %r\n' % shalists)
while len(parts) > 1:
    _pop()
#log('parts out: %r\n' % parts)
#log('stk out: %r\n' % shalists)
assert(len(shalists) == 1)
tree = w.new_tree(shalists[-1])
if opt.tree:
    print tree.encode('hex')
if opt.commit or opt.name:
    msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
    ref = opt.name and ('refs/heads/%s' % opt.name) or None
    commit = w.new_commit(oldref, tree, msg)
    if opt.commit:
        print commit.encode('hex')

w.close()  # must close before we can update the ref

if opt.name:
    if cli:
        cli.update_ref(refname, commit, oldref)
    else:
        git.update_ref(refname, commit, oldref)

if cli:
    cli.close()

if saved_errors:
    log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
<|endoftext|>


<commit_before>from abc import ABC, abstractmethod

import cvxpy as cvx
import numpy as np
import scipy.stats
from cvxpy import SolverError

class MLProblem(ABC):

    def __init__(self, **kwargs):

        self.chosen_parameters_ = {}

        for p in self.parameters():
            if p in kwargs:
                if kwargs[p] is not None:
                    self.chosen_parameters_[p] = kwargs[p]

        self.relax_factors_ = {}
        for p in self.relax_factors():
            if p in kwargs:
                if kwargs[p] is not None:
                    self.relax_factors_[p] = kwargs[p]

    @classmethod
    @abstractmethod
    def parameters(cls):
        raise NotImplementedError

    def get_chosen_parameter(self, p):
        try:
            return [self.chosen_parameters_[p]]  # We return list for param search function
        except:
            return scipy.stats.reciprocal(a=1e-3, b=1e3)

    def get_all_parameters(self):
        return {p: self.get_chosen_parameter(p) for p in self.parameters()}

    @classmethod
    @abstractmethod
    def relax_factors(cls):
        raise NotImplementedError

    def get_chosen_relax_factors(self, p):
        try:
            factor = self.relax_factors_[p]
        except KeyError:
            try:
                factor = self.relax_factors_[p + "_slack"]
            except KeyError:
                factor = 0.01
        assert factor > 0
        return factor

    def get_all_relax_factors(self):
        return {p: self.get_chosen_relax_factors(p) for p in self.relax_factors()}

    @classmethod
    @abstractmethod
    def get_init_model(cls):
        pass

    @classmethod
    @abstractmethod
    def get_bound_model(cls):
        pass

    @abstractmethod
    def preprocessing(self, data, lupi_features=None):
        return data

    def postprocessing(self, bounds):
        return bounds

    def get_relaxed_constraints(self, constraints):
        return {c: self.relax_constraint(c, v) for c, v in constraints.items()}

    def relax_constraint(self, key, value):
        return value * (1 + self.get_chosen_relax_factors(key))

class Relevance_CVXProblem(ABC):

    def __str__(self) -> str:
        if self.isLowerBound:
            lower = "Lower"
        else:
            lower = "Upper"
        if self.sign == True:
            sign = "(+)"
        elif self.sign == False:
            sign = "(-)"
        else:
            sign = ""
        name = f"{sign}{lower}_{self.current_feature}_{self.__class__.__name__}"
        state = ""
        for s in self.init_hyperparameters.items():
            state += f"{s[0]}:{s[1]}, "
        for s in self.init_model_constraints.items():
            state += f"{s[0]}:{s[1]}, "
        state = "(" + state[:-2] + ")"
        if self.isProbe:
            prefix = "Probe_"
        else:
            prefix = ""
        return prefix + name + state

    def __init__(self, isLowerBound: bool, current_feature: int, data: tuple, hyperparameters, best_model_constraints,
                 sign: bool = False, preset_model=None, best_model_state=None, isProbe=False) -> None:
        self.isLowerBound = isLowerBound
        self.isProbe = isProbe

        # General data
        self.sign = sign
        self.current_feature = current_feature
        self.preset_model = preset_model
        self.best_model_state = best_model_state

        self.preprocessing_data(data, best_model_state)

        # Initialize constraints
        self._constraints = []
        self._objective = None
        self._is_solved = False
        self._init_constraints(hyperparameters, best_model_constraints)
        self._init_objective(isLowerBound)

        if self.preset_model is not None:
            self._add_preset_constraints(self.preset_model, self.best_model_state, best_model_constraints)

        self.init_hyperparameters = hyperparameters
        self.init_model_constraints = best_model_constraints

    def preprocessing_data(self, data, best_model_state):
        X, y = data
        self.n = X.shape[0]
        self.d = X.shape[1]
        self.X = X
        self.y = np.array(y)

    @property
    def constraints(self):
        return self._constraints

    def add_constraint(self, new):
        self._constraints.append(new)

    @property
    def objective(self):
        return self._objective

    @abstractmethod
    def _init_constraints(self, parameters, init_model_constraints):
        pass

    def _init_objective(self, isLowerBound):
        if isLowerBound:
            self._init_objective_LB()
        else:
            self._init_objective_UB()

    @abstractmethod
    def _init_objective_UB(self):
        pass

    @abstractmethod
    def _init_objective_LB(self):
        pass

    @property
    def cvx_problem(self):
        return self._cvx_problem

    @property
    def is_solved(self):
        return self._is_solved

    def solve(self) -> object:
        # We init cvx problem here because pickling LP solver objects is problematic
        # by deferring it to here, worker threads do the problem building themselves and we spare the serialization
        self._cvx_problem = cvx.Problem(objective=self.objective, constraints=self.constraints)
        try:
            self._cvx_problem.solve(**self.solver_kwargs)
        except SolverError:
            # We ignore Solver Errors, which are common with our framework:
            # We solve multiple problems per bound and choose a feasible solution later (see '_create_interval')
            pass

        status = self._cvx_problem.status
        if status is "optimal":
            # TODO: add other stati
            self._is_solved = True

        self._solver_status = status
        self._cvx_problem = None
        return self

    def _retrieve_result(self):
        return self.current_feature, self.objective

    @property
    def solver_kwargs(self):
        return {"verbose": False, "solver": "ECOS", "max_iters": 1000}

    def _add_preset_constraints(self, preset_model: dict, best_model_state, best_model_constraints):
        w = best_model_state["w"]
        assert w is not None

        for feature, current_preset in preset_model.items():
            # Skip current feature
            if feature == self.current_feature:
                continue

            # Skip unset values
            if all(np.isnan(current_preset)):
                continue

            # a weight bigger than the optimal model L1 makes no sense
            assert abs(current_preset[0]) <= best_model_constraints["w_l1"]
            assert abs(current_preset[1]) <= best_model_constraints["w_l1"]

            # We add a pair of constraints depending on sign of known coefficient
            # this makes it possible to solve this as a convex problem
            if current_preset[0] >= 0:
                self.add_constraint(
                    w[feature] >= current_preset[0]
                )
                self.add_constraint(
                    w[feature] <= current_preset[1]
                )
            else:
                self.add_constraint(
                    w[feature] <= current_preset[0]
                )
                self.add_constraint(
                    w[feature] >= current_preset[1]
                )
<commit_msg>Added optimal_innacurate to allowed status Added new parameter ranges for lupi problems<commit_after>from abc import ABC, abstractmethod

import cvxpy as cvx
import numpy as np
import scipy.stats
from cvxpy import SolverError

class MLProblem(ABC):

    def __init__(self, **kwargs):

        self.chosen_parameters_ = {}

        for p in self.parameters():
            if p in kwargs:
                if kwargs[p] is not None:
                    self.chosen_parameters_[p] = kwargs[p]

        self.relax_factors_ = {}
        for p in self.relax_factors():
            if p in kwargs:
                if kwargs[p] is not None:
                    self.relax_factors_[p] = kwargs[p]

    @classmethod
    @abstractmethod
    def parameters(cls):
        raise NotImplementedError

    def get_chosen_parameter(self, p):
        try:
            return [self.chosen_parameters_[p]]  # We return list for param search function
        except:
            # TODO: rewrite the parameter logic
            # TODO: move this to subclass
            if p == "scaling_lupi_w":
                return scipy.stats.reciprocal(a=1e-10, b=1e1)
            if p == "scaling_lupi_loss":
                return scipy.stats.reciprocal(a=1e1, b=1e10)
            else:
                return scipy.stats.reciprocal(a=1e-5, b=1e5)

    def get_all_parameters(self):
        return {p: self.get_chosen_parameter(p) for p in self.parameters()}

    @classmethod
    @abstractmethod
    def relax_factors(cls):
        raise NotImplementedError

    def get_chosen_relax_factors(self, p):
        try:
            factor = self.relax_factors_[p]
        except KeyError:
            try:
                factor = self.relax_factors_[p + "_slack"]
            except KeyError:
                factor = 0.01
        assert factor > 0
        return factor

    def get_all_relax_factors(self):
        return {p: self.get_chosen_relax_factors(p) for p in self.relax_factors()}

    @classmethod
    @abstractmethod
    def get_init_model(cls):
        pass

    @classmethod
    @abstractmethod
    def get_bound_model(cls):
        pass

    @abstractmethod
    def preprocessing(self, data, lupi_features=None):
        return data

    def postprocessing(self, bounds):
        return bounds

    def get_relaxed_constraints(self, constraints):
        return {c: self.relax_constraint(c, v) for c, v in constraints.items()}

    def relax_constraint(self, key, value):
        return value * (1 + self.get_chosen_relax_factors(key))

class Relevance_CVXProblem(ABC):

    def __str__(self) -> str:
        if self.isLowerBound:
            lower = "Lower"
        else:
            lower = "Upper"
        if self.sign == True:
            sign = "(+)"
        elif self.sign == False:
            sign = "(-)"
        else:
            sign = ""
        name = f"{sign}{lower}_{self.current_feature}_{self.__class__.__name__}"
        state = ""
        for s in self.init_hyperparameters.items():
            state += f"{s[0]}:{s[1]}, "
        for s in self.init_model_constraints.items():
            state += f"{s[0]}:{s[1]}, "
        state = "(" + state[:-2] + ")"
        if self.isProbe:
            prefix = "Probe_"
        else:
            prefix = ""
        return prefix + name + state

    def __init__(self, isLowerBound: bool, current_feature: int, data: tuple, hyperparameters, best_model_constraints,
                 sign: bool = False, preset_model=None, best_model_state=None, isProbe=False) -> None:
        self.isLowerBound = isLowerBound
        self.isProbe = isProbe

        # General data
        self.sign = sign
        self.current_feature = current_feature
        self.preset_model = preset_model
        self.best_model_state = best_model_state

        self.preprocessing_data(data, best_model_state)

        # Initialize constraints
        self._constraints = []
        self._objective = None
        self._is_solved = False
        self._init_constraints(hyperparameters, best_model_constraints)
        self._init_objective(isLowerBound)

        if self.preset_model is not None:
            self._add_preset_constraints(self.preset_model, self.best_model_state, best_model_constraints)

        self.init_hyperparameters = hyperparameters
        self.init_model_constraints = best_model_constraints

    def preprocessing_data(self, data, best_model_state):
        X, y = data
        self.n = X.shape[0]
        self.d = X.shape[1]
        self.X = X
        self.y = np.array(y)

    @property
    def constraints(self):
        return self._constraints

    def add_constraint(self, new):
        self._constraints.append(new)

    @property
    def objective(self):
        return self._objective

    @abstractmethod
    def _init_constraints(self, parameters, init_model_constraints):
        pass

    def _init_objective(self, isLowerBound):
        if isLowerBound:
            self._init_objective_LB()
        else:
            self._init_objective_UB()

    @abstractmethod
    def _init_objective_UB(self):
        pass

    @abstractmethod
    def _init_objective_LB(self):
        pass

    @property
    def cvx_problem(self):
        return self._cvx_problem

    @property
    def is_solved(self):
        return self._is_solved

    @property
    def accepted_status(self):
        return ["optimal", "optimal_inaccurate"]

    def solve(self) -> object:
        # We init cvx problem here because pickling LP solver objects is problematic
        # by deferring it to here, worker threads do the problem building themselves and we spare the serialization
        self._cvx_problem = cvx.Problem(objective=self.objective, constraints=self.constraints)
        try:
            self._cvx_problem.solve(**self.solver_kwargs)
        except SolverError:
            # We ignore Solver Errors, which are common with our framework:
            # We solve multiple problems per bound and choose a feasible solution later (see '_create_interval')
            pass

        status = self._cvx_problem.status
        if status in self.accepted_status:
            self._is_solved = True

        self._solver_status = status
        self._cvx_problem = None
        return self

    def _retrieve_result(self):
        return self.current_feature, self.objective

    @property
    def solver_kwargs(self):
        return {"verbose": False, "solver": "ECOS"}

    def _add_preset_constraints(self, preset_model: dict, best_model_state, best_model_constraints):
        w = best_model_state["w"]
        assert w is not None

        for feature, current_preset in preset_model.items():
            # Skip current feature
            if feature == self.current_feature:
                continue

            # Skip unset values
            if all(np.isnan(current_preset)):
                continue

            # a weight bigger than the optimal model L1 makes no sense
            assert abs(current_preset[0]) <= best_model_constraints["w_l1"]
            assert abs(current_preset[1]) <= best_model_constraints["w_l1"]

            # We add a pair of constraints depending on sign of known coefficient
            # this makes it possible to solve this as a convex problem
            if current_preset[0] >= 0:
                self.add_constraint(
                    w[feature] >= current_preset[0]
                )
                self.add_constraint(
                    w[feature] <= current_preset[1]
                )
            else:
                self.add_constraint(
                    w[feature] <= current_preset[0]
                )
                self.add_constraint(
                    w[feature] >= current_preset[1]
                )
<|endoftext|>


<commit_before>import anki
import anki.utils
from anki.notes import Note as AnkiNote

from ..config.config_settings import ConfigSettings
from .json_serializable import JsonSerializableAnkiObject
from .note_model import NoteModel
from ..anki.overrides.change_model_dialog import ChangeModelDialog
from ..utils.constants import UUID_FIELD_NAME

class Note(JsonSerializableAnkiObject):
    export_filter_set = JsonSerializableAnkiObject.export_filter_set | \
                        {"col",  # Don't need collection
                         "_fmap",  # Generated data
                         "_model",  # Card model. Would be handled by deck.
                         "mid",  # -> uuid
                         "scm",  # todo: clarify
                         "config"
                         }

    def __init__(self, anki_note=None, config: ConfigSettings = None):
        super(Note, self).__init__(anki_note)
        self.note_model_uuid = None
        self.config = config or ConfigSettings.get_instance()

    @staticmethod
    def get_notes_from_collection(collection, deck_id, note_models):
        note_ids = collection.decks.get_note_ids(deck_id, include_from_dynamic=True)
        return [Note.from_collection(collection, note_id, note_models) for note_id in note_ids]

    @classmethod
    def from_collection(cls, collection, note_id, note_models):
        anki_note = AnkiNote(collection, id=note_id)
        note = Note(anki_note=anki_note)

        note_model = NoteModel.from_collection(collection, note.anki_object.mid)
        note_models.setdefault(note_model.get_uuid(), note_model)

        note.note_model_uuid = note_model.get_uuid()

        return note

    @classmethod
    def from_json(cls, json_dict):
        note = Note()
        note.anki_object_dict = json_dict
        note.note_model_uuid = json_dict["note_model_uuid"]
        return note

    def get_uuid(self):
        return self.anki_object.guid if self.anki_object else self.anki_object_dict.get("guid")

    def handle_model_update(self, collection, model_map_cache):
        """
        Update note's cards if note's model has changed
        """
        old_model_uuid = self.anki_object.model().get(UUID_FIELD_NAME)
        if self.note_model_uuid == old_model_uuid:
            return

        # todo if models semantically identical - create map without calling dialog

        new_model = NoteModel.from_json(collection.models.get_by_uuid(self.note_model_uuid))
        mapping = model_map_cache[old_model_uuid].get(self.note_model_uuid)
        if mapping:
            collection.models.change(self.anki_object.model(),
                                     [self.anki_object.id],
                                     new_model.anki_dict,
                                     mapping.field_map,
                                     mapping.template_map)
        else:
            new_model.make_current(collection)
            # todo signals instead of direct dialog creation?
            dialog = ChangeModelDialog(collection, [self.anki_object.id], self.anki_object.model())

            def on_accepted():
                model_map_cache[old_model_uuid][self.note_model_uuid] = \
                    NoteModel.ModelMap(dialog.get_field_map(), dialog.get_template_map())

            dialog.accepted.connect(on_accepted)
            dialog.exec_()
            # todo process cancel

        # To get an updated note to work with
        self.anki_object = AnkiNote.get_by_uuid(collection, self.get_uuid())

    def move_cards_to_deck(self, deck_id, move_from_dynamic_decks=False):
        """
        Move all cards for note with given id to specified deck.
        :param deck_id:
        :param move_from_dynamic_decks:
        :return:
        """
        # Todo: consider move only when majority of cards are in a different deck.
        for card in self.anki_object.cards():
            card.move_to_deck(deck_id, move_from_dynamic_decks)
            card.flush()

    def save_to_collection(self, collection, deck, model_map_cache):
        # Todo uuid match on existing notes

        note_model = deck.metadata.models[self.note_model_uuid]
        # You may ask WTF? Well, it seems anki has 2 ways to identify deck where to place card:
        # 1) Looking for existing cards of this note
        # 2) Looking at model->did and to add new cards to correct deck we need to modify the did each time
        # ;(
        note_model.anki_dict["did"] = deck.anki_dict["id"]

        self.anki_object = AnkiNote.get_by_uuid(collection, self.get_uuid())
        new_note = self.anki_object is None
        if new_note:
            self.anki_object = AnkiNote(collection, note_model.anki_dict)
        else:
            self.handle_model_update(collection, model_map_cache)

        self.anki_object.__dict__.update(self.anki_object_dict)
        self.anki_object.mid = note_model.anki_dict["id"]
        self.anki_object.mod = anki.utils.intTime()
        self.anki_object.flush()

        if new_note:
            collection.addNote(self.anki_object)
        elif not self.config.import_notes_ignore_deck_movement:
            self.move_cards_to_deck(deck.anki_dict["id"])
<commit_msg>Personal Fields - First proof of concept<commit_after>import anki
import anki.utils
from anki.notes import Note as AnkiNote

from ..config.config_settings import ConfigSettings
from .json_serializable import JsonSerializableAnkiObject
from .note_model import NoteModel
from ..anki.overrides.change_model_dialog import ChangeModelDialog
from ..utils.constants import UUID_FIELD_NAME

class Note(JsonSerializableAnkiObject):
    export_filter_set = JsonSerializableAnkiObject.export_filter_set | \
                        {"col",  # Don't need collection
                         "_fmap",  # Generated data
                         "_model",  # Card model. Would be handled by deck.
                         "mid",  # -> uuid
                         "scm",  # todo: clarify
                         "config"
                         }

    def __init__(self, anki_note=None, config: ConfigSettings = None):
        super(Note, self).__init__(anki_note)
        self.note_model_uuid = None
        self.config = config or ConfigSettings.get_instance()

    @staticmethod
    def get_notes_from_collection(collection, deck_id, note_models):
        note_ids = collection.decks.get_note_ids(deck_id, include_from_dynamic=True)
        return [Note.from_collection(collection, note_id, note_models) for note_id in note_ids]

    @classmethod
    def from_collection(cls, collection, note_id, note_models):
        anki_note = AnkiNote(collection, id=note_id)
        note = Note(anki_note=anki_note)

        note_model = NoteModel.from_collection(collection, note.anki_object.mid)
        note_models.setdefault(note_model.get_uuid(), note_model)

        note.note_model_uuid = note_model.get_uuid()

        return note

    @classmethod
    def from_json(cls, json_dict):
        note = Note()
        note.anki_object_dict = json_dict
        note.note_model_uuid = json_dict["note_model_uuid"]
        return note

    def get_uuid(self):
        return self.anki_object.guid if self.anki_object else self.anki_object_dict.get("guid")

    def handle_model_update(self, collection, model_map_cache):
        """
        Update note's cards if note's model has changed
        """
        old_model_uuid = self.anki_object.model().get(UUID_FIELD_NAME)
        if self.note_model_uuid == old_model_uuid:
            return

        # todo if models semantically identical - create map without calling dialog

        new_model = NoteModel.from_json(collection.models.get_by_uuid(self.note_model_uuid))
        mapping = model_map_cache[old_model_uuid].get(self.note_model_uuid)
        if mapping:
            collection.models.change(self.anki_object.model(),
                                     [self.anki_object.id],
                                     new_model.anki_dict,
                                     mapping.field_map,
                                     mapping.template_map)
        else:
            new_model.make_current(collection)
            # todo signals instead of direct dialog creation?
            dialog = ChangeModelDialog(collection, [self.anki_object.id], self.anki_object.model())

            def on_accepted():
                model_map_cache[old_model_uuid][self.note_model_uuid] = \
                    NoteModel.ModelMap(dialog.get_field_map(), dialog.get_template_map())

            dialog.accepted.connect(on_accepted)
            dialog.exec_()
            # todo process cancel

        # To get an updated note to work with
        self.anki_object = AnkiNote.get_by_uuid(collection, self.get_uuid())

    def move_cards_to_deck(self, deck_id, move_from_dynamic_decks=False):
        """
        Move all cards for note with given id to specified deck.
        :param deck_id:
        :param move_from_dynamic_decks:
        :return:
        """
        # Todo: consider move only when majority of cards are in a different deck.
        for card in self.anki_object.cards():
            card.move_to_deck(deck_id, move_from_dynamic_decks)
            card.flush()

    def save_to_collection(self, collection, deck, model_map_cache):
        # Todo uuid match on existing notes

        note_model = deck.metadata.models[self.note_model_uuid]
        # You may ask WTF? Well, it seems anki has 2 ways to identify deck where to place card:
        # 1) Looking for existing cards of this note
        # 2) Looking at model->did and to add new cards to correct deck we need to modify the did each time
        # ;(
        note_model.anki_dict["did"] = deck.anki_dict["id"]

        self.anki_object = AnkiNote.get_by_uuid(collection, self.get_uuid())
        new_note = self.anki_object is None
        if new_note:
            self.anki_object = AnkiNote(collection, note_model.anki_dict)
        else:
            self.handle_model_update(collection, model_map_cache)

        self.handle_dictionary_update()
        self.anki_object.mid = note_model.anki_dict["id"]
        self.anki_object.mod = anki.utils.intTime()
        self.anki_object.flush()

        if new_note:
            collection.addNote(self.anki_object)
        elif not self.config.import_notes_ignore_deck_movement:
            self.move_cards_to_deck(deck.anki_dict["id"])

    def handle_dictionary_update(self):
        fields_personal = self.anki_object_dict["fields"]
        for num, field in enumerate(fields_personal):
            if field is False:
                fields_personal[num] = self.anki_object.fields[num]

        self.anki_object_dict["fields"] = fields_personal
        self.anki_object.__dict__.update(self.anki_object_dict)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models

from rest_framework.mixins import ListModelMixin
from rest_framework.renderers import BrowsableAPIRenderer
from rest_framework.settings import api_settings
from drf_haystack.generics import HaystackGenericAPIView

from shop.models.product import ProductModel
from shop.rest.filters import CMSPagesFilterBackend
from shop.rest.renderers import CMSPageRenderer
from shop.rest.money import JSONRenderer
from shop.views.catalog import ProductListView, AddFilterContextMixin

class SearchView(ListModelMixin, HaystackGenericAPIView):
    """
    A generic view to be used for rendering the result list while searching.
    """
    renderer_classes = (CMSPageRenderer, JSONRenderer, BrowsableAPIRenderer)
    serializer_class = None  # to be set by SearchView.as_view(serializer_class=...)

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)

    def get_template_names(self):
        return [self.request.current_page.get_template()]

class CMSPageSearchMixin(object):
    """
    Mixin to restrict search results to products associated with the current CMS page only
    """
    def get_queryset(self, index_models=[]):
        search_qs = super(SearchView, self).get_queryset(index_models)
        catalog_qs = self.product_model.objects.filter(self.limit_choices_to)
        catalog_qs = CMSPagesFilterBackend().filter_queryset(self.request, catalog_qs, self)
        primary_keys = [e[0] for e in catalog_qs.values_list('pk')]
        return search_qs.filter(id__in=primary_keys)

class AddSearchContextMixin(object):
    """
    A mixin that can be used to enrich the render context by ``autocomplete``, so that
    templates can decide weather to add a search field or not.
    """
    def get_renderer_context(self):
        renderer_context = super(AddSearchContextMixin, self).get_renderer_context()
        if renderer_context['request'].accepted_renderer.format == 'html':
            renderer_context['search_autocomplete'] = True
        return renderer_context

class CMSPageCatalogWrapper(object):
    """
    Wraps the view classes :class:`shop.views.catalog.ProductListView` and
    :class:`shop.search.views.SearchView` into a callable which behaves like a
    Django View class, but dispatches requests between both of them depending on
    the query params.

    Usage: Add it to the urlpatterns responsible for rendering the catalog's list view:
    ```
    urlpatterns = [
        url(r'^$', CMSPageCatalogWrapper.as_view(
            filter_class=CustomFilterSet,
        )),
    ```

    In addition to ``filter_class`` you may override other attributes:

    :param renderer_classes: A list or tuple of REST renderer classes.

    :param product_model: The product model onto which the filter set is applied.

    :param limit_choices_to: Limit the queryset of product models to these choices.

    :param filter_class: A filter set which must be inherit from :class:`django_filters.FilterSet`.

    :param search_serializer_class: The serializer class used to process the queryset returned
        by Haystack, while performing an autocomplete lookup.

    :param model_serializer_class: The serializer class used to process the queryset returned
        by the catalog's product list view, while using a filter set.

    :param filter_backends: The filter backends used to narrow down the catalog's product list view.

    :param cms_pages_fields: Specify the fields used by the Product model to assign them to
        categories.
    """
    renderer_classes = (CMSPageRenderer, JSONRenderer, BrowsableAPIRenderer)
    product_model = ProductModel
    limit_choices_to = models.Q()
    search_serializer_class = None  # must be overridden by CMSPageCatalogWrapper.as_view()
    model_serializer_class = None  # must be overridden by CMSPageCatalogWrapper.as_view()
    filter_class = None  # may be overridden by CMSPageCatalogWrapper.as_view()
    filter_backends = [CMSPagesFilterBackend] + list(api_settings.DEFAULT_FILTER_BACKENDS)
    cms_pages_fields = ['cms_pages']

    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)

    @classmethod
    def as_view(cls, **initkwargs):
        for key in initkwargs:
            if not hasattr(cls, key):
                msg = "{0}() received an invalid keyword {1}. Method as_view() only " \
                      " accepts arguments that are already attributes of the class."
                raise TypeError(msg.format(cls.__name__, key))

        self = cls(**initkwargs)

        bases = (AddFilterContextMixin, AddSearchContextMixin, CMSPageSearchMixin, SearchView)
        attrs = dict(renderer_classes=self.renderer_classes, product_model=self.product_model,
                     limit_choices_to=self.limit_choices_to, filter_class=self.filter_class)
        self.search_view = type(str('CatalogSearchView'), bases, attrs).as_view(
            serializer_class=self.search_serializer_class,
        )

        bases = (AddFilterContextMixin, AddSearchContextMixin, ProductListView)
        attrs.update(filter_backends=self.filter_backends, cms_pages_fields=self.cms_pages_fields)
        self.list_view = type(str('CatalogListView'), bases, attrs).as_view(
            serializer_class=self.model_serializer_class,
        )

        return self

    def __call__(self, request):
        if request.GET.get('q'):
            return self.search_view(request)
        return self.list_view(request)
<commit_msg>code cleaned<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models

from rest_framework.mixins import ListModelMixin
from rest_framework.renderers import BrowsableAPIRenderer
from rest_framework.settings import api_settings
from drf_haystack.generics import HaystackGenericAPIView

from shop.models.product import ProductModel
from shop.rest.filters import CMSPagesFilterBackend
from shop.rest.renderers import CMSPageRenderer
from shop.rest.money import JSONRenderer
from shop.views.catalog import ProductListView, AddFilterContextMixin

class SearchView(ListModelMixin, HaystackGenericAPIView):
    """
    A generic view to be used for rendering the result list while searching.
    """
    renderer_classes = (CMSPageRenderer, JSONRenderer, BrowsableAPIRenderer)
    serializer_class = None  # to be set by SearchView.as_view(serializer_class=...)

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)

    def get_template_names(self):
        return [self.request.current_page.get_template()]

class CMSPageSearchMixin(object):
    """
    Mixin to restrict search results to products associated with the current CMS page only
    """
    def get_queryset(self, index_models=[]):
        catalog_qs = self.product_model.objects.filter(self.limit_choices_to)
        catalog_qs = CMSPagesFilterBackend().filter_queryset(self.request, catalog_qs, self)
        primary_keys = [e[0] for e in catalog_qs.values_list('pk')]
        search_qs = super(SearchView, self).get_queryset(index_models).filter(id__in=primary_keys)
        return search_qs

class AddSearchContextMixin(object):
    """
    A mixin that can be used to enrich the render context by ``autocomplete``, so that
    templates can decide weather to add a search field or not.
    """
    def get_renderer_context(self):
        renderer_context = super(AddSearchContextMixin, self).get_renderer_context()
        if renderer_context['request'].accepted_renderer.format == 'html':
            renderer_context['search_autocomplete'] = True
        return renderer_context

class CMSPageCatalogWrapper(object):
    """
    Wraps the view classes :class:`shop.views.catalog.ProductListView` and
    :class:`shop.search.views.SearchView` into a callable which behaves like a
    Django View class, but dispatches requests between both of them depending on
    the query params.

    Usage: Add it to the urlpatterns responsible for rendering the catalog's list view:
    ```
    urlpatterns = [
        url(r'^$', CMSPageCatalogWrapper.as_view(
            filter_class=CustomFilterSet,
        )),
    ```

    In addition to ``filter_class`` you may override other attributes:

    :param renderer_classes: A list or tuple of REST renderer classes.

    :param product_model: The product model onto which the filter set is applied.

    :param limit_choices_to: Limit the queryset of product models to these choices.

    :param filter_class: A filter set which must be inherit from :class:`django_filters.FilterSet`.

    :param search_serializer_class: The serializer class used to process the queryset returned
        by Haystack, while performing an autocomplete lookup.

    :param model_serializer_class: The serializer class used to process the queryset returned
        by the catalog's product list view, while using a filter set.

    :param filter_backends: The filter backends used to narrow down the catalog's product list view.

    :param cms_pages_fields: Specify the fields used by the Product model to assign them to
        categories.
    """
    renderer_classes = (CMSPageRenderer, JSONRenderer, BrowsableAPIRenderer)
    product_model = ProductModel
    limit_choices_to = models.Q()
    search_serializer_class = None  # must be overridden by CMSPageCatalogWrapper.as_view()
    model_serializer_class = None  # must be overridden by CMSPageCatalogWrapper.as_view()
    filter_class = None  # may be overridden by CMSPageCatalogWrapper.as_view()
    filter_backends = [CMSPagesFilterBackend] + list(api_settings.DEFAULT_FILTER_BACKENDS)
    cms_pages_fields = ['cms_pages']

    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)

    @classmethod
    def as_view(cls, **initkwargs):
        for key in initkwargs:
            if not hasattr(cls, key):
                msg = "{0}() received an invalid keyword {1}. Method as_view() only " \
                      " accepts arguments that are already attributes of the class."
                raise TypeError(msg.format(cls.__name__, key))

        self = cls(**initkwargs)

        bases = (AddFilterContextMixin, AddSearchContextMixin, CMSPageSearchMixin, SearchView)
        attrs = dict(renderer_classes=self.renderer_classes, product_model=self.product_model,
                     limit_choices_to=self.limit_choices_to, filter_class=self.filter_class)
        self.search_view = type(str('CatalogSearchView'), bases, attrs).as_view(
            serializer_class=self.search_serializer_class,
        )

        bases = (AddFilterContextMixin, AddSearchContextMixin, ProductListView)
        attrs.update(filter_backends=self.filter_backends, cms_pages_fields=self.cms_pages_fields)
        self.list_view = type(str('CatalogListView'), bases, attrs).as_view(
            serializer_class=self.model_serializer_class,
        )

        return self

    def __call__(self, request):
        if request.GET.get('q'):
            return self.search_view(request)
        return self.list_view(request)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2014, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import Gaffer
import GafferUI
import GafferScene

##########################################################################
# Metadata
##########################################################################

Gaffer.Metadata.registerNode(

	GafferScene.Camera,

	"description",
	"""
	Produces scenes containing a camera. To choose which camera is
	used for rendering, use a StandardOptions node.
	""",

	plugs = {

		"projection" : [

			"description",
			"""
			The basic camera type.
			""",

			"preset:Perspective", "perspective",
			"preset:Orthographic", "orthographic",

			"plugValueWidget:type", "GafferUI.PresetsPlugValueWidget",

		],

		"fieldOfView" : [

			"description",
			"""
			The field of view, specified in degrees, and interpreted
			as defined in the RenderMan specification. This is only
			relevant for perspective cameras.
			""",

		],

		"clippingPlanes" : [

			"description",
			"""
			The near and far clipping planes.
			""",

		],

	}

)
<commit_msg>CameraUI : Add "Copy From Viewer" item to NodeEditor tool menu.<commit_after>##########################################################################
#
#  Copyright (c) 2014, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import math
import functools

import Gaffer
import GafferUI
import GafferScene
import GafferSceneUI

##########################################################################
# Metadata
##########################################################################

Gaffer.Metadata.registerNode(

	GafferScene.Camera,

	"description",
	"""
	Produces scenes containing a camera. To choose which camera is
	used for rendering, use a StandardOptions node.
	""",

	plugs = {

		"projection" : [

			"description",
			"""
			The basic camera type.
			""",

			"preset:Perspective", "perspective",
			"preset:Orthographic", "orthographic",

			"plugValueWidget:type", "GafferUI.PresetsPlugValueWidget",

		],

		"fieldOfView" : [

			"description",
			"""
			The field of view, specified in degrees, and interpreted
			as defined in the RenderMan specification. This is only
			relevant for perspective cameras.
			""",

		],

		"clippingPlanes" : [

			"description",
			"""
			The near and far clipping planes.
			""",

		],

	}

)

##########################################################################
# NodeEditor tool menu
##########################################################################

def __copyCamera( node, camera ) :

	with Gaffer.UndoContext( node.scriptNode() ) :

		s, h, r, t = camera.getTransform().transform().extractSHRT()
		node["transform"]["translate"].setValue( t )
		node["transform"]["rotate"].setValue( r * 180.0 / math.pi )
		node["transform"]["scale"].setValue( s )

def __nodeEditorToolMenu( nodeEditor, node, menuDefinition ) :

	if not isinstance( node, GafferScene.Camera ) :
		return

	layout = nodeEditor.ancestor( GafferUI.CompoundEditor )
	if layout is None :
		return

	viewers = [ v for v in layout.editors( GafferUI.Viewer ) if isinstance( v.view(), GafferSceneUI.SceneView ) ]
	if not viewers :
		return

	for viewer in viewers :

		menuDefinition.append(

			"/Copy From Viewer" + ( "/" + viewer.getTitle() if len( viewers ) > 1 else "" ),
			{
				"command" : functools.partial( __copyCamera, node, viewer.view().viewportGadget().getCamera() ),
				"active" : not Gaffer.readOnly( node["transform"] ),
			}

		)

__nodeEditorToolMenuConnection = GafferUI.NodeEditor.toolMenuSignal().connect( __nodeEditorToolMenu )
<|endoftext|>


<commit_before>from data_importers.github_importer import BaseGitHubImporter

class Command(BaseGitHubImporter):

    srid = 4326
    districts_srid = 4326
    council_id = "E08000017"
    elections = ["parl.2019-12-12"]
    scraper_name = "wdiv-scrapers/DC-PollingStations-Doncaster"
    geom_type = "geojson"

    def district_record_to_dict(self, record):
        poly = self.extract_geometry(record, self.geom_type, self.get_srid("districts"))
        code = record["CODE"].strip()
        return {
            "internal_council_id": code,
            "name": code,
            "area": poly,
            "polling_station_id": code,
        }

    def station_record_to_dict(self, record):
        location = self.extract_geometry(
            record, self.geom_type, self.get_srid("stations")
        )
        return {
            "internal_council_id": record["POLLING_DI"].strip(),
            "postcode": "",
            "address": record["ADDRESS"].strip(),
            "location": location,
        }
<commit_msg>Import script for Doncaster<commit_after>from data_importers.github_importer import BaseGitHubImporter

class Command(BaseGitHubImporter):

    srid = 4326
    districts_srid = 4326
    council_id = "DNC"
    elections = ["2021-05-06"]
    scraper_name = "wdiv-scrapers/DC-PollingStations-Doncaster"
    geom_type = "geojson"

    def district_record_to_dict(self, record):
        poly = self.extract_geometry(record, self.geom_type, self.get_srid("districts"))
        code = record["CODE"].strip()
        return {
            "internal_council_id": code,
            "name": code,
            "area": poly,
            "polling_station_id": code,
        }

    def station_record_to_dict(self, record):
        location = self.extract_geometry(
            record, self.geom_type, self.get_srid("stations")
        )
        return {
            "internal_council_id": record["POLLING_DI"].strip(),
            "postcode": "",
            "address": record["ADDRESS"].strip(),
            "location": location,
        }
<|endoftext|>


<commit_before># shrunk - Rutgers University URL Shortener

"""Flask application for the link server."""

from flask import Flask, render_template, request, redirect, g

import shrunk.client
import shrunk.util

# Create application
app = Flask(__name__)

# Import settings in config.py
app.config.from_pyfile("config.py", silent=True)
app.secret_key = app.config['SECRET_KEY']

# Initialize logging
shrunk.util.set_logger(app)

### Views ###
@app.route("/<short_url>")
def redirect_link(short_url):
    """Redirects to the short URL's true destination.

    This looks up the short URL's destination in the database and performs a
    redirect, logging some information at the same time. If no such link exists,
    a not found page is shown.

    :Parameters:
      - `short_url`: A string containing a shrunk-ified URL.
    """
    client = shrunk.util.get_db_client(app, g)
    app.logger.info("{} requests {}".format(request.remote_addr, short_url))

    # Perform a lookup and redirect
    long_url = client.get_long_url(short_url)
    if long_url is None:
        return render_template("link-404.html", short_url=short_url)
    else:
        client.visit(short_url, request.remote_addr)

        # Check if a protocol exists
        if "://" in long_url:
            return redirect(long_url)
        else:
            return redirect("http://{}".format(long_url))

@app.route("/")
def render_index():
    """Renders the homepage.

    The link server will redirect to the URL for the public URL manager.
    """
    app.logger.info("Redirecting {} to shrunk".format(request.remote_addr))
    return redirect(app.config["SHRUNK_URL"])
<commit_msg>use shrunk flask so we dont have repeated code<commit_after># shrunk - Rutgers University URL Shortener

"""Flask application for the link server."""

from flask import Flask, render_template, request, redirect, g

import shrunk.client
import shrunk.util
from shrunk.app_decorate import ShrunkFlaskMini

# Create application
app = ShrunkFlaskMini(__name__)

### Views ###
@app.route("/<short_url>")
def redirect_link(short_url):
    """Redirects to the short URL's true destination.

    This looks up the short URL's destination in the database and performs a
    redirect, logging some information at the same time. If no such link exists,
    a not found page is shown.

    :Parameters:
      - `short_url`: A string containing a shrunk-ified URL.
    """
    client = shrunk.util.get_db_client(app, g)
    app.logger.info("{} requests {}".format(request.remote_addr, short_url))

    # Perform a lookup and redirect
    long_url = client.get_long_url(short_url)
    if long_url is None:
        return render_template("link-404.html", short_url=short_url)
    else:
        client.visit(short_url, request.remote_addr)

        # Check if a protocol exists
        if "://" in long_url:
            return redirect(long_url)
        else:
            return redirect("http://{}".format(long_url))

@app.route("/")
def render_index():
    """Renders the homepage.

    The link server will redirect to the URL for the public URL manager.
    """
    app.logger.info("Redirecting {} to shrunk".format(request.remote_addr))
    return redirect(app.config["SHRUNK_URL"])
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
from mvtrialanderror import *

def xCutsLine(x1Line, x2Line, y1Line, y2Line, xCutLocs):
	xLine = []
	yLine = []

	for cutLoc in xCutLocs:
		xLine.append(cutLoc)
		yLine.append(y1Line - float(x1Line - cutLoc) / float(x1Line - x2Line) * (y1Line - y2Line))

	return xLine, yLine

true = 1
false = 0
valueToTargetSum = 0
xPoints = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]
yPoints = [749.0, 614.0, 518.0, 412.0, 398.0, 347.0, 291.0, 238.0, 338.0]
topMostValue = max(yPoints)
bottomMostValue = min(yPoints)
xValues = [min(xPoints), max(xPoints)]
yValues = []

for iteration in range(1, 50):

	yValues = guessValues(topMostValue, bottomMostValue, len(xValues), valueToTargetSum, iteration)

	xLinePts, yLinePts = xCutsLine(xValues[0], xValues[1], yValues[0], yValues[1], xPoints)

	if iteration == 1:
		closerFurtherIni(yLinePts)

	closerFurtherRetData = closerFurther(yLinePts, yPoints, len(yLinePts), 0.001, true)
	#valueIsCloserArray = closerFurtherRetData[0]
	#valueIsMostClosestArray = closerFurtherRetData[1]
	#numOfValuesWithinTolerance = closerFurtherRetData[2]
	valueToTargetSum = closerFurtherRetData[3]

	print("valueToTargetSum: %f" % (valueToTargetSum))

	#plt.clf()
	plt.plot(xPoints, yPoints, 'ro')
	plt.plot([xValues[0], xValues[1]], [yValues[0], yValues[1]])
	plt.draw()

plt.savefig('graph.png')
<commit_msg>Delete graph1.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import
import argparse
import logging
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.animation as manimation
from astropy.io import fits
from astropy import wcs

logging.basicConfig(level='INFO', format='%(levelname)7s %(message)s')
logger = logging.getLogger(__name__)

def get_data(filename, ra, dec, half_width):
    header = fits.getheader(filename)
    w = wcs.WCS(header)
    x, y = w.all_world2pix(ra, dec, 0)
    x_offset, y_offset = x % 1, y % 1
    xlim = (x - half_width, x + half_width)
    ylim = (y - half_width, y + half_width)
    vignette = np.log10(fits.getdata(filename)[ylim[0]:ylim[1], xlim[0]:xlim[1]] + 1)
    return vignette, x_offset, y_offset

def fetch_ra_dec_from_catalogue(catalogue, index):
    with fits.open(catalogue) as infile:
        if 'catalogue' in infile:
            cat = infile['catalogue'].data
            ra, dec = cat['ra'][index], cat['dec'][index]
        else:
            cat = infile[1].data
            ra, dec = (np.degrees(cat['ra'][index]),
                    np.degrees(cat['dec'][index]))
    return ra, dec

def main(args):
    if args.verbose:
        logger.setLevel('DEBUG')
    logger.debug(args)

    if not args.ra and not args.dec:
        ra, dec = fetch_ra_dec_from_catalogue(args.catalogue, args.index)
    else:
        ra, dec = args.ra, args.dec

    logger.info('RA: %s, DEC: %s', ra, dec)

    files = sorted(args.filename)
    nfiles = len(files)
    half_width = args.half_width

    fig = plt.figure()
    initial_data, initial_x_offset, initial_y_offset = get_data(files[0], ra, dec,
                                                                half_width)

    im = plt.imshow(initial_data, interpolation='None',
            origin='lower')
    ax = plt.gca()
    circle = plt.Circle(
        (half_width, +initial_x_offset, half_width + initial_y_offset), 3,
        edgecolor='g',
        facecolor='None')

    ax.add_patch(circle)

    def update(i):
        fname = files[i]
        vignette, offset_x, offset_y = get_data(fname, ra, dec, half_width)
        im.set_data(vignette)
        med_vignette = np.median(vignette)
        im.set_clim(0.99 * med_vignette, 1.2 * med_vignette)
        circle.center = (half_width + offset_x, half_width + offset_y)

        print('\r{}/{}'.format(i + 1, nfiles), end='')
        return im, circle

    interval = 1000. / args.fps

    ani = manimation.FuncAnimation(fig, update, frames=nfiles, interval=interval, blit=True)
    if args.output is not None:
        ani.save(args.output, writer='ffmpeg')
    else:
        plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('filename', nargs='+')
    parser.add_argument('-r', '--ra', type=float)
    parser.add_argument('-d', '--dec', type=float)
    parser.add_argument('-i', '--index', required=False, type=int)
    parser.add_argument('-c', '--catalogue', required=False)
    parser.add_argument('-o', '--output', required=False)
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('-w', '--half-width', required=False, type=int, default=20)
    parser.add_argument('--fps', required=False, default=24, type=int)
    main(parser.parse_args())
<commit_msg>Use average instead of median<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import
import argparse
import logging
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.animation as manimation
from astropy.io import fits
from astropy import wcs

logging.basicConfig(level='INFO', format='%(levelname)7s %(message)s')
logger = logging.getLogger(__name__)

def get_data(filename, ra, dec, half_width):
    header = fits.getheader(filename)
    w = wcs.WCS(header)
    x, y = w.all_world2pix(ra, dec, 0)
    x_offset, y_offset = x % 1, y % 1
    xlim = (x - half_width, x + half_width)
    ylim = (y - half_width, y + half_width)
    vignette = np.log10(fits.getdata(filename)[ylim[0]:ylim[1], xlim[0]:xlim[1]] + 1)
    return vignette, x_offset, y_offset

def fetch_ra_dec_from_catalogue(catalogue, index):
    with fits.open(catalogue) as infile:
        if 'catalogue' in infile:
            cat = infile['catalogue'].data
            ra, dec = cat['ra'][index], cat['dec'][index]
        else:
            cat = infile[1].data
            ra, dec = (np.degrees(cat['ra'][index]),
                    np.degrees(cat['dec'][index]))
    return ra, dec

def main(args):
    if args.verbose:
        logger.setLevel('DEBUG')
    logger.debug(args)

    if not args.ra and not args.dec:
        ra, dec = fetch_ra_dec_from_catalogue(args.catalogue, args.index)
    else:
        ra, dec = args.ra, args.dec

    logger.info('RA: %s, DEC: %s', ra, dec)

    files = sorted(args.filename)
    nfiles = len(files)
    half_width = args.half_width

    fig = plt.figure()
    initial_data, initial_x_offset, initial_y_offset = get_data(files[0], ra, dec,
                                                                half_width)

    im = plt.imshow(initial_data, interpolation='None',
            origin='lower')
    ax = plt.gca()
    circle = plt.Circle(
        (half_width, +initial_x_offset, half_width + initial_y_offset), 3,
        edgecolor='g',
        facecolor='None')

    ax.add_patch(circle)

    def update(i):
        fname = files[i]
        vignette, offset_x, offset_y = get_data(fname, ra, dec, half_width)
        im.set_data(vignette)
        med_vignette = np.average(vignette)
        im.set_clim(0.99 * med_vignette, 1.2 * med_vignette)
        circle.center = (half_width + offset_x, half_width + offset_y)

        print('\r{}/{}'.format(i + 1, nfiles), end='')
        return im, circle

    interval = 1000. / args.fps

    ani = manimation.FuncAnimation(fig, update, frames=nfiles, interval=interval, blit=True)
    if args.output is not None:
        ani.save(args.output, writer='ffmpeg')
    else:
        plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('filename', nargs='+')
    parser.add_argument('-r', '--ra', type=float)
    parser.add_argument('-d', '--dec', type=float)
    parser.add_argument('-i', '--index', required=False, type=int)
    parser.add_argument('-c', '--catalogue', required=False)
    parser.add_argument('-o', '--output', required=False)
    parser.add_argument('-v', '--verbose', action='store_true')
    parser.add_argument('-w', '--half-width', required=False, type=int, default=20)
    parser.add_argument('--fps', required=False, default=24, type=int)
    main(parser.parse_args())
<|endoftext|>