<commit_before>##
# Copyright (c) 2008-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

class AdapterError(Exception):
    pass

class Adapter(type):
    adapters = {}

    def __new__(mcls, name, bases, clsdict, *, adapts=None, **kwargs):
        if adapts is not None:
            bases = bases + (adapts,)

        result = super().__new__(mcls, name, bases, clsdict, **kwargs)
        if adapts is not None:
            assert issubclass(mcls, Adapter) and mcls is not Adapter

            adapters = Adapter.adapters.get(mcls)
            if not adapters:
                Adapter.adapters[mcls] = adapters = {}

            assert adapts not in adapters
            adapters[adapts] = result

        result.__sx_adaptee__ = adapts

        return result

    def __init__(cls, name, bases, clsdict, *, adapts=None, **kwargs):
        super().__init__(name, bases, clsdict, **kwargs)

    @classmethod
    def get_adapter(mcls, cls):
        for mc in [mcls] + mcls.__subclasses__(mcls):
            adapters = Adapter.adapters.get(mc)

            if adapters is not None:
                for adaptee, adapter in adapters.items():
                    for c in cls.mro():
                        if issubclass(c, adapter):
                            return c
                        elif issubclass(c, adaptee):
                            return adapter
            elif mc is not mcls:
                adapter = mc.get_adapter(cls)
                if adapter:
                    return adapter

    @classmethod
    def adapt(mcls, obj):
        adapter = mcls.get_adapter(obj.__class__)
        if adapter is None:
            raise AdapterError('could not find %s.%s adapter for %s' % \
                               (mcls.__module__, mcls.__name__, obj.__class__.__name__))
        elif adapter is not obj.__class__:
            return adapter.adapt(obj)
        else:
            return obj

    def get_adaptee(cls):
        return cls.__sx_adaptee__
<commit_msg>utils.functional.adapter: Extend Adapter functionality<commit_after>##
# Copyright (c) 2008-2013 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##

class AdapterError(Exception):
    pass

class Adapter(type):
    adapters = {}

    def __new__(mcls, name, bases, clsdict, *, adapts=None, adapts_instances_of=None, pure=False,
                                                            adapterargs=None, **kwargs):

        if adapts is not None and adapts_instances_of is not None:
            msg = 'adapter class: adapts and adapts_instances_of args are mutually exclusive'
            raise AdapterError(msg)

        if adapts_instances_of is not None:
            pure = True
            adapts = adapts_instances_of

        if adapts is not None and not pure:
            bases = bases + (adapts,)

        result = super().__new__(mcls, name, bases, clsdict, **kwargs)
        if adapts is not None:
            assert issubclass(mcls, Adapter) and mcls is not Adapter

            registry_key = mcls.get_registry_key(adapterargs)

            try:
                adapters = Adapter.adapters[registry_key]
            except KeyError:
                adapters = Adapter.adapters[registry_key] = {}

            mcls.register_adapter(adapters, adapts, adapts_instances_of is not None, result)

        result.__sx_adaptee__ = adapts

        return result

    def __init__(cls, name, bases, clsdict, *, adapts=None, adapts_instances_of=None, pure=False,
                                                            adapterargs=None, **kwargs):
        super().__init__(name, bases, clsdict, **kwargs)

    @classmethod
    def register_adapter(mcls, registry, adaptee, adapting_instances, adapter):
        assert (adaptee, adapting_instances) not in registry
        registry[(adaptee, adapting_instances)] = adapter

    @classmethod
    def match_adapter(mcls, obj, adaptee, adapting_instances, adapter):
        if adapting_instances:
            if isinstance(obj, adaptee):
                return adapter
        elif isinstance(obj, type):
            if issubclass(obj, adapter) and obj is not adapter:
                return obj
            elif issubclass(obj, adaptee):
                return adapter

    @classmethod
    def get_adapter(mcls, cls, **kwargs):
        for mc in [mcls] + mcls.__subclasses__(mcls):
            registry_key = mc.get_registry_key(kwargs)
            adapters = Adapter.adapters.get(registry_key)

            if adapters is not None:
                for (adaptee, adapting_instances), adapter in adapters.items():
                    if adapting_instances:
                        result = mcls.match_adapter(cls, adaptee, adapting_instances, adapter)
                        if result is not None:
                            return result
                    else:
                        mro = getattr(cls, '__mro__', (cls,))

                        for c in mro:
                            result = mcls.match_adapter(c, adaptee, adapting_instances, adapter)
                            if result is not None:
                                return result

            elif mc is not mcls:
                adapter = mc.get_adapter(cls, **kwargs)
                if adapter:
                    return adapter

    @classmethod
    def adapt(mcls, obj):
        adapter = mcls.get_adapter(obj.__class__)
        if adapter is None:
            raise AdapterError('could not find %s.%s adapter for %s' % \
                               (mcls.__module__, mcls.__name__, obj.__class__.__name__))
        elif adapter is not obj.__class__:
            return adapter.adapt(obj)
        else:
            return obj

    @classmethod
    def get_registry_key(mcls, adapterargs):
        if adapterargs:
            return (mcls, frozenset(adapterargs.items()))
        else:
            return mcls

    def get_adaptee(cls):
        return cls.__sx_adaptee__

class MultiAdapter(Adapter):
    @classmethod
    def register_adapter(mcls, registry, adaptee, adapting_instances, adapter):
        try:
            registry[adaptee, adapting_instances] += (adapter,)
        except KeyError:
            registry[adaptee, adapting_instances] = (adapter,)

    @classmethod
    def match_adapter(mcls, obj, adaptee, adapting_instances, adapter):
        if adapting_instances:
            if isinstance(obj, adaptee):
                return adapter
        elif isinstance(obj, type):
            if issubclass(obj, adapter) and obj not in adapter:
                return (obj,)
            elif issubclass(obj, adaptee):
                return adapter
<|endoftext|>


<commit_before>import os
import json
import zipfile
import tempfile
from decimal import Decimal

from django.db import migrations, models
from django.core.files import File

# Some Fidus Writer 3.9.0 and 3.9.1 installations will have empty table cells
FW_DOCUMENT_VERSION = 3.3

ID_COUNTER = 0

def update_initial_node(node):
    global ID_COUNTER
    if "type" in node:
        if node["type"] in ["bullet_list", "ordered_list"]:
            if not "attrs" in node:
                node["attrs"] = {}
            ID_COUNTER += 1
            if not "id" in node["attrs"]:
                node["attrs"]["id"] = "{}{:0>8d}".format("L", ID_COUNTER)
        elif node["type"] == "table" and "content" in node:
            if not "attrs" in node:
                node["attrs"] = {}
            ID_COUNTER += 1
            if not "id" in node["attrs"]:
                node["attrs"]["id"] = "{}{:0>8d}".format("T", ID_COUNTER)
                node["attrs"]["caption"] = False
                if "content" in node:
                    node["content"] = [
                        {
                            "type": "table_caption"
                        },
                        {
                            "type": "table_body",
                            "content" : node["content"]
                        }
                    ]
        elif (
            node["type"] == "table_cell" and
            (
                not "content" in node or
                len(node["content"]) == 0
            )
        ):
            node["content"] = [{"type": "paragraph"}]
        elif node["type"] == "figure":
            if not "attrs" in node:
                node["attrs"] = {}
            attrs = node["attrs"]
            if "category" in attrs:
                # Already updated
                return
            if "figureCategory" in attrs:
                attrs["category"] = attrs["figureCategory"]
                del attrs["figureCategory"]
            else:
                attrs["category"] = "none"
            node["content"] = []
            if "image" in attrs and attrs["image"] is not False:
                node["content"].append({
                    "type": "image",
                    "attrs": {
                        "image": attrs["image"]
                    }
                })
            else:
                equation = ""
                if "equation" in attrs:
                    equation = attrs["equation"]
                node["content"].append({
                    "type": "figure_equation",
                    "attrs": {
                        "equation": equation
                    }
                })
            if "image" in attrs:
                del attrs["image"]
            if "equation" in attrs:
                del attrs["equation"]
            caption = {
                "type": "figure_caption"
            }
            if (
                "caption" in attrs and
                attrs["caption"] is not False and
                len(attrs["caption"]) > 0
            ):
                caption["content"] = [{
                    "type": "text",
                    "text": attrs["caption"]
                }]
                attrs["caption"] = True
            else:
                attrs["caption"] = False
            if attrs["category"] == "table":
                node["content"].insert(0, caption)
            else:
                node["content"].append(caption)
            node["attrs"] = attrs
    if "content" in node:
        for sub_node in node["content"]:
            update_initial_node(sub_node)

def update_node(node):
    if "contents" in node:  # revision
        update_node(node["contents"])
    if "type" in node:
        if (
            node["type"] == "table_cell" and
            (
                not "content" in node or
                len(node["content"]) == 0
            )
        ):
            node["content"] = [{"type": "paragraph"}]
    if "content" in node:
        for sub_node in node["content"]:
            update_node(sub_node)
    if (
        "attrs" in node and
        "initial" in node["attrs"] and
        bool(node["attrs"]["initial"])
    ):
        for sub_node in node["attrs"]["initial"]:
            update_initial_node(sub_node)

# from https://stackoverflow.com/questions/25738523/how-to-update-one-file-inside-zip-file-using-python
def update_revision_zip(file_field, file_name):
    # generate a temp file
    tmpfd, tmpname = tempfile.mkstemp()
    os.close(tmpfd)
    # create a temp copy of the archive without filename
    with zipfile.ZipFile(file_field.open(), 'r') as zin:
        with zipfile.ZipFile(tmpname, 'w') as zout:
            zout.comment = zin.comment # preserve the comment
            for item in zin.infolist():
                if item.filename == 'document.json':
                    doc_string = zin.read(item.filename)
                    doc = json.loads(doc_string)
                    update_node(doc)
                    zout.writestr(
                        item,
                        json.dumps(doc)
                    )
                else:
                    zout.writestr(item, zin.read(item.filename))
    # replace with the temp archive
    with open(tmpname, 'rb') as tmp_file:
        file_field.save(file_name, File(tmp_file))
    os.remove(tmpname)

def update_documents(apps, schema_editor):
    Document = apps.get_model('document', 'Document')
    documents = Document.objects.all().iterator()
    for document in documents:
        if document.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_node(document.content)
            document.save()

    DocumentTemplate = apps.get_model('document', 'DocumentTemplate')
    templates = DocumentTemplate.objects.all()
    for template in templates:
        if template.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_node(template.content)
            template.save()

    DocumentRevision = apps.get_model('document', 'DocumentRevision')
    revisions = DocumentRevision.objects.all()
    for revision in revisions:
        if not revision.file_object:
            revision.delete()
            continue
        if revision.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_revision_zip(revision.file_object, revision.file_name)

class Migration(migrations.Migration):

    dependencies = [
        ('document', '0006_auto_20201209_1610'),
    ]

    operations = [
        migrations.RunPython(update_documents),
    ]
<commit_msg>modify migration<commit_after>import os
import json
import zipfile
import tempfile
from decimal import Decimal

from django.db import migrations, models
from django.core.files import File

# Some Fidus Writer 3.9.0 and 3.9.1 installations will have empty table cells
FW_DOCUMENT_VERSION = 3.3

ID_COUNTER = 0

def update_initial_node(node):
    global ID_COUNTER
    if "type" in node:
        if node["type"] in ["bullet_list", "ordered_list"]:
            if not "attrs" in node:
                node["attrs"] = {}
            ID_COUNTER += 1
            if not "id" in node["attrs"]:
                node["attrs"]["id"] = "{}{:0>8d}".format("L", ID_COUNTER)
        elif node["type"] == "table" and "content" in node:
            if not "attrs" in node:
                node["attrs"] = {}
            ID_COUNTER += 1
            if not "id" in node["attrs"]:
                node["attrs"]["id"] = "{}{:0>8d}".format("T", ID_COUNTER)
                node["attrs"]["caption"] = False
                if "content" in node:
                    node["content"] = [
                        {
                            "type": "table_caption"
                        },
                        {
                            "type": "table_body",
                            "content" : node["content"]
                        }
                    ]
        elif (
            node["type"] == "table_cell" and
            (
                not "content" in node or
                len(node["content"]) == 0
            )
        ):
            node["content"] = [{"type": "paragraph"}]
        elif node["type"] == "figure":
            if not "attrs" in node:
                node["attrs"] = {}
            attrs = node["attrs"]
            if "category" in attrs:
                # Already updated
                return
            if "figureCategory" in attrs:
                attrs["category"] = attrs["figureCategory"]
                del attrs["figureCategory"]
            else:
                attrs["category"] = "none"
            node["content"] = []
            if "image" in attrs and attrs["image"] is not False:
                node["content"].append({
                    "type": "image",
                    "attrs": {
                        "image": attrs["image"]
                    }
                })
            else:
                equation = ""
                if "equation" in attrs:
                    equation = attrs["equation"]
                node["content"].append({
                    "type": "figure_equation",
                    "attrs": {
                        "equation": equation
                    }
                })
            if "image" in attrs:
                del attrs["image"]
            if "equation" in attrs:
                del attrs["equation"]
            caption = {
                "type": "figure_caption"
            }
            if (
                "caption" in attrs and
                attrs["caption"] is not False and
                len(attrs["caption"]) > 0
            ):
                caption["content"] = [{
                    "type": "text",
                    "text": attrs["caption"]
                }]
                attrs["caption"] = True
            else:
                attrs["caption"] = False
            if attrs["category"] == "table":
                node["content"].insert(0, caption)
            else:
                node["content"].append(caption)
            node["attrs"] = attrs
    if "content" in node:
        for sub_node in node["content"]:
            update_initial_node(sub_node)

def update_node(node):
    if "contents" in node:  # revision
        update_node(node["contents"])
    if "type" in node:
        if (
            node["type"] == "table_cell" and
            (
                not "content" in node or
                len(node["content"]) == 0
            )
        ):
            node["content"] = [{"type": "paragraph"}]
    if "content" in node:
        for sub_node in node["content"]:
            update_node(sub_node)
    if (
        "attrs" in node and
        "initial" in node["attrs"] and
        bool(node["attrs"]["initial"])
    ):
        for sub_node in node["attrs"]["initial"]:
            update_initial_node(sub_node)

# from https://stackoverflow.com/questions/25738523/how-to-update-one-file-inside-zip-file-using-python
def update_revision_zip(file_field, file_name):
    # generate a temp file
    tmpfd, tmpname = tempfile.mkstemp()
    os.close(tmpfd)
    # create a temp copy of the archive without filename
    with zipfile.ZipFile(file_field.open(), 'r') as zin:
        with zipfile.ZipFile(tmpname, 'w') as zout:
            zout.comment = zin.comment # preserve the comment
            for item in zin.infolist():
                if item.filename == 'document.json':
                    doc_string = zin.read(item.filename)
                    doc = json.loads(doc_string)
                    if 'contents' in doc:
                        doc['content'] = doc['contents']
                        del doc['contents']
                    update_node(doc['content'])
                    zout.writestr(
                        item,
                        json.dumps(doc)
                    )
                else:
                    zout.writestr(item, zin.read(item.filename))
    # replace with the temp archive
    with open(tmpname, 'rb') as tmp_file:
        file_field.save(file_name, File(tmp_file))
    os.remove(tmpname)

def update_documents(apps, schema_editor):
    Document = apps.get_model('document', 'Document')
    documents = Document.objects.all().iterator()
    for document in documents:
        if document.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_node(document.content)
            document.save()

    DocumentTemplate = apps.get_model('document', 'DocumentTemplate')
    templates = DocumentTemplate.objects.all()
    for template in templates:
        if template.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_node(template.content)
            template.save()

    DocumentRevision = apps.get_model('document', 'DocumentRevision')
    revisions = DocumentRevision.objects.all()
    for revision in revisions:
        if not revision.file_object:
            revision.delete()
            continue
        if revision.doc_version == Decimal(str(FW_DOCUMENT_VERSION)):
            update_revision_zip(revision.file_object, revision.file_name)

class Migration(migrations.Migration):

    dependencies = [
        ('document', '0006_auto_20201209_1610'),
    ]

    operations = [
        migrations.RunPython(update_documents),
    ]
<|endoftext|>


<commit_before>import re, os, sys, errno
from argparse import ArgumentParser

def main():
    args = marshall_arguments()
    in_file_or_dir = args.file

    if is_dir(in_file_or_dir):
        process_directory(in_file_or_dir)
    else:
        in_file = in_file_or_dir
        rules = process_rules_file(in_file)
        do_line_replacements(rules, in_file)

class ccolor:
    END = '\033[0m'
    BOLD = '\033[1m'
    COLOR = '\033[4m'

class cmsg:
    DOTTEDLINE = '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '

class Replacement(object):
    def __init__(self, replacement):
        self.replacement = replacement
        self.matched = None
        self.replaced = None

    def __call__(self, match):
        self.matched = match.group(0)
        self.replaced = match.expand(self.replacement)
        return self.replaced

def is_valid_file_or_dir(parser, arg):
    if not os.path.exists(arg):
        parser.error("%s does not exist!" % arg)
    else:
        if is_dir(arg):
            return arg
        else:
            return open(arg, 'r')  # return an open file handle

def write_output_file(file_name, result):
    text_file = open(file_name, "w")
    text_file.write(result)
    text_file.close()

def replace_lines(lines_to_match, replacement_lines, targetString):
    regex_to_match = "(\n\s*)"
    replacement_text = "\g<1>"
    regex_group_count = 2

    print cmsg.DOTTEDLINE
    print ccolor.BOLD + "All occurrences of:" + ccolor.END

    for step in lines_to_match:
        regex_to_match = regex_to_match + step + "(\s*\n\s*)"
        print "\t" + step

    print ccolor.BOLD + "have been replaced with:" + ccolor.END

    for step in replacement_lines:
        replacement_text = replacement_text + step + "\g<" + str(regex_group_count) + ">"
        print "\t" + step
        regex_group_count += 1

    processed_text = re.subn(regex_to_match, replacement_text, targetString, flags=re.IGNORECASE | re.DOTALL | re.MULTILINE)

    print ccolor.BOLD + (str(processed_text[1]) + " replacements performed in this stage") + ccolor.END
    print cmsg.DOTTEDLINE

    return processed_text[0]

def marshall_arguments():
    global parser
    parser = ArgumentParser(description="The Axiomatic Text Recomposer v0.0.1")
    parser.add_argument(dest="file",
                        help="name of the *.feature file (or directory) to process.  \n(NB - Directory MUST contain a *.rules file.)",
                        metavar="FEATURE_FILE",
                        type=lambda x: is_valid_file_or_dir(parser, x))

    group = parser.add_mutually_exclusive_group()
    group.add_argument('-o', '--output',
                       help='specify output file.  Default: modify input file(s) in place')

    return parser.parse_args()

def get_extension(file):
    if not type(file) is str:
        file = file.name

    return os.path.splitext(file)[1]

def is_dir(arg):
    if not type(arg) is str:
        arg = arg.name

    return os.path.isdir(arg)

def process_rules_file(ff):
    if is_dir(ff):
        directory_containing_ff = ff
    else:
        directory_containing_ff = os.path.dirname(os.path.realpath(ff.name))

    rules_file = ""
    missing_rules_file = True

    for root, dirs, files in os.walk(directory_containing_ff):
        for file in files:
            if file.endswith(".rules"):
                print("Using Rules file: " + os.path.join(root, file))
                missing_rules_file = False
                rules_file = open(os.path.join(root, file))

    if missing_rules_file:
        print "Error: No rules list was found.\n"
        parser.print_help()
        sys.exit(errno.ENOENT)

    rules_list = []
    add = []
    sub = []
    for line in rules_file:
        if line.lstrip().startswith("-"):
            add.append(line.lstrip().lstrip("-").lstrip().strip())
        elif line.lstrip().startswith("+"):
            sub.append(line.lstrip().lstrip("+").lstrip().strip())
        else:
            # only add the rules to the rules list if there is a rule to add (catches multiple empty lines)
            if add:
                rules_list.append([add, sub])
                add = []
                sub = []
    if add:
        rules_list.append([add, sub])

    return rules_list

def do_line_replacements(master_rule_list, ff):
    ff_name = ff.name
    feature_text = ff.read()
    for lrr in master_rule_list:
        old_steps = lrr[0]
        new_steps = lrr[1]
        feature_text = replace_lines(old_steps, new_steps, feature_text)
    write_output_file(ff_name, feature_text)

def process_directory(dir):
    for root, dirs, files in os.walk(dir):
        for file in files:
            if file.endswith(".feature"):
                in_file = dir
                rules = process_rules_file(in_file)
                do_line_replacements(rules, in_file)

main()
<commit_msg>implement all basic rule processing functions, perform major refactor<commit_after>import re, os, sys, errno
from argparse import ArgumentParser

parser = ArgumentParser

def main():
    args = marshall_arguments()
    path = args.path

    if is_dir(path):
        process_directory(path, ".feature")
    else:
        process_file(path, ".feature")

class ccolor:
    END = '\033[0m'
    BOLD = '\033[1m'
    COLOR = '\033[4m'

class cmsg:
    DOTTEDLINE = '- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - '
    DOTTEDLINE_RGX = '- - REGEX RULE  - - - - - - - - - - - - - - - - - - - - - - - - - - '
    DOTTEDLINE_LRR = '- - LINE REPLACEMENT RULE - - - - - - - - - - - - - - - - - - - - - '
    DOTTEDLINE_WRR = '- - WORD REPLACEMENT RULE - - - - - - - - - - - - - - - - - - - - - '
    BLANKLINE = '\n'

def is_valid_file_or_dir(parser, arg):
    if not os.path.exists(arg):
        parser.error("%s does not exist!" % arg)
    else:
        if is_dir(arg):
            return arg
        else:
            return open(arg, 'r')  # return an open file handle

def write_output_file(file_name, result):
    text_file = open(file_name, "w")
    text_file.write(result)
    text_file.close()

def replace_lines(lines_to_match, replacement_lines, target_string):
    regex_to_match = "(\n\s*)"
    replacement_text = "\g<1>"
    regex_group_count = 2

    print cmsg.DOTTEDLINE_LRR
    print ccolor.BOLD + "All occurrences of:" + ccolor.END

    for step in lines_to_match:
        regex_to_match += step + "(\s*\n\s*)"
        print "\t" + step

    print ccolor.BOLD + "have been replaced with:" + ccolor.END

    for step in replacement_lines:
        replacement_text += step + "\g<" + str(regex_group_count) + ">"
        print "\t" + step
        regex_group_count += 1

    processed_text = re.subn(regex_to_match, replacement_text, target_string, flags=re.IGNORECASE | re.DOTALL | re.MULTILINE)

    print ccolor.BOLD + (str(processed_text[1]) + " replacement(s) performed in this stage") + ccolor.END
    print cmsg.BLANKLINE

    return processed_text[0]

def replace_regex(regex_rules, replacement_string, target_string):
    print cmsg.DOTTEDLINE_RGX
    print ccolor.BOLD + "Each and every match of the pattern:" + ccolor.END
    print "\t" + regex_rules[0].pattern

    print ccolor.BOLD + "has been replaced with:" + ccolor.END
    print "\t" + replacement_string[0]

    processed_text = re.subn(regex_rules[0], replacement_string[0], target_string)

    print ccolor.BOLD + (str(processed_text[1]) + " replacement(s) performed in this stage") + ccolor.END
    print cmsg.BLANKLINE

    return processed_text[0]

def replace_word(words_to_match, replacement_word, target_string):
    print cmsg.DOTTEDLINE_WRR
    print ccolor.BOLD + "Each and every occurrence of:" + ccolor.END
    print "\t" + words_to_match[0].pattern

    print ccolor.BOLD + "has been replaced with:" + ccolor.END
    print "\t" + replacement_word[0]

    processed_text = re.subn(words_to_match[0], replacement_word[0], target_string)

    print ccolor.BOLD + (str(processed_text[1]) + " replacement(s) performed in this stage") + ccolor.END
    print cmsg.BLANKLINE

    return processed_text[0]

def marshall_arguments():
    global parser
    parser = ArgumentParser(description="The Axiomatic Text Recomposer v0.0.1")
    parser.add_argument(dest="path",
                        help="path to file or directory to process.  \n(NB - Directory MUST contain a *.rules file.)",
                        metavar="PATH",
                        type=lambda x: is_valid_file_or_dir(parser, x))

    group = parser.add_mutually_exclusive_group()
    group.add_argument('-o', '--output',
                       help='specify output file.  Default: modify input file(s) in place')

    return parser.parse_args()

def get_extension(fi):
    if not type(fi) is str:
        fi = fi.name

    return os.path.splitext(fi)[1]

def is_dir(arg):
    if not type(arg) is str:
        arg = arg.name

    return os.path.isdir(arg)

def process_rules_file_in_dir_containing(ff):
    if is_dir(ff):
        directory_containing_ff = ff
    else:
        directory_containing_ff = os.path.dirname(os.path.realpath(ff.name))

    rules_file = ""
    missing_rules_file = True

    for root, dirs, files in os.walk(directory_containing_ff):
        for fi in files:
            if fi.endswith(".rules"):
                print("Using Rules file: " + os.path.join(root, fi))
                missing_rules_file = False
                rules_file = open(os.path.join(root, fi))

    if missing_rules_file:
        print "Error: No rules list was found.\n"
        parser.print_help()
        sys.exit(errno.ENOENT)
    previous_rule_type = ""
    rules_list, line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule = [], [], [], [], [], [], []
    # TODO consider a better pattern for this. At minimum, replace "get_rule_type(rule), add_rule(rule_type) and a dictionary
    for rule in rules_file:
        rule_type = get_rule_type(rule)
        rule = clean_rule(rule, rule_type)
        # ignore comments
        if rule_type == "##":
            continue
        # line replacement rules
        elif rule_type == "l-":
            line_add_rule.append(rule)
        elif rule_type == "l+":
            line_sub_rule.append(rule)
        # word replacement rules
        elif rule_type == "w-":
            escaped_string = re.escape(rule)
            word_sub_rule.append(re.compile(escaped_string))
        elif rule_type == "w+":
            escaped_string = re.escape(rule)
            word_add_rule.append(escaped_string)
        # regex rules
        elif rule_type == "r-":
            regx_mat_rule.append(re.compile(rule, flags=re.IGNORECASE))
        elif rule_type == "r+":
            regx_rpl_rule.append(rule)
        elif rule_type == "ri-":
            regx_mat_rule.append(re.compile(rule))
        elif rule_type == "r-\i":
            regx_mat_rule.append(re.compile(rule.rstrip("\i")))
        elif rule_type == "ri+":
            regx_rpl_rule.append(rule)
        # blank lines
        elif rule_type == "blank":
            if previous_rule_type != "blank":
                rules_list.append([line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule])
                line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule = [], [], [], [], [], []
        else:
            # only add the rule to the rules list if there is a rule to add (catches multiple empty lines)
            if len(line_add_rule) > 0 or len(word_add_rule) > 0 or len(regx_mat_rule) > 0:
                rules_list.append([line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule])
                line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule = [], [], [], [], [], []
        previous_rule_type = rule_type

    # add final rule to rule list
    if len(line_add_rule) > 0 or len(word_add_rule) > 0 or len(regx_mat_rule) > 0:
        rules_list.append([line_add_rule, line_sub_rule, word_sub_rule, word_add_rule, regx_mat_rule, regx_rpl_rule])

    return rules_list

def clean_rule(rule, rule_type):
    return rule.lstrip().lstrip("-").lstrip("+").lstrip(rule_type).lstrip().strip()

def get_rule_type(rule):
    rule = rule.lstrip()
    if len(rule) == 0 or rule.isspace():
        rule_type = "blank"
    elif rule[0] == "#":
        rule_type = "##"
    elif rule[0] == "-" or rule[0] == "+":
        rule_type = "l" + rule[0]
    elif len(rule) > 2 and rule[0:3] == "ri-":
        rule_type = "ri-"
    elif len(rule) > 2 and rule[0:3] == "ri+":
        rule_type = "ri+"
    elif len(rule) > 4 and rule[0:2] == "r-" and rule[-2:] == "\i":
        rule_type = "r-\i"
    else:
        if len(rule) < 2:
            rule_type = "??"
        else:
            rule_type = rule[0:2]
    return rule_type

def process_line_replacement_rules(master_rule_list, text):
    num_rules_processed = 0
    for rule in master_rule_list:
        lines_to_delete = rule[0]
        replacement_lines = rule[1]
        if len(lines_to_delete) > 0:
            text = replace_lines(lines_to_delete, replacement_lines, text)
            num_rules_processed += 1
    return text

def process_regex_rules(master_rule_list, text):
    num_rules_processed = 0
    for rule in master_rule_list:
        match_regex = rule[4]
        replacement_string = rule[5]
        if len(match_regex) > 0:
            text = replace_regex(match_regex, replacement_string, text)
            num_rules_processed += 1
    return text

def process_word_replacement_rules(master_rule_list, text):
    num_rules_processed = 0
    for rule in master_rule_list:
        target_word = rule[2]
        replacement_word = rule[3]
        if len(target_word) > 0:
            text = replace_word(target_word, replacement_word, text)
            num_rules_processed += 1
    return text

def process_directory(direc, file_type_to_process):
    for root, dirs, files in os.walk(direc):
        for f in files:
            process_file(open(direc + "/" + f, "r"), file_type_to_process)

def process_file(fi, fi_type):
    if fi.name.endswith(fi_type):
        in_file = fi
        # TODO: The rules file should be processed once for each unique directory, not once per file
        rules = process_rules_file_in_dir_containing(in_file)
        text = in_file.read()
        text = process_regex_rules(rules, text)
        text = process_line_replacement_rules(rules, text)
        text = process_word_replacement_rules(rules, text)
        write_output_file(fi.name, text)

main()
<|endoftext|>


<commit_before>import datetime
import logging
import subprocess
import time

import util

class Screenshotter(object):
    """
    Implements all screenshot-related logic.
    """

    def __init__(self, config):
        self.config = config
        self.screenshot_filenames = []

    def take_screenshots(self):
        """
        Runs through the process of taking all screenshots.
        """
        self.current_datetime_string = util.get_current_datetime_string()
        images = []
        for image in self.config.get('images'):
            self.add_filepaths(image)
            try:
                self.depict(
                    image.get('url'),
                    image.get('selector'),
                    image.get('local_filepath'),
                    str(int(self.config.get('page_load_delay', 2) * 1000))
                )
                images.append({
                    'slug': image.get('slug'),
                    'filepath': image.get('filepath'),
                    'local_filepath': image.get('local_filepath'),
                    'aws_filepath': image.get('aws_filepath'),
                    'aws_latest_filepath': image.get('aws_latest_filepath'),
                })
            except RuntimeError as e:
                logging.error(e)
        return images

    def add_filepaths(self, image):
        image['filepath'] = util.generate_image_filepath(
                self.current_datetime_string, image.get('slug'))
        image['local_filepath'] = util.generate_image_local_filepath(
                self.config.get('local_image_directory'),
                self.current_datetime_string, image.get('slug'))
        image['aws_filepath'] = util.generate_image_aws_filepath(
                self.config.get('aws_subpath'),
                self.current_datetime_string, image.get('slug'))
        image['aws_latest_filepath'] = util.generate_image_aws_latest_filepath(
                self.config.get('aws_subpath'), image.get('slug'))

    def depict(self, url, selector, destination, page_load_delay):
        """
        Runs the command-line utility `depict`.
        """
        args = ['depict', url, destination, '-s', selector,
                '--delay', page_load_delay]

        override_css_file = self.config.get('override_css_file')
        if override_css_file:
            args = args + ['--css', override_css_file]

        if self.config.get('wait_for_js_signal', False):
            args = args + ['--call-phantom']

        logging.info('Running shell command: %s' % (args))

        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)

        # TODO Don't terminate if failure_timeout is set to 0
        deadline = time.time() + self.config.get('failure_timeout', 30)
        while time.time() < deadline and p.poll() == None:
            time.sleep(1)

        if p.poll() == None:
            p.terminate()
            raise RuntimeError('Terminated shell command: %s' % (args))

        out, err = p.communicate()
        if p.returncode != 0:
            raise RuntimeError('depict error: %s' % err)
<commit_msg>Don't terminate if failure_timeout is 0<commit_after>import datetime
import logging
import subprocess
import time

import util

class Screenshotter(object):
    """
    Implements all screenshot-related logic.
    """

    def __init__(self, config):
        self.config = config
        self.screenshot_filenames = []

    def take_screenshots(self):
        """
        Runs through the process of taking all screenshots.
        """
        self.current_datetime_string = util.get_current_datetime_string()
        images = []
        for image in self.config.get('images'):
            self.add_filepaths(image)
            try:
                self.depict(
                    image.get('url'),
                    image.get('selector', 'body'),
                    image.get('local_filepath'),
                    # Depict's delay argument is defined in milliseconds
                    str(int(self.config.get('page_load_delay', 2)) * 1000)
                )
                images.append({
                    'slug': image.get('slug'),
                    'filepath': image.get('filepath'),
                    'local_filepath': image.get('local_filepath'),
                    'aws_filepath': image.get('aws_filepath'),
                    'aws_latest_filepath': image.get('aws_latest_filepath'),
                })
            except RuntimeError as e:
                logging.error(e)
        return images

    def add_filepaths(self, image):
        image['filepath'] = util.generate_image_filepath(
                self.current_datetime_string, image.get('slug'))
        image['local_filepath'] = util.generate_image_local_filepath(
                self.config.get('local_image_directory'),
                self.current_datetime_string, image.get('slug'))
        image['aws_filepath'] = util.generate_image_aws_filepath(
                self.config.get('aws_subpath'),
                self.current_datetime_string, image.get('slug'))
        image['aws_latest_filepath'] = util.generate_image_aws_latest_filepath(
                self.config.get('aws_subpath'), image.get('slug'))

    def depict(self, url, selector, destination, page_load_delay):
        """
        Runs the command-line utility `depict`.
        """
        args = ['depict', url, destination, '-s', selector,
                '--delay', page_load_delay]

        override_css_file = self.config.get('override_css_file')
        if override_css_file:
            args = args + ['--css', override_css_file]

        if self.config.get('wait_for_js_signal', False):
            args = args + ['--call-phantom']

        logging.info('Running shell command: %s' % (args))

        p = subprocess.Popen(args, stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)

        # Wait at most `failure_timeout` seconds for the process to finish
        failure_timeout = self.config.get('failure_timeout', 30)
        deadline = time.time() + failure_timeout
        while time.time() < deadline and p.poll() == None:
            time.sleep(1)

        # If the failure timeout is more than 0 and the process still hasn't
        # finished, terminate it.
        if failure_timeout > 0 and p.poll() == None:
            p.terminate()
            raise RuntimeError('Terminated shell command: %s' % (args))

        out, err = p.communicate()
        if p.returncode != 0:
            raise RuntimeError('depict error: %s' % err)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.core.management.base import BaseCommand

import json
from lxml import objectify

from candidates.models import invalidate_cache_entries_from_person_data
from candidates.popit import create_popit_api_object
from candidates.update import PersonParseMixin, PersonUpdateMixin
from candidates.views.version_data import get_change_metadata

CONSTITUENCIES_JSON_FILE = 'data/mapit-WMC-generation-22.json'
ALL_MEMBERS_XML_FILE = 'data/parlparse/members/all-members-2010.xml'
PEOPLE_XML_FILE = 'data/parlparse/members/people.xml'

PARTY_MAPPING = {
    'Alliance': 'Alliance - Alliance Party of Northern Ireland',
    'Con': 'Conservative Party',
    'DUP': 'Democratic Unionist Party - D.U.P.',
    'Green': 'Green Party',
    'Ind': 'Independent',
    'LDem': 'Liberal Democrats',
    'Lab': 'Labour Party',
    'PC': 'Plaid Cymru - The Party of Wales',
    'Res': 'The Respect Party',
    'SDLP': 'SDLP (Social Democratic & Labour Party)',
    'SF': u'Sinn Féin',
    'SNP': 'Scottish National Party (SNP)',
    'SPK': 'Speaker seeking re-election',
    'UKIP': 'UK Independence Party (UKIP)',
}

class Command(PersonParseMixin, PersonUpdateMixin, BaseCommand):
    help = "Update candidates' parlparse id"

    def handle(self, *args, **options):
        self.verbosity = int(options.get('verbosity', 1))
        self.popit = create_popit_api_object()
        with open(CONSTITUENCIES_JSON_FILE) as f:
            constituencies = json.load(f).values()
        xml = objectify.parse(ALL_MEMBERS_XML_FILE).getroot()
        self.members = [member.attrib for member in xml.findall('member')]
        people_xml = objectify.parse(PEOPLE_XML_FILE).getroot()
        people = people_xml.findall('person')
        self.id_mapping = {p.get('latestname'): p.get('id') for p in people}
        self.update_candidates(constituencies)

    def update_candidates(self, constituencies):
        for constituency in constituencies:
            for member in self.members_for(constituency['name']):
                self.add_parlparse_id_to_member(member)

    def members_for(self, constituency_name):
        constituency_members = []
        for member in self.members:
            if member['constituency'] == constituency_name:
                constituency_members.append(member)
        return constituency_members

    def add_parlparse_id_to_member(self, member):
        # Try and find the corresponding person in the popit instance
        name = u'{} {}'.format(member['firstname'], member['lastname'])
        query = u'"{}"'.format(name)
        result = self.popit.api.search.persons.get(q=query)['result']
        # Have we not got any results?
        if len(result) == 0:
            if self.verbosity > 0:
                msg = u'No results for {}, skipping'
                self.stderr.write(msg.format(name))
            return
        # Find the person with a matching constituency
        person = self.person_match(member=member, people=result)
        if not person:
            if self.verbosity > 0:
                msg = u'{} result but no matches found for {}'
                self.stderr.write(msg.format(len(result), name))
            return
        parlparse_id = self.id_mapping[name]
        person_data, _ = self.get_person(person['id'])
        previous_versions = person_data.pop('versions')
        identifiers = person_data.get('identifiers', [])
        existing_identifiers = [i['identifier'] for i in identifiers]
        # Does that person already have this identifier?
        if parlparse_id in existing_identifiers:
            if self.verbosity > 1:
                msg = u'Found existing identifier for {} ({}), skipping'
                self.stderr.write(msg.format(name, parlparse_id))
            return
        identifier = {
            'identifier': parlparse_id,
            'scheme': 'uk.org.publicwhip'
        }
        identifiers.append(identifier)
        person_data['identifiers'] = identifiers
        change_metadata = get_change_metadata(
            None,
            'Updated candidate with parlparse person id',
        )
        self.update_person(
            person_data,
            change_metadata,
            previous_versions,
        )
        invalidate_cache_entries_from_person_data(person_data)
        if self.verbosity > 0:
            msg = u"Successfully updated {} with {}"
            self.stdout.write(msg.format(name, parlparse_id))

    def person_match(self, people, member):
        match = None
        for person in people:
            constituency_name = person.get('standing_in') and \
                person['standing_in'].get('2010', {}).get('name', '')
            party_name = person.get('party_memberships') and \
                person['party_memberships'].get('2010', {}).get('name', '')
            if constituency_name == member['constituency'] and \
                    party_name == PARTY_MAPPING[member['party']]:
                match = person
                break
        return match
<commit_msg>Remove the now unneeded candidates_add_parlparse_id command<commit_after><|endoftext|>


<commit_before>from xmppd import filter
from xmpp  import *
from spade import *

class Component(filter.Filter):

	def __init__(self,router):

		filter.Filter.__init__(self,router)

		router.server.Dispatcher.RegisterNamespaceHandler(NS_COMPONENT_ACCEPT, router.routerHandler)
	        router.server.Dispatcher.RegisterHandler('handshake', self.componentHandler, xmlns=NS_COMPONENT_ACCEPT)
        	router.server.Dispatcher.RegisterHandler('message', router.routerHandler, xmlns=NS_COMPONENT_ACCEPT)

	def test(self,stanza):
		#Counter measure for component originated messages
		the_from = stanza['from']
		simple_from = str(the_from)
		to = str(stanza['to'])

		if not('@' in simple_from):  # Component-originated
			if stanza.getNamespace()==NS_COMPONENT_ACCEPT and stanza.getName()=='message':
				self._router.DEBUG("Message for a component",'info')
				return True
		if not ('@' in to): return True
		self._router.DEBUG("Message NOT for a component",'error')
		return False

	def filter(self,session,stanza):

			s = False
			to = str(stanza['to'])
			ffrom = str(stanza['from'])

			if not('@' in ffrom):
                        	# Fake the namespace. Pose as a client
	                        stanza.setNamespace(NS_CLIENT)
        	                #self._router.DEBUG("NS faked: " + str(stanza.getNamespace()), 'info')

			if not('@' in to):
				s=self._router._owner.getsession(to)
				#self._router.DEBUG("Component getsession("+str(to)+") ->" + str(s), 'info')
				if s:
					s.enqueue(stanza)
					self._router.DEBUG("message for a component delivered", 'info')
					return None

			return stanza

	def componentHandler(self, session, stanza):
                name = stanza.getName()
                if name == 'handshake':
			#self._router.DEBUG("HANDSHAKE received", 'info')
                        # Reply handshake
                        rep = Node('handshake')
                        session.send(rep)
                        # Identify component
                        host,port = session._sock.getsockname()
                        #print "HOST: " + str(host) + " PORT: " + str(port)
                        primary_name = self._router.server.servernames[0]
                        if port == 9000:  # ACC
                                component_name = 'acc.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                        elif port == 9001:  # AMS
                                component_name = 'ams.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                        elif port == 9002:  # DF
                                component_name = 'df.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                elif name == 'message':
                        #"Component sends a MESSAGE"
                        to=stanza['to']
                        simple_to = str(to)
			s=False
                        s=self._router._owner.getsession(to)
                        if s:
                                self._router.DEBUG( "Found session for to: %s %d" % (str(to), s._session_state),'info')
                                s.enqueue(stanza)
                        raise NodeProcessed
                else:
                        if session._session_state >= SESSION_AUTHED:
                                self._router.DEBUG( str(stanza), 'send')

<commit_msg>packaging<commit_after>from xmppd import filter
from xmpp  import *
from spade import *

class Component(filter.Filter):

	def __init__(self,router):

		filter.Filter.__init__(self,router)

		router.server.Dispatcher.RegisterNamespaceHandler(NS_COMPONENT_ACCEPT, router.routerHandler)
	        router.server.Dispatcher.RegisterHandler('handshake', self.componentHandler, xmlns=NS_COMPONENT_ACCEPT)
        	router.server.Dispatcher.RegisterHandler('message', router.routerHandler, xmlns=NS_COMPONENT_ACCEPT)

	def test(self,stanza):
		#Counter measure for component originated messages
		the_from = stanza['from']
		simple_from = str(the_from)
		to = str(stanza['to'])

		if not('@' in simple_from):  # Component-originated
			if stanza.getNamespace()==NS_COMPONENT_ACCEPT and stanza.getName()=='message':
				self._router.DEBUG("Message for a component",'info')
				return True
		if not ('@' in to) and (to not in self._router._owner.servernames): return True
		self._router.DEBUG("Message NOT for a component",'error')
		return False

	def filter(self,session,stanza):

			s = False
			to = str(stanza['to'])
			ffrom = str(stanza['from'])

			if not('@' in ffrom):
                        	# Fake the namespace. Pose as a client
	                        stanza.setNamespace(NS_CLIENT)
        	                #self._router.DEBUG("NS faked: " + str(stanza.getNamespace()), 'info')

			if not('@' in to):
				s=self._router._owner.getsession(to)
				#self._router.DEBUG("Component getsession("+str(to)+") ->" + str(s), 'info')
				if s:
					s.enqueue(stanza)
					self._router.DEBUG("message for a component delivered", 'info')
					return None

			return stanza

	def componentHandler(self, session, stanza):
                name = stanza.getName()
                if name == 'handshake':
			#self._router.DEBUG("HANDSHAKE received", 'info')
                        # Reply handshake
                        rep = Node('handshake')
                        session.send(rep)
                        # Identify component
                        host,port = session._sock.getsockname()
                        #print "HOST: " + str(host) + " PORT: " + str(port)
                        primary_name = self._router.server.servernames[0]
                        if port == 9000:  # ACC
                                component_name = 'acc.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                        elif port == 9001:  # AMS
                                component_name = 'ams.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                        elif port == 9002:  # DF
                                component_name = 'df.' + primary_name
                                session.peer = component_name
                                self._router.server.activatesession(session, component_name)
                                session.set_session_state(SESSION_AUTHED)
                                session.set_session_state(SESSION_OPENED)
                                raise NodeProcessed
                elif name == 'message':
                        #"Component sends a MESSAGE"
                        to=stanza['to']
                        simple_to = str(to)
			s=False
                        s=self._router._owner.getsession(to)
                        if s:
                                self._router.DEBUG( "Found session for to: %s %d" % (str(to), s._session_state),'info')
                                s.enqueue(stanza)
                        raise NodeProcessed
                else:
                        if session._session_state >= SESSION_AUTHED:
                                self._router.DEBUG( str(stanza), 'send')

<|endoftext|>


<commit_before>from django.db import models
from core.models import PlCoreBase,SingletonModel,PlCoreBaseManager
from core.models.plcorebase import StrippedCharField
from xos.exceptions import *
import json

class Service(PlCoreBase):
    # when subclassing a service, redefine KIND to describe the new service
    KIND = "generic"

    description = models.TextField(max_length=254,null=True, blank=True,help_text="Description of Service")
    enabled = models.BooleanField(default=True)
    kind = StrippedCharField(max_length=30, help_text="Kind of service", default=KIND)
    name = StrippedCharField(max_length=30, help_text="Service Name")
    versionNumber = StrippedCharField(max_length=30, help_text="Version of Service Definition")
    published = models.BooleanField(default=True)
    view_url = StrippedCharField(blank=True, null=True, max_length=1024)
    icon_url = StrippedCharField(blank=True, null=True, max_length=1024)
    public_key = models.TextField(null=True, blank=True, max_length=1024, help_text="Public key string")

    def __init__(self, *args, **kwargs):
        # for subclasses, set the default kind appropriately
        self._meta.get_field("kind").default = self.KIND
        super(Service, self).__init__(*args, **kwargs)

    @classmethod
    def get_service_objects(cls):
        return cls.objects.filter(kind = cls.KIND)

    def __unicode__(self): return u'%s' % (self.name)

class ServiceAttribute(PlCoreBase):
    name = models.SlugField(help_text="Attribute Name", max_length=128)
    value = StrippedCharField(help_text="Attribute Value", max_length=1024)
    service = models.ForeignKey(Service, related_name='serviceattributes', help_text="The Service this attribute is associated with")

class Tenant(PlCoreBase):
    """ A tenant is a relationship between two entities, a subscriber and a
        provider.

        The subscriber can be a User, a Service, or a Tenant.

        The provider is always a Service.
    """

    CONNECTIVITY_CHOICES = (('public', 'Public'), ('private', 'Private'), ('na', 'Not Applicable'))

    # when subclassing a service, redefine KIND to describe the new service
    KIND = "generic"

    kind = StrippedCharField(max_length=30, default=KIND)
    provider_service = models.ForeignKey(Service, related_name='tenants')
    subscriber_service = models.ForeignKey(Service, related_name='subscriptions', blank=True, null=True)
    subscriber_tenant = models.ForeignKey("Tenant", related_name='subscriptions', blank=True, null=True)
    subscriber_user = models.ForeignKey("User", related_name='subscriptions', blank=True, null=True)
    service_specific_id = StrippedCharField(max_length=30, blank=True, null=True)
    service_specific_attribute = models.TextField(blank=True, null=True)
    connect_method = models.CharField(null=False, blank=False, max_length=30, choices=CONNECTIVITY_CHOICES, default="na")

    def __init__(self, *args, **kwargs):
        # for subclasses, set the default kind appropriately
        self._meta.get_field("kind").default = self.KIND
        super(Tenant, self).__init__(*args, **kwargs)

    def __unicode__(self):
        if not hasattr(self, "provider_service"):
           # When the REST API does a POST on a CordSubscriber object, for
           # some reason there is no provider_service field. All of the other
           # fields are there. Provider_service is even in the dir(). However,
           # trying to getattr() on it will fail.
           return "confused-tenant-object"

        if self.subscriber_service:
            return u'%s service %s on service %s' % (str(self.kind), str(self.subscriber_service.id), str(self.provider_service.id))
        elif self.subscriber_tenant:
            return u'%s tenant %s on service %s' % (str(self.kind), str(self.subscriber_tenant.id), str(self.provider_service.id))
        else:
            return u'%s on service %s' % (str(self.kind), str(self.provider_service.id))

    # helper for extracting things from a json-encoded service_specific_attribute
    def get_attribute(self, name, default=None):
        if self.service_specific_attribute:
            attributes = json.loads(self.service_specific_attribute)
        else:
            attributes = {}
        return attributes.get(name, default)

    def set_attribute(self, name, value):
        if self.service_specific_attribute:
            attributes = json.loads(self.service_specific_attribute)
        else:
            attributes = {}
        attributes[name]=value
        self.service_specific_attribute = json.dumps(attributes)

    @classmethod
    def get_tenant_objects(cls):
        return cls.objects.filter(kind = cls.KIND)

    @classmethod
    def get_deleted_tenant_objects(cls):
        return cls.deleted_objects.filter(kind = cls.KIND)

    # helper function to be used in subclasses that want to ensure service_specific_id is unique
    def validate_unique_service_specific_id(self):
        if self.pk is None:
            if self.service_specific_id is None:
                raise XOSMissingField("subscriber_specific_id is None, and it's a required field", fields={"service_specific_id": "cannot be none"})

            conflicts = self.get_tenant_objects().filter(service_specific_id=self.service_specific_id)
            if conflicts:
                raise XOSDuplicateKey("service_specific_id %s already exists" % self.service_specific_id, fields={"service_specific_id": "duplicate key"})

class CoarseTenant(Tenant):
    class Meta:
        proxy = True

    KIND = "coarse"

    def save(self, *args, **kwargs):
        if (not self.subscriber_service):
            raise XOSValidationError("subscriber_service cannot be null")
        if (self.subscriber_tenant or self.subscriber_user):
            raise XOSValidationError("subscriber_tenant and subscriber_user must be null")

        super(CoarseTenant,self).save()
<commit_msg>add ability for services to scale<commit_after>from django.db import models
from core.models import PlCoreBase,SingletonModel,PlCoreBaseManager
from core.models.plcorebase import StrippedCharField
from xos.exceptions import *
import json

class Service(PlCoreBase):
    # when subclassing a service, redefine KIND to describe the new service
    KIND = "generic"

    description = models.TextField(max_length=254,null=True, blank=True,help_text="Description of Service")
    enabled = models.BooleanField(default=True)
    kind = StrippedCharField(max_length=30, help_text="Kind of service", default=KIND)
    name = StrippedCharField(max_length=30, help_text="Service Name")
    versionNumber = StrippedCharField(max_length=30, help_text="Version of Service Definition")
    published = models.BooleanField(default=True)
    view_url = StrippedCharField(blank=True, null=True, max_length=1024)
    icon_url = StrippedCharField(blank=True, null=True, max_length=1024)
    public_key = models.TextField(null=True, blank=True, max_length=1024, help_text="Public key string")

    def __init__(self, *args, **kwargs):
        # for subclasses, set the default kind appropriately
        self._meta.get_field("kind").default = self.KIND
        super(Service, self).__init__(*args, **kwargs)

    @classmethod
    def get_service_objects(cls):
        return cls.objects.filter(kind = cls.KIND)

    def __unicode__(self): return u'%s' % (self.name)

    def get_scalable_nodes(self, slice, max_per_node=None, exclusive_slices=[]):
        """
             Get a list of nodes that can be used to scale up a slice.

                slice - slice to scale up
                max_per_node - maximum numbers of slivers that 'slice' can have on a single node
                exclusive_slices - list of slices that must have no nodes in common with 'slice'.
        """

        from core.models import Node, Sliver # late import to get around order-of-imports constraint in __init__.py

        nodes = list(Node.objects.all())

        conflicting_slivers = Sliver.objects.filter(slice__in = exclusive_slices)
        conflicting_nodes = Node.objects.filter(slivers__in = conflicting_slivers)

        nodes = [x for x in nodes if x not in conflicting_nodes]

        # If max_per_node is set, then limit the number of slivers this slice
        # can have on a single node.
        if max_per_node:
            acceptable_nodes = []
            for node in nodes:
                existing_count = node.slivers.filter(slice=slice).count()
                if existing_count < max_per_node:
                    acceptable_nodes.append(node)
            nodes = acceptable_nodes

        return nodes

    def pick_node(self, slice, max_per_node=None, exclusive_slices=[]):
        # Pick the best node to scale up a slice.

        nodes = self.get_scalable_nodes(slice, max_per_node, exclusive_slices)
        nodes = sorted(nodes, key=lambda node: node.slivers.all().count())
        if not nodes:
            return None
        return nodes[0]

    def adjust_scale(self, slice_hint, scale, max_per_node=None, exclusive_slices=[]):
        from core.models import Sliver # late import to get around order-of-imports constraint in __init__.py

        slices = [x for x in self.slices.all() if slice_hint in x.name]
        for slice in slices:
            while slice.slivers.all().count() > scale:
                s = slice.slivers.all()[0]
                # print "drop sliver", s
                s.delete()

            while slice.slivers.all().count() < scale:
                node = self.pick_node(slice, max_per_node, exclusive_slices)
                if not node:
                    # no more available nodes
                    break

                image = slice.default_image
                if not image:
                    raise XOSConfigurationError("No default_image for slice %s" % slice.name)

                flavor = slice.default_flavor
                if not flavor:
                    raise XOSConfigurationError("No default_flavor for slice %s" % slice.name)

                s = Sliver(slice=slice,
                           node=node,
                           creator=slice.creator,
                           image=image,
                           flavor=flavor,
                           deployment=node.site_deployment.deployment)
                s.save()

                # print "add sliver", s

class ServiceAttribute(PlCoreBase):
    name = models.SlugField(help_text="Attribute Name", max_length=128)
    value = StrippedCharField(help_text="Attribute Value", max_length=1024)
    service = models.ForeignKey(Service, related_name='serviceattributes', help_text="The Service this attribute is associated with")

class Tenant(PlCoreBase):
    """ A tenant is a relationship between two entities, a subscriber and a
        provider.

        The subscriber can be a User, a Service, or a Tenant.

        The provider is always a Service.
    """

    CONNECTIVITY_CHOICES = (('public', 'Public'), ('private', 'Private'), ('na', 'Not Applicable'))

    # when subclassing a service, redefine KIND to describe the new service
    KIND = "generic"

    kind = StrippedCharField(max_length=30, default=KIND)
    provider_service = models.ForeignKey(Service, related_name='tenants')
    subscriber_service = models.ForeignKey(Service, related_name='subscriptions', blank=True, null=True)
    subscriber_tenant = models.ForeignKey("Tenant", related_name='subscriptions', blank=True, null=True)
    subscriber_user = models.ForeignKey("User", related_name='subscriptions', blank=True, null=True)
    service_specific_id = StrippedCharField(max_length=30, blank=True, null=True)
    service_specific_attribute = models.TextField(blank=True, null=True)
    connect_method = models.CharField(null=False, blank=False, max_length=30, choices=CONNECTIVITY_CHOICES, default="na")

    def __init__(self, *args, **kwargs):
        # for subclasses, set the default kind appropriately
        self._meta.get_field("kind").default = self.KIND
        super(Tenant, self).__init__(*args, **kwargs)

    def __unicode__(self):
        if not hasattr(self, "provider_service"):
           # When the REST API does a POST on a CordSubscriber object, for
           # some reason there is no provider_service field. All of the other
           # fields are there. Provider_service is even in the dir(). However,
           # trying to getattr() on it will fail.
           return "confused-tenant-object"

        if self.subscriber_service:
            return u'%s service %s on service %s' % (str(self.kind), str(self.subscriber_service.id), str(self.provider_service.id))
        elif self.subscriber_tenant:
            return u'%s tenant %s on service %s' % (str(self.kind), str(self.subscriber_tenant.id), str(self.provider_service.id))
        else:
            return u'%s on service %s' % (str(self.kind), str(self.provider_service.id))

    # helper for extracting things from a json-encoded service_specific_attribute
    def get_attribute(self, name, default=None):
        if self.service_specific_attribute:
            attributes = json.loads(self.service_specific_attribute)
        else:
            attributes = {}
        return attributes.get(name, default)

    def set_attribute(self, name, value):
        if self.service_specific_attribute:
            attributes = json.loads(self.service_specific_attribute)
        else:
            attributes = {}
        attributes[name]=value
        self.service_specific_attribute = json.dumps(attributes)

    @classmethod
    def get_tenant_objects(cls):
        return cls.objects.filter(kind = cls.KIND)

    @classmethod
    def get_deleted_tenant_objects(cls):
        return cls.deleted_objects.filter(kind = cls.KIND)

    # helper function to be used in subclasses that want to ensure service_specific_id is unique
    def validate_unique_service_specific_id(self):
        if self.pk is None:
            if self.service_specific_id is None:
                raise XOSMissingField("subscriber_specific_id is None, and it's a required field", fields={"service_specific_id": "cannot be none"})

            conflicts = self.get_tenant_objects().filter(service_specific_id=self.service_specific_id)
            if conflicts:
                raise XOSDuplicateKey("service_specific_id %s already exists" % self.service_specific_id, fields={"service_specific_id": "duplicate key"})

class CoarseTenant(Tenant):
    class Meta:
        proxy = True

    KIND = "coarse"

    def save(self, *args, **kwargs):
        if (not self.subscriber_service):
            raise XOSValidationError("subscriber_service cannot be null")
        if (self.subscriber_tenant or self.subscriber_user):
            raise XOSValidationError("subscriber_tenant and subscriber_user must be null")

        super(CoarseTenant,self).save()
<|endoftext|>


<commit_before>import numpy as np
import os

import chainer
from chainer.dataset import download

from chainercv import utils

root = 'pfnet/chainercv/online_products'
url = 'ftp://cs.stanford.edu/cs/cvgl/Stanford_Online_Products.zip'

def _get_online_products():
    data_root = download.get_dataset_directory(root)
    base_path = os.path.join(data_root, 'Stanford_Online_Products')
    if os.path.exists(base_path):
        # skip downloading
        return base_path

    download_file_path = utils.cached_download(url)
    ext = os.path.splitext(url)[1]
    utils.extractall(download_file_path, data_root, ext)
    return base_path

class OnlineProductsDataset(chainer.dataset.DatasetMixin):

    """Simple class to load data from `Stanford Online Products Dataset`_.

    .. _`Stanford Online Products Dataset`:
        http://cvgl.stanford.edu/projects/lifted_struct

    The :obj:`mode` selects train and test split of the dataset as done in
    [Song]_. The train split contains the first 11318 classes and the test
    split contains the remaining 11316 classes.

    .. [Song] Hyun Oh Song, Yu Xiang, Stefanie Jegelka, Silvio Savarese.
        Deep Metric Learning via Lifted Structured Feature Embedding.
        https://arxiv.org/abs/1511.06452.

    All returned images are in RGB format.

    Args:
        data_dir (string): Path to the root of the training data. If this is
            :obj:`auto`, this class will automatically download data for you
            under :obj:`$CHAINER_DATASET_ROOT/pfnet/chainercv/online_products`.
        mode ({'train', 'test'}): Mode of the dataset.

    """

    def __init__(self, data_dir='auto', mode='train'):
        if data_dir == 'auto':
            data_dir = _get_online_products()
        self.data_dir = data_dir

        self.class_ids = []
        self.super_class_ids = []
        self.paths = []
        # for mode in ['train', 'test']:
        id_list_file = os.path.join(data_dir, 'Ebay_{}.txt'.format(mode))
        ids_tmp = [id_.strip().split() for id_ in open(id_list_file)][1:]
        self.class_ids += [int(id_[1]) for id_ in ids_tmp]
        self.super_class_ids += [int(id_[2]) for id_ in ids_tmp]
        self.paths += [os.path.join(data_dir, id_[3]) for id_ in ids_tmp]

        self.class_ids_dict = self._list_to_dict(self.class_ids)
        self.super_class_ids_dict = self._list_to_dict(self.super_class_ids)

    def _list_to_dict(self, l):
        dict_ = {}
        for i, v in enumerate(l):
            if v not in dict_:
                dict_[v] = []
            dict_[v].append(i)
        return dict_

    def __len__(self):
        return len(self.paths)

    def get_example(self, i):
        """Returns the i-th example.

        Returns a color image, class_id and super_class_id. The image is in CHW
        format.
        The returned image is BGR.

        Args:
            i (int): The index of the example.
        Returns:
            i-th example
        """

        class_id = np.array(self.class_ids[i], np.int32)
        super_class_id = np.array(self.super_class_ids[i], np.int32)

        img = utils.read_image_as_array(self.paths[i])

        if img.ndim == 2:
            img = utils.gray2rgb(img)
        img = img[:, :, ::-1]  # RGB to BGR
        img = img.transpose(2, 0, 1).astype(np.float32)
        return img, class_id, super_class_id
<commit_msg>delete unncessary functions<commit_after>import numpy as np
import os

import chainer
from chainer.dataset import download

from chainercv import utils

root = 'pfnet/chainercv/online_products'
url = 'ftp://cs.stanford.edu/cs/cvgl/Stanford_Online_Products.zip'

def _get_online_products():
    data_root = download.get_dataset_directory(root)
    base_path = os.path.join(data_root, 'Stanford_Online_Products')
    if os.path.exists(base_path):
        # skip downloading
        return base_path

    download_file_path = utils.cached_download(url)
    ext = os.path.splitext(url)[1]
    utils.extractall(download_file_path, data_root, ext)
    return base_path

class OnlineProductsDataset(chainer.dataset.DatasetMixin):

    """Simple class to load data from `Stanford Online Products Dataset`_.

    .. _`Stanford Online Products Dataset`:
        http://cvgl.stanford.edu/projects/lifted_struct

    The :obj:`mode` selects train and test split of the dataset as done in
    [Song]_. The train split contains the first 11318 classes and the test
    split contains the remaining 11316 classes.

    .. [Song] Hyun Oh Song, Yu Xiang, Stefanie Jegelka, Silvio Savarese.
        Deep Metric Learning via Lifted Structured Feature Embedding.
        https://arxiv.org/abs/1511.06452.

    All returned images are in RGB format.

    Args:
        data_dir (string): Path to the root of the training data. If this is
            :obj:`auto`, this class will automatically download data for you
            under :obj:`$CHAINER_DATASET_ROOT/pfnet/chainercv/online_products`.
        mode ({'train', 'test'}): Mode of the dataset.

    """

    def __init__(self, data_dir='auto', mode='train'):
        if data_dir == 'auto':
            data_dir = _get_online_products()
        self.data_dir = data_dir

        self.class_ids = []
        self.super_class_ids = []
        self.paths = []
        # for mode in ['train', 'test']:
        id_list_file = os.path.join(data_dir, 'Ebay_{}.txt'.format(mode))
        ids_tmp = [id_.strip().split() for id_ in open(id_list_file)][1:]
        self.class_ids += [int(id_[1]) for id_ in ids_tmp]
        self.super_class_ids += [int(id_[2]) for id_ in ids_tmp]
        self.paths += [os.path.join(data_dir, id_[3]) for id_ in ids_tmp]

    def __len__(self):
        return len(self.paths)

    def get_example(self, i):
        """Returns the i-th example.

        Returns a color image, class_id and super_class_id. The image is in CHW
        format.
        The returned image is BGR.

        Args:
            i (int): The index of the example.
        Returns:
            i-th example
        """

        class_id = np.array(self.class_ids[i], np.int32)
        super_class_id = np.array(self.super_class_ids[i], np.int32)

        img = utils.read_image_as_array(self.paths[i])

        if img.ndim == 2:
            img = utils.gray2rgb(img)
        img = img[:, :, ::-1]  # RGB to BGR
        img = img.transpose(2, 0, 1).astype(np.float32)
        return img, class_id, super_class_id
<|endoftext|>


<commit_before>import logging
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

LIST_URL = 'http://code.google.com/p/dart/issues/list'
NEW_URL = 'http://code.google.com/p/dart/issues/entry'
SHOW_URL = 'http://code.google.com/p/dart/issues/detail?id='
USER_ME_URL = 'http://code.google.com/p/dart/issues/list?can=2&q=owner%3Ame'
USER_URL = 'http://code.google.com/p/dart/issues/list?can=2&q=owner%3A'

class ListIssues(webapp.RequestHandler):
  def get(self):
    self.redirect(LIST_URL)

class NewIssue(webapp.RequestHandler):
  def get(self):
    self.redirect(NEW_URL)

class ShowIssue(webapp.RequestHandler):
  def get(self, issue_id):
    try:
      issue_num = int(issue_id) # validation
      self.redirect(SHOW_URL + str(issue_num))
    except:
      self.redirect(NEW_URL)

class UserMeIssue(webapp.RequestHandler):
  def get(self, issue_id):
    self.redirect(USER_ME_URL)

class UserNameIssue(webapp.RequestHandler):
  def get(self, username):
    self.redirect(USER_ME_URL + username + '%40google.com')

application = webapp.WSGIApplication([(r'/([0-9]+)', ShowIssue),
                                      ('/new', NewIssue),
                                      ('/me', UserMeIssue),
                                      (r'/([a-zA-Z]+)', UserNameIssue),
                                      ('/', ListIssues)],
                                     debug=True)

def main():
  run_wsgi_app(application)

if __name__ == '__main__':
  main()<commit_msg>fix me redirect<commit_after>import logging
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

LIST_URL = 'http://code.google.com/p/dart/issues/list'
NEW_URL = 'http://code.google.com/p/dart/issues/entry'
SHOW_URL = 'http://code.google.com/p/dart/issues/detail?id='
USER_ME_URL = 'http://code.google.com/p/dart/issues/list?can=2&q=owner%3Ame'
USER_URL = 'http://code.google.com/p/dart/issues/list?can=2&q=owner%3A'

class ListIssues(webapp.RequestHandler):
  def get(self):
    self.redirect(LIST_URL)

class NewIssue(webapp.RequestHandler):
  def get(self):
    self.redirect(NEW_URL)

class ShowIssue(webapp.RequestHandler):
  def get(self, issue_id):
    try:
      issue_num = int(issue_id) # validation
      self.redirect(SHOW_URL + str(issue_num))
    except:
      self.redirect(NEW_URL)

class UserMeIssue(webapp.RequestHandler):
  def get(self):
    self.redirect(USER_ME_URL)

class UserNameIssue(webapp.RequestHandler):
  def get(self, username):
    self.redirect(USER_ME_URL + username + '%40google.com')

application = webapp.WSGIApplication([(r'/([0-9]+)', ShowIssue),
                                      ('/new', NewIssue),
                                      ('/me', UserMeIssue),
                                      (r'/([a-zA-Z]+)', UserNameIssue),
                                      ('/', ListIssues)],
                                     debug=True)

def main():
  run_wsgi_app(application)

if __name__ == '__main__':
  main()<|endoftext|>


<commit_before>from __future__ import division, print_function
import logging
import numpy as np

from .utils import (_ensure_strings_from_expressions,
    _ensure_string_from_expression,
    _ensure_list,
    _is_limit,
    _isnumber,
    _issequence,
    _is_string,
    _parse_reduction,
    _parse_n,
    _normalize_selection_name,
    _normalize,
    _parse_f,
    _expand,
    _expand_shape,
    _expand_limits,
    as_flat_float,
    as_flat_array,
    _split_and_combine_mask)
from .expression import expression_namespace
import vaex.expression

logger = logging.getLogger('vaex.scopes')

class UnitScope(object):
    def __init__(self, df, value=None):
        self.df = df
        self.value = value

    def __getitem__(self, variable):
        import astropy.units
        if variable in self.df.units:
            unit = self.df.units[variable]
            return (self.value * unit) if self.value is not None else unit
        elif variable in self.df.virtual_columns:
            return eval(self.df.virtual_columns[variable], expression_namespace, self)
        elif variable in self.df.variables:
            return astropy.units.dimensionless_unscaled  # TODO units for variables?
        else:
            raise KeyError("unkown variable %s" % variable)

class _BlockScope(object):
    def __init__(self, df, i1, i2, mask=None, **variables):
        """

        :param DataFrameLocal DataFrame: the *local*  DataFrame
        :param i1: start index
        :param i2: end index
        :param values:
        :return:
        """
        self.df = df
        self.i1 = int(i1)
        self.i2 = int(i2)
        self.variables = variables
        self.values = dict(self.variables)
        self.buffers = {}
        self.mask = mask if mask is not None else slice(None, None, None)

    def move(self, i1, i2):
        length_new = i2 - i1
        length_old = self.i2 - self.i1
        if length_new > length_old:  # old buffers are too small, discard them
            self.buffers = {}
        else:
            for name in list(self.buffers.keys()):
                self.buffers[name] = self.buffers[name][:length_new]
        self.i1 = int(i1)
        self.i2 = int(i2)
        self.values = dict(self.variables)

    def _ensure_buffer(self, column):
        if column not in self.buffers:
            logger.debug("creating column for: %s", column)
            self.buffers[column] = np.zeros(self.i2 - self.i1)

    def evaluate(self, expression, out=None):
        if isinstance(expression, vaex.expression.Expression):
            expression = expression.expression
        try:
            # logger.debug("try avoid evaluating: %s", expression)
            result = self[expression]
        except KeyError:
            # logger.debug("no luck, eval: %s", expression)
            # result = ne.evaluate(expression, local_dict=self, out=out)
            # logger.debug("in eval")
            # eval("def f(")
            result = eval(expression, expression_namespace, self)
            self.values[expression] = result
            # if out is not None:
            #   out[:] = result
            #   result = out
            # logger.debug("out eval")
        # logger.debug("done with eval of %s", expression)
        return result

    def __getitem__(self, variable):
        # logger.debug("get " + variable)
        # return self.df.columns[variable][self.i1:self.i2]
        if variable in expression_namespace:
            return expression_namespace[variable]
        try:
            if variable in self.values:
                return self.values[variable]
            elif variable in self.df.get_column_names(virtual=False, hidden=True):
                offset = self.df._index_start
                # if self.df._needs_copy(variable):
                    # self._ensure_buffer(variable)
                    # self.values[variable] = self.buffers[variable] = self.df.columns[variable][self.i1:self.i2].astype(np.float64)
                    # Previously we casted anything to .astype(np.float64), this led to rounding off of int64, when exporting
                    # self.values[variable] = self.df.columns[variable][offset+self.i1:offset+self.i2][:]
                # else:
                self.values[variable] = self.df.columns[variable][offset+self.i1:offset+self.i2]
                if self.mask is not None:
                    self.values[variable] = self.values[variable][self.mask]
            elif variable in list(self.df.virtual_columns.keys()):
                expression = self.df.virtual_columns[variable]
                if isinstance(expression, dict):
                    function = expression['function']
                    arguments = [self.evaluate(k) for k in expression['arguments']]
                    self.values[variable] = function(*arguments)
                else:
                    # self._ensure_buffer(variable)
                    self.values[variable] = self.evaluate(expression)  # , out=self.buffers[variable])
                    # self.values[variable] = self.buffers[variable]
            elif variable in self.df.functions:
                return self.df.functions[variable].f
            if variable not in self.values:
                raise KeyError("Unknown variables or column: %r" % (variable,))

            return self.values[variable]
        except:
            # logger.exception("error in evaluating: %r" % variable)
            raise

class _BlockScopeSelection(object):
    def __init__(self, df, i1, i2, selection=None, cache=False):
        self.df = df
        self.i1 = i1
        self.i2 = i2
        self.selection = selection
        self.store_in_cache = cache

    def evaluate(self, expression):
        if expression is True:
            expression = "default"
        try:
            expression = _ensure_string_from_expression(expression)
            return eval(expression, expression_namespace, self)
        except:
            import traceback as tb
            tb.print_stack()
            raise

    def __getitem__(self, variable):
        # logger.debug("getitem for selection: %s", variable)
        try:
            selection = self.selection
            if selection is None and self.df.has_selection(variable):
                selection = self.df.get_selection(variable)
            # logger.debug("selection for %r: %s %r", variable, selection, self.df.selection_histories)
            key = (self.i1, self.i2)
            if selection:
                assert variable in self.df._selection_masks, "%s mask not found" % (variable, )
                cache = self.df._selection_mask_caches[variable]
                # logger.debug("selection cache: %r" % cache)
                full_mask = self.df._selection_masks[variable]
                selection_in_cache, mask = cache.get(key, (None, None))

                # logger.debug("mask for %r is %r", variable, mask)
                if selection_in_cache == selection:
                    return mask
                # logger.debug("was not cached")
                if variable in self.df.variables:
                    return self.df.variables[variable]
                mask_values = selection.evaluate(self.df, variable, self.i1, self.i2)
                # get a view on a subset of the mask
                sub_mask = full_mask.view(self.i1, self.i2)
                sub_mask_array = np.asarray(sub_mask)
                # and update it
                sub_mask_array[:] = mask_values
                # logger.debug("put selection in mask with key %r" % (key,))
                if self.store_in_cache:
                    cache[key] = selection, sub_mask_array
                    # cache[key] = selection, mask_values
                return sub_mask_array
                # return mask_values
            else:
                offset = self.df._index_start
                if variable in expression_namespace:
                    return expression_namespace[variable]
                elif variable in self.df.get_column_names(hidden=True, virtual=False):
                    return self.df.columns[variable][offset+self.i1:offset+self.i2]
                elif variable in self.df.variables:
                    return self.df.variables[variable]
                elif variable in list(self.df.virtual_columns.keys()):
                    expression = self.df.virtual_columns[variable]
                    # self._ensure_buffer(variable)
                    return self.evaluate(expression)  # , out=self.buffers[variable])
                    # self.values[variable] = self.buffers[variable]
                elif variable in self.df.functions:
                    return self.df.functions[variable].f
                raise KeyError("Unknown variables or column: %r" % (variable,))
        except:
            import traceback as tb
            tb.print_exc()
            logger.exception("error in evaluating: %r" % variable)
            raise

<commit_msg>fix(core): pytest + pdb crashed since it uses BlockScope as frame, mock it<commit_after>from __future__ import division, print_function
import logging
import numpy as np

from .utils import (_ensure_strings_from_expressions,
    _ensure_string_from_expression,
    _ensure_list,
    _is_limit,
    _isnumber,
    _issequence,
    _is_string,
    _parse_reduction,
    _parse_n,
    _normalize_selection_name,
    _normalize,
    _parse_f,
    _expand,
    _expand_shape,
    _expand_limits,
    as_flat_float,
    as_flat_array,
    _split_and_combine_mask)
from .expression import expression_namespace
import vaex.expression

logger = logging.getLogger('vaex.scopes')

class UnitScope(object):
    def __init__(self, df, value=None):
        self.df = df
        self.value = value

    def __getitem__(self, variable):
        import astropy.units
        if variable in self.df.units:
            unit = self.df.units[variable]
            return (self.value * unit) if self.value is not None else unit
        elif variable in self.df.virtual_columns:
            return eval(self.df.virtual_columns[variable], expression_namespace, self)
        elif variable in self.df.variables:
            return astropy.units.dimensionless_unscaled  # TODO units for variables?
        else:
            raise KeyError("unkown variable %s" % variable)

class _BlockScope(object):
    def __init__(self, df, i1, i2, mask=None, **variables):
        """

        :param DataFrameLocal DataFrame: the *local*  DataFrame
        :param i1: start index
        :param i2: end index
        :param values:
        :return:
        """
        self.df = df
        self.i1 = int(i1)
        self.i2 = int(i2)
        self.variables = variables
        self.values = dict(self.variables)
        self.buffers = {}
        self.mask = mask if mask is not None else slice(None, None, None)

    def move(self, i1, i2):
        length_new = i2 - i1
        length_old = self.i2 - self.i1
        if length_new > length_old:  # old buffers are too small, discard them
            self.buffers = {}
        else:
            for name in list(self.buffers.keys()):
                self.buffers[name] = self.buffers[name][:length_new]
        self.i1 = int(i1)
        self.i2 = int(i2)
        self.values = dict(self.variables)

    def get(self, attr, default=None):  # otherwise pdb crashes during pytest
        if attr == "__tracebackhide__":
            return False
        return default

    def __contains__(self, name):  # otherwise pdb crashes during pytest
        return name in self.buffers  # not sure this should also include varibles, columns and virtual columns

    def _ensure_buffer(self, column):
        if column not in self.buffers:
            logger.debug("creating column for: %s", column)
            self.buffers[column] = np.zeros(self.i2 - self.i1)

    def evaluate(self, expression, out=None):
        if isinstance(expression, vaex.expression.Expression):
            expression = expression.expression
        try:
            # logger.debug("try avoid evaluating: %s", expression)
            result = self[expression]
        except KeyError:
            # logger.debug("no luck, eval: %s", expression)
            # result = ne.evaluate(expression, local_dict=self, out=out)
            # logger.debug("in eval")
            # eval("def f(")
            result = eval(expression, expression_namespace, self)
            self.values[expression] = result
            # if out is not None:
            #   out[:] = result
            #   result = out
            # logger.debug("out eval")
        # logger.debug("done with eval of %s", expression)
        return result

    def __getitem__(self, variable):
        # logger.debug("get " + variable)
        # return self.df.columns[variable][self.i1:self.i2]
        if variable in expression_namespace:
            return expression_namespace[variable]
        try:
            if variable in self.values:
                return self.values[variable]
            elif variable in self.df.get_column_names(virtual=False, hidden=True):
                offset = self.df._index_start
                # if self.df._needs_copy(variable):
                    # self._ensure_buffer(variable)
                    # self.values[variable] = self.buffers[variable] = self.df.columns[variable][self.i1:self.i2].astype(np.float64)
                    # Previously we casted anything to .astype(np.float64), this led to rounding off of int64, when exporting
                    # self.values[variable] = self.df.columns[variable][offset+self.i1:offset+self.i2][:]
                # else:
                self.values[variable] = self.df.columns[variable][offset+self.i1:offset+self.i2]
                if self.mask is not None:
                    self.values[variable] = self.values[variable][self.mask]
            elif variable in list(self.df.virtual_columns.keys()):
                expression = self.df.virtual_columns[variable]
                if isinstance(expression, dict):
                    function = expression['function']
                    arguments = [self.evaluate(k) for k in expression['arguments']]
                    self.values[variable] = function(*arguments)
                else:
                    # self._ensure_buffer(variable)
                    self.values[variable] = self.evaluate(expression)  # , out=self.buffers[variable])
                    # self.values[variable] = self.buffers[variable]
            elif variable in self.df.functions:
                return self.df.functions[variable].f
            if variable not in self.values:
                raise KeyError("Unknown variables or column: %r" % (variable,))

            return self.values[variable]
        except:
            # logger.exception("error in evaluating: %r" % variable)
            raise

class _BlockScopeSelection(object):
    def __init__(self, df, i1, i2, selection=None, cache=False):
        self.df = df
        self.i1 = i1
        self.i2 = i2
        self.selection = selection
        self.store_in_cache = cache

    def evaluate(self, expression):
        if expression is True:
            expression = "default"
        try:
            expression = _ensure_string_from_expression(expression)
            return eval(expression, expression_namespace, self)
        except:
            import traceback as tb
            tb.print_stack()
            raise

    def __getitem__(self, variable):
        # logger.debug("getitem for selection: %s", variable)
        try:
            selection = self.selection
            if selection is None and self.df.has_selection(variable):
                selection = self.df.get_selection(variable)
            # logger.debug("selection for %r: %s %r", variable, selection, self.df.selection_histories)
            key = (self.i1, self.i2)
            if selection:
                assert variable in self.df._selection_masks, "%s mask not found" % (variable, )
                cache = self.df._selection_mask_caches[variable]
                # logger.debug("selection cache: %r" % cache)
                full_mask = self.df._selection_masks[variable]
                selection_in_cache, mask = cache.get(key, (None, None))

                # logger.debug("mask for %r is %r", variable, mask)
                if selection_in_cache == selection:
                    return mask
                # logger.debug("was not cached")
                if variable in self.df.variables:
                    return self.df.variables[variable]
                mask_values = selection.evaluate(self.df, variable, self.i1, self.i2)
                # get a view on a subset of the mask
                sub_mask = full_mask.view(self.i1, self.i2)
                sub_mask_array = np.asarray(sub_mask)
                # and update it
                sub_mask_array[:] = mask_values
                # logger.debug("put selection in mask with key %r" % (key,))
                if self.store_in_cache:
                    cache[key] = selection, sub_mask_array
                    # cache[key] = selection, mask_values
                return sub_mask_array
                # return mask_values
            else:
                offset = self.df._index_start
                if variable in expression_namespace:
                    return expression_namespace[variable]
                elif variable in self.df.get_column_names(hidden=True, virtual=False):
                    return self.df.columns[variable][offset+self.i1:offset+self.i2]
                elif variable in self.df.variables:
                    return self.df.variables[variable]
                elif variable in list(self.df.virtual_columns.keys()):
                    expression = self.df.virtual_columns[variable]
                    # self._ensure_buffer(variable)
                    return self.evaluate(expression)  # , out=self.buffers[variable])
                    # self.values[variable] = self.buffers[variable]
                elif variable in self.df.functions:
                    return self.df.functions[variable].f
                raise KeyError("Unknown variables or column: %r" % (variable,))
        except:
            import traceback as tb
            tb.print_exc()
            logger.exception("error in evaluating: %r" % variable)
            raise

<|endoftext|>