<commit_before>#!/usr/bin/env python
import fileinput
import re
import os
import glob

fileDir = os.path.dirname(os.path.realpath("__file__"))
# print(fileDir)

output_file = "examples.dox"
read_mes = [
    "..\\examples\\ReadMe.md",
    "..\\examples\\a_wild_card\\ReadMe.md",
    "..\\examples\\b_address_change\\ReadMe.md",
    "..\\examples\\c_check_all_addresses\\ReadMe.md",
    "..\\examples\\d_simple_logger\\ReadMe.md",
    "..\\examples\\e_simple_parsing\\ReadMe.md",
    "..\\examples\\f_basic_data_request\\ReadMe.md",
    "..\\examples\\g_terminal_window\\ReadMe.md",
    "..\\examples\\h_SDI-12_slave_implementation\\ReadMe.md",
    "..\\examples\\i_SDI-12_interface\\ReadMe.md",
    "..\\examples\\j_external_pcint_library\\ReadMe.md",
    "..\\examples\\k_concurrent_logger\\ReadMe.md",
]

if not os.path.exists(os.path.join(fileDir, "examples")):
    os.makedirs(os.path.join(fileDir, "examples"))

for filename in read_mes:
    out_path = os.path.join(fileDir, "examples")
    out_dir = filename.split("\\")[2]
    out_name = out_dir + ".dox"
    if out_name == "ReadMe.md.dox":
        out_name = "examples.dox"
    abs_out = os.path.join(out_path, out_name)
    # print(abs_out)
    # with open(output_file, 'w+') as out_file:
    with open(abs_out, "w+") as out_file:

        abs_file_path = os.path.join(fileDir, filename)
        abs_file_path = os.path.abspath(os.path.realpath(abs_file_path))
        # print(abs_file_path)

        with open(abs_file_path, "r") as in_file:  # open in readonly mode
            out_file.write("/**\n")
            if out_name != "examples.dox":
                # out_file.write(
                #     "@example{{lineno}} {} @m_examplenavigation{{examples_page,{}/}} @m_footernavigation \n\n".format(
                #         filename.replace("..\\examples\\", "").replace(
                #             "\\ReadMe.md", ".ino"
                #         ), out_dir
                #     )
                # )
                out_file.write(
                    "@example{{lineno}} {} @m_examplenavigation{{examples_page,}} @m_footernavigation \n\n".format(
                        filename.replace("..\\examples\\", "").replace(
                            "\\ReadMe.md", ".ino"
                        )
                    )
                )
                # out_file.write(
                #     "@example{{lineno}} {} \n\n".format(
                #         filename.replace("..\\examples\\", "").replace(
                #             "\\ReadMe.md", ".ino"
                #         )
                #     )
                # )
            # out_file.write('\n@tableofcontents\n\n')

            print_me = True
            skip_me = False
            i = 1
            lines = in_file.readlines()
            for line in lines:
                # print(i, print_me, skip_me, line)

                # Remove markdown comment tags from doxygen commands within the markdown
                if print_me and not skip_me:
                    new_line = (
                        re.sub(r"\[//\]: # \( @(\w+?.*) \)", r"@\1", line)
                        .replace("```ini", "@code{.ini}")
                        .replace("```cpp", "@code{.cpp}")
                        .replace("```", "@endcode")
                    )
                    if out_name != "examples.dox":
                        new_line = new_line.replace("@page", "@section")
                        #    .replace('@section', '')
                        #    .replace('@subsection', '')
                        #    .replace('@subsubsection', '')
                        #    .replace('@paragraph', '')
                        #    .replace('@par', '')
                    out_file.write(new_line)

                # using skip_me to skip single lines, so unset it after reading a line
                if skip_me:
                    skip_me = False

                # a page, section, subsection, or subsubsection commands followed
                # immediately with by a markdown header leads to that section appearing
                # twice in the doxygen html table of contents.
                # I'm putting the section markers right above the header and then will skip the header.
                if re.match(r"\[//\]: # \( @mainpage", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @page", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @.*section", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @paragraph", line) is not None:
                    skip_me = True

                # I'm using these comments to fence off content that is only intended for
                # github mardown rendering
                if "[//]: # ( Start GitHub Only )" in line:
                    print_me = False

                if "[//]: # ( End GitHub Only )" in line:
                    print_me = True

                i += 1

            out_file.write("\n*/\n\n")
<commit_msg>Change slash direction<commit_after>#!/usr/bin/env python
import fileinput
import re
import os
import glob

fileDir = os.path.dirname(os.path.realpath("__file__"))
# print(fileDir)

output_file = "examples.dox"
read_mes = [
    "../examples/ReadMe.md",
    "../examples/a_wild_card/ReadMe.md",
    "../examples/b_address_change/ReadMe.md",
    "../examples/c_check_all_addresses/ReadMe.md",
    "../examples/d_simple_logger/ReadMe.md",
    "../examples/e_simple_parsing/ReadMe.md",
    "../examples/f_basic_data_request/ReadMe.md",
    "../examples/g_terminal_window/ReadMe.md",
    "../examples/h_SDI-12_slave_implementation/ReadMe.md",
    "../examples/i_SDI-12_interface/ReadMe.md",
    "../examples/j_external_pcint_library/ReadMe.md",
    "../examples/k_concurrent_logger/ReadMe.md",
]

if not os.path.exists(os.path.join(fileDir, "examples")):
    os.makedirs(os.path.join(fileDir, "examples"))

for filename in read_mes:
    out_path = os.path.join(fileDir, "examples")
    out_dir = filename.split("/")[2]
    out_name = out_dir + ".dox"
    if out_name == "ReadMe.md.dox":
        out_name = "examples.dox"
    abs_out = os.path.join(out_path, out_name)
    # print(abs_out)
    # with open(output_file, 'w+') as out_file:
    with open(abs_out, "w+") as out_file:

        abs_file_path = os.path.join(fileDir, filename)
        abs_file_path = os.path.abspath(os.path.realpath(abs_file_path))
        # print(abs_file_path)

        with open(abs_file_path, "r") as in_file:  # open in readonly mode
            out_file.write("/**\n")
            if out_name != "examples.dox":
                # out_file.write(
                #     "@example{{lineno}} {} @m_examplenavigation{{examples_page,{}/}} @m_footernavigation \n\n".format(
                #         filename.replace("..\\examples\\", "").replace(
                #             "\\ReadMe.md", ".ino"
                #         ), out_dir
                #     )
                # )
                out_file.write(
                    "@example{{lineno}} {} @m_examplenavigation{{examples_page,}} @m_footernavigation \n\n".format(
                        filename.replace("../examples/", "").replace(
                            "/ReadMe.md", ".ino"
                        )
                    )
                )
                # out_file.write(
                #     "@example{{lineno}} {} \n\n".format(
                #         filename.replace("..\\examples\\", "").replace(
                #             "\\ReadMe.md", ".ino"
                #         )
                #     )
                # )
            # out_file.write('\n@tableofcontents\n\n')

            print_me = True
            skip_me = False
            i = 1
            lines = in_file.readlines()
            for line in lines:
                # print(i, print_me, skip_me, line)

                # Remove markdown comment tags from doxygen commands within the markdown
                if print_me and not skip_me:
                    new_line = (
                        re.sub(r"\[//\]: # \( @(\w+?.*) \)", r"@\1", line)
                        .replace("```ini", "@code{.ini}")
                        .replace("```cpp", "@code{.cpp}")
                        .replace("```", "@endcode")
                    )
                    if out_name != "examples.dox":
                        new_line = new_line.replace("@page", "@section")
                        #    .replace('@section', '')
                        #    .replace('@subsection', '')
                        #    .replace('@subsubsection', '')
                        #    .replace('@paragraph', '')
                        #    .replace('@par', '')
                    out_file.write(new_line)

                # using skip_me to skip single lines, so unset it after reading a line
                if skip_me:
                    skip_me = False

                # a page, section, subsection, or subsubsection commands followed
                # immediately with by a markdown header leads to that section appearing
                # twice in the doxygen html table of contents.
                # I'm putting the section markers right above the header and then will skip the header.
                if re.match(r"\[//\]: # \( @mainpage", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @page", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @.*section", line) is not None:
                    skip_me = True
                if re.match(r"\[//\]: # \( @paragraph", line) is not None:
                    skip_me = True

                # I'm using these comments to fence off content that is only intended for
                # github mardown rendering
                if "[//]: # ( Start GitHub Only )" in line:
                    print_me = False

                if "[//]: # ( End GitHub Only )" in line:
                    print_me = True

                i += 1

            out_file.write("\n*/\n\n")
<|endoftext|>


<commit_before># djangoshop/checkout.py

from djangoshop.payment_base import PaymentBase
from djangoshop.shipper_base import ShipperBase

from django.views.generic import TemplateView

class CheckoutView(TemplateView):
    template_name = "checkout.html"

class CheckoutSite(object):

    checkout_view = CheckoutView.as_view()

    def __init__(self, name=None, app_name='django_shop'):
        self._payment_registry = []
        self._shipper_registry = []
         # model_class class -> admin_class instance
        self.root_path = None
        if name is None:
            self.name = 'checkout'
        else:
            self.name = name
        self.app_name = app_name

    def register(self, registry, classtype, class_or_iterable):
        """
        Registers the given model(s) with the checkoutsite
        """

        if isinstance(cls, classtype):
            class_or_iterable = [class_or_iterable]
        for cls in class_or_iterable:
            if cls in getattr(self, '_%s_registry' % registry):
                raise AlreadyRegistered('The %s class %s is already registered' % (registry, cls.__name__))
            # Instantiate the class to save in the registry
            getattr(self, '_%s_registry' % registry).append(cls(self))

     def register(self, registry, classtype, class_or_iterable):
        """
        Unregisters the given model(s).

        If a class isn't already registered, this will raise NotRegistered.
        """
        if isinstance(cls, classtype):
            class_or_iterable = [class_or_iterable]
        for cls in class_or_iterable:
            if cls in getattr(self, '_%s_registry' % registry):
                raise AlreadyRegistered('The %s class %s is alrea
            if cls in getattr(self, '_%s_registry' % registry):
                raise NotRegistered('The %s class %s is not registered' % (registry, cls.__name__))
            del self._registry[cls] # not a dict anymore how to remove

    def register_shipper(self, shipper):
        self.register(self, 'shipper', ShipperBase, shipper)

    def unregister_shipper(self, shipper):
        self.unregister(self, 'shipper', ShipperBase, shipper)

    def register_payment(self, payment):
        self.register(self, 'payment', PaymentBase, payment)

    def unregister_payment(self, shipper):
        self.unregister(self, 'payment', PaymentBase, payment)

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url, include

        # Checkout-site-wide views.
        urlpatterns = patterns('',
            url(r'^$', self.checkout_view, name='checkout'),
        )

        # Add in each model's views.
        for payment in self._payment_registry:
            if hasattr(payment, 'urls'):
                urlpatterns += patterns('',
                    url(r'^shipment/%s/%s/' % payment.url_prefix,
                        include(payment.urls))
                )
        for shipper in self._shippers_registry:
            if hasattr(shipper, 'urls'):
                urlpatterns += patterns('',
                    url(r'^payment/%s/' % payment.url_prefix,
                        include(shipper.urls))
                )
        return urlpatterns

    @property
    def urls(self):
        return self.get_urls(), self.app_name, self.name

def autodiscover():
    pass

# djangoshop/__init__.py

from djangoshop.checkout import CheckoutSite
checkoutsite = CheckoutSite()

# djangoshop/shipper_base.py

class ShipperBase(object)
    pass

# djangoshop/payment_base.py
from djangoshop. import RegisterAbleClass

class PaymentBase(object)

  def __init__(self, checkout_site):
    self.checkout_site = checkout_site
    super(PaymentBase, self).__init__()

# app/djangoshop_shipper.py

from djangoshop.shipper_base import ShipperBase

class ShipmentClass(ShipperBase):

  def __init__(self, checkout_site):
    self.checkout_site = checkout_site
    super(PaymentBase, self).__init__()

checkoutsite.register_shipment(ShipmentClass)

# app/djangoshop_payment.py

from django.views.generic import TemplateView
from djangoshop.payment_base import PaymentBase

class PaymentView(TemplateView):
    template_name = "payment.html"

class PaymentClass(PaymentBase, UrlMixin):

    url_prefix = 'payment'

    payment_view = PaymentView.as_view()

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url

        urlpatterns = patterns('',
            url(r'^$', self.payment_view,
                name='%s_payment' % self.url_prefix),
        )
        return urlpatterns

    def urls(self):
        return self.get_urls()
    urls = property(urls)

checkoutsite.register_payment(PaymentClass)
<commit_msg>fixed some code in docs/snippets/modules.py<commit_after># djangoshop/checkout.py

from djangoshop.payment_base import PaymentBase
from djangoshop.shipper_base import ShipperBase

from django.views.generic import TemplateView

class CheckoutView(TemplateView):
    template_name = "checkout.html"

class CheckoutSite(object):

    checkout_view = CheckoutView.as_view()

    def __init__(self, name=None, app_name='django_shop'):
        self._registry = {
            'shipper': {},
            'payment': {}
        }
         # model_class class -> admin_class instance
        self.root_path = None
        if name is None:
            self.name = 'checkout'
        else:
            self.name = name
        self.app_name = app_name

    def register(self, registry, classtype, class_or_iterable):
        """
        Registers the given model(s) with the checkoutsite
        """
        if isinstance(cls, classtype):
            class_or_iterable = [class_or_iterable]
        for cls in class_or_iterable:
            if cls in self._registry[registry].keys():
                raise AlreadyRegistered('The %s class %s is already registered' % (registry, cls.__name__))
            # Instantiate the class to save in the registry
            self._registry[registry][cls] = cls(self)

     def unregister(self, registry, classtype, class_or_iterable):
        """
        Unregisters the given classes(s).

        If a class isn't already registered, this will raise NotRegistered.
        """
        if isinstance(cls, classtype):
            class_or_iterable = [class_or_iterable]
        for cls in class_or_iterable:
            if cls not in self._registry[registry].keys():
                raise NotRegistered('The %s class %s is not registered' % (registry, cls.__name__))
            del self._registry[cls]

    def register_shipper(self, shipper):
        self.register(self, 'shipper', ShipperBase, shipper)

    def unregister_shipper(self, shipper):
        self.unregister(self, 'shipper', ShipperBase, shipper)

    def register_payment(self, payment):
        self.register(self, 'payment', PaymentBase, payment)

    def unregister_payment(self, payment):
        self.unregister(self, 'payment', PaymentBase, payment)

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url, include

        # Checkout-site-wide views.
        urlpatterns = patterns('',
            url(r'^$', self.checkout_view, name='checkout'),
        )

        # Add in each model's views.
        for payment in self._payment_registry:
            if hasattr(payment, 'urls'):
                urlpatterns += patterns('',
                    url(r'^shipment/%s/%s/' % payment.url_prefix,
                        include(payment.urls))
                )
        for shipper in self._shippers_registry:
            if hasattr(shipper, 'urls'):
                urlpatterns += patterns('',
                    url(r'^payment/%s/' % payment.url_prefix,
                        include(shipper.urls))
                )
        return urlpatterns

    @property
    def urls(self):
        return self.get_urls(), self.app_name, self.name

def autodiscover():
    pass

# djangoshop/__init__.py

from djangoshop.checkout import CheckoutSite
checkoutsite = CheckoutSite()

# djangoshop/shipper_base.py

class ShipperBase(object)
    pass

# djangoshop/payment_base.py
from djangoshop. import RegisterAbleClass

class PaymentBase(object)

  def __init__(self, checkout_site):
    self.checkout_site = checkout_site
    super(PaymentBase, self).__init__()

# app/djangoshop_shipper.py

from djangoshop.shipper_base import ShipperBase

class ShipmentClass(ShipperBase):

  def __init__(self, checkout_site):
    self.checkout_site = checkout_site
    super(PaymentBase, self).__init__()

checkoutsite.register_shipment(ShipmentClass)

# app/djangoshop_payment.py

from django.views.generic import TemplateView
from djangoshop.payment_base import PaymentBase

class PaymentView(TemplateView):
    template_name = "payment.html"

class PaymentClass(PaymentBase, UrlMixin):

    url_prefix = 'payment'

    payment_view = PaymentView.as_view()

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url

        urlpatterns = patterns('',
            url(r'^$', self.payment_view,
                name='%s_payment' % self.url_prefix),
        )
        return urlpatterns

    def urls(self):
        return self.get_urls()
    urls = property(urls)

checkoutsite.register_payment(PaymentClass)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding=utf-8
"""Download all Glottolog Newick Files"""

import os
import re
import codecs

try:
    import requests
except ImportError:
    raise ImportError("please install the requests library")

try:
    from bs4 import BeautifulSoup as bs
except ImportError:
    raise ImportError("please install the beautifulsoup4 library")

try:
    from ete2 import Tree
except ImportError:
    raise ImportError("please install the ete2 library")

IS_ISOCODE = re.compile(r"""'.* <\w{4}\d{4}><(\w{3})>'$""")

def clean_tree(tree):
    """Removes taxa with no ISO-639 codes"""
    to_keep = []
    for leaf in tree.iter_leaves():
        iso = IS_ISOCODE.findall(leaf.name)
        if len(iso) == 1:
            leaf.name = iso[0]  # rename to ISO code
            to_keep.append(leaf.name)

    if len(to_keep) <= 3:  # not worth keeping.
        return None
    tree.prune(to_keep)
    return tree

GLOTTOLOG_FAMILIES = "http://glottolog.org/glottolog/language.atom?type=families&sSearch_1=Top-level+unit"

if __name__ == '__main__':
    urls = {}

    for entry in bs(requests.get(GLOTTOLOG_FAMILIES).text).find_all('entry'):
        urls[entry.find('title').text] = entry.find('id').text

    for family in sorted(urls):
        url = urls[family]
        filename = "%s.glotto.trees" % family
        if not os.path.isfile(filename):
            print("%30s <- %s" % (family, url))
            newick = requests.get(url + '.newick.txt').text.encode('utf-8')
            newick = newick.replace("[", "<").replace("]", ">")  # hack to keep ISO code in leaf name
            tree = Tree(newick)
            tree = clean_tree(tree)
            if tree is not None:
                with codecs.open(filename, 'w', encoding="utf-8") as handle:
                    handle.write("#NEXUS\nBegin taxa;\n") #write taxa to file
                    for leaf in tree.iter_leaves():
                        handle.write(leaf.name)
                        handle.write("\n")
                    handle.write(";\nend;")
                    handle.write("\nBegin trees;\ntree UNTITLED = ") #write newick string to file
                    handle.write(tree.write())
                    handle.write("\nend;")
        else:
            print("%30s <- %s" % (family, '[ok]'))
<commit_msg>Editing get-1.py to get glottolog trees with glottocodes at the tips<commit_after>#!/usr/bin/env python
#coding=utf-8
"""Download all Glottolog Newick Files"""

import os
import re
import codecs

try:
    import requests
except ImportError:
    raise ImportError("please install the requests library")

try:
    from bs4 import BeautifulSoup as bs
except ImportError:
    raise ImportError("please install the beautifulsoup4 library")

try:
    from ete2 import Tree
except ImportError:
    raise ImportError("please install the ete2 library")

IS_ISOCODE = re.compile(r"""'.* <\w{4}\d{4}><(\w{3})>'$""")
IS_GLOTTOCODE = re.compile(r"""'.* <(\w{4}\d{4})><\w{3}>'$""")

def clean_tree(tree):
    """Previously, removed taxa with no ISO-639 codes.
    Now renames taxa to Glottocodes."""
    to_keep = []
    for leaf in tree.iter_leaves():
        #iso = IS_ISOCODE.findall(leaf.name)
        glotto = IS_GLOTTOCODE.findall(leaf.name)
        if len(glotto) == 1:
            leaf.name = glotto[0]  # rename to glotto code
            to_keep.append(leaf.name)

    if len(to_keep) <= 3:  # not worth keeping.
        return None
    tree.prune(to_keep)
    return tree

GLOTTOLOG_FAMILIES = "http://glottolog.org/glottolog/language.atom?type=families&sSearch_1=Top-level+unit"

if __name__ == '__main__':
    urls = {}

    for entry in bs(requests.get(GLOTTOLOG_FAMILIES).text).find_all('entry'):
        urls[entry.find('title').text] = entry.find('id').text

    for family in sorted(urls):
        url = urls[family]
        filename = "%s.glotto.trees" % family
        if not os.path.isfile(filename):
            print("%30s <- %s" % (family, url))
            newick = requests.get(url + '.newick.txt').text.encode('utf-8')
            newick = newick.replace("[", "<").replace("]", ">")  # hack to keep ISO code in leaf name
            tree = Tree(newick)
            tree = clean_tree(tree)
            if tree is not None:
                with codecs.open(filename, 'w', encoding="utf-8") as handle:
                    handle.write("#NEXUS\nBegin taxa;\n") #write taxa to file
                    for leaf in tree.iter_leaves():
                        handle.write(leaf.name)
                        handle.write("\n")
                    handle.write(";\nend;")
                    handle.write("\nBegin trees;\ntree UNTITLED = ") #write newick string to file
                    handle.write(tree.write())
                    handle.write("\nend;")
        else:
            print("%30s <- %s" % (family, '[ok]'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2013 Brett Slatkin
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility for doing incremental diffs for a live website."""

import HTMLParser
import Queue
import datetime
import json
import logging
import os
import re
import sys
import urlparse

# Local Libraries
import gflags
FLAGS = gflags.FLAGS

# Local modules
from dpxdt.client import release_worker
from dpxdt.client import fetch_worker
from dpxdt.client import workers
import flags

gflags.DEFINE_integer(
    'crawl_depth', -1,
    'How deep to crawl. Depth of 0 means only the given page. 1 means pages '
    'that are one click away, 2 means two clicks, and so on. Set to -1 to '
    'scan every URL with the supplied prefix.')

gflags.DEFINE_spaceseplist(
    'ignore_prefixes', [],
    'URL prefixes that should not be crawled.')

# URL regex rewriting code originally from mirrorrr
# http://code.google.com/p/mirrorrr/source/browse/trunk/transform_content.py

# URLs that have absolute addresses
ABSOLUTE_URL_REGEX = r"(?P<url>(http(s?):)?//[^\"'> \t]+)"
# URLs that are relative to the base of the current hostname.
BASE_RELATIVE_URL_REGEX = (
    r"/(?!(/)|(http(s?)://)|(url\())(?P<url>[^\"'> \t]*)")
# URLs that have '../' or './' to start off their paths.
TRAVERSAL_URL_REGEX = (
    r"(?P<relative>\.(\.)?)/(?!(/)|"
    r"(http(s?)://)|(url\())(?P<url>[^\"'> \t]*)")
# URLs that are in the same directory as the requested URL.
SAME_DIR_URL_REGEX = r"(?!(/)|(http(s?)://)|(#)|(url\())(?P<url>[^\"'> \t]+)"
# URL matches the root directory.
ROOT_DIR_URL_REGEX = r"(?!//(?!>))/(?P<url>)(?=[ \t\n]*[\"'> /])"
# Start of a tag using 'src' or 'href'
TAG_START = (
    r"(?i)(?P<tag>\ssrc|href|action|url|background)"
    r"(?P<equals>[\t ]*=[\t ]*)(?P<quote>[\"']?)")
# Potential HTML document URL with no fragments.
MAYBE_HTML_URL_REGEX = (
    TAG_START + r"(?P<absurl>(http(s?):)?//[^\"'> \t]+)")

REPLACEMENT_REGEXES = [
    (TAG_START + SAME_DIR_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(accessed_dir)s\g<url>"),
    (TAG_START + TRAVERSAL_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(accessed_dir)s/\g<relative>/\g<url>"),
    (TAG_START + BASE_RELATIVE_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(base)s/\g<url>"),
    (TAG_START + ROOT_DIR_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(base)s/"),
    (TAG_START + ABSOLUTE_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>\g<url>"),
]

def clean_url(url, force_scheme=None):
    """Cleans the given URL."""
    # Collapse ../../ and related
    url_parts = urlparse.urlparse(url)
    path_parts = []
    for part in url_parts.path.split('/'):
        if part == '.':
            continue
        elif part == '..':
            if path_parts:
                path_parts.pop()
        else:
            path_parts.append(part)

    url_parts = list(url_parts)
    if force_scheme:
        url_parts[0] = force_scheme
    url_parts[2] = '/'.join(path_parts)
    url_parts[4] = ''    # No query string
    url_parts[5] = ''    # No path

    # Always have a trailing slash
    if not url_parts[2]:
        url_parts[2] = '/'

    return urlparse.urlunparse(url_parts)

def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """Extracts the URLs from an HTML document."""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result

IGNORE_SUFFIXES = frozenset([
    'jpg', 'jpeg', 'png', 'css', 'js', 'xml', 'json', 'gif', 'ico', 'doc'])

def prune_urls(url_set, start_url, allowed_list, ignored_list):
    """Prunes URLs that should be ignored."""
    result = set()

    for url in url_set:
        allowed = False
        for allow_url in allowed_list:
            if url.startswith(allow_url):
                allowed = True
                break

        if not allowed:
            continue

        ignored = False
        for ignore_url in ignored_list:
            if url.startswith(ignore_url):
                ignored = True
                break

        if ignored:
            continue

        prefix, suffix = (url.rsplit('.', 1) + [''])[:2]
        if suffix.lower() in IGNORE_SUFFIXES:
            continue

        result.add(url)

    return result

class SiteDiff(workers.WorkflowItem):
    """Workflow for coordinating the site diff.

    Args:
        start_url: URL to begin the site diff scan.
        ignore_prefixes: Optional. List of URL prefixes to ignore during
            the crawl; start_url should be a common prefix with all of these.
        upload_build_id: Build ID of the site being compared.
        upload_release_name: Optional. Release name to use for the build. When
            not supplied, a new release based on the current time will be
            created.
        heartbeat: Function to call with progress status.
    """

    def run(self,
            start_url,
            ignore_prefixes,
            upload_build_id,
            upload_release_name=None,
            heartbeat=None):
        if not ignore_prefixes:
            ignore_prefixes = []

        pending_urls = set([clean_url(start_url)])
        seen_urls = set()
        good_urls = set()

        yield heartbeat('Scanning for content')

        limit_depth = FLAGS.crawl_depth >= 0
        depth = 0
        while (not limit_depth or depth <= FLAGS.crawl_depth) and pending_urls:
            # TODO: Enforce a job-wide timeout on the whole process of
            # URL discovery, to make sure infinitely deep sites do not
            # cause this job to never stop.
            seen_urls.update(pending_urls)
            yield heartbeat(
                'Scanning %d pages for good urls' % len(pending_urls))
            output = yield [fetch_worker.FetchItem(u) for u in pending_urls]
            pending_urls.clear()

            for item in output:
                if not item.data:
                    logging.debug('No data from url=%r', item.url)
                    continue

                if item.headers.gettype() != 'text/html':
                    logging.debug('Skipping non-HTML document url=%r',
                                  item.url)
                    continue

                good_urls.add(item.url)
                found = extract_urls(item.url, item.data)
                pruned = prune_urls(
                    found, start_url, [start_url], ignore_prefixes)
                new = pruned - seen_urls
                pending_urls.update(new)
                yield heartbeat('Found %d new URLs from %s' % (
                                len(new), item.url))

            yield heartbeat('Finished crawl at depth %d' % depth)
            depth += 1

        yield heartbeat(
            'Found %d total URLs, %d good HTML pages; starting '
            'screenshots' % (len(seen_urls), len(good_urls)))

        # TODO: Make the default release name prettier.
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, start_url)

        run_requests = []
        for url in good_urls:
            yield heartbeat('Requesting run for %s' % url)
            parts = urlparse.urlparse(url)
            run_name = parts.path

            config_dict = {
                'viewportSize': {
                    'width': 1280,
                    'height': 1024,
                }
            }
            if FLAGS.inject_css:
                config_dict['injectCss'] = FLAGS.inject_css
            if FLAGS.inject_js:
                config_dict['injectJs'] = FLAGS.inject_js
            if FLAGS.cookies:
                config_dict['cookies'] = json.loads(
                    open(FLAGS.cookies).read())
            config_data = json.dumps(config_dict)

            run_requests.append(release_worker.RequestRunWorkflow(
                upload_build_id, upload_release_name, release_number,
                run_name, url, config_data))

        yield run_requests

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)

class PrintWorkflow(workers.WorkflowItem):
    """Prints a message to stdout."""

    def run(self, message):
        yield []  # Make this into a generator
        print message

def real_main(start_url=None,
              ignore_prefixes=None,
              upload_build_id=None,
              upload_release_name=None):
    """Runs the site_diff."""
    coordinator = workers.get_coordinator()
    fetch_worker.register(coordinator)
    coordinator.start()

    item = SiteDiff(
        start_url=start_url,
        ignore_prefixes=ignore_prefixes,
        upload_build_id=upload_build_id,
        upload_release_name=upload_release_name,
        heartbeat=PrintWorkflow)
    item.root = True

    coordinator.input_queue.put(item)
    coordinator.wait_one()

def main(argv):
    try:
        argv = FLAGS(argv)
    except gflags.FlagsError, e:
        print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], FLAGS)
        sys.exit(1)

    if len(argv) != 2:
        print 'Must supply a website URL as the first argument.'
        sys.exit(1)

    assert FLAGS.upload_build_id
    assert FLAGS.release_server_prefix

    if FLAGS.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    real_main(
        start_url=argv[1],
        ignore_prefixes=FLAGS.ignore_prefixes,
        upload_build_id=FLAGS.upload_build_id,
        upload_release_name=FLAGS.upload_release_name)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Fixes #32<commit_after>#!/usr/bin/env python
# Copyright 2013 Brett Slatkin
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility for doing incremental diffs for a live website."""

import HTMLParser
import Queue
import datetime
import json
import logging
import os
import re
import sys
import urlparse

# Local Libraries
import gflags
FLAGS = gflags.FLAGS

# Local modules
from dpxdt.client import release_worker
from dpxdt.client import fetch_worker
from dpxdt.client import workers
import flags

gflags.DEFINE_integer(
    'crawl_depth', -1,
    'How deep to crawl. Depth of 0 means only the given page. 1 means pages '
    'that are one click away, 2 means two clicks, and so on. Set to -1 to '
    'scan every URL with the supplied prefix.')

gflags.DEFINE_spaceseplist(
    'ignore_prefixes', [],
    'URL prefixes that should not be crawled.')

# URL regex rewriting code originally from mirrorrr
# http://code.google.com/p/mirrorrr/source/browse/trunk/transform_content.py

# URLs that have absolute addresses
ABSOLUTE_URL_REGEX = r"(?P<url>(http(s?):)?//[^\"'> \t]+)"
# URLs that are relative to the base of the current hostname.
BASE_RELATIVE_URL_REGEX = (
    r"/(?!(/)|(http(s?)://)|(url\())(?P<url>[^\"'> \t]*)")
# URLs that have '../' or './' to start off their paths.
TRAVERSAL_URL_REGEX = (
    r"(?P<relative>\.(\.)?)/(?!(/)|"
    r"(http(s?)://)|(url\())(?P<url>[^\"'> \t]*)")
# URLs that are in the same directory as the requested URL.
SAME_DIR_URL_REGEX = r"(?!(/)|(http(s?)://)|(#)|(url\())(?P<url>[^\"'> \t]+)"
# URL matches the root directory.
ROOT_DIR_URL_REGEX = r"(?!//(?!>))/(?P<url>)(?=[ \t\n]*[\"'> /])"
# Start of a tag using 'src' or 'href'
TAG_START = (
    r"(?i)(?P<tag>\ssrc|href|action|url|background)"
    r"(?P<equals>[\t ]*=[\t ]*)(?P<quote>[\"']?)")
# Potential HTML document URL with no fragments.
MAYBE_HTML_URL_REGEX = (
    TAG_START + r"(?P<absurl>(http(s?):)?//[^\"'> \t]+)")

REPLACEMENT_REGEXES = [
    (TAG_START + SAME_DIR_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(accessed_dir)s\g<url>"),
    (TAG_START + TRAVERSAL_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(accessed_dir)s/\g<relative>/\g<url>"),
    (TAG_START + BASE_RELATIVE_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(base)s/\g<url>"),
    (TAG_START + ROOT_DIR_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>%(base)s/"),
    (TAG_START + ABSOLUTE_URL_REGEX,
     "\g<tag>\g<equals>\g<quote>\g<url>"),
]

def clean_url(url, force_scheme=None):
    """Cleans the given URL."""
    # URL should be ASCII according to RFC 3986
    url = str(url)
    # Collapse ../../ and related
    url_parts = urlparse.urlparse(url)
    path_parts = []
    for part in url_parts.path.split('/'):
        if part == '.':
            continue
        elif part == '..':
            if path_parts:
                path_parts.pop()
        else:
            path_parts.append(part)

    url_parts = list(url_parts)
    if force_scheme:
        url_parts[0] = force_scheme
    url_parts[2] = '/'.join(path_parts)
    url_parts[4] = ''    # No query string
    url_parts[5] = ''    # No path

    # Always have a trailing slash
    if not url_parts[2]:
        url_parts[2] = '/'

    return urlparse.urlunparse(url_parts)

def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """Extracts the URLs from an HTML document."""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result

IGNORE_SUFFIXES = frozenset([
    'jpg', 'jpeg', 'png', 'css', 'js', 'xml', 'json', 'gif', 'ico', 'doc'])

def prune_urls(url_set, start_url, allowed_list, ignored_list):
    """Prunes URLs that should be ignored."""
    result = set()

    for url in url_set:
        allowed = False
        for allow_url in allowed_list:
            if url.startswith(allow_url):
                allowed = True
                break

        if not allowed:
            continue

        ignored = False
        for ignore_url in ignored_list:
            if url.startswith(ignore_url):
                ignored = True
                break

        if ignored:
            continue

        prefix, suffix = (url.rsplit('.', 1) + [''])[:2]
        if suffix.lower() in IGNORE_SUFFIXES:
            continue

        result.add(url)

    return result

class SiteDiff(workers.WorkflowItem):
    """Workflow for coordinating the site diff.

    Args:
        start_url: URL to begin the site diff scan.
        ignore_prefixes: Optional. List of URL prefixes to ignore during
            the crawl; start_url should be a common prefix with all of these.
        upload_build_id: Build ID of the site being compared.
        upload_release_name: Optional. Release name to use for the build. When
            not supplied, a new release based on the current time will be
            created.
        heartbeat: Function to call with progress status.
    """

    def run(self,
            start_url,
            ignore_prefixes,
            upload_build_id,
            upload_release_name=None,
            heartbeat=None):
        if not ignore_prefixes:
            ignore_prefixes = []

        pending_urls = set([clean_url(start_url)])
        seen_urls = set()
        good_urls = set()

        yield heartbeat('Scanning for content')

        limit_depth = FLAGS.crawl_depth >= 0
        depth = 0
        while (not limit_depth or depth <= FLAGS.crawl_depth) and pending_urls:
            # TODO: Enforce a job-wide timeout on the whole process of
            # URL discovery, to make sure infinitely deep sites do not
            # cause this job to never stop.
            seen_urls.update(pending_urls)
            yield heartbeat(
                'Scanning %d pages for good urls' % len(pending_urls))
            output = yield [fetch_worker.FetchItem(u) for u in pending_urls]
            pending_urls.clear()

            for item in output:
                if not item.data:
                    logging.debug('No data from url=%r', item.url)
                    continue

                if item.headers.gettype() != 'text/html':
                    logging.debug('Skipping non-HTML document url=%r',
                                  item.url)
                    continue

                good_urls.add(item.url)
                found = extract_urls(item.url, item.data)
                pruned = prune_urls(
                    found, start_url, [start_url], ignore_prefixes)
                new = pruned - seen_urls
                pending_urls.update(new)
                yield heartbeat('Found %d new URLs from %s' % (
                                len(new), item.url))

            yield heartbeat('Finished crawl at depth %d' % depth)
            depth += 1

        yield heartbeat(
            'Found %d total URLs, %d good HTML pages; starting '
            'screenshots' % (len(seen_urls), len(good_urls)))

        # TODO: Make the default release name prettier.
        if not upload_release_name:
            upload_release_name = str(datetime.datetime.utcnow())

        release_number = yield release_worker.CreateReleaseWorkflow(
            upload_build_id, upload_release_name, start_url)

        run_requests = []
        for url in good_urls:
            yield heartbeat('Requesting run for %s' % url)
            parts = urlparse.urlparse(url)
            run_name = parts.path

            config_dict = {
                'viewportSize': {
                    'width': 1280,
                    'height': 1024,
                }
            }
            if FLAGS.inject_css:
                config_dict['injectCss'] = FLAGS.inject_css
            if FLAGS.inject_js:
                config_dict['injectJs'] = FLAGS.inject_js
            if FLAGS.cookies:
                config_dict['cookies'] = json.loads(
                    open(FLAGS.cookies).read())
            config_data = json.dumps(config_dict)

            run_requests.append(release_worker.RequestRunWorkflow(
                upload_build_id, upload_release_name, release_number,
                run_name, url, config_data))

        yield run_requests

        yield heartbeat('Marking runs as complete')
        release_url = yield release_worker.RunsDoneWorkflow(
            upload_build_id, upload_release_name, release_number)

        yield heartbeat('Results viewable at: %s' % release_url)

class PrintWorkflow(workers.WorkflowItem):
    """Prints a message to stdout."""

    def run(self, message):
        yield []  # Make this into a generator
        print message

def real_main(start_url=None,
              ignore_prefixes=None,
              upload_build_id=None,
              upload_release_name=None):
    """Runs the site_diff."""
    coordinator = workers.get_coordinator()
    fetch_worker.register(coordinator)
    coordinator.start()

    item = SiteDiff(
        start_url=start_url,
        ignore_prefixes=ignore_prefixes,
        upload_build_id=upload_build_id,
        upload_release_name=upload_release_name,
        heartbeat=PrintWorkflow)
    item.root = True

    coordinator.input_queue.put(item)
    coordinator.wait_one()

def main(argv):
    try:
        argv = FLAGS(argv)
    except gflags.FlagsError, e:
        print '%s\nUsage: %s ARGS\n%s' % (e, sys.argv[0], FLAGS)
        sys.exit(1)

    if len(argv) != 2:
        print 'Must supply a website URL as the first argument.'
        sys.exit(1)

    assert FLAGS.upload_build_id
    assert FLAGS.release_server_prefix

    if FLAGS.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    real_main(
        start_url=argv[1],
        ignore_prefixes=FLAGS.ignore_prefixes,
        upload_build_id=FLAGS.upload_build_id,
        upload_release_name=FLAGS.upload_release_name)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before># pylint: disable=E1101
# -*- coding: utf-8 -*-
"""
The internal API for VAL
"""
import logging

from edxval.models import Video
from edxval.serializers import VideoSerializer, ProfileSerializer

logger = logging.getLogger(__name__)  # pylint: disable=C0103

class ValError(Exception):
    """
    An error that occurs during VAL actions.

    This error is raised when the VAL API cannot perform a requested
    action.

    """
    pass

class ValInternalError(ValError):
    """
    An error internal to the VAL API has occurred.

    This error is raised when an error occurs that is not caused by incorrect
    use of the API, but rather internal implementation of the underlying
    services.

    """
    pass

class ValVideoNotFoundError(ValError):
    """
    This error is raised when a video is not found

    If a state is specified in a call to the API that results in no matching
    entry in database, this error may be raised.

    """
    pass

class ValCannotCreateError(ValError):
    """
    This error is raised when an object cannot be created
    """
    pass

def create_video(video_data):
    """
    Called on to create Video objects in the database

    create_video is used to create Video objects whose children are EncodedVideo
    objects which are linked to Profile objects. This is an alternative to the HTTP
    requests so it can be used internally. The VideoSerializer is used to
    deserialize this object. If there are duplicate profile_names, the entire
    creation will be rejected. If the profile is not found in the database, the
    video will not be created.
    Args:
        data (dict):
         {
                url: api url to the video
                edx_video_id: ID of the video
                duration: Length of video in seconds
                client_video_id: client ID of video
                encoded_video: a list of EncodedVideo dicts
                    url: url of the video
                    file_size: size of the video in bytes
                    profile: a dict of encoding details
                        profile_name: ID of the profile
                        extension: 3 letter extension of video
                        width: horizontal pixel resolution
                        height: vertical pixel resolution
         }
    """
    serializer = VideoSerializer(data=video_data)
    if serializer.is_valid():
        serializer.save()
        return video_data.get("edx_video_id")
    else:
        raise ValCannotCreateError(serializer.errors)

def create_profile(profile_data):
    """
    Used to create Profile objects in the database

    A profile needs to exists before an EncodedVideo object can be created.

    Args:
        data (dict):
        {
            profile_name: ID of the profile
            extension: 3 letter extension of video
            width: horizontal pixel resolution
            height: vertical pixel resolution
        }

    Returns:
        new_object.id (int): id of the newly created object

    Raises:
        ValCannotCreateError: Raised if the serializer throws an error
    """
    serializer = ProfileSerializer(data=profile_data)
    if serializer.is_valid():
        serializer.save()
        return profile_data.get("profile_name")
    else:
        raise ValCannotCreateError(serializer.errors)

def get_video_info(edx_video_id, location=None):  # pylint: disable=W0613
    """
    Retrieves all encoded videos of a video found with given video edx_video_id

    Args:
        location (str): geographic locations used determine CDN
        edx_video_id (str): id for video content.

    Returns:
        result (dict): Deserialized Video Object with related field EncodedVideo
            Returns all the Video object fields, and it's related EncodedVideo
            objects in a list.
            {
                url: api url to the video
                edx_video_id: ID of the video
                duration: Length of video in seconds
                client_video_id: client ID of video
                encoded_video: a list of EncodedVideo dicts
                    url: url of the video
                    file_size: size of the video in bytes
                    profile: a dict of encoding details
                        profile_name: ID of the profile
                        extension: 3 letter extension of video
                        width: horizontal pixel resolution
                        height: vertical pixel resolution
                subtitles: a list of Subtitle dicts
                    fmt: file format (SRT or SJSON)
                    language: language code
                    content_url: url of file
                    url: api url to subtitle
            }

    Raises:
        ValVideoNotFoundError: Raised if video doesn't exist
        ValInternalError: Raised for unknown errors

    Example:
        Given one EncodedVideo with edx_video_id "example"
        >>> get_video_info("example")
        Returns (dict):
        {
            'url' : '/edxval/videos/example',
            'edx_video_id': u'example',
            'duration': 111.0,
            'client_video_id': u'The example video',
            'encoded_videos': [
                {
                    'url': u'http://www.meowmix.com',
                    'file_size': 25556,
                    'bitrate': 9600,
                    'profile': u'mobile'
                 }
            ]
        }
    """
    try:
        video = Video.objects.prefetch_related("encoded_videos", "courses").get(edx_video_id=edx_video_id)
        result = VideoSerializer(video)
    except Video.DoesNotExist:
        error_message = u"Video not found for edx_video_id: {0}".format(edx_video_id)
        raise ValVideoNotFoundError(error_message)
    except Exception:
        error_message = u"Could not get edx_video_id: {0}".format(edx_video_id)
        logger.exception(error_message)
        raise ValInternalError(error_message)
    return result.data  # pylint: disable=E1101

def get_urls_for_profiles(edx_video_id, profiles):
    """Returns a dict mapping profiles to URLs.

    If the profiles or video is not found, urls will be blank.
    """
    profiles_to_urls = {profile: "" for profile in profiles}
    try:
        video_info = get_video_info(edx_video_id)
    except ValVideoNotFoundError:
        return profiles_to_urls

    for encoded_video in video_info["encoded_videos"]:
        if encoded_video["profile"] in profiles:
            profiles_to_urls[encoded_video["profile"]] = encoded_video["url"]

    return profiles_to_urls

def get_url_for_profile(edx_video_id, profile):
    return get_urls_for_profiles(edx_video_id, [profile])[profile]

def get_videos_for_course(course_id):
    """
    Returns an iterator of videos for the given course id
    """
    videos = Video.objects.filter(courses__course_id=course_id)
    return (VideoSerializer(video).data for video in videos)
<commit_msg>Add api.get_video_info_for_course_and_profile()<commit_after># pylint: disable=E1101
# -*- coding: utf-8 -*-
"""
The internal API for VAL. This is not yet stable
"""
import logging

from edxval.models import Video, EncodedVideo
from edxval.serializers import VideoSerializer, ProfileSerializer

logger = logging.getLogger(__name__)  # pylint: disable=C0103

class ValError(Exception):
    """
    An error that occurs during VAL actions.

    This error is raised when the VAL API cannot perform a requested
    action.

    """
    pass

class ValInternalError(ValError):
    """
    An error internal to the VAL API has occurred.

    This error is raised when an error occurs that is not caused by incorrect
    use of the API, but rather internal implementation of the underlying
    services.

    """
    pass

class ValVideoNotFoundError(ValError):
    """
    This error is raised when a video is not found

    If a state is specified in a call to the API that results in no matching
    entry in database, this error may be raised.

    """
    pass

class ValCannotCreateError(ValError):
    """
    This error is raised when an object cannot be created
    """
    pass

def create_video(video_data):
    """
    Called on to create Video objects in the database

    create_video is used to create Video objects whose children are EncodedVideo
    objects which are linked to Profile objects. This is an alternative to the HTTP
    requests so it can be used internally. The VideoSerializer is used to
    deserialize this object. If there are duplicate profile_names, the entire
    creation will be rejected. If the profile is not found in the database, the
    video will not be created.
    Args:
        data (dict):
         {
                url: api url to the video
                edx_video_id: ID of the video
                duration: Length of video in seconds
                client_video_id: client ID of video
                encoded_video: a list of EncodedVideo dicts
                    url: url of the video
                    file_size: size of the video in bytes
                    profile: a dict of encoding details
                        profile_name: ID of the profile
                        extension: 3 letter extension of video
                        width: horizontal pixel resolution
                        height: vertical pixel resolution
         }
    """
    serializer = VideoSerializer(data=video_data)
    if serializer.is_valid():
        serializer.save()
        return video_data.get("edx_video_id")
    else:
        raise ValCannotCreateError(serializer.errors)

def create_profile(profile_data):
    """
    Used to create Profile objects in the database

    A profile needs to exists before an EncodedVideo object can be created.

    Args:
        data (dict):
        {
            profile_name: ID of the profile
            extension: 3 letter extension of video
            width: horizontal pixel resolution
            height: vertical pixel resolution
        }

    Returns:
        new_object.id (int): id of the newly created object

    Raises:
        ValCannotCreateError: Raised if the serializer throws an error
    """
    serializer = ProfileSerializer(data=profile_data)
    if serializer.is_valid():
        serializer.save()
        return profile_data.get("profile_name")
    else:
        raise ValCannotCreateError(serializer.errors)

def get_video_info(edx_video_id, location=None):  # pylint: disable=W0613
    """
    Retrieves all encoded videos of a video found with given video edx_video_id

    Args:
        location (str): geographic locations used determine CDN
        edx_video_id (str): id for video content.

    Returns:
        result (dict): Deserialized Video Object with related field EncodedVideo
            Returns all the Video object fields, and it's related EncodedVideo
            objects in a list.
            {
                url: api url to the video
                edx_video_id: ID of the video
                duration: Length of video in seconds
                client_video_id: client ID of video
                encoded_video: a list of EncodedVideo dicts
                    url: url of the video
                    file_size: size of the video in bytes
                    profile: a dict of encoding details
                        profile_name: ID of the profile
                        extension: 3 letter extension of video
                        width: horizontal pixel resolution
                        height: vertical pixel resolution
                subtitles: a list of Subtitle dicts
                    fmt: file format (SRT or SJSON)
                    language: language code
                    content_url: url of file
                    url: api url to subtitle
            }

    Raises:
        ValVideoNotFoundError: Raised if video doesn't exist
        ValInternalError: Raised for unknown errors

    Example:
        Given one EncodedVideo with edx_video_id "example"
        >>> get_video_info("example")
        Returns (dict):
        {
            'url' : '/edxval/videos/example',
            'edx_video_id': u'example',
            'duration': 111.0,
            'client_video_id': u'The example video',
            'encoded_videos': [
                {
                    'url': u'http://www.meowmix.com',
                    'file_size': 25556,
                    'bitrate': 9600,
                    'profile': u'mobile'
                 }
            ]
        }
    """
    try:
        video = Video.objects.prefetch_related("encoded_videos", "courses").get(edx_video_id=edx_video_id)
        result = VideoSerializer(video)
    except Video.DoesNotExist:
        error_message = u"Video not found for edx_video_id: {0}".format(edx_video_id)
        raise ValVideoNotFoundError(error_message)
    except Exception:
        error_message = u"Could not get edx_video_id: {0}".format(edx_video_id)
        logger.exception(error_message)
        raise ValInternalError(error_message)
    return result.data  # pylint: disable=E1101

def get_urls_for_profiles(edx_video_id, profiles):
    """Returns a dict mapping profiles to URLs.

    If the profiles or video is not found, urls will be blank.
    """
    profiles_to_urls = {profile: "" for profile in profiles}
    try:
        video_info = get_video_info(edx_video_id)
    except ValVideoNotFoundError:
        return profiles_to_urls

    for encoded_video in video_info["encoded_videos"]:
        if encoded_video["profile"] in profiles:
            profiles_to_urls[encoded_video["profile"]] = encoded_video["url"]

    return profiles_to_urls

def get_url_for_profile(edx_video_id, profile):
    return get_urls_for_profiles(edx_video_id, [profile])[profile]

def get_videos_for_course(course_id):
    """
    Returns an iterator of videos for the given course id
    """
    videos = Video.objects.filter(courses__course_id=course_id)
    return (VideoSerializer(video).data for video in videos)

def get_video_info_for_course_and_profile(course_id, profile_name):
    """Returns a dict mapping profiles to URLs.

    If the profiles or video is not found, urls will be blank.
    """
    # In case someone passes in a key (VAL doesn't really understand opaque keys)
    course_id = unicode(course_id)
    try:
        encoded_videos = EncodedVideo.objects.filter(
            profile__profile_name=profile_name,
            video__courses__course_id=course_id
        ).select_related()
    except Exception:
        error_message = u"Could not get encoded videos for course: {0}".format(course_id)
        logger.exception(error_message)
        raise ValInternalError(error_message)

    # DRF serializers were causing extra queries for some reason...
    return {
        enc_vid.video.edx_video_id: {
            "url": enc_vid.url,
            "file_size": enc_vid.file_size,
            "duration": enc_vid.video.duration,
        }
        for enc_vid in encoded_videos
    }
<|endoftext|>


<commit_before>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from flask import Flask, render_template, flash, request, Markup, \
    session, redirect, url_for, escape, Response, abort, send_file
import StringIO
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import (LoginManager, current_user, login_required,
                            login_user, logout_user, UserMixin, AnonymousUser,
                            confirm_login, fresh_login_required)
from sqlalchemy import func
from datetime import datetime

from iatidataquality import app
from iatidataquality import db

import os
import sys
import json

current = os.path.dirname(os.path.abspath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from iatidq import dqdownload, dqregistry, dqindicators, dqorganisations, dqpackages
import aggregation

from iatidq.models import *

import StringIO
import unicodecsv
import tempfile
import spreadsheet

@app.route("/packages/manage/", methods=['GET', 'POST'])
def packages_manage():
    if request.method != 'POST':
        pkgs = Package.query.order_by(Package.package_name).all()
        return render_template("packages_manage.html", pkgs=pkgs)

    if "refresh" in request.form:
        dqregistry.refresh_packages()
        flash("Refreshed packages from Registry", "success")
    else:
        data = []
        for package in request.form.getlist('package'):
            try:
                request.form["active_"+package]
                active=True
            except Exception:
                active=False
            data.append((package, active))
        dqregistry.activate_packages(data)
        flash("Updated packages", "success")
    return redirect(url_for('packages_manage'))

def package_aggregation(p, latest_runtime):
    return db.session.query(
        Indicator,
        Test,
        AggregateResult.results_data,
        AggregateResult.results_num,
        AggregateResult.result_hierarchy
        ).filter(
        AggregateResult.package_id==p[0].id,
        AggregateResult.runtime_id==latest_runtime.id
        ).group_by(
            AggregateResult.result_hierarchy,
            Test,
            AggregateResult.results_data,
            AggregateResult.results_num,
            Indicator
            ).join(
            IndicatorTest
            ).join(
                Test
                ).join(
                AggregateResult
                ).all()

@app.route("/packages/")
@app.route("/packages/<package_name>/")
@app.route("/packages/<package_name>/runtimes/<runtime_id>/")
def packages(package_name=None, runtime_id=None):
    if package_name is None:
        pkgs = Package.query.filter_by(active=True).order_by(
            Package.package_name).all()
        return render_template("packages.html", pkgs=pkgs)

    # Get package data
    p = db.session.query(Package,
                         PackageGroup
                         ).filter(Package.package_name == package_name
                                  ).join(PackageGroup).first()

    def get_pconditions():
        return {}
        # Publisher conditions have been removed in favour
        #  of organisation conditions. need to consider how to
        #  possibly include this again here.
        """if p is None:
            p = db.session.query(Package).\
                filter(Package.package_name == id).first()
            return {}
        else:
        # Get publisher-specific conditions.
            return PublisherCondition.query.filter_by(
                publisher_id=p[1].id).all()"""

    pconditions = get_pconditions()

    # Get list of runtimes
    try:
        runtimes = db.session.query(Result.runtime_id,
                                    Runtime.runtime_datetime
            ).filter(Result.package_id==p[0].id
            ).distinct(
            ).join(Runtime
            ).all()
    except Exception:
        return abort(404)

    def get_latest_runtime():
        if runtime_id:
            # If a runtime is specified in the request, get the data
            return (db.session.query(Runtime
                        ).filter(Runtime.id==runtime_id
                        ).first(), False)
        else:
            # Select the highest runtime; then get data for that one
            return (db.session.query(Runtime
                        ).filter(Result.package_id==p[0].id
                        ).join(Result
                        ).order_by(Runtime.id.desc()).first(),
                    True)

    latest_runtime, latest = get_latest_runtime()

    if latest_runtime:
        aggregate_results = package_aggregation(p, latest_runtime)

        flat_results = aggregate_results

        aggregate_results = aggregation.agr_results(aggregate_results,
                                                    pconditions)
    else:
        aggregate_results = None
        pconditions = None
        flat_results = None
        latest_runtime = None

    organisations = dqpackages.packageOrganisations(p[0].id)

    return render_template("package.html", p=p, runtimes=runtimes,
                           results=aggregate_results,
                           latest_runtime=latest_runtime, latest=latest,
                           pconditions=pconditions, flat_results=flat_results,
organisations=organisations)
<commit_msg>Improve get_latest_runtime lookup performance<commit_after>
#  IATI Data Quality, tools for Data QA on IATI-formatted  publications
#  by Mark Brough, Martin Keegan, Ben Webb and Jennifer Smith
#
#  Copyright (C) 2013  Publish What You Fund
#
#  This programme is free software; you may redistribute and/or modify
#  it under the terms of the GNU Affero General Public License v3.0

from flask import Flask, render_template, flash, request, Markup, \
    session, redirect, url_for, escape, Response, abort, send_file
import StringIO
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.login import (LoginManager, current_user, login_required,
                            login_user, logout_user, UserMixin, AnonymousUser,
                            confirm_login, fresh_login_required)
from sqlalchemy import func
from datetime import datetime

from iatidataquality import app
from iatidataquality import db

import os
import sys
import json

current = os.path.dirname(os.path.abspath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from iatidq import dqdownload, dqregistry, dqindicators, dqorganisations, dqpackages
import aggregation

from iatidq.models import *

import StringIO
import unicodecsv
import tempfile
import spreadsheet

@app.route("/packages/manage/", methods=['GET', 'POST'])
def packages_manage():
    if request.method != 'POST':
        pkgs = Package.query.order_by(Package.package_name).all()
        return render_template("packages_manage.html", pkgs=pkgs)

    if "refresh" in request.form:
        dqregistry.refresh_packages()
        flash("Refreshed packages from Registry", "success")
    else:
        data = []
        for package in request.form.getlist('package'):
            try:
                request.form["active_"+package]
                active=True
            except Exception:
                active=False
            data.append((package, active))
        dqregistry.activate_packages(data)
        flash("Updated packages", "success")
    return redirect(url_for('packages_manage'))

def package_aggregation(p, latest_runtime):
    return db.session.query(
        Indicator,
        Test,
        AggregateResult.results_data,
        AggregateResult.results_num,
        AggregateResult.result_hierarchy
        ).filter(
        AggregateResult.package_id==p[0].id,
        AggregateResult.runtime_id==latest_runtime.id
        ).group_by(
            AggregateResult.result_hierarchy,
            Test,
            AggregateResult.results_data,
            AggregateResult.results_num,
            Indicator
            ).join(
            IndicatorTest
            ).join(
                Test
                ).join(
                AggregateResult
                ).all()

@app.route("/packages/")
@app.route("/packages/<package_name>/")
@app.route("/packages/<package_name>/runtimes/<runtime_id>/")
def packages(package_name=None, runtime_id=None):
    if package_name is None:
        pkgs = Package.query.filter_by(active=True).order_by(
            Package.package_name).all()
        return render_template("packages.html", pkgs=pkgs)

    # Get package data
    p = db.session.query(Package,
                         PackageGroup
                         ).filter(Package.package_name == package_name
                                  ).join(PackageGroup).first()

    def get_pconditions():
        return {}
        # Publisher conditions have been removed in favour
        #  of organisation conditions. need to consider how to
        #  possibly include this again here.
        """if p is None:
            p = db.session.query(Package).\
                filter(Package.package_name == id).first()
            return {}
        else:
        # Get publisher-specific conditions.
            return PublisherCondition.query.filter_by(
                publisher_id=p[1].id).all()"""

    pconditions = get_pconditions()

    # Get list of runtimes
    try:
        runtimes = db.session.query(Result.runtime_id,
                                    Runtime.runtime_datetime
            ).filter(Result.package_id==p[0].id
            ).distinct(
            ).join(Runtime
            ).all()
    except Exception:
        return abort(404)

    def get_latest_runtime():
        if runtime_id:
            # If a runtime is specified in the request, get the data
            return (db.session.query(Runtime
                        ).filter(Runtime.id==runtime_id
                        ).first(), False)
        else:
            # Select the highest runtime; then get data for that one
            runtime = db.session.query(Runtime,
                                    func.max(Runtime.id)
                    ).join(AggregateResult
                    ).group_by(Runtime.id
                    ).filter(AggregateResult.package_id==p[0].id
                    ).first()
            return runtime[0], True

    latest_runtime, latest = get_latest_runtime()

    if latest_runtime:
        aggregate_results = package_aggregation(p, latest_runtime)

        flat_results = aggregate_results

        aggregate_results = aggregation.agr_results(aggregate_results,
                                                    pconditions)
    else:
        aggregate_results = None
        pconditions = None
        flat_results = None
        latest_runtime = None

    organisations = dqpackages.packageOrganisations(p[0].id)

    return render_template("package.html", p=p, runtimes=runtimes,
                           results=aggregate_results,
                           latest_runtime=latest_runtime, latest=latest,
                           pconditions=pconditions, flat_results=flat_results,
organisations=organisations)
<|endoftext|>


<commit_before>import functools
import logging
import traceback
from datetime import datetime

from ibis.backends.pandas.dispatcher import TwoLevelDispatcher
from ibis.config import get_option, set_option
from ibis.expr import types as ir

"""Module that adds tracing to dask execution.
With tracing enabled, this module will log time and call stack information of
the executed expression. Call stack information is presented with indentation
level.
For example:
import dask.dataframe as dd
import pandas as pd
import logging
import ibis.expr.datatypes as dt
import ibis.dask
from ibis.udf.vectorized import elementwise
from ibis.dask import trace
logging.basicConfig()
trace.enable()
df = dd.from_pandas(
    pd.DataFrame({'a': [1, 2, 3]})
)
con = ibis.dask.connect({"table1": df})
@elementwise(
    input_type=[dt.double],
    output_type=dt.double
)
def add_one(v):
    import time
    time.sleep(5)
    return v + 1
table = con.table("table1")
table = table.mutate(b=add_one(table['a']))
table.execute()
Output:
DEBUG:ibis.dask.trace: main_execute Selection
DEBUG:ibis.dask.trace:   execute_until_in_scope Selection
DEBUG:ibis.dask.trace:     execute_until_in_scope DaskTable
DEBUG:ibis.dask.trace:       execute_database_table_client DaskTable
DEBUG:ibis.dask.trace:       execute_database_table_client DaskTable 0:00:00.000085
DEBUG:ibis.dask.trace:     execute_until_in_scope DaskTable 0:00:00.000362
DEBUG:ibis.dask.trace:     execute_selection_dataframe Selection
DEBUG:ibis.dask.trace:       main_execute ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:         execute_until_in_scope ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:           execute_until_in_scope TableColumn
DEBUG:ibis.dask.trace:             execute_until_in_scope DaskTable
DEBUG:ibis.dask.trace:             execute_until_in_scope DaskTable 0:00:00.000061
DEBUG:ibis.dask.trace:             execute_table_column_df_or_df_groupby TableColumn
DEBUG:ibis.dask.trace:             execute_table_column_df_or_df_groupby TableColumn 0:00:00.000304  # noqa: E501
DEBUG:ibis.dask.trace:           execute_until_in_scope TableColumn 0:00:00.000584
DEBUG:ibis.dask.trace:           execute_udf_node ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:           execute_udf_node ElementWiseVectorizedUDF 0:00:05.019173
DEBUG:ibis.dask.trace:         execute_until_in_scope ElementWiseVectorizedUDF 0:00:05.052604  # noqa: E501
DEBUG:ibis.dask.trace:       main_execute ElementWiseVectorizedUDF 0:00:05.052819
DEBUG:ibis.dask.trace:     execute_selection_dataframe Selection 0:00:05.054894
DEBUG:ibis.dask.trace:   execute_until_in_scope Selection 0:00:05.055662
DEBUG:ibis.dask.trace: main_execute Selection 0:00:05.056556
"""

_logger = logging.getLogger('ibis.dask.trace')

# A list of funcs that is traced
_trace_funcs = set()
_trace_root = "main_execute"
_TRACE_CONFIG = 'dask.enable_trace'

def enable():
    """Enable tracing."""
    set_option(_TRACE_CONFIG, True)
    logging.getLogger('ibis.dask.trace').setLevel(logging.DEBUG)

def _log_trace(func, start=None):
    level = 0
    current_frame = None

    # Increase the current level for each traced function in the stackframe
    # This way we can visualize the call stack.
    for frame, _ in traceback.walk_stack(None):
        current_frame = current_frame if current_frame is not None else frame
        func_name = frame.f_code.co_name
        if func_name in _trace_funcs:
            level += 1

    # We can assume we have 'args' because we only call _log_trace inside
    # trace or TraceDispatcher.resgister
    current_op = current_frame.f_locals['args'][0]

    # If the first argument is a Expr, we print its op because it's more
    # informative.
    if isinstance(current_op, ir.Expr):
        current_op = current_op.op()

    _logger.debug(
        f"{'  ' * level} {func.__name__} {type(current_op).__qualname__} "
        f"{datetime.now() - start if start else ''}"
    )

def trace(func):
    """Return a function decorator that wraped the decorated function with
    tracing.
    """
    _trace_funcs.add(func.__name__)

    @functools.wraps(func)
    def traced_func(*args, **kwargs):
        import ibis

        # Similar to the pandas backend, it is possible to call this function
        # without having initialized the configuration option. This can happen
        # when tests are distributed across multiple processes, for example.
        ibis.dask

        trace_enabled = get_option(_TRACE_CONFIG)

        if not trace_enabled:
            return func(*args, **kwargs)
        else:
            start = datetime.now()
            _log_trace(func)
            res = func(*args, **kwargs)
            _log_trace(func, start)
            return res

    return traced_func

class TraceTwoLevelDispatcher(TwoLevelDispatcher):
    """A Dispatcher that also wraps the registered function with tracing."""

    def __init__(self, name, doc=None):
        super().__init__(name, doc)

    def register(self, *types, **kwargs):
        """Register a function with this Dispatcher.
        The function will also be wrapped with tracing information.
        """

        def _(func):
            trace_func = trace(func)
            TwoLevelDispatcher.register(self, *types, **kwargs)(trace_func)
            # return func instead trace_func here so that
            # chained register didn't get wrapped multiple
            # times
            return func

        return _
<commit_msg>DOC: Update dask.trace import comment (#3102)<commit_after>import functools
import logging
import traceback
from datetime import datetime

from ibis.backends.pandas.dispatcher import TwoLevelDispatcher
from ibis.config import get_option, set_option
from ibis.expr import types as ir

"""Module that adds tracing to dask execution.
With tracing enabled, this module will log time and call stack information of
the executed expression. Call stack information is presented with indentation
level.
For example:
import dask.dataframe as dd
import pandas as pd
import logging
import ibis.expr.datatypes as dt
import ibis.dask
from ibis.udf.vectorized import elementwise
from ibis.backends.dask import trace
logging.basicConfig()
trace.enable()
df = dd.from_pandas(
    pd.DataFrame({'a': [1, 2, 3]})
)
con = ibis.dask.connect({"table1": df})
@elementwise(
    input_type=[dt.double],
    output_type=dt.double
)
def add_one(v):
    import time
    time.sleep(5)
    return v + 1
table = con.table("table1")
table = table.mutate(b=add_one(table['a']))
table.execute()
Output:
DEBUG:ibis.dask.trace: main_execute Selection
DEBUG:ibis.dask.trace:   execute_until_in_scope Selection
DEBUG:ibis.dask.trace:     execute_until_in_scope DaskTable
DEBUG:ibis.dask.trace:       execute_database_table_client DaskTable
DEBUG:ibis.dask.trace:       execute_database_table_client DaskTable 0:00:00.000085
DEBUG:ibis.dask.trace:     execute_until_in_scope DaskTable 0:00:00.000362
DEBUG:ibis.dask.trace:     execute_selection_dataframe Selection
DEBUG:ibis.dask.trace:       main_execute ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:         execute_until_in_scope ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:           execute_until_in_scope TableColumn
DEBUG:ibis.dask.trace:             execute_until_in_scope DaskTable
DEBUG:ibis.dask.trace:             execute_until_in_scope DaskTable 0:00:00.000061
DEBUG:ibis.dask.trace:             execute_table_column_df_or_df_groupby TableColumn
DEBUG:ibis.dask.trace:             execute_table_column_df_or_df_groupby TableColumn 0:00:00.000304  # noqa: E501
DEBUG:ibis.dask.trace:           execute_until_in_scope TableColumn 0:00:00.000584
DEBUG:ibis.dask.trace:           execute_udf_node ElementWiseVectorizedUDF
DEBUG:ibis.dask.trace:           execute_udf_node ElementWiseVectorizedUDF 0:00:05.019173
DEBUG:ibis.dask.trace:         execute_until_in_scope ElementWiseVectorizedUDF 0:00:05.052604  # noqa: E501
DEBUG:ibis.dask.trace:       main_execute ElementWiseVectorizedUDF 0:00:05.052819
DEBUG:ibis.dask.trace:     execute_selection_dataframe Selection 0:00:05.054894
DEBUG:ibis.dask.trace:   execute_until_in_scope Selection 0:00:05.055662
DEBUG:ibis.dask.trace: main_execute Selection 0:00:05.056556
"""

_logger = logging.getLogger('ibis.dask.trace')

# A list of funcs that is traced
_trace_funcs = set()
_trace_root = "main_execute"
_TRACE_CONFIG = 'dask.enable_trace'

def enable():
    """Enable tracing."""
    set_option(_TRACE_CONFIG, True)
    logging.getLogger('ibis.dask.trace').setLevel(logging.DEBUG)

def _log_trace(func, start=None):
    level = 0
    current_frame = None

    # Increase the current level for each traced function in the stackframe
    # This way we can visualize the call stack.
    for frame, _ in traceback.walk_stack(None):
        current_frame = current_frame if current_frame is not None else frame
        func_name = frame.f_code.co_name
        if func_name in _trace_funcs:
            level += 1

    # We can assume we have 'args' because we only call _log_trace inside
    # trace or TraceDispatcher.resgister
    current_op = current_frame.f_locals['args'][0]

    # If the first argument is a Expr, we print its op because it's more
    # informative.
    if isinstance(current_op, ir.Expr):
        current_op = current_op.op()

    _logger.debug(
        f"{'  ' * level} {func.__name__} {type(current_op).__qualname__} "
        f"{datetime.now() - start if start else ''}"
    )

def trace(func):
    """Return a function decorator that wraped the decorated function with
    tracing.
    """
    _trace_funcs.add(func.__name__)

    @functools.wraps(func)
    def traced_func(*args, **kwargs):
        import ibis

        # Similar to the pandas backend, it is possible to call this function
        # without having initialized the configuration option. This can happen
        # when tests are distributed across multiple processes, for example.
        ibis.dask

        trace_enabled = get_option(_TRACE_CONFIG)

        if not trace_enabled:
            return func(*args, **kwargs)
        else:
            start = datetime.now()
            _log_trace(func)
            res = func(*args, **kwargs)
            _log_trace(func, start)
            return res

    return traced_func

class TraceTwoLevelDispatcher(TwoLevelDispatcher):
    """A Dispatcher that also wraps the registered function with tracing."""

    def __init__(self, name, doc=None):
        super().__init__(name, doc)

    def register(self, *types, **kwargs):
        """Register a function with this Dispatcher.
        The function will also be wrapped with tracing information.
        """

        def _(func):
            trace_func = trace(func)
            TwoLevelDispatcher.register(self, *types, **kwargs)(trace_func)
            # return func instead trace_func here so that
            # chained register didn't get wrapped multiple
            # times
            return func

        return _
<|endoftext|>