<commit_before># -*- coding: utf-8 -*-
#
# bigmetadata documentation build configuration file, created by
# sphinx-quickstart on Mon Jan 11 21:23:39 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

from docutils import nodes
from sphinx import addnodes

class env(nodes.literal):
    pass

sys.setrecursionlimit(2500)

# "Inline" extension to allow use of Sphinx-specific (vs docutils/ReST)
# methods.
# def setup(app):
#    app.add_node(env)
#    # 'env' object type for Fabric env var referencing
#    # (not to be confused with Sphinx' builtin :envvar:)
#    app.add_object_type(
#        directivename='column',
#        rolename='column',
#        ref_nodeclass=env,
#        objname='Data Column',
#    )
#
#    app.add_object_type(
#        directivename='table',
#        rolename='table',
#        ref_nodeclass=env,
#        objname='Data Table',
#    )

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['rst2pdf.pdfbuilder', 'sphinx_tabs.tabs']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'observatory'
copyright = u'2017, CARTO'
author = u'CARTO'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'0.0.1'
# The full version, including alpha/beta/rc tags.
release = u'0.0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['**/.git']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    'show_related': True
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['../_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'bigmetadatadoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'bigmetadata.tex', u'Observatory Catalog',
   u'CARTO', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'observatory', u'Observatory Catalog',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'observatory', u'Observatory Catalog',
   author, 'observatory', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

pdf_documents = [
    ('index', u'observatory', u'The Data Observatory', u'CARTO', {
    }),
]

# A list of folders to search for stylesheets. Example:
pdf_style_path = ['.', 'stylesheets']
pdf_stylesheets = ['style.yml']
#pdf_stylesheets = ['sphinx', 'kerning', 'a4']

pdf_use_toc = True
pdf_use_numbered_links = True
pdf_toc_depth = 3
pdf_verbosity = 2
pdf_break_level = 3
pdf_breakside = "any"
pdf_section_header_depth = 3

pdf_smartquotes = 1
pdf_font_path = ["/usr/share/texlive/texmf-dist/fonts/"]

pdf_cover_template = 'cover.tmpl'
pdf_use_index = True
<commit_msg>Py3 catalog config, disable broken PDF generator (rst2pdf bug)<commit_after># -*- coding: utf-8 -*-
#
# bigmetadata documentation build configuration file, created by
# sphinx-quickstart on Mon Jan 11 21:23:39 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

from docutils import nodes
from sphinx import addnodes

class env(nodes.literal):
    pass

sys.setrecursionlimit(2500)

# "Inline" extension to allow use of Sphinx-specific (vs docutils/ReST)
# methods.
# def setup(app):
#    app.add_node(env)
#    # 'env' object type for Fabric env var referencing
#    # (not to be confused with Sphinx' builtin :envvar:)
#    app.add_object_type(
#        directivename='column',
#        rolename='column',
#        ref_nodeclass=env,
#        objname='Data Column',
#    )
#
#    app.add_object_type(
#        directivename='table',
#        rolename='table',
#        ref_nodeclass=env,
#        objname='Data Table',
#    )

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx_tabs.tabs']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'observatory'
copyright = '2017, CARTO'
author = 'CARTO'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.0.1'
# The full version, including alpha/beta/rc tags.
release = '0.0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['**/.git']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
html_theme_options = {
    'show_related': True
}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['../_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'bigmetadatadoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'bigmetadata.tex', 'Observatory Catalog',
   'CARTO', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'observatory', 'Observatory Catalog',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'observatory', 'Observatory Catalog',
   author, 'observatory', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

pdf_documents = [
    ('index', 'observatory', 'The Data Observatory', 'CARTO', {
    }),
]

# A list of folders to search for stylesheets. Example:
pdf_style_path = ['.', 'stylesheets']
pdf_stylesheets = ['style.yml']
#pdf_stylesheets = ['sphinx', 'kerning', 'a4']

pdf_use_toc = True
pdf_use_numbered_links = True
pdf_toc_depth = 3
pdf_verbosity = 2
pdf_break_level = 3
pdf_breakside = "any"
pdf_section_header_depth = 3

pdf_smartquotes = 1
pdf_font_path = ["/usr/share/texlive/texmf-dist/fonts/"]

pdf_cover_template = 'cover.tmpl'
pdf_use_index = True
<|endoftext|>


<commit_before>"""
Models backed by SQL using SQLAlchemy.

Note that all genomic positions in this module are one-based and inclusive.

.. moduleauthor:: Martijn Vermaat <martijn@vermaat.name>

.. todo:: Perhaps add some delete cascade rules.

.. Licensed under the MIT license, see the LICENSE file.
"""

from datetime import date
import gzip
import os
import uuid

from flask import current_app
from sqlalchemy import Index
from sqlalchemy.orm.exc import DetachedInstanceError
import bcrypt

from . import db
from .region_binning import assign_bin

# Todo: Use the types for which we have validators.
DATA_SOURCE_FILETYPES = ('bed', 'vcf')

# Note: Add new roles at the end.
USER_ROLES = (
    'admin',       # Can do anything.
    'importer',    # Can import samples.
    'annotator',   # Can annotate samples.
    'trader'       # Can annotate samples if they are also imported.
)

class InvalidDataSource(Exception):
    """
    Exception thrown if data source validation failed.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(InvalidDataSource, self).__init__(code, message)

class DataUnavailable(Exception):
    """
    Exception thrown if reading from a data source which data is not cached
    anymore (in case of local storage) or does not exist anymore (in case of
    a URL resource.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(DataUnavailable, self).__init__(code, message)

class User(db.Model):
    """
    User in the system.

    For the roles column we use a bitstring where the leftmost role in the
    :data:`USER_ROLES` tuple is defined by the least-significant bit.
    Essentially, this creates a set of roles.

    .. todo:: Login should really be validated to only contain alphanums.
    .. todo:: The bitstring encoding/decoding can probably be implemented more
        efficiently.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(200))
    login = db.Column(db.String(200), index=True, unique=True)
    password_hash = db.Column(db.String(200))
    roles_bitstring = db.Column(db.Integer)
    added = db.Column(db.Date)

    def __init__(self, name, login, password, roles=[]):
        self.name = name
        self.login = login
        self.password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
        self.roles_bitstring = sum(pow(2, i) for i, role in enumerate(USER_ROLES)
                                   if role in roles)
        self.added = date.today()

    def __repr__(self):
        return 'User(%r, %r, %r, %r)' % (self.name, self.login, '***', list(self.roles))

    def check_password(self, password):
        return bcrypt.hashpw(password, self.password_hash) == self.password_hash

    @property
    def roles(self):
        return {role for i, role in enumerate(USER_ROLES)
                if self.roles_bitstring & pow(2, i)}

class Sample(db.Model):
    """
    Sample.

    ``coverage_profile`` is essentially ``not is_population_study`` and should
    always be True iff the sample has one or more Coverage entries (so perhaps
    we should not store it, but have it as a calculated property).
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    name = db.Column(db.String(200))
    pool_size = db.Column(db.Integer)
    added = db.Column(db.Date)
    active = db.Column(db.Boolean, default=False)
    coverage_profile = db.Column(db.Boolean)
    public = db.Column(db.Boolean)

    user = db.relationship(User, backref=db.backref('samples', lazy='dynamic'))

    def __init__(self, user, name, pool_size=1, coverage_profile=True, public=False):
        self.user = user
        self.name = name
        self.pool_size = pool_size
        self.added = date.today()
        self.coverage_profile = coverage_profile
        self.public = public

    def __repr__(self):
        return '<Sample "%s" of %d individuals added %s>' % (self.name, self.pool_size, str(self.added))

class DataSource(db.Model):
    """
    Data source (probably uploaded as a file). E.g. VCF file to be imported, or
    BED track from which Region entries are created.

    .. todo:: We can now provide data as an uploaded file or as a path to a
        local file. We also want to be able to give a link to an internet
        resource.

    .. note:: Data source checksums are not forced to be unique, since several
        users might upload the same data source and do different things with
        it.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    name = db.Column(db.String(200))
    filename = db.Column(db.String(50))
    filetype = db.Column(db.Enum(*DATA_SOURCE_FILETYPES, name='filetype'))
    gzipped = db.Column(db.Boolean)
    added = db.Column(db.Date)
    checksum = db.Column(db.String(40))
    records = db.Column(db.Integer)

    user = db.relationship(User, backref=db.backref('data_sources', lazy='dynamic'))

    def __init__(self, user, name, filetype, upload=None, local_path=None, empty=False, gzipped=False):
        if not filetype in DATA_SOURCE_FILETYPES:
            raise InvalidDataSource('unknown_filetype',
                                    'Data source filetype "%s" is unknown'
                                    % filetype)

        self.user = user
        self.name = name
        self.filename = str(uuid.uuid4())
        self.filetype = filetype
        self.gzipped = gzipped
        self.added = date.today()

        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)

        if upload is not None:
            if gzipped:
                upload.save(filepath)
            else:
                data = gzip.open(filepath, 'wb')
                data.write(upload.read())
                data.close()
            self.gzipped = True
        elif local_path is not None:
            os.symlink(local_path, filepath)

        if not empty and not self.is_valid():
            os.unlink(filepath)
            raise InvalidDataSource('invalid_data', 'Data source cannot be read')

    def __repr__(self):
        # Todo: If CELERY_ALWAYS_EAGER=True, the worker can end up with a
        #     detached session when printing its log after an error. This
        #     is a hacky workaround, we might implement it as a decorator
        #     on the __repr__ method or the model itself, but it will still
        #     be a hack. I think this is something that could be fixed in
        #     celery itself.
        try:
            return '<DataSource "%s" as %s added %s>' % (self.name, self.filetype, str(self.added))
        except DetachedInstanceError:
            return '<DataSource ...>'

    def data(self):
        """
        Get open file-like handle to data contained in this data source for
        reading.

        .. note:: Be sure to close after calling this.
        """
        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)
        try:
            if self.gzipped:
                return gzip.open(filepath)
            else:
                return open(filepath)
        except EnvironmentError:
            raise DataUnavailable('data_source_not_cached', 'Data source is not in the cache')

    def data_writer(self):
        """
        Get open file-like handle to data contained in this data source for
        writing.

        .. note:: Be sure to close after calling this.
        """
        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)
        try:
            if self.gzipped:
                return gzip.open(filepath, 'wb')
            else:
                return open(filepath, 'wb')
        except EnvironmentError:
            raise DataUnavailable('data_source_not_cached', 'Data source is not in the cache')

    def empty(self):
        """
        Remove all data from this data source.
        """
        with self.data_writer():
            pass

    def local_path(self):
        """
        Get a local filepath for the data.
        """
        return os.path.join(current_app.config['FILES_DIR'], self.filename)

    def is_valid(self):
        """
        Peek into the file and determine if it is of the given filetype.
        """
        data = self.data()

        def is_bed():
            # Todo.
            return True

        def is_vcf():
            return 'fileformat=VCFv4.1' in data.readline()

        validators = {'bed': is_bed,
                      'vcf': is_vcf}
        with data as data:
            return validators[self.filetype]()

class Variation(db.Model):
    """
    Coupling between a Sample, a DataSource, and Observations.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    sample_id = db.Column(db.Integer, db.ForeignKey('sample.id'))
    data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    imported = db.Column(db.Boolean, default=False)
    import_task_uuid = db.Column(db.String(36))

    sample = db.relationship(Sample, backref=db.backref('variations', lazy='dynamic'))
    data_source = db.relationship(DataSource, backref=db.backref('variations', lazy='dynamic'))

    def __init__(self, sample, data_source):
        self.sample = sample
        self.data_source = data_source

    def __repr__(self):
        return '<Variation ...>'

class Coverage(db.Model):
    """
    Coupling between a Sample, a DataSource, and Regions.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    sample_id = db.Column(db.Integer, db.ForeignKey('sample.id'))
    data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    imported = db.Column(db.Boolean, default=False)
    import_task_uuid = db.Column(db.String(36))

    sample = db.relationship(Sample, backref=db.backref('coverages', lazy='dynamic'))
    data_source = db.relationship(DataSource, backref=db.backref('coverages', lazy='dynamic'))

    def __init__(self, sample, data_source):
        self.sample = sample
        self.data_source = data_source

    def __repr__(self):
        return '<Coverage ...>'

class Annotation(db.Model):
    """
    Annotated data source.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    original_data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    annotated_data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    written = db.Column(db.Boolean, default=False)
    write_task_uuid = db.Column(db.String(36))

    original_data_source = db.relationship(DataSource, primaryjoin='DataSource.id==Annotation.original_data_source_id', backref=db.backref('annotations', lazy='dynamic'))
    annotated_data_source = db.relationship(DataSource, primaryjoin='DataSource.id==Annotation.annotated_data_source_id', backref=db.backref('annotation', uselist=False, lazy='dynamic'))

    def __init__(self, original_data_source, annotated_data_source):
        self.original_data_source = original_data_source
        self.annotated_data_source = annotated_data_source

    def __repr__(self):
        return '<Annotation ...>'

class Observation(db.Model):
    """
    Observation in a sample.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    variation_id = db.Column(db.Integer, db.ForeignKey('variation.id'))

    chromosome = db.Column(db.String(30))
    position = db.Column(db.Integer)
    reference = db.Column(db.String(200))
    observed = db.Column(db.String(200))
    bin = db.Column(db.Integer)

    # Number of individuals.
    support = db.Column(db.Integer)

    variation = db.relationship(Variation, backref=db.backref('observations', lazy='dynamic'))

    def __init__(self, variation, chromosome, position, reference, observed, support=1):
        self.variation = variation
        self.chromosome = chromosome
        self.position = position
        self.reference = reference
        self.observed = observed
        # We choose the 'region' of the reference covered by an insertion to
        # be the base next to it.
        self.bin = assign_bin(self.position,
                              self.position + max(1, len(self.reference)) - 1)
        self.support = support

    def __repr__(self):
        return 'Observation<%r, %r, %r, %r>' % (
            self.chromosome, self.position, self.reference, self.observed)

    def is_deletion(self):
        return self.observed == ''

    def is_insertion(self):
        return self.reference == ''

    def is_snv(self):
        return len(self.observed) == len(self.reference) == 1

    def is_indel(self):
        return not (self.is_deletion() or
                    self.is_insertion() or
                    self.is_snv())

Index('observation_location',
      Observation.bin, Observation.chromosome, Observation.position)

class Region(db.Model):
    """
    Covered region for a sample.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    coverage_id = db.Column(db.Integer, db.ForeignKey('coverage.id'))
    chromosome = db.Column(db.String(30))
    begin = db.Column(db.Integer)
    end = db.Column(db.Integer)
    bin = db.Column(db.Integer)

    coverage = db.relationship(Coverage, backref=db.backref('regions', lazy='dynamic'))

    def __init__(self, coverage, chromosome, begin, end):
        self.coverage = coverage
        self.chromosome = chromosome
        self.begin = begin
        self.end = end
        self.bin = assign_bin(self.begin, self.end)

    def __repr__(self):
        return '<Region chr%s:%i-%i>' % (self.chromosome, self.begin, self.end)

Index('region_location',
      Region.bin, Region.chromosome, Region.begin)
<commit_msg>Use date and time instead of just date<commit_after>"""
Models backed by SQL using SQLAlchemy.

Note that all genomic positions in this module are one-based and inclusive.

.. moduleauthor:: Martijn Vermaat <martijn@vermaat.name>

.. todo:: Perhaps add some delete cascade rules.

.. Licensed under the MIT license, see the LICENSE file.
"""

from datetime import datetime
import gzip
import os
import uuid

from flask import current_app
from sqlalchemy import Index
from sqlalchemy.orm.exc import DetachedInstanceError
import bcrypt

from . import db
from .region_binning import assign_bin

# Todo: Use the types for which we have validators.
DATA_SOURCE_FILETYPES = ('bed', 'vcf')

# Note: Add new roles at the end.
USER_ROLES = (
    'admin',       # Can do anything.
    'importer',    # Can import samples.
    'annotator',   # Can annotate samples.
    'trader'       # Can annotate samples if they are also imported.
)

class InvalidDataSource(Exception):
    """
    Exception thrown if data source validation failed.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(InvalidDataSource, self).__init__(code, message)

class DataUnavailable(Exception):
    """
    Exception thrown if reading from a data source which data is not cached
    anymore (in case of local storage) or does not exist anymore (in case of
    a URL resource.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(DataUnavailable, self).__init__(code, message)

class User(db.Model):
    """
    User in the system.

    For the roles column we use a bitstring where the leftmost role in the
    :data:`USER_ROLES` tuple is defined by the least-significant bit.
    Essentially, this creates a set of roles.

    .. todo:: Login should really be validated to only contain alphanums.
    .. todo:: The bitstring encoding/decoding can probably be implemented more
        efficiently.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(200))
    login = db.Column(db.String(200), index=True, unique=True)
    password_hash = db.Column(db.String(200))
    roles_bitstring = db.Column(db.Integer)
    added = db.Column(db.DateTime)

    def __init__(self, name, login, password, roles=[]):
        self.name = name
        self.login = login
        self.password_hash = bcrypt.hashpw(password, bcrypt.gensalt())
        self.roles_bitstring = sum(pow(2, i) for i, role in enumerate(USER_ROLES)
                                   if role in roles)
        self.added = datetime.now()

    def __repr__(self):
        return 'User(%r, %r, %r, %r)' % (self.name, self.login, '***', list(self.roles))

    def check_password(self, password):
        return bcrypt.hashpw(password, self.password_hash) == self.password_hash

    @property
    def roles(self):
        return {role for i, role in enumerate(USER_ROLES)
                if self.roles_bitstring & pow(2, i)}

class Sample(db.Model):
    """
    Sample.

    ``coverage_profile`` is essentially ``not is_population_study`` and should
    always be True iff the sample has one or more Coverage entries (so perhaps
    we should not store it, but have it as a calculated property).
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    name = db.Column(db.String(200))
    pool_size = db.Column(db.Integer)
    added = db.Column(db.DateTime)
    active = db.Column(db.Boolean, default=False)
    coverage_profile = db.Column(db.Boolean)
    public = db.Column(db.Boolean)

    user = db.relationship(User, backref=db.backref('samples', lazy='dynamic'))

    def __init__(self, user, name, pool_size=1, coverage_profile=True, public=False):
        self.user = user
        self.name = name
        self.pool_size = pool_size
        self.added = datetime.now()
        self.coverage_profile = coverage_profile
        self.public = public

    def __repr__(self):
        return '<Sample "%s" of %d individuals added %s>' % (self.name, self.pool_size, str(self.added))

class DataSource(db.Model):
    """
    Data source (probably uploaded as a file). E.g. VCF file to be imported, or
    BED track from which Region entries are created.

    .. todo:: We can now provide data as an uploaded file or as a path to a
        local file. We also want to be able to give a link to an internet
        resource.

    .. note:: Data source checksums are not forced to be unique, since several
        users might upload the same data source and do different things with
        it.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
    name = db.Column(db.String(200))
    filename = db.Column(db.String(50))
    filetype = db.Column(db.Enum(*DATA_SOURCE_FILETYPES, name='filetype'))
    gzipped = db.Column(db.Boolean)
    added = db.Column(db.DateTime)
    checksum = db.Column(db.String(40))
    records = db.Column(db.Integer)

    user = db.relationship(User, backref=db.backref('data_sources', lazy='dynamic'))

    def __init__(self, user, name, filetype, upload=None, local_path=None, empty=False, gzipped=False):
        if not filetype in DATA_SOURCE_FILETYPES:
            raise InvalidDataSource('unknown_filetype',
                                    'Data source filetype "%s" is unknown'
                                    % filetype)

        self.user = user
        self.name = name
        self.filename = str(uuid.uuid4())
        self.filetype = filetype
        self.gzipped = gzipped
        self.added = datetime.now()

        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)

        if upload is not None:
            if gzipped:
                upload.save(filepath)
            else:
                data = gzip.open(filepath, 'wb')
                data.write(upload.read())
                data.close()
            self.gzipped = True
        elif local_path is not None:
            os.symlink(local_path, filepath)

        if not empty and not self.is_valid():
            os.unlink(filepath)
            raise InvalidDataSource('invalid_data', 'Data source cannot be read')

    def __repr__(self):
        # Todo: If CELERY_ALWAYS_EAGER=True, the worker can end up with a
        #     detached session when printing its log after an error. This
        #     is a hacky workaround, we might implement it as a decorator
        #     on the __repr__ method or the model itself, but it will still
        #     be a hack. I think this is something that could be fixed in
        #     celery itself.
        try:
            return '<DataSource "%s" as %s added %s>' % (self.name, self.filetype, str(self.added))
        except DetachedInstanceError:
            return '<DataSource ...>'

    def data(self):
        """
        Get open file-like handle to data contained in this data source for
        reading.

        .. note:: Be sure to close after calling this.
        """
        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)
        try:
            if self.gzipped:
                return gzip.open(filepath)
            else:
                return open(filepath)
        except EnvironmentError:
            raise DataUnavailable('data_source_not_cached', 'Data source is not in the cache')

    def data_writer(self):
        """
        Get open file-like handle to data contained in this data source for
        writing.

        .. note:: Be sure to close after calling this.
        """
        filepath = os.path.join(current_app.config['FILES_DIR'], self.filename)
        try:
            if self.gzipped:
                return gzip.open(filepath, 'wb')
            else:
                return open(filepath, 'wb')
        except EnvironmentError:
            raise DataUnavailable('data_source_not_cached', 'Data source is not in the cache')

    def empty(self):
        """
        Remove all data from this data source.
        """
        with self.data_writer():
            pass

    def local_path(self):
        """
        Get a local filepath for the data.
        """
        return os.path.join(current_app.config['FILES_DIR'], self.filename)

    def is_valid(self):
        """
        Peek into the file and determine if it is of the given filetype.
        """
        data = self.data()

        def is_bed():
            # Todo.
            return True

        def is_vcf():
            return 'fileformat=VCFv4.1' in data.readline()

        validators = {'bed': is_bed,
                      'vcf': is_vcf}
        with data as data:
            return validators[self.filetype]()

class Variation(db.Model):
    """
    Coupling between a Sample, a DataSource, and Observations.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    sample_id = db.Column(db.Integer, db.ForeignKey('sample.id'))
    data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    imported = db.Column(db.Boolean, default=False)
    import_task_uuid = db.Column(db.String(36))

    sample = db.relationship(Sample, backref=db.backref('variations', lazy='dynamic'))
    data_source = db.relationship(DataSource, backref=db.backref('variations', lazy='dynamic'))

    def __init__(self, sample, data_source):
        self.sample = sample
        self.data_source = data_source

    def __repr__(self):
        return '<Variation ...>'

class Coverage(db.Model):
    """
    Coupling between a Sample, a DataSource, and Regions.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    sample_id = db.Column(db.Integer, db.ForeignKey('sample.id'))
    data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    imported = db.Column(db.Boolean, default=False)
    import_task_uuid = db.Column(db.String(36))

    sample = db.relationship(Sample, backref=db.backref('coverages', lazy='dynamic'))
    data_source = db.relationship(DataSource, backref=db.backref('coverages', lazy='dynamic'))

    def __init__(self, sample, data_source):
        self.sample = sample
        self.data_source = data_source

    def __repr__(self):
        return '<Coverage ...>'

class Annotation(db.Model):
    """
    Annotated data source.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    original_data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    annotated_data_source_id = db.Column(db.Integer, db.ForeignKey('data_source.id'))
    written = db.Column(db.Boolean, default=False)
    write_task_uuid = db.Column(db.String(36))

    original_data_source = db.relationship(DataSource, primaryjoin='DataSource.id==Annotation.original_data_source_id', backref=db.backref('annotations', lazy='dynamic'))
    annotated_data_source = db.relationship(DataSource, primaryjoin='DataSource.id==Annotation.annotated_data_source_id', backref=db.backref('annotation', uselist=False, lazy='dynamic'))

    def __init__(self, original_data_source, annotated_data_source):
        self.original_data_source = original_data_source
        self.annotated_data_source = annotated_data_source

    def __repr__(self):
        return '<Annotation ...>'

class Observation(db.Model):
    """
    Observation in a sample.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    variation_id = db.Column(db.Integer, db.ForeignKey('variation.id'))

    chromosome = db.Column(db.String(30))
    position = db.Column(db.Integer)
    reference = db.Column(db.String(200))
    observed = db.Column(db.String(200))
    bin = db.Column(db.Integer)

    # Number of individuals.
    support = db.Column(db.Integer)

    variation = db.relationship(Variation, backref=db.backref('observations', lazy='dynamic'))

    def __init__(self, variation, chromosome, position, reference, observed, support=1):
        self.variation = variation
        self.chromosome = chromosome
        self.position = position
        self.reference = reference
        self.observed = observed
        # We choose the 'region' of the reference covered by an insertion to
        # be the base next to it.
        self.bin = assign_bin(self.position,
                              self.position + max(1, len(self.reference)) - 1)
        self.support = support

    def __repr__(self):
        return 'Observation<%r, %r, %r, %r>' % (
            self.chromosome, self.position, self.reference, self.observed)

    def is_deletion(self):
        return self.observed == ''

    def is_insertion(self):
        return self.reference == ''

    def is_snv(self):
        return len(self.observed) == len(self.reference) == 1

    def is_indel(self):
        return not (self.is_deletion() or
                    self.is_insertion() or
                    self.is_snv())

Index('observation_location',
      Observation.bin, Observation.chromosome, Observation.position)

class Region(db.Model):
    """
    Covered region for a sample.
    """
    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}

    id = db.Column(db.Integer, primary_key=True)
    coverage_id = db.Column(db.Integer, db.ForeignKey('coverage.id'))
    chromosome = db.Column(db.String(30))
    begin = db.Column(db.Integer)
    end = db.Column(db.Integer)
    bin = db.Column(db.Integer)

    coverage = db.relationship(Coverage, backref=db.backref('regions', lazy='dynamic'))

    def __init__(self, coverage, chromosome, begin, end):
        self.coverage = coverage
        self.chromosome = chromosome
        self.begin = begin
        self.end = end
        self.bin = assign_bin(self.begin, self.end)

    def __repr__(self):
        return '<Region chr%s:%i-%i>' % (self.chromosome, self.begin, self.end)

Index('region_location',
      Region.bin, Region.chromosome, Region.begin)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.db import models

from model_utils import Choices
from model_utils.fields import StatusField

class Entry(models.Model):
    STATUS = Choices('new', 'checked', 'deleted')

    status = StatusField()
    shown_name = models.CharField(max_length=255, blank=True, default='')

    first_name = models.CharField(max_length=255, blank=True, default='')
    last_name = models.CharField(max_length=255, blank=True, default='')

    email = models.EmailField(blank=True, default='')
    phone = models.CharField(max_length=32, blank=True, default='')
    address = models.CharField(max_length=255, blank=True, default='')
    postnummer = models.ForeignKey('Postnummer', null=True)
    place = models.CharField(max_length=64, blank=True, default='')

    notes = models.TextField(blank=True, default='')
    groups = models.ManyToManyField('Group')

    @property
    def printed_name(self):
        if self.shown_name:
            return self.shown_name
        return self.contact_full_name

    @property
    def contact_full_name(self):
        return "{s.first_name} {s.last_name}".format(s=self)

    class Meta:
        verbose_name_plural = 'entries'

    def __unicode__(self):
        return "{s.printed_name} ({s.status})".format(s=self)

class Postnummer(models.Model):
    postnr = models.CharField(max_length=6)
    poststad = models.CharField(max_length=50)
    bruksomrade = models.CharField(max_length=50)
    folketal = models.SmallIntegerField(null=True, blank=True)
    bydel = models.CharField(max_length=50)
    kommnr = models.CharField(max_length=50)
    kommune = models.CharField(max_length=50)
    fylke = models.CharField(max_length=50)
    lat = models.FloatField()
    lon = models.FloatField()
    datakvalitet = models.SmallIntegerField()
    sist_oppdatert = models.DateTimeField(null=True, blank=True)

    def __unicode__(self):
        return "{0} {1}".format(self.postnr, self.poststad)

class Group(models.Model):
    category = models.SlugField()
    name = models.SlugField()

    @property
    def slug(self):
        return "{s.category}:{s.name}".format(s=self)

    def __unicode__(self):
        return self.slug
<commit_msg>Don't require Group for Entry<commit_after>from __future__ import unicode_literals

from django.db import models

from model_utils import Choices
from model_utils.fields import StatusField

class Entry(models.Model):
    STATUS = Choices('new', 'checked', 'deleted')

    status = StatusField()
    shown_name = models.CharField(max_length=255, blank=True, default='')

    first_name = models.CharField(max_length=255, blank=True, default='')
    last_name = models.CharField(max_length=255, blank=True, default='')

    email = models.EmailField(blank=True, default='')
    phone = models.CharField(max_length=32, blank=True, default='')
    address = models.CharField(max_length=255, blank=True, default='')
    postnummer = models.ForeignKey('Postnummer', null=True)
    place = models.CharField(max_length=64, blank=True, default='')

    notes = models.TextField(blank=True, default='')
    groups = models.ManyToManyField('Group', blank=True)

    @property
    def printed_name(self):
        if self.shown_name:
            return self.shown_name
        return self.contact_full_name

    @property
    def contact_full_name(self):
        return "{s.first_name} {s.last_name}".format(s=self)

    class Meta:
        verbose_name_plural = 'entries'

    def __unicode__(self):
        return "{s.printed_name} ({s.status})".format(s=self)

class Postnummer(models.Model):
    postnr = models.CharField(max_length=6)
    poststad = models.CharField(max_length=50)
    bruksomrade = models.CharField(max_length=50)
    folketal = models.SmallIntegerField(null=True, blank=True)
    bydel = models.CharField(max_length=50)
    kommnr = models.CharField(max_length=50)
    kommune = models.CharField(max_length=50)
    fylke = models.CharField(max_length=50)
    lat = models.FloatField()
    lon = models.FloatField()
    datakvalitet = models.SmallIntegerField()
    sist_oppdatert = models.DateTimeField(null=True, blank=True)

    def __unicode__(self):
        return "{0} {1}".format(self.postnr, self.poststad)

class Group(models.Model):
    category = models.SlugField()
    name = models.SlugField()

    @property
    def slug(self):
        return "{s.category}:{s.name}".format(s=self)

    def __unicode__(self):
        return self.slug
<|endoftext|>


<commit_before>from os import path
import requests
import threading
import io

from entities.dialogue import *
from entities.files import *

from async.workers import WorkerPool
from async.timed import Signaller

class APIException(Exception):
    def __init__(self, reason, request_url = None):
        self.reason = reason
        self.request_url = request_url

    def __str__(self):
        msg = ""
        if self.request_url != None: msg += "Error while fetching {0}:\n".format(self.request_url)
        msg += self.reason
        return msg

class BotAPI(object):
    ENDPOINT_URL = "https://api.telegram.org/bot{0}/{1}"
    DOWNLOAD_URL = "https://api.telegram.org/file/bot{0}/{1}"

    def __init__(self, token):
        self.token = token
        self.passed_api = None

    def request(self, method_name, method="post", params = {}):
        url = self.ENDPOINT_URL.format(self.token, method_name)
        payload = {}
        files = {}
        for k, v in params.iteritems():
            if v == None: continue
            if issubclass(type(v), io.IOBase):
                files[k] = v
            else:
                payload[k] = v
        try:
            response = requests.request(method, url, params = payload, files = files, timeout = 600, headers = {"User-Agent": "tgbot"})
        finally:
            # Close all files
            for v in files.values():
                try:
                    v.close()
                except: pass
        try:
            data = response.json()
        except:
            raise APIException("Telegram returned invalid JSON", url)
        if not data["ok"]:
            raise APIException(data["description"] if "description" in data else "A Telegram API error occurred", url)
        return data

    def get_updates(self, **kwargs):
        return self.request("getUpdates", "post", kwargs)

    def set_webhook(self, **kwargs):
        return self.request("setWebhook", "post", kwargs)

    def remove_webhook(self):
        return self.set_webhook(url = "")

    def forward_message(self, pass_api = None, **kwargs):
        return Message.build(self.request("forwardMessage", "post", kwargs)["result"], self.passed_api)

    def send_message(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendMessage", "post", kwargs)["result"], self.passed_api)

    def edit_message_text(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageText", "post", kwargs)["result"], self.passed_api)

    def edit_message_caption(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageCaption", "post", kwargs)["result"], self.passed_api)

    def edit_message_reply_markup(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageReplyMarkup", "post", kwargs)["result"], self.passed_api)

    def send_photo(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendPhoto", "post", kwargs)["result"], self.passed_api)

    def send_audio(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendAudio", "post", kwargs)["result"], self.passed_api)

    def send_document(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendDocument", "post", kwargs)["result"], self.passed_api)

    def send_sticker(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendSticker", "post", kwargs)["result"], self.passed_api)

    def send_video(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendVideo", "post", kwargs)["result"], self.passed_api)

    def send_voice(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendVoice", "post", kwargs)["result"], self.passed_api)

    def download_file(self, file_id, name):
        file_data = self.request("getFile", "post", {"file_id": file_id})["result"]
        if "file_path" not in file_data:
            raise APIException("File expired")

        parts = file_data["file_path"].split(".")
        ext = parts[-1] if len(parts) > 0 else ""
        target_path = name + "." + ext

        url = self.DOWNLOAD_URL.format(self.token, file_data["file_path"])
        response = requests.get(url, stream = True)
        with open(target_path, "wb") as fp:
            for chunk in response.iter_content(chunk_size = 1024):
                fp.write(chunk)
        response.close()

        return target_path

class LimitedBotAPI(BotAPI):
    REQUEST_SIGNAL = 0

    def __init__(self, token, requests_per_second = 5, messages_to_chat_per_second = 1):
        BotAPI.__init__(self, token)
        self.signaller = Signaller()
        self.signaller.add_signal(self.REQUEST_SIGNAL, 1. / requests_per_second)
        self._mstps = messages_to_chat_per_second

        self.mutex = threading.Lock()
        self.chat_signals_added = []

    def request(self, method_name, method = "post", params = {}):
        self.signaller.wait_for(self.REQUEST_SIGNAL)
        return BotAPI.request(self, method_name, method, params)

    def wait_till_chat_signal(self, chat_id):
        self.mutex.acquire()
        if chat_id not in self.chat_signals_added:
            self.signaller.add_signal(chat_id, self._mstps)
            self.chat_signals_added.append(chat_id)
        self.mutex.release()
        self.signaller.wait_for(chat_id)

    def send_message(self, pass_api = None, **kwargs):
        if "chat_id" not in kwargs:
            raise KeyError("'chat_id' was not passed to 'send_message'")
        self.wait_till_chat_signal(kwargs["chat_id"])
        return BotAPI.send_message(self, pass_api, **kwargs)

    def forward_message(self, pass_api = None, **kwargs):
        if "chat_id" not in kwargs:
            raise KeyError("'chat_id' was not passed to 'forward_message'")
        self.wait_till_chat_signal(kwargs["chat_id"])
        return BotAPI.forward_message(self, pass_api, **kwargs)

    def start(self):
        self.signaller.start()

    def stop(self):
        self.signaller.stop()

class AsyncifiedAPI(object):
    def __init__(self, api, worker_pool):
        self.worker_pool = worker_pool
        self._api = api
        self.token = self._api.token
        self._api.passed_api = self #Pass this object as API to the instantiated entities

    def underlying_api(self):
        return self._api

    def __getattr__(self, key):
        if not hasattr(self._api, key): raise NameError()
        return self.worker_pool.asyncify(getattr(self._api, key))
<commit_msg>Fixed sending files bug<commit_after>from os import path
import requests
import threading
import io

from entities.dialogue import *
from entities.files import *

from async.workers import WorkerPool
from async.timed import Signaller

class APIException(Exception):
    def __init__(self, reason, request_url = None):
        self.reason = reason
        self.request_url = request_url

    def __str__(self):
        msg = ""
        if self.request_url != None: msg += "Error while fetching {0}:\n".format(self.request_url)
        msg += self.reason
        return msg

class BotAPI(object):
    ENDPOINT_URL = "https://api.telegram.org/bot{0}/{1}"
    DOWNLOAD_URL = "https://api.telegram.org/file/bot{0}/{1}"

    def __init__(self, token):
        self.token = token
        self.passed_api = None

    def request(self, method_name, method="post", params = {}):
        url = self.ENDPOINT_URL.format(self.token, method_name)
        payload = {}
        files = {}
        for k, v in params.iteritems():
            if v == None: continue
            if issubclass(type(v), io.IOBase) or type(v) == file:
                files[k] = v
            else:
                payload[k] = v
        try:
            response = requests.request(method, url, params = payload, files = files, timeout = 600, headers = {"User-Agent": "tgbot"})
        finally:
            # Close all files
            for v in files.values():
                try:
                    v.close()
                except: pass
        try:
            data = response.json()
        except:
            raise APIException("Telegram returned invalid JSON", url)
        if not data["ok"]:
            raise APIException(data["description"] if "description" in data else "A Telegram API error occurred", url)
        return data

    def get_updates(self, **kwargs):
        return self.request("getUpdates", "post", kwargs)

    def set_webhook(self, **kwargs):
        return self.request("setWebhook", "post", kwargs)

    def remove_webhook(self):
        return self.set_webhook(url = "")

    def forward_message(self, pass_api = None, **kwargs):
        return Message.build(self.request("forwardMessage", "post", kwargs)["result"], self.passed_api)

    def send_message(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendMessage", "post", kwargs)["result"], self.passed_api)

    def edit_message_text(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageText", "post", kwargs)["result"], self.passed_api)

    def edit_message_caption(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageCaption", "post", kwargs)["result"], self.passed_api)

    def edit_message_reply_markup(self, pass_api = None, **kwargs):
        return Message.build(self.request("editMessageReplyMarkup", "post", kwargs)["result"], self.passed_api)

    def send_photo(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendPhoto", "post", kwargs)["result"], self.passed_api)

    def send_audio(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendAudio", "post", kwargs)["result"], self.passed_api)

    def send_document(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendDocument", "post", kwargs)["result"], self.passed_api)

    def send_sticker(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendSticker", "post", kwargs)["result"], self.passed_api)

    def send_video(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendVideo", "post", kwargs)["result"], self.passed_api)

    def send_voice(self, pass_api = None, **kwargs):
        return Message.build(self.request("sendVoice", "post", kwargs)["result"], self.passed_api)

    def download_file(self, file_id, name):
        file_data = self.request("getFile", "post", {"file_id": file_id})["result"]
        if "file_path" not in file_data:
            raise APIException("File expired")

        parts = file_data["file_path"].split(".")
        ext = parts[-1] if len(parts) > 0 else ""
        target_path = name + "." + ext

        url = self.DOWNLOAD_URL.format(self.token, file_data["file_path"])
        response = requests.get(url, stream = True)
        with open(target_path, "wb") as fp:
            for chunk in response.iter_content(chunk_size = 1024):
                fp.write(chunk)
        response.close()

        return target_path

class LimitedBotAPI(BotAPI):
    REQUEST_SIGNAL = 0

    def __init__(self, token, requests_per_second = 5, messages_to_chat_per_second = 1):
        BotAPI.__init__(self, token)
        self.signaller = Signaller()
        self.signaller.add_signal(self.REQUEST_SIGNAL, 1. / requests_per_second)
        self._mstps = messages_to_chat_per_second

        self.mutex = threading.Lock()
        self.chat_signals_added = []

    def request(self, method_name, method = "post", params = {}):
        self.signaller.wait_for(self.REQUEST_SIGNAL)
        return BotAPI.request(self, method_name, method, params)

    def wait_till_chat_signal(self, chat_id):
        self.mutex.acquire()
        if chat_id not in self.chat_signals_added:
            self.signaller.add_signal(chat_id, self._mstps)
            self.chat_signals_added.append(chat_id)
        self.mutex.release()
        self.signaller.wait_for(chat_id)

    def send_message(self, pass_api = None, **kwargs):
        if "chat_id" not in kwargs:
            raise KeyError("'chat_id' was not passed to 'send_message'")
        self.wait_till_chat_signal(kwargs["chat_id"])
        return BotAPI.send_message(self, pass_api, **kwargs)

    def forward_message(self, pass_api = None, **kwargs):
        if "chat_id" not in kwargs:
            raise KeyError("'chat_id' was not passed to 'forward_message'")
        self.wait_till_chat_signal(kwargs["chat_id"])
        return BotAPI.forward_message(self, pass_api, **kwargs)

    def start(self):
        self.signaller.start()

    def stop(self):
        self.signaller.stop()

class AsyncifiedAPI(object):
    def __init__(self, api, worker_pool):
        self.worker_pool = worker_pool
        self._api = api
        self.token = self._api.token
        self._api.passed_api = self #Pass this object as API to the instantiated entities

    def underlying_api(self):
        return self._api

    def __getattr__(self, key):
        if not hasattr(self._api, key): raise NameError()
        return self.worker_pool.asyncify(getattr(self._api, key))
<|endoftext|>


<commit_before>from settings_base import *
<commit_msg>Add database for tests.<commit_after>import os

from settings_base import *

APP_ROOT = os.path.dirname(os.path.abspath(__file__))
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.dirname(APP_ROOT)+'/newsdiffs.db',
    }
}

<|endoftext|>


<commit_before># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import re
import subprocess
import os

try:
  from internal import internal_presubmit
except ImportError:
  internal_presubmit = None

SOURCE_FILE_EXTENSIONS = [
        '.c', '.cc', '.cpp', '.h', '.m', '.mm', '.py', '.mk', '.am', '.json',
        '.gyp', '.gypi'
        ]

EXCLUDED_PATHS = []

# Finds what seem to be definitions of DllRegisterServer.
DLL_REGISTER_SERVER_RE = re.compile('\s+DllRegisterServer\s*\(')

# Matches a Tracker story URL
story_url_re = re.compile('https?://tracker.+/[0-9]+')

# Matches filenames in which we allow tabs
tabs_ok_re = re.compile('.*\.(vcproj|vsprops|sln)$')

# Matches filenames of source files
source_files_re = re.compile('.*\.(cc|h|py|js)$')

# The top-level source directory.
_SRC_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

def CheckChange(input_api, output_api, committing, is_chrome_frame=False):
  results = []

  do_not_submit_errors = input_api.canned_checks.CheckDoNotSubmit(input_api,
                                                                  output_api)
  if committing:
    results += do_not_submit_errors
  elif do_not_submit_errors:
    results += [output_api.PresubmitNotifyResult(
        'There is a DO-NOT-SUBMIT issue')]

  results += CheckChangeHasNoTabs(input_api, output_api)
  results += CheckLongLines(input_api, output_api)
  results += CheckHasStoryOrBug(input_api, output_api)
  results += LocalChecks(input_api, output_api)
  results += WarnOnAtlSmartPointers(input_api, output_api)
  if not is_chrome_frame:
    results += CheckNoDllRegisterServer(input_api, output_api)
    results += CheckUnittestsRan(input_api, output_api, committing)
    if internal_presubmit:
      results += internal_presubmit.InternalChecks(input_api, output_api,
                                                   committing)
  return results

def CheckHasStoryOrBug(input_api, output_api):
  """We require either BUG= to be present and non-empty.  For
  a completely trivial change, use BUG=none.
  """
  if not ('BUG' in input_api.change.tags):
    return [output_api.PresubmitError('A BUG= tag is required. For trivial '
                                      'changes you can use BUG=none.')]
  if ('BUG' in input_api.change.tags and
      len(input_api.change.tags['BUG']) == 0):
    return [output_api.PresubmitError('A non-empty BUG= is required. For '
                                      'trivial changes you can use BUG=none.')]
  return []

def CheckChangeHasNoTabs(input_api, output_api):
  """Slightly modified version of the canned check with the same name.

  This version ignores certain file types in which we allow tabs.
  """
  for f, line_num, line in input_api.RightHandSideLines():
    if tabs_ok_re.match(f.LocalPath()):
      continue
    if '\t' in line:
      return [output_api.PresubmitError(
          "Found a tab character in %s, line %s" %
          (f.LocalPath(), line_num))]
  return []

def CheckLongLines(input_api, output_api, maxlen=80):
  """Checks that there aren't any lines longer than maxlen characters in any of
  the text files to be submitted.
  """
  basename = input_api.basename

  bad = []
  for f, line_num, line in input_api.RightHandSideLines():
    if not source_files_re.match(f.LocalPath()):
      continue
    if line.find('http://') != -1:
      # Exemption for long URLs
      continue
    if line.endswith('\n'):
      line = line[:-1]
    if len(line) > maxlen:
      bad.append(
          '%s, line %s, %s chars' %
          (basename(f.LocalPath()), line_num, len(line)))
      if len(bad) == 5:  # Just show the first 5 errors.
        break

  if bad:
    msg = "Found lines longer than %s characters (first 5 shown)." % maxlen
    return [output_api.PresubmitPromptWarning(msg, items=bad)]
  else:
    return []

_UNITTEST_MESSAGE = '''\
You must build and run the CEEE smoke tests before submitting. To clear this
error, run the script "smoke_test.bat" in the CEEE directory.
'''

def CheckUnittestsRan(input_api, output_api, committing):
  '''Checks that the unittests success file is newer than any modified file'''
  # But only if there were IE files modified, since we only have unit tests
  # for CEEE IE.
  files = []
  ie_paths_re = re.compile('ceee[\\/](ie|common)[\\/]')
  for f in input_api.AffectedFiles(include_deletes = False):
    path = f.LocalPath()
    if (ie_paths_re.match(path)):
      files.append(f)

  if not files:
    return []

  def MakeResult(message, modified_files=[]):
    if committing:
      return output_api.PresubmitError(message, modified_files)
    else:
      return output_api.PresubmitNotifyResult(message, modified_files)
  os_path = input_api.os_path

  success_files = [
      os_path.join(input_api.PresubmitLocalPath(),
                    '../chrome/Debug/ceee.success'),
      os_path.join(input_api.PresubmitLocalPath(),
                   '../chrome/Release/ceee.success')]

  if (not os_path.exists(success_files[0]) or
      not os_path.exists(success_files[1])):
    return [MakeResult(_UNITTEST_MESSAGE)]

  success_time = min(os.stat(success_files[0]).st_mtime,
                     os.stat(success_files[1]).st_mtime)
  modified_files = []
  for f in modified_files:
    file_time = os.stat(f.AbsoluteLocalPath()).st_mtime
    if file_time > success_time:
      modified_files.append(f.LocalPath())

  result = []
  if modified_files:
    result.append(MakeResult('These files have been modified since Debug and/or'
                             ' Release unittests were built.', modified_files))

  return result

def CheckNoDllRegisterServer(input_api, output_api):
  for f, line_num, line in input_api.RightHandSideLines():
    if DLL_REGISTER_SERVER_RE.search(line):
      file_name = os.path.basename(f.LocalPath())
      if file_name not in ['install_utils.h', 'install_utils_unittest.cc']:
        return [output_api.PresubmitError(
            '%s seems to contain a definition of DllRegisterServer.\n'
            'Please search for CEEE_DEFINE_DLL_REGISTER_SERVER.' %
            f.LocalPath())]
  return []

def WarnOnAtlSmartPointers(input_api, output_api):
  smart_pointer_re = re.compile(r'\bCCom(Ptr|BSTR|Variant)\b')
  bad_files = []
  for f in input_api.AffectedFiles(include_deletes=False):
    contents = input_api.ReadFile(f, 'r')
    if smart_pointer_re.search(contents):
      bad_files.append(f.LocalPath())
  if bad_files:
    return [output_api.PresubmitPromptWarning(
        'The following files use CComPtr, CComBSTR and/or CComVariant.\n'
        'Please consider switching to ScopedComPtr, ScopedBstr and/or\n'
        'ScopedVariant as per team policy. (NOTE: Will soon be an error.)',
        items=bad_files)]
  else:
    return []

def LocalChecks(input_api, output_api, max_cols=80):
  """Reports an error if for any source file in SOURCE_FILE_EXTENSIONS:
     - uses CR (or CRLF)
     - contains a TAB
     - has a line that ends with whitespace
     - contains a line >|max_cols| cols unless |max_cols| is 0.
     - File does not end in a newline, or ends in more than one.

  Note that the whole file is checked, not only the changes.
  """
  C_SOURCE_FILE_EXTENSIONS = ('.c', '.cc', '.cpp', '.h', '.inl')
  cr_files = []
  eof_files = []
  results = []
  excluded_paths = [input_api.re.compile(x) for x in EXCLUDED_PATHS]
  files = input_api.AffectedFiles(include_deletes=False)
  for f in files:
    path = f.LocalPath()
    root, ext = input_api.os_path.splitext(path)
    # Look for unsupported extensions.
    if not ext in SOURCE_FILE_EXTENSIONS:
      continue
    # Look for excluded paths.
    found = False
    for item in excluded_paths:
      if item.match(path):
        found = True
        break
    if found:
      continue

    # Need to read the file ourselves since AffectedFile.NewContents()
    # will normalize line endings.
    contents = input_api.ReadFile(f, 'rb')
    if '\r' in contents:
      cr_files.append(path)

    # Check that the file ends in one and only one newline character.
    if len(contents) > 0 and (contents[-1:] != "\n" or contents[-2:-1] == "\n"):
      eof_files.append(path)

    local_errors = []
    # Remove end of line character.
    lines = contents.splitlines()
    line_num = 1
    for line in lines:
      if line.endswith(' '):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s ends with whitespaces.' %
            (path, line_num)))
      # Accept lines with http://, https:// and C #define/#pragma/#include to
      # exceed the max_cols rule.
      if (max_cols and
          len(line) > max_cols and
          not 'http://' in line and
          not 'https://' in line and
          not (line[0] == '#' and ext in C_SOURCE_FILE_EXTENSIONS)):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s has %s chars, please reduce to %d chars.' %
            (path, line_num, len(line), max_cols)))
      if '\t' in line:
        local_errors.append(output_api.PresubmitError(
            "%s, line %s contains a tab character." %
            (path, line_num)))
      line_num += 1
      # Just show the first 5 errors.
      if len(local_errors) == 6:
        local_errors.pop()
        local_errors.append(output_api.PresubmitError("... and more."))
        break
    results.extend(local_errors)

  if cr_files:
    results.append(output_api.PresubmitError(
        'Found CR (or CRLF) line ending in these files, please use only LF:',
        items=cr_files))
  if eof_files:
    results.append(output_api.PresubmitError(
        'These files should end in one (and only one) newline character:',
        items=eof_files))
  return results
<commit_msg>Fix smoke_test presubmit match expression to work non-cygwin.<commit_after># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import re
import subprocess
import os

try:
  from internal import internal_presubmit
except ImportError:
  internal_presubmit = None

SOURCE_FILE_EXTENSIONS = [
        '.c', '.cc', '.cpp', '.h', '.m', '.mm', '.py', '.mk', '.am', '.json',
        '.gyp', '.gypi'
        ]

EXCLUDED_PATHS = []

# Finds what seem to be definitions of DllRegisterServer.
DLL_REGISTER_SERVER_RE = re.compile('\s+DllRegisterServer\s*\(')

# Matches a Tracker story URL
story_url_re = re.compile('https?://tracker.+/[0-9]+')

# Matches filenames in which we allow tabs
tabs_ok_re = re.compile('.*\.(vcproj|vsprops|sln)$')

# Matches filenames of source files
source_files_re = re.compile('.*\.(cc|h|py|js)$')

# The top-level source directory.
_SRC_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

def CheckChange(input_api, output_api, committing, is_chrome_frame=False):
  results = []

  do_not_submit_errors = input_api.canned_checks.CheckDoNotSubmit(input_api,
                                                                  output_api)
  if committing:
    results += do_not_submit_errors
  elif do_not_submit_errors:
    results += [output_api.PresubmitNotifyResult(
        'There is a DO-NOT-SUBMIT issue')]

  results += CheckChangeHasNoTabs(input_api, output_api)
  results += CheckLongLines(input_api, output_api)
  results += CheckHasStoryOrBug(input_api, output_api)
  results += LocalChecks(input_api, output_api)
  results += WarnOnAtlSmartPointers(input_api, output_api)
  if not is_chrome_frame:
    results += CheckNoDllRegisterServer(input_api, output_api)
    results += CheckUnittestsRan(input_api, output_api, committing)
    if internal_presubmit:
      results += internal_presubmit.InternalChecks(input_api, output_api,
                                                   committing)
  return results

def CheckHasStoryOrBug(input_api, output_api):
  """We require either BUG= to be present and non-empty.  For
  a completely trivial change, use BUG=none.
  """
  if not ('BUG' in input_api.change.tags):
    return [output_api.PresubmitError('A BUG= tag is required. For trivial '
                                      'changes you can use BUG=none.')]
  if ('BUG' in input_api.change.tags and
      len(input_api.change.tags['BUG']) == 0):
    return [output_api.PresubmitError('A non-empty BUG= is required. For '
                                      'trivial changes you can use BUG=none.')]
  return []

def CheckChangeHasNoTabs(input_api, output_api):
  """Slightly modified version of the canned check with the same name.

  This version ignores certain file types in which we allow tabs.
  """
  for f, line_num, line in input_api.RightHandSideLines():
    if tabs_ok_re.match(f.LocalPath()):
      continue
    if '\t' in line:
      return [output_api.PresubmitError(
          "Found a tab character in %s, line %s" %
          (f.LocalPath(), line_num))]
  return []

def CheckLongLines(input_api, output_api, maxlen=80):
  """Checks that there aren't any lines longer than maxlen characters in any of
  the text files to be submitted.
  """
  basename = input_api.basename

  bad = []
  for f, line_num, line in input_api.RightHandSideLines():
    if not source_files_re.match(f.LocalPath()):
      continue
    if line.find('http://') != -1:
      # Exemption for long URLs
      continue
    if line.endswith('\n'):
      line = line[:-1]
    if len(line) > maxlen:
      bad.append(
          '%s, line %s, %s chars' %
          (basename(f.LocalPath()), line_num, len(line)))
      if len(bad) == 5:  # Just show the first 5 errors.
        break

  if bad:
    msg = "Found lines longer than %s characters (first 5 shown)." % maxlen
    return [output_api.PresubmitPromptWarning(msg, items=bad)]
  else:
    return []

_UNITTEST_MESSAGE = '''\
You must build and run the CEEE smoke tests before submitting. To clear this
error, run the script "smoke_test.bat" in the CEEE directory.
'''

def CheckUnittestsRan(input_api, output_api, committing):
  '''Checks that the unittests success file is newer than any modified file'''
  # But only if there were IE files modified, since we only have unit tests
  # for CEEE IE.
  files = []
  ie_paths_re = re.compile('ceee[\\\\/](ie|common)[\\\\/]')
  for f in input_api.AffectedFiles(include_deletes = False):
    path = f.LocalPath()
    if (ie_paths_re.match(path)):
      files.append(f)

  if not files:
    return []

  def MakeResult(message, modified_files=[]):
    if committing:
      return output_api.PresubmitError(message, modified_files)
    else:
      return output_api.PresubmitNotifyResult(message, modified_files)
  os_path = input_api.os_path

  success_files = [
      os_path.join(input_api.PresubmitLocalPath(),
                    '../chrome/Debug/ceee.success'),
      os_path.join(input_api.PresubmitLocalPath(),
                   '../chrome/Release/ceee.success')]

  if (not os_path.exists(success_files[0]) or
      not os_path.exists(success_files[1])):
    return [MakeResult(_UNITTEST_MESSAGE)]

  success_time = min(os.stat(success_files[0]).st_mtime,
                     os.stat(success_files[1]).st_mtime)
  modified_files = []
  for f in modified_files:
    file_time = os.stat(f.AbsoluteLocalPath()).st_mtime
    if file_time > success_time:
      modified_files.append(f.LocalPath())

  result = []
  if modified_files:
    result.append(MakeResult('These files have been modified since Debug and/or'
                             ' Release unittests were built.', modified_files))

  return result

def CheckNoDllRegisterServer(input_api, output_api):
  for f, line_num, line in input_api.RightHandSideLines():
    if DLL_REGISTER_SERVER_RE.search(line):
      file_name = os.path.basename(f.LocalPath())
      if file_name not in ['install_utils.h', 'install_utils_unittest.cc']:
        return [output_api.PresubmitError(
            '%s seems to contain a definition of DllRegisterServer.\n'
            'Please search for CEEE_DEFINE_DLL_REGISTER_SERVER.' %
            f.LocalPath())]
  return []

def WarnOnAtlSmartPointers(input_api, output_api):
  smart_pointer_re = re.compile(r'\bCCom(Ptr|BSTR|Variant)\b')
  bad_files = []
  for f in input_api.AffectedFiles(include_deletes=False):
    contents = input_api.ReadFile(f, 'r')
    if smart_pointer_re.search(contents):
      bad_files.append(f.LocalPath())
  if bad_files:
    return [output_api.PresubmitPromptWarning(
        'The following files use CComPtr, CComBSTR and/or CComVariant.\n'
        'Please consider switching to ScopedComPtr, ScopedBstr and/or\n'
        'ScopedVariant as per team policy. (NOTE: Will soon be an error.)',
        items=bad_files)]
  else:
    return []

def LocalChecks(input_api, output_api, max_cols=80):
  """Reports an error if for any source file in SOURCE_FILE_EXTENSIONS:
     - uses CR (or CRLF)
     - contains a TAB
     - has a line that ends with whitespace
     - contains a line >|max_cols| cols unless |max_cols| is 0.
     - File does not end in a newline, or ends in more than one.

  Note that the whole file is checked, not only the changes.
  """
  C_SOURCE_FILE_EXTENSIONS = ('.c', '.cc', '.cpp', '.h', '.inl')
  cr_files = []
  eof_files = []
  results = []
  excluded_paths = [input_api.re.compile(x) for x in EXCLUDED_PATHS]
  files = input_api.AffectedFiles(include_deletes=False)
  for f in files:
    path = f.LocalPath()
    root, ext = input_api.os_path.splitext(path)
    # Look for unsupported extensions.
    if not ext in SOURCE_FILE_EXTENSIONS:
      continue
    # Look for excluded paths.
    found = False
    for item in excluded_paths:
      if item.match(path):
        found = True
        break
    if found:
      continue

    # Need to read the file ourselves since AffectedFile.NewContents()
    # will normalize line endings.
    contents = input_api.ReadFile(f, 'rb')
    if '\r' in contents:
      cr_files.append(path)

    # Check that the file ends in one and only one newline character.
    if len(contents) > 0 and (contents[-1:] != "\n" or contents[-2:-1] == "\n"):
      eof_files.append(path)

    local_errors = []
    # Remove end of line character.
    lines = contents.splitlines()
    line_num = 1
    for line in lines:
      if line.endswith(' '):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s ends with whitespaces.' %
            (path, line_num)))
      # Accept lines with http://, https:// and C #define/#pragma/#include to
      # exceed the max_cols rule.
      if (max_cols and
          len(line) > max_cols and
          not 'http://' in line and
          not 'https://' in line and
          not (line[0] == '#' and ext in C_SOURCE_FILE_EXTENSIONS)):
        local_errors.append(output_api.PresubmitError(
            '%s, line %s has %s chars, please reduce to %d chars.' %
            (path, line_num, len(line), max_cols)))
      if '\t' in line:
        local_errors.append(output_api.PresubmitError(
            "%s, line %s contains a tab character." %
            (path, line_num)))
      line_num += 1
      # Just show the first 5 errors.
      if len(local_errors) == 6:
        local_errors.pop()
        local_errors.append(output_api.PresubmitError("... and more."))
        break
    results.extend(local_errors)

  if cr_files:
    results.append(output_api.PresubmitError(
        'Found CR (or CRLF) line ending in these files, please use only LF:',
        items=cr_files))
  if eof_files:
    results.append(output_api.PresubmitError(
        'These files should end in one (and only one) newline character:',
        items=eof_files))
  return results
<|endoftext|>


<commit_before>import numpy
from scipy import signal

from ..io.annotation import Annotation

def resample_ann(resampled_t, ann_sample):
    """
    Compute the new annotation indices

    Parameters
    ----------
    resampled_t : numpy array
        Array of signal locations as returned by scipy.signal.resample
    ann_sample : numpy array
        Array of annotation locations

    Returns
    -------
    resampled_ann_sample : numpy array
        Array of resampled annotation locations

    """
    tmp = numpy.zeros(len(resampled_t), dtype='int16')
    j = 0
    tprec = resampled_t[j]
    for i, v in enumerate(ann_sample):
        while True:
            d = False
            if v < tprec:
                j -= 1
                tprec = resampled_t[j]

            if j+1 == len(resampled_t):
                tmp[j] += 1
                break

            tnow = resampled_t[j+1]
            if tprec <= v and v <= tnow:
                if v-tprec < tnow-v:
                    tmp[j] += 1
                else:
                    tmp[j+1] += 1
                d = True
            j += 1
            tprec = tnow
            if d:
                break

    idx = numpy.where(tmp>0)[0].astype('int64')
    res = []
    for i in idx:
        for j in range(tmp[i]):
            res.append(i)
    assert len(res) == len(ann_sample)

    return numpy.asarray(res, dtype='int64')

def resample_sig(x, fs, fs_target):
    """
    Resample a signal to a different frequency.

    Parameters
    ----------
    x : numpy array
        Array containing the signal
    fs : int, or float
        The original sampling frequency
    fs_target : int, or float
        The target frequency

    Returns
    -------
    resampled_x : numpy array
        Array of the resampled signal values
    resampled_t : numpy array
        Array of the resampled signal locations

    """

    t = numpy.arange(x.shape[0]).astype('float64')

    if fs == fs_target:
        return x, t

    new_length = int(x.shape[0]*fs_target/fs)
    resampled_x, resampled_t = signal.resample(x, num=new_length, t=t)
    assert resampled_x.shape == resampled_t.shape and resampled_x.shape[0] == new_length
    assert numpy.all(numpy.diff(resampled_t) > 0)

    return resampled_x, resampled_t

def resample_singlechan(x, ann, fs, fs_target):
    """
    Resample a single-channel signal with its annotations

    Parameters
    ----------
    x: numpy array
        The signal array
    ann : wfdb Annotation
        The wfdb annotation object
    fs : int, or float
        The original frequency
    fs_target : int, or float
        The target frequency

    Returns
    -------
    resampled_x : numpy array
        Array of the resampled signal values
    resampled_ann : wfdb Annotation
        Annotation containing resampled annotation locations

    """

    resampled_x, resampled_t = resample_sig(x, fs, fs_target)

    new_sample = resample_ann(resampled_t, ann.sample)
    assert ann.sample.shape == new_sample.shape

    resampled_ann = Annotation(ann.record_name, ann.extension, new_sample,
        ann.symbol, ann.num, ann.subtype, ann.chan, ann.aux_note, ann.fs)

    return resampled_x, resampled_ann

def resample_multichan(xs, ann, fs, fs_target, resamp_ann_chan=0):
    """
    Resample multiple channels with their annotations

    Parameters
    ----------
    xs: numpy array
        The signal array
    ann : wfdb Annotation
        The wfdb annotation object
    fs : int, or float
        The original frequency
    fs_target : int, or float
        The target frequency
    resample_ann_channel : int, optional
        The signal channel used to compute new annotation indices

    Returns
    -------
    resampled_xs : numpy array
        Array of the resampled signal values
    resampled_ann : wfdb Annotation
        Annotation containing resampled annotation locations

    """
    assert resamp_ann_chan < xs.shape[1]

    lx = []
    lt = None
    for chan in range(xs.shape[1]):
        resampled_x, resampled_t = resample_sig(xs[:, chan], fs, fs_target)
        lx.append(resampled_x)
        if chan == resamp_ann_chan:
            lt = resampled_t

    new_sample = resample_ann(lt, ann.sample)
    assert ann.sample.shape == new_sample.shape

    resampled_ann = Annotation(ann.record_name, ann.extension, new_sample, ann.symbol,
        ann.num, ann.subtype, ann.chan, ann.aux_note, ann.fs)

    return numpy.column_stack(lx), resampled_ann

def normalize(x, lb=0, ub=1):
    """
    Normalize a signal between the lower and upper bound

    Parameters
    ----------
    x : numpy array
        Original signal to be normalized
    lb : int, or float
        Lower bound
    ub : int, or float
        Upper bound

    Returns
    -------
    x_normalized : numpy array
        Normalized signal

    """

    mid = ub - (ub - lb) / 2
    min_v = numpy.min(x)
    max_v = numpy.max(x)
    mid_v =  max_v - (max_v - min_v) / 2
    coef = (ub - lb) / (max_v - min_v)
    return x * coef - (mid_v * coef) + mid

def smooth(x, window_size):
    """
    """
    box = numpy.ones(window_size)/window_size
    return numpy.convolve(x, box, mode='same')
<commit_msg>resampling: ann.fs -> fs_target<commit_after>import numpy
from scipy import signal

from ..io.annotation import Annotation

def resample_ann(resampled_t, ann_sample):
    """
    Compute the new annotation indices

    Parameters
    ----------
    resampled_t : numpy array
        Array of signal locations as returned by scipy.signal.resample
    ann_sample : numpy array
        Array of annotation locations

    Returns
    -------
    resampled_ann_sample : numpy array
        Array of resampled annotation locations

    """
    tmp = numpy.zeros(len(resampled_t), dtype='int16')
    j = 0
    tprec = resampled_t[j]
    for i, v in enumerate(ann_sample):
        while True:
            d = False
            if v < tprec:
                j -= 1
                tprec = resampled_t[j]

            if j+1 == len(resampled_t):
                tmp[j] += 1
                break

            tnow = resampled_t[j+1]
            if tprec <= v and v <= tnow:
                if v-tprec < tnow-v:
                    tmp[j] += 1
                else:
                    tmp[j+1] += 1
                d = True
            j += 1
            tprec = tnow
            if d:
                break

    idx = numpy.where(tmp>0)[0].astype('int64')
    res = []
    for i in idx:
        for j in range(tmp[i]):
            res.append(i)
    assert len(res) == len(ann_sample)

    return numpy.asarray(res, dtype='int64')

def resample_sig(x, fs, fs_target):
    """
    Resample a signal to a different frequency.

    Parameters
    ----------
    x : numpy array
        Array containing the signal
    fs : int, or float
        The original sampling frequency
    fs_target : int, or float
        The target frequency

    Returns
    -------
    resampled_x : numpy array
        Array of the resampled signal values
    resampled_t : numpy array
        Array of the resampled signal locations

    """

    t = numpy.arange(x.shape[0]).astype('float64')

    if fs == fs_target:
        return x, t

    new_length = int(x.shape[0]*fs_target/fs)
    resampled_x, resampled_t = signal.resample(x, num=new_length, t=t)
    assert resampled_x.shape == resampled_t.shape and resampled_x.shape[0] == new_length
    assert numpy.all(numpy.diff(resampled_t) > 0)

    return resampled_x, resampled_t

def resample_singlechan(x, ann, fs, fs_target):
    """
    Resample a single-channel signal with its annotations

    Parameters
    ----------
    x: numpy array
        The signal array
    ann : wfdb Annotation
        The wfdb annotation object
    fs : int, or float
        The original frequency
    fs_target : int, or float
        The target frequency

    Returns
    -------
    resampled_x : numpy array
        Array of the resampled signal values
    resampled_ann : wfdb Annotation
        Annotation containing resampled annotation locations

    """

    resampled_x, resampled_t = resample_sig(x, fs, fs_target)

    new_sample = resample_ann(resampled_t, ann.sample)
    assert ann.sample.shape == new_sample.shape

    resampled_ann = Annotation(ann.record_name, ann.extension, new_sample,
        ann.symbol, ann.num, ann.subtype, ann.chan, ann.aux_note, fs_target)

    return resampled_x, resampled_ann

def resample_multichan(xs, ann, fs, fs_target, resamp_ann_chan=0):
    """
    Resample multiple channels with their annotations

    Parameters
    ----------
    xs: numpy array
        The signal array
    ann : wfdb Annotation
        The wfdb annotation object
    fs : int, or float
        The original frequency
    fs_target : int, or float
        The target frequency
    resample_ann_channel : int, optional
        The signal channel used to compute new annotation indices

    Returns
    -------
    resampled_xs : numpy array
        Array of the resampled signal values
    resampled_ann : wfdb Annotation
        Annotation containing resampled annotation locations

    """
    assert resamp_ann_chan < xs.shape[1]

    lx = []
    lt = None
    for chan in range(xs.shape[1]):
        resampled_x, resampled_t = resample_sig(xs[:, chan], fs, fs_target)
        lx.append(resampled_x)
        if chan == resamp_ann_chan:
            lt = resampled_t

    new_sample = resample_ann(lt, ann.sample)
    assert ann.sample.shape == new_sample.shape

    resampled_ann = Annotation(ann.record_name, ann.extension, new_sample, ann.symbol,
        ann.num, ann.subtype, ann.chan, ann.aux_note, fs_target)

    return numpy.column_stack(lx), resampled_ann

def normalize(x, lb=0, ub=1):
    """
    Normalize a signal between the lower and upper bound

    Parameters
    ----------
    x : numpy array
        Original signal to be normalized
    lb : int, or float
        Lower bound
    ub : int, or float
        Upper bound

    Returns
    -------
    x_normalized : numpy array
        Normalized signal

    """

    mid = ub - (ub - lb) / 2
    min_v = numpy.min(x)
    max_v = numpy.max(x)
    mid_v =  max_v - (max_v - min_v) / 2
    coef = (ub - lb) / (max_v - min_v)
    return x * coef - (mid_v * coef) + mid

def smooth(x, window_size):
    """
    """
    box = numpy.ones(window_size)/window_size
    return numpy.convolve(x, box, mode='same')
<|endoftext|>