<commit_before>"""Utility to help with conducting analyses with two-phase sampled data.
Uses calls to the osDesign and survival packages

Example
-------

from quickr import runRscript

matchCols = ['infected', 'sex', 'white', 'black', 'hispanic', 'asian', 'otherrace']

tmpPhaseII = df.loc[(adf['visit_type'] == 'trough') & (df['treatment'] == 1)].drop_duplicates('ptid')
tmpPhaseI = vxDf.loc[(vxDf['valid'] == 1) & (vxDf['treatment'] == 1)]

wcorDf, controlN, caseN = joinIPWeights(tmpPhaseII, tmpPhaseI, matchCols=matchCols)
plotDf = wcorDf.loc[(wcorDf['male'] == 1) & (wcorDf['visit_days'] <= wcorDf['daysrnaneg'])]

controlN = controlN.set_index('stratumID')
caseN = caseN.set_index('stratumID')

Rcmd = cchCmd.format(formula='Surv(daysinfectiondiagnosis, infected) ~ reading_cal')
res, resDf = runRscript(Rcmd, inDf=[plotDf, controlN], outputFile=True)
print(res)
"""

import pandas as pd
import numpy as np

__all__ = ['joinIPWeights',
            'cchCmd',
            'tpsCmd',
            'logisticCmd',
            'printCCH',
            'printTPS',
            'printLogistic']

def joinIPWeights(twoDf, oneDf, matchCols, caseCol='infected'):
    """Compute inverse probability weights for each sample in twoDf,
    limited to the specified visit type (e.g. peak, trough, addtional_peak).
    Strata are defined by matchCols, which default to the sex and race variables
    used in the sampling design.

    Weights are computed by counting the individuals in each strata in the target
    population (i.e. oneDf controls, Phase I sample) and comparing to the Phase II
    sampled population (i.e. twoDf controls). Cases are given a weight of 1 since all
    cases were sampled.

    Parameters
    ----------
    twoDf : pd.DataFrame
        Phase II data from cases and controls.
        Data will be subsetted on visit_type
    oneDf : pd.DataFrame
        Phase I sample including all participants. Constitutes the target population.
    caseCol : str
        Column that defines cases which are fully sampled and given weight one
    matchCols : list
        List of columns that define the Phase II sampling strata

    Returns
    -------
    twoDf : pd.DataFrame
        A copy of twoDf, after subsetting on visit_type and
        joining columns IPWeights (inverse probability weights)
        stratumID and insubcohort (all controls eligible for sampling)
    controlN : pd.DataFrame
        Number of controls in each strata with a column for stratumID
        Index contains variables that define the strata.
    caseN : pd.DataFrame
        Number of cases in each strata with a column for stratumID
        Index contains variables that define the strata."""

    """Subset on visit_type"""
    targetPop = oneDf.groupby(matchCols)['itt'].agg(np.sum)
    sampledPop = twoDf.groupby(matchCols)['itt'].agg(np.sum)

    """Inverse probability weighting"""
    ipwS = targetPop/sampledPop
    ipwS.name = 'IPWeights'

    """Strata IDs: label the strata in controls, will be same for cases"""
    stratumID = targetPop.copy()
    stratumID.name = 'stratumID'
    stratumID[:] = np.arange(targetPop.shape[0])
    stratumID = stratumID.xs(0, level=caseCol)

    """N of each strata, excluding cases"""
    controlN = targetPop.copy()
    controlN.name = 'controlN'
    controlN = controlN.xs(0, level=caseCol)
    controlN = pd.concat((controlN, stratumID), axis=1)

    """N of each strata, excluding controls"""
    caseN = targetPop.copy()
    caseN.name = 'caseN'
    caseN = caseN.xs(1, level=caseCol)
    caseN = pd.concat((caseN, stratumID), axis=1)

    """Set case weights to 1: cases should not be reweighted."""
    ipwS.loc[[1]] = 1.
    twoDf = twoDf.copy().set_index(matchCols).join(ipwS)
    """Assign cases the same strata IDs as controls"""
    notInfCols = [c for c in matchCols if not c == caseCol]
    twoDf = twoDf.reset_index().set_index(notInfCols).join(stratumID)
    twoDf = twoDf.reset_index()

    """Only controls are in the subcohort eligible for sampling"""
    twoDf.loc[:, 'insubcohort'] = (twoDf[caseCol] == 0).astype(int)

    """Optional check since the reweighted controls should equal  the target pop"""
    reweightedPop = twoDf.groupby(matchCols)['IPWeights'].agg(np.sum).xs(0, level=caseCol)
    success = np.all(np.isclose(targetPop.xs(0, level=caseCol).values, reweightedPop.values))
    if not success:
        print('Warning: reweighted controls do not match the target population!')
        print(reweightedPop)
        print(targetPop.xs(0, level=caseCol))

    return twoDf, controlN, caseN

cchCmd = """library(survival)
INPUTDF0$stratumID <- factor(INPUTDF0$stratumID)
cohortSize = table(INPUTDF1$stratumID) * INPUTDF1$controlN

fit <- cch({formula},
             data=INPUTDF0,
             subcoh=INPUTDF0$insubcohort,
             stratum=INPUTDF0$stratumID,
             id=INPUTDF0$ptid,
             cohort.size=cohortSize,
             method="II.Borgan")
summary(fit)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

tpsCmd = """library(osDesign)
INPUTDF0$stratumID <- factor(INPUTDF0$stratumID)
nn0 = table(INPUTDF1$stratumID) * INPUTDF1$controlN
nn1 = table(INPUTDF2$stratumID) * INPUTDF2$caseN
print(nn0)
print(nn1)
fit <- tps({formula},
             data=INPUTDF0,
             nn0=nn0,
             nn1=nn1,
             group=INPUTDF0$stratumID)
summary(fit)
output <- as.matrix(cbind(exp(fit$coef[2]),exp(fit$coef[2] - sqrt(fit$cove[2,2])*1.96),
exp(fit$coef[2] + sqrt(fit$cove[2,2])*1.96),min(2*(1-pnorm(abs(fit$coef[2]/sqrt(fit$cove[2,2])))),1.0)))
print(output)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

logisticCmd = """fit <- glm({formula},
                         data=INPUTDF0,
                         family=binomial(link='logit'))
summary(fit)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

def printCCH(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Value']
        se = resDf.loc[i, 'SE']
        pvalue = resDf.loc[i, 'p']
        print('%s HR = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, np.exp(v), np.exp(v-se*1.96), np.exp(v+se*1.96), pvalue))

def printTPS(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Value']
        se = resDf.loc[i, 'Mod.SE']
        pvalue = resDf.loc[i, 'Mod.p']
        print('%s coef = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, v, v-se*1.96, v+se*1.96, pvalue))

def printLogistic(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Estimate']
        se = resDf.loc[i, 'Std..Error']
        pvalue = resDf.loc[i, 'Pr...z..']
        print('%s coef = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, v, v-se*1.96, v+se*1.96, pvalue))

<commit_msg>Fixed check<commit_after>"""Utility to help with conducting analyses with two-phase sampled data.
Uses calls to the osDesign and survival packages

Example
-------

from quickr import runRscript

matchCols = ['infected', 'sex', 'white', 'black', 'hispanic', 'asian', 'otherrace']

tmpPhaseII = df.loc[(adf['visit_type'] == 'trough') & (df['treatment'] == 1)].drop_duplicates('ptid')
tmpPhaseI = vxDf.loc[(vxDf['valid'] == 1) & (vxDf['treatment'] == 1)]

wcorDf, controlN, caseN = joinIPWeights(tmpPhaseII, tmpPhaseI, matchCols=matchCols)
plotDf = wcorDf.loc[(wcorDf['male'] == 1) & (wcorDf['visit_days'] <= wcorDf['daysrnaneg'])]

controlN = controlN.set_index('stratumID')
caseN = caseN.set_index('stratumID')

Rcmd = cchCmd.format(formula='Surv(daysinfectiondiagnosis, infected) ~ reading_cal')
res, resDf = runRscript(Rcmd, inDf=[plotDf, controlN], outputFile=True)
print(res)
"""

import pandas as pd
import numpy as np

__all__ = ['joinIPWeights',
            'cchCmd',
            'tpsCmd',
            'logisticCmd',
            'printCCH',
            'printTPS',
            'printLogistic']

def joinIPWeights(twoDf, oneDf, matchCols, caseCol='infected'):
    """Compute inverse probability weights for each sample in twoDf,
    limited to the specified visit type (e.g. peak, trough, addtional_peak).
    Strata are defined by matchCols, which default to the sex and race variables
    used in the sampling design.

    Weights are computed by counting the individuals in each strata in the target
    population (i.e. oneDf controls, Phase I sample) and comparing to the Phase II
    sampled population (i.e. twoDf controls). Cases are given a weight of 1 since all
    cases were sampled.

    Parameters
    ----------
    twoDf : pd.DataFrame
        Phase II data from cases and controls.
        Data will be subsetted on visit_type
    oneDf : pd.DataFrame
        Phase I sample including all participants. Constitutes the target population.
    caseCol : str
        Column that defines cases which are fully sampled and given weight one
    matchCols : list
        List of columns that define the Phase II sampling strata

    Returns
    -------
    twoDf : pd.DataFrame
        A copy of twoDf, after subsetting on visit_type and
        joining columns IPWeights (inverse probability weights)
        stratumID and insubcohort (all controls eligible for sampling)
    controlN : pd.DataFrame
        Number of controls in each strata with a column for stratumID
        Index contains variables that define the strata.
    caseN : pd.DataFrame
        Number of cases in each strata with a column for stratumID
        Index contains variables that define the strata."""

    """Subset on visit_type"""
    targetPop = oneDf.groupby(matchCols)['itt'].agg(np.sum)
    sampledPop = twoDf.groupby(matchCols)['itt'].agg(np.sum)

    """Inverse probability weighting"""
    ipwS = targetPop/sampledPop
    ipwS.name = 'IPWeights'

    """Strata IDs: label the strata in controls, will be same for cases"""
    stratumID = targetPop.copy()
    stratumID.name = 'stratumID'
    stratumID[:] = np.arange(targetPop.shape[0])
    stratumID = stratumID.xs(0, level=caseCol)

    """N of each strata, excluding cases"""
    controlN = targetPop.copy()
    controlN.name = 'controlN'
    controlN = controlN.xs(0, level=caseCol)
    controlN = pd.concat((controlN, stratumID), axis=1)

    """N of each strata, excluding controls"""
    caseN = targetPop.copy()
    caseN.name = 'caseN'
    caseN = caseN.xs(1, level=caseCol)
    caseN = pd.concat((caseN, stratumID), axis=1)

    """Set case weights to 1: cases should not be reweighted."""
    ipwS.loc[[1]] = 1.
    twoDf = twoDf.copy().set_index(matchCols).join(ipwS)
    """Assign cases the same strata IDs as controls"""
    notInfCols = [c for c in matchCols if not c == caseCol]
    twoDf = twoDf.reset_index().set_index(notInfCols).join(stratumID)
    twoDf = twoDf.reset_index()

    """Only controls are in the subcohort eligible for sampling"""
    twoDf.loc[:, 'insubcohort'] = (twoDf[caseCol] == 0).astype(int)

    """Optional check since the reweighted controls should equal  the target pop"""
    reweightedPop = twoDf.groupby(matchCols)['IPWeights'].agg(np.sum).xs(0, level=caseCol)
    tmp = pd.concat((targetPop.xs(0, level=caseCol), reweightedPop), axis=1)
    success = np.all(np.isclose(tmp.iloc[:, 0].values, tmp.iloc[:, 1].values))
    if not success:
        print('Warning: reweighted controls do not match the target population!')
        print(reweightedPop)
        print(targetPop.xs(0, level=caseCol))

    return twoDf, controlN, caseN

cchCmd = """library(survival)
INPUTDF0$stratumID <- factor(INPUTDF0$stratumID)
cohortSize = table(INPUTDF1$stratumID) * INPUTDF1$controlN

fit <- cch({formula},
             data=INPUTDF0,
             subcoh=INPUTDF0$insubcohort,
             stratum=INPUTDF0$stratumID,
             id=INPUTDF0$ptid,
             cohort.size=cohortSize,
             method="II.Borgan")
summary(fit)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

tpsCmd = """library(osDesign)
INPUTDF0$stratumID <- factor(INPUTDF0$stratumID)
nn0 = table(INPUTDF1$stratumID) * INPUTDF1$controlN
nn1 = table(INPUTDF2$stratumID) * INPUTDF2$caseN
print(nn0)
print(nn1)
fit <- tps({formula},
             data=INPUTDF0,
             nn0=nn0,
             nn1=nn1,
             group=INPUTDF0$stratumID)
summary(fit)
output <- as.matrix(cbind(exp(fit$coef[2]),exp(fit$coef[2] - sqrt(fit$cove[2,2])*1.96),
exp(fit$coef[2] + sqrt(fit$cove[2,2])*1.96),min(2*(1-pnorm(abs(fit$coef[2]/sqrt(fit$cove[2,2])))),1.0)))
print(output)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

logisticCmd = """fit <- glm({formula},
                         data=INPUTDF0,
                         family=binomial(link='logit'))
summary(fit)
write.csv(data.frame(summary(fit)$coeff), OUTPUTFN)"""

def printCCH(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Value']
        se = resDf.loc[i, 'SE']
        pvalue = resDf.loc[i, 'p']
        print('%s HR = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, np.exp(v), np.exp(v-se*1.96), np.exp(v+se*1.96), pvalue))

def printTPS(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Value']
        se = resDf.loc[i, 'Mod.SE']
        pvalue = resDf.loc[i, 'Mod.p']
        print('%s coef = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, v, v-se*1.96, v+se*1.96, pvalue))

def printLogistic(resDf):
    for i in resDf.index:
        var = resDf.loc[i, 'Unnamed: 0']
        v = resDf.loc[i, 'Estimate']
        se = resDf.loc[i, 'Std..Error']
        pvalue = resDf.loc[i, 'Pr...z..']
        print('%s coef = %1.2f [%1.2f, %1.2f]; p = %1.3f' % (var, v, v-se*1.96, v+se*1.96, pvalue))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import fnmatch
import imp
import logging
import os
import re

logger = logging.getLogger(__name__)

# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
SCHEMA_VERSION = 47

dir_path = os.path.abspath(os.path.dirname(__file__))

class PrepareDatabaseException(Exception):
    pass

class UpgradeDatabaseException(PrepareDatabaseException):
    pass

def prepare_database(db_conn, database_engine, config):
    """Prepares a database for usage. Will either create all necessary tables
    or upgrade from an older schema version.

    If `config` is None then prepare_database will assert that no upgrade is
    necessary, *or* will create a fresh database if the database is empty.

    Args:
        db_conn:
        database_engine:
        config (synapse.config.homeserver.HomeServerConfig|None):
            application config, or None if we are connecting to an existing
            database which we expect to be configured already
    """
    try:
        cur = db_conn.cursor()
        version_info = _get_or_create_schema_state(cur, database_engine)

        if version_info:
            user_version, delta_files, upgraded = version_info

            if config is None:
                if user_version != SCHEMA_VERSION:
                    # If we don't pass in a config file then we are expecting to
                    # have already upgraded the DB.
                    raise UpgradeDatabaseException("Database needs to be upgraded")
            else:
                _upgrade_existing_database(
                    cur, user_version, delta_files, upgraded, database_engine, config
                )
        else:
            _setup_new_database(cur, database_engine)

        # check if any of our configured dynamic modules want a database
        if config is not None:
            _apply_module_schemas(cur, database_engine, config)

        cur.close()
        db_conn.commit()
    except Exception:
        db_conn.rollback()
        raise

def _setup_new_database(cur, database_engine):
    """Sets up the database by finding a base set of "full schemas" and then
    applying any necessary deltas.

    The "full_schemas" directory has subdirectories named after versions. This
    function searches for the highest version less than or equal to
    `SCHEMA_VERSION` and executes all .sql files in that directory.

    The function will then apply all deltas for all versions after the base
    version.

    Example directory structure:

        schema/
            delta/
                ...
            full_schemas/
                3/
                    test.sql
                    ...
                11/
                    foo.sql
                    bar.sql
                ...

    In the example foo.sql and bar.sql would be run, and then any delta files
    for versions strictly greater than 11.
    """
    current_dir = os.path.join(dir_path, "schema", "full_schemas")
    directory_entries = os.listdir(current_dir)

    valid_dirs = []
    pattern = re.compile(r"^\d+(\.sql)?$")
    for filename in directory_entries:
        match = pattern.match(filename)
        abs_path = os.path.join(current_dir, filename)
        if match and os.path.isdir(abs_path):
            ver = int(match.group(0))
            if ver <= SCHEMA_VERSION:
                valid_dirs.append((ver, abs_path))
        else:
            logger.warn("Unexpected entry in 'full_schemas': %s", filename)

    if not valid_dirs:
        raise PrepareDatabaseException(
            "Could not find a suitable base set of full schemas"
        )

    max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])

    logger.debug("Initialising schema v%d", max_current_ver)

    directory_entries = os.listdir(sql_dir)

    for filename in fnmatch.filter(directory_entries, "*.sql"):
        sql_loc = os.path.join(sql_dir, filename)
        logger.debug("Applying schema %s", sql_loc)
        executescript(cur, sql_loc)

    cur.execute(
        database_engine.convert_param_style(
            "INSERT INTO schema_version (version, upgraded)"
            " VALUES (?,?)"
        ),
        (max_current_ver, False,)
    )

    _upgrade_existing_database(
        cur,
        current_version=max_current_ver,
        applied_delta_files=[],
        upgraded=False,
        database_engine=database_engine,
        config=None,
        is_empty=True,
    )

def _upgrade_existing_database(cur, current_version, applied_delta_files,
                               upgraded, database_engine, config, is_empty=False):
    """Upgrades an existing database.

    Delta files can either be SQL stored in *.sql files, or python modules
    in *.py.

    There can be multiple delta files per version. Synapse will keep track of
    which delta files have been applied, and will apply any that haven't been
    even if there has been no version bump. This is useful for development
    where orthogonal schema changes may happen on separate branches.

    Different delta files for the same version *must* be orthogonal and give
    the same result when applied in any order. No guarantees are made on the
    order of execution of these scripts.

    This is a no-op of current_version == SCHEMA_VERSION.

    Example directory structure:

        schema/
            delta/
                11/
                    foo.sql
                    ...
                12/
                    foo.sql
                    bar.py
                ...
            full_schemas/
                ...

    In the example, if current_version is 11, then foo.sql will be run if and
    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
    some arbitrary order.

    Args:
        cur (Cursor)
        current_version (int): The current version of the schema.
        applied_delta_files (list): A list of deltas that have already been
            applied.
        upgraded (bool): Whether the current version was generated by having
            applied deltas or from full schema file. If `True` the function
            will never apply delta files for the given `current_version`, since
            the current_version wasn't generated by applying those delta files.
    """

    if current_version > SCHEMA_VERSION:
        raise ValueError(
            "Cannot use this database as it is too " +
            "new for the server to understand"
        )

    start_ver = current_version
    if not upgraded:
        start_ver += 1

    logger.debug("applied_delta_files: %s", applied_delta_files)

    for v in range(start_ver, SCHEMA_VERSION + 1):
        logger.info("Upgrading schema to v%d", v)

        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))

        try:
            directory_entries = os.listdir(delta_dir)
        except OSError:
            logger.exception("Could not open delta dir for version %d", v)
            raise UpgradeDatabaseException(
                "Could not open delta dir for version %d" % (v,)
            )

        directory_entries.sort()
        for file_name in directory_entries:
            relative_path = os.path.join(str(v), file_name)
            logger.debug("Found file: %s", relative_path)
            if relative_path in applied_delta_files:
                continue

            absolute_path = os.path.join(
                dir_path, "schema", "delta", relative_path,
            )
            root_name, ext = os.path.splitext(file_name)
            if ext == ".py":
                # This is a python upgrade module. We need to import into some
                # package and then execute its `run_upgrade` function.
                module_name = "synapse.storage.v%d_%s" % (
                    v, root_name
                )
                with open(absolute_path) as python_file:
                    module = imp.load_source(
                        module_name, absolute_path, python_file
                    )
                logger.info("Running script %s", relative_path)
                module.run_create(cur, database_engine)
                if not is_empty:
                    module.run_upgrade(cur, database_engine, config=config)
            elif ext == ".pyc":
                # Sometimes .pyc files turn up anyway even though we've
                # disabled their generation; e.g. from distribution package
                # installers. Silently skip it
                pass
            elif ext == ".sql":
                # A plain old .sql file, just read and execute it
                logger.info("Applying schema %s", relative_path)
                executescript(cur, absolute_path)
            else:
                # Not a valid delta file.
                logger.warn(
                    "Found directory entry that did not end in .py or"
                    " .sql: %s",
                    relative_path,
                )
                continue

            # Mark as done.
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO applied_schema_deltas (version, file)"
                    " VALUES (?,?)",
                ),
                (v, relative_path)
            )

            cur.execute("DELETE FROM schema_version")
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO schema_version (version, upgraded)"
                    " VALUES (?,?)",
                ),
                (v, True)
            )

def _apply_module_schemas(txn, database_engine, config):
    """Apply the module schemas for the dynamic modules, if any

    Args:
        cur: database cursor
        database_engine: synapse database engine class
        config (synapse.config.homeserver.HomeServerConfig):
            application config
    """
    for (mod, _config) in config.password_providers:
        if not hasattr(mod, 'get_db_schema_files'):
            continue
        modname = ".".join((mod.__module__, mod.__name__))
        _apply_module_schema_files(
            txn, database_engine, modname, mod.get_db_schema_files(),
        )

def _apply_module_schema_files(cur, database_engine, modname, names_and_streams):
    """Apply the module schemas for a single module

    Args:
        cur: database cursor
        database_engine: synapse database engine class
        modname (str): fully qualified name of the module
        names_and_streams (Iterable[(str, file)]): the names and streams of
            schemas to be applied
    """
    cur.execute(
        database_engine.convert_param_style(
            "SELECT file FROM applied_module_schemas WHERE module_name = ?"
        ),
        (modname,)
    )
    applied_deltas = set(d for d, in cur)
    for (name, stream) in names_and_streams:
        if name in applied_deltas:
            continue

        root_name, ext = os.path.splitext(name)
        if ext != '.sql':
            raise PrepareDatabaseException(
                "only .sql files are currently supported for module schemas",
            )

        logger.info("applying schema %s for %s", name, modname)
        for statement in get_statements(stream):
            cur.execute(statement)

        # Mark as done.
        cur.execute(
            database_engine.convert_param_style(
                "INSERT INTO applied_module_schemas (module_name, file)"
                " VALUES (?,?)",
            ),
            (modname, name)
        )

def get_statements(f):
    statement_buffer = ""
    in_comment = False  # If we're in a /* ... */ style comment

    for line in f:
        line = line.strip()

        if in_comment:
            # Check if this line contains an end to the comment
            comments = line.split("*/", 1)
            if len(comments) == 1:
                continue
            line = comments[1]
            in_comment = False

        # Remove inline block comments
        line = re.sub(r"/\*.*\*/", " ", line)

        # Does this line start a comment?
        comments = line.split("/*", 1)
        if len(comments) > 1:
            line = comments[0]
            in_comment = True

        # Deal with line comments
        line = line.split("--", 1)[0]
        line = line.split("//", 1)[0]

        # Find *all* semicolons. We need to treat first and last entry
        # specially.
        statements = line.split(";")

        # We must prepend statement_buffer to the first statement
        first_statement = "%s %s" % (
            statement_buffer.strip(),
            statements[0].strip()
        )
        statements[0] = first_statement

        # Every entry, except the last, is a full statement
        for statement in statements[:-1]:
            yield statement.strip()

        # The last entry did *not* end in a semicolon, so we store it for the
        # next semicolon we find
        statement_buffer = statements[-1].strip()

def executescript(txn, schema_path):
    with open(schema_path, 'r') as f:
        for statement in get_statements(f):
            txn.execute(statement)

def _get_or_create_schema_state(txn, database_engine):
    # Bluntly try creating the schema_version tables.
    schema_path = os.path.join(
        dir_path, "schema", "schema_version.sql",
    )
    executescript(txn, schema_path)

    txn.execute("SELECT version, upgraded FROM schema_version")
    row = txn.fetchone()
    current_version = int(row[0]) if row else None
    upgraded = bool(row[1]) if row else None

    if current_version:
        txn.execute(
            database_engine.convert_param_style(
                "SELECT file FROM applied_schema_deltas WHERE version >= ?"
            ),
            (current_version,)
        )
        applied_deltas = [d for d, in txn]
        return current_version, applied_deltas, upgraded

    return None
<commit_msg>bump schema version<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 - 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import fnmatch
import imp
import logging
import os
import re

logger = logging.getLogger(__name__)

# Remember to update this number every time a change is made to database
# schema files, so the users will be informed on server restarts.
SCHEMA_VERSION = 48

dir_path = os.path.abspath(os.path.dirname(__file__))

class PrepareDatabaseException(Exception):
    pass

class UpgradeDatabaseException(PrepareDatabaseException):
    pass

def prepare_database(db_conn, database_engine, config):
    """Prepares a database for usage. Will either create all necessary tables
    or upgrade from an older schema version.

    If `config` is None then prepare_database will assert that no upgrade is
    necessary, *or* will create a fresh database if the database is empty.

    Args:
        db_conn:
        database_engine:
        config (synapse.config.homeserver.HomeServerConfig|None):
            application config, or None if we are connecting to an existing
            database which we expect to be configured already
    """
    try:
        cur = db_conn.cursor()
        version_info = _get_or_create_schema_state(cur, database_engine)

        if version_info:
            user_version, delta_files, upgraded = version_info

            if config is None:
                if user_version != SCHEMA_VERSION:
                    # If we don't pass in a config file then we are expecting to
                    # have already upgraded the DB.
                    raise UpgradeDatabaseException("Database needs to be upgraded")
            else:
                _upgrade_existing_database(
                    cur, user_version, delta_files, upgraded, database_engine, config
                )
        else:
            _setup_new_database(cur, database_engine)

        # check if any of our configured dynamic modules want a database
        if config is not None:
            _apply_module_schemas(cur, database_engine, config)

        cur.close()
        db_conn.commit()
    except Exception:
        db_conn.rollback()
        raise

def _setup_new_database(cur, database_engine):
    """Sets up the database by finding a base set of "full schemas" and then
    applying any necessary deltas.

    The "full_schemas" directory has subdirectories named after versions. This
    function searches for the highest version less than or equal to
    `SCHEMA_VERSION` and executes all .sql files in that directory.

    The function will then apply all deltas for all versions after the base
    version.

    Example directory structure:

        schema/
            delta/
                ...
            full_schemas/
                3/
                    test.sql
                    ...
                11/
                    foo.sql
                    bar.sql
                ...

    In the example foo.sql and bar.sql would be run, and then any delta files
    for versions strictly greater than 11.
    """
    current_dir = os.path.join(dir_path, "schema", "full_schemas")
    directory_entries = os.listdir(current_dir)

    valid_dirs = []
    pattern = re.compile(r"^\d+(\.sql)?$")
    for filename in directory_entries:
        match = pattern.match(filename)
        abs_path = os.path.join(current_dir, filename)
        if match and os.path.isdir(abs_path):
            ver = int(match.group(0))
            if ver <= SCHEMA_VERSION:
                valid_dirs.append((ver, abs_path))
        else:
            logger.warn("Unexpected entry in 'full_schemas': %s", filename)

    if not valid_dirs:
        raise PrepareDatabaseException(
            "Could not find a suitable base set of full schemas"
        )

    max_current_ver, sql_dir = max(valid_dirs, key=lambda x: x[0])

    logger.debug("Initialising schema v%d", max_current_ver)

    directory_entries = os.listdir(sql_dir)

    for filename in fnmatch.filter(directory_entries, "*.sql"):
        sql_loc = os.path.join(sql_dir, filename)
        logger.debug("Applying schema %s", sql_loc)
        executescript(cur, sql_loc)

    cur.execute(
        database_engine.convert_param_style(
            "INSERT INTO schema_version (version, upgraded)"
            " VALUES (?,?)"
        ),
        (max_current_ver, False,)
    )

    _upgrade_existing_database(
        cur,
        current_version=max_current_ver,
        applied_delta_files=[],
        upgraded=False,
        database_engine=database_engine,
        config=None,
        is_empty=True,
    )

def _upgrade_existing_database(cur, current_version, applied_delta_files,
                               upgraded, database_engine, config, is_empty=False):
    """Upgrades an existing database.

    Delta files can either be SQL stored in *.sql files, or python modules
    in *.py.

    There can be multiple delta files per version. Synapse will keep track of
    which delta files have been applied, and will apply any that haven't been
    even if there has been no version bump. This is useful for development
    where orthogonal schema changes may happen on separate branches.

    Different delta files for the same version *must* be orthogonal and give
    the same result when applied in any order. No guarantees are made on the
    order of execution of these scripts.

    This is a no-op of current_version == SCHEMA_VERSION.

    Example directory structure:

        schema/
            delta/
                11/
                    foo.sql
                    ...
                12/
                    foo.sql
                    bar.py
                ...
            full_schemas/
                ...

    In the example, if current_version is 11, then foo.sql will be run if and
    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
    some arbitrary order.

    Args:
        cur (Cursor)
        current_version (int): The current version of the schema.
        applied_delta_files (list): A list of deltas that have already been
            applied.
        upgraded (bool): Whether the current version was generated by having
            applied deltas or from full schema file. If `True` the function
            will never apply delta files for the given `current_version`, since
            the current_version wasn't generated by applying those delta files.
    """

    if current_version > SCHEMA_VERSION:
        raise ValueError(
            "Cannot use this database as it is too " +
            "new for the server to understand"
        )

    start_ver = current_version
    if not upgraded:
        start_ver += 1

    logger.debug("applied_delta_files: %s", applied_delta_files)

    for v in range(start_ver, SCHEMA_VERSION + 1):
        logger.info("Upgrading schema to v%d", v)

        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))

        try:
            directory_entries = os.listdir(delta_dir)
        except OSError:
            logger.exception("Could not open delta dir for version %d", v)
            raise UpgradeDatabaseException(
                "Could not open delta dir for version %d" % (v,)
            )

        directory_entries.sort()
        for file_name in directory_entries:
            relative_path = os.path.join(str(v), file_name)
            logger.debug("Found file: %s", relative_path)
            if relative_path in applied_delta_files:
                continue

            absolute_path = os.path.join(
                dir_path, "schema", "delta", relative_path,
            )
            root_name, ext = os.path.splitext(file_name)
            if ext == ".py":
                # This is a python upgrade module. We need to import into some
                # package and then execute its `run_upgrade` function.
                module_name = "synapse.storage.v%d_%s" % (
                    v, root_name
                )
                with open(absolute_path) as python_file:
                    module = imp.load_source(
                        module_name, absolute_path, python_file
                    )
                logger.info("Running script %s", relative_path)
                module.run_create(cur, database_engine)
                if not is_empty:
                    module.run_upgrade(cur, database_engine, config=config)
            elif ext == ".pyc":
                # Sometimes .pyc files turn up anyway even though we've
                # disabled their generation; e.g. from distribution package
                # installers. Silently skip it
                pass
            elif ext == ".sql":
                # A plain old .sql file, just read and execute it
                logger.info("Applying schema %s", relative_path)
                executescript(cur, absolute_path)
            else:
                # Not a valid delta file.
                logger.warn(
                    "Found directory entry that did not end in .py or"
                    " .sql: %s",
                    relative_path,
                )
                continue

            # Mark as done.
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO applied_schema_deltas (version, file)"
                    " VALUES (?,?)",
                ),
                (v, relative_path)
            )

            cur.execute("DELETE FROM schema_version")
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO schema_version (version, upgraded)"
                    " VALUES (?,?)",
                ),
                (v, True)
            )

def _apply_module_schemas(txn, database_engine, config):
    """Apply the module schemas for the dynamic modules, if any

    Args:
        cur: database cursor
        database_engine: synapse database engine class
        config (synapse.config.homeserver.HomeServerConfig):
            application config
    """
    for (mod, _config) in config.password_providers:
        if not hasattr(mod, 'get_db_schema_files'):
            continue
        modname = ".".join((mod.__module__, mod.__name__))
        _apply_module_schema_files(
            txn, database_engine, modname, mod.get_db_schema_files(),
        )

def _apply_module_schema_files(cur, database_engine, modname, names_and_streams):
    """Apply the module schemas for a single module

    Args:
        cur: database cursor
        database_engine: synapse database engine class
        modname (str): fully qualified name of the module
        names_and_streams (Iterable[(str, file)]): the names and streams of
            schemas to be applied
    """
    cur.execute(
        database_engine.convert_param_style(
            "SELECT file FROM applied_module_schemas WHERE module_name = ?"
        ),
        (modname,)
    )
    applied_deltas = set(d for d, in cur)
    for (name, stream) in names_and_streams:
        if name in applied_deltas:
            continue

        root_name, ext = os.path.splitext(name)
        if ext != '.sql':
            raise PrepareDatabaseException(
                "only .sql files are currently supported for module schemas",
            )

        logger.info("applying schema %s for %s", name, modname)
        for statement in get_statements(stream):
            cur.execute(statement)

        # Mark as done.
        cur.execute(
            database_engine.convert_param_style(
                "INSERT INTO applied_module_schemas (module_name, file)"
                " VALUES (?,?)",
            ),
            (modname, name)
        )

def get_statements(f):
    statement_buffer = ""
    in_comment = False  # If we're in a /* ... */ style comment

    for line in f:
        line = line.strip()

        if in_comment:
            # Check if this line contains an end to the comment
            comments = line.split("*/", 1)
            if len(comments) == 1:
                continue
            line = comments[1]
            in_comment = False

        # Remove inline block comments
        line = re.sub(r"/\*.*\*/", " ", line)

        # Does this line start a comment?
        comments = line.split("/*", 1)
        if len(comments) > 1:
            line = comments[0]
            in_comment = True

        # Deal with line comments
        line = line.split("--", 1)[0]
        line = line.split("//", 1)[0]

        # Find *all* semicolons. We need to treat first and last entry
        # specially.
        statements = line.split(";")

        # We must prepend statement_buffer to the first statement
        first_statement = "%s %s" % (
            statement_buffer.strip(),
            statements[0].strip()
        )
        statements[0] = first_statement

        # Every entry, except the last, is a full statement
        for statement in statements[:-1]:
            yield statement.strip()

        # The last entry did *not* end in a semicolon, so we store it for the
        # next semicolon we find
        statement_buffer = statements[-1].strip()

def executescript(txn, schema_path):
    with open(schema_path, 'r') as f:
        for statement in get_statements(f):
            txn.execute(statement)

def _get_or_create_schema_state(txn, database_engine):
    # Bluntly try creating the schema_version tables.
    schema_path = os.path.join(
        dir_path, "schema", "schema_version.sql",
    )
    executescript(txn, schema_path)

    txn.execute("SELECT version, upgraded FROM schema_version")
    row = txn.fetchone()
    current_version = int(row[0]) if row else None
    upgraded = bool(row[1]) if row else None

    if current_version:
        txn.execute(
            database_engine.convert_param_style(
                "SELECT file FROM applied_schema_deltas WHERE version >= ?"
            ),
            (current_version,)
        )
        applied_deltas = [d for d, in txn]
        return current_version, applied_deltas, upgraded

    return None
<|endoftext|>


<commit_before>from .helpers import public_api
from .twitter import TwitterBasePlugin
from ircbot.plugin import BotPlugin

class UrlShow(TwitterBasePlugin):
    github_repo = r'.*https?://github\.com/([\w\d_\.-]+)/([\w\d_\.-]+)'
    urlab_url = r'.*https?://urlab\.be'

    @BotPlugin.command(r'.*https?://twitter.com/[^/]+/status/(\d+)')
    def twitter_status(self, msg):
        url = 'statuses/show/{}.json'.format(msg.args[0])
        tweet = yield from self.twitter_request('GET', url)
        msg.reply(self.format_tweet(tweet))

    @BotPlugin.command(github_repo + r'/?(\s|$)')
    def github(self, msg):
        url = "https://api.github.com/repos/{}/{}".format(*msg.args)
        repo = yield from public_api(url)
        repo['name'] = self.bot.text.bold(repo['name'])
        repo['language'] = self.bot.text.purple('[', repo['language'], ']')
        repo['stars'] = self.bot.text.yellow('(', repo['stargazers_count'], '*)')
        fmt = "{name} {language} {stars}: «{description}»"
        msg.reply(fmt.format(**repo))

    @BotPlugin.command(github_repo + r'/(issues|pull)/(\d+)')
    def github_issue(self, msg):
        user, repo, kind, id = msg.args
        args = user, repo, id
        url = "https://api.github.com/repos/{}/{}/issues/{}".format(*args)
        issue = yield from public_api(url)
        issue['author'] = self.bot.text.bold('@' + issue['user']['login'])
        issue['labels'] = ' '.join(self.bot.text.grey('(%s)') % x['name'] for x in issue['labels'])
        if issue['state'] == 'open':
            issue['number'] = self.bot.text.green('[#', issue['number'], ' (open)]')
        elif issue['state'] == 'closed':
            issue['number'] = self.bot.text.red('[#', issue['number'], ' (closed)]')
        fmt = "{author} {number}: «{title}» {labels}"
        msg.reply(fmt.format(**issue))

    @BotPlugin.command(r'.*https?://www\.reddit\.com/r/([\w\d_\.-]+)/comments/([\w\d_\.-]+)')
    def reddit(self, msg):
        url = "https://api.bot.text.reddit.com/r/{}/comments/{}".format(*msg.args[:2])
        data = yield from public_api(url)
        post = data[0]['data']['children'][0]['data']
        post['author'] = self.bot.text.bold('@' + post['author'])
        post['upvote_ratio'] = self.bot.text.yellow('(', post['upvote_ratio'], '+)')
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{author} {upvote_ratio}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(r'.*https?://news\.ycombinator\.com/item\?id=(\d+)')
    def hackernews(self, msg):
        url = "https://hacker-news.firebaseio.com/v0/item/"
        post = yield from public_api(url + "{}.json".format(msg.args[0]))
        post['by'] = self.bot.text.bold('@', post['by'])
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{by}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(urlab_url + r'/(projects/\d+)')
    def urlab_project(self, msg):
        project_status = {
            'p': self.bot.text.yellow,  # Proposition
            'i': self.bot.text.blue,    # In progress
            'f': self.bot.text.green,   # Finished
        }

        proj = yield from public_api(msg.args[0])
        color = project_status.get(proj['status'], self.bot.text.grey)
        proj['desc'] = color(proj['short_description'])
        proj['title'] = self.bot.text.bold(proj['title'])
        fmt = "{title}: {desc}"
        msg.reply(fmt.format(**proj))

    @BotPlugin.command(urlab_url + r'/(events/\d+)')
    def urlab_event(self, msg):
        event_status = {
            'r': self.bot.text.bold,    # Ready
            'i': self.bot.text.yellow,  # Incubation
        }

        evt = yield from public_api(msg.args[0])
        if evt.get('start', None):
            evt['when'] = self.bot.text.yellow(self.bot.naturaltime(evt['start']))
        else:
            evt['when'] = "pas de date"
        color = event_status.get(evt['status'], self.bot.text.grey)
        evt['title'] = color(evt['title'])
        evt['place'] = self.bot.text.purple(evt['place'])
        fmt = "{title} ({when} :: {place})"
        msg.reply(fmt.format(**evt))
<commit_msg>[add] Github commit url<commit_after>from .helpers import public_api
from .twitter import TwitterBasePlugin
from ircbot.plugin import BotPlugin

class UrlShow(TwitterBasePlugin):
    github_repo = r'.*https?://github\.com/([\w\d_\.-]+)/([\w\d_\.-]+)'
    urlab_url = r'.*https?://urlab\.be'

    @BotPlugin.command(r'.*https?://twitter.com/[^/]+/status/(\d+)')
    def twitter_status(self, msg):
        url = 'statuses/show/{}.json'.format(msg.args[0])
        tweet = yield from self.twitter_request('GET', url)
        msg.reply(self.format_tweet(tweet))

    @BotPlugin.command(github_repo + r'/?(\s|$)')
    def github(self, msg):
        url = "https://api.github.com/repos/{}/{}".format(*msg.args)
        repo = yield from public_api(url)
        repo['name'] = self.bot.text.bold(repo['name'])
        repo['language'] = self.bot.text.purple('[', repo['language'], ']')
        repo['stars'] = self.bot.text.yellow('(', repo['stargazers_count'], '*)')
        fmt = "{name} {language} {stars}: «{description}»"
        msg.reply(fmt.format(**repo))

    @BotPlugin.command(github_repo + r'/(issues|pull)/(\d+)')
    def github_issue(self, msg):
        user, repo, kind, id = msg.args
        args = user, repo, id
        url = "https://api.github.com/repos/{}/{}/issues/{}".format(*args)
        issue = yield from public_api(url)
        issue['author'] = self.bot.text.bold('@' + issue['user']['login'])
        issue['labels'] = ' '.join(self.bot.text.grey('(%s)') % x['name'] for x in issue['labels'])
        if issue['state'] == 'open':
            issue['number'] = self.bot.text.green('[#', issue['number'], ' (open)]')
        elif issue['state'] == 'closed':
            issue['number'] = self.bot.text.red('[#', issue['number'], ' (closed)]')
        fmt = "{author} {number}: «{title}» {labels}"
        msg.reply(fmt.format(**issue))

    @BotPlugin.command(github_repo + r'/commit/([0-9a-fA-F]{,40})')
    def github_commit(self, msg):
        url = "https://api.github.com/repos/{}/{}/commits/{}".format(*msg.args)
        commit = yield from public_api(url)
        commit['author'] = self.bot.text.bold('@' + commit['author']['login'])
        commit['title'] = commit['commit']['message']
        additions = self.bot.text.green("%d+" % commit['stats']['additions'])
        deletions = self.bot.text.red("%d-" % commit['stats']['deletions'])
        files_changed = self.bot.text.yellow("%d files" % len(commit['files']))
        commit['stats'] = " ".join([additions, deletions, files_changed])
        msg.reply("{author} «{title}» ({stats})".format(**commit))

    @BotPlugin.command(r'.*https?://www\.reddit\.com/r/([\w\d_\.-]+)/comments/([\w\d_\.-]+)')
    def reddit(self, msg):
        url = "https://api.bot.text.reddit.com/r/{}/comments/{}".format(*msg.args[:2])
        data = yield from public_api(url)
        post = data[0]['data']['children'][0]['data']
        post['author'] = self.bot.text.bold('@' + post['author'])
        post['upvote_ratio'] = self.bot.text.yellow('(', post['upvote_ratio'], '+)')
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{author} {upvote_ratio}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(r'.*https?://news\.ycombinator\.com/item\?id=(\d+)')
    def hackernews(self, msg):
        url = "https://hacker-news.firebaseio.com/v0/item/"
        post = yield from public_api(url + "{}.json".format(msg.args[0]))
        post['by'] = self.bot.text.bold('@', post['by'])
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{by}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(urlab_url + r'/(projects/\d+)')
    def urlab_project(self, msg):
        project_status = {
            'p': self.bot.text.yellow,  # Proposition
            'i': self.bot.text.blue,    # In progress
            'f': self.bot.text.green,   # Finished
        }

        proj = yield from public_api(msg.args[0])
        color = project_status.get(proj['status'], self.bot.text.grey)
        proj['desc'] = color(proj['short_description'])
        proj['title'] = self.bot.text.bold(proj['title'])
        fmt = "{title}: {desc}"
        msg.reply(fmt.format(**proj))

    @BotPlugin.command(urlab_url + r'/(events/\d+)')
    def urlab_event(self, msg):
        event_status = {
            'r': self.bot.text.bold,    # Ready
            'i': self.bot.text.yellow,  # Incubation
        }

        evt = yield from public_api(msg.args[0])
        if evt.get('start', None):
            evt['when'] = self.bot.text.yellow(self.bot.naturaltime(evt['start']))
        else:
            evt['when'] = "pas de date"
        color = event_status.get(evt['status'], self.bot.text.grey)
        evt['title'] = color(evt['title'])
        evt['place'] = self.bot.text.purple(evt['place'])
        fmt = "{title} ({when} :: {place})"
        msg.reply(fmt.format(**evt))
<|endoftext|>


<commit_before># encoding: utf-8
"""
resume.py is a pre-processor for Markdown resumes, targeting the pandoc
document processor.

Pandoc extended Markdown supports embedded HTML (like all compliant Markdown
parser) and a subset of LaTeX, but when outputting LaTeX any unrecognized
LaTeX commands will simply be passed through.

This means you can keep your resume in pure markdown and define pre-processing
functions that do different things with different parts of the input depending
on the target output format.

Currently, the main feature is extraction of contact details.  They are
expected to begin on the fourth line, following the header and a blank line,
and extend until the next blank line.  Lines with bullets (•) will be split
into separate lines.

    Michael White
    =============

    72 Bower St. #1 • Medford, MA, 02155
    617-899-1621

You can then define a function for an output format like this:

    def tex(lines, contact_lines, *args):
        '''
        Returns the pre-processed Markdown output suitable for tex processing,
        as a string.

        lines -- a list of lines, without the contact lines
        contact_lines -- the extracted contact lines
        args -- any extra command-line arguments
        '''

And finally run it like this:

    python resume.py tex < resume.md

"""
import sys
import re

class Processor(object):
    handlers = {}

    def register(self, fn):
        self.handlers[fn.__name__] = fn
        return fn

    def process(self, format, lines, contact_lines, *args):
        try:
            handler = self.handlers[format]
        except KeyError:
            raise Exception("Unknown format: %s" % format)

        return handler(lines, contact_lines, *args)

processor = Processor()

@processor.register
def tex(lines, contact_lines, *args):
    # pandoc doesn't seem to support markdown inside latex blocks, so we're
    # just going to hardcode the two most common link formats for now so people
    # can put links in their contact info
    def replace_links(line):
        line = re.sub(r"<([^:]+@.+?)>", r"\href{mailto:\1}{\1}", line)
        line = re.sub(r"<(http.+?)>", r"\url{\1}", line)
        return re.sub(r"\[([^\]]+)\]\(([^\)]+)\)", r"\href{\2}{\1}", line)

    lines.insert(0, "\\begin{nospace}\\begin{flushright}\n" +
                    "\n\n".join(map(replace_links, contact_lines)) +
                    "\n\\end{flushright}\\end{nospace}\n")
    replace = {
        '~': '$\sim$',
    }

    escape = ['#']

    out = "".join(lines)
    for search in replace:
        out = re.sub(search, replace[search], out)

    for c in escape:
        out = re.sub(r'([^\\])\%s' % c, r'\1\%s' % c, out)

    return out

@processor.register
def html(lines, contact_lines, *args):
    untex = ['LaTeX']

    for word in untex:
        # yuck
        replace = lambda l: l.replace(r"\%s" % word, word)
        lines = map(replace, lines)
        contact_lines = map(replace, contact_lines)

    lines.insert(0, "

%s
\n" %
                         ("
" + "
".join(contact_lines)))
    lines.insert(1, "
")
    lines.append("
")

    return "".join(lines)

def main():
    try:
        format = sys.argv[1]
    except IndexError:
        raise Exception("No format specified")

    lines = sys.stdin.readlines()

    contact_lines = []
    for line in lines[3:]:
        lines.remove(line)
        parts = [x.strip() for x in line.split("•")]
        if parts == ['']:
            break

        contact_lines.extend(parts)

    print processor.process(format, lines, contact_lines, *sys.argv[1:])

if __name__ == '__main__':
    main()
<commit_msg>better html style<commit_after># encoding: utf-8
"""
resume.py is a pre-processor for Markdown resumes, targeting the pandoc
document processor.

Pandoc extended Markdown supports embedded HTML (like all compliant Markdown
parser) and a subset of LaTeX, but when outputting LaTeX any unrecognized
LaTeX commands will simply be passed through.

This means you can keep your resume in pure markdown and define pre-processing
functions that do different things with different parts of the input depending
on the target output format.

Currently, the main feature is extraction of contact details.  They are
expected to begin on the fourth line, following the header and a blank line,
and extend until the next blank line.  Lines with bullets (•) will be split
into separate lines.

    Michael White
    =============

    72 Bower St. #1 • Medford, MA, 02155
    617-899-1621

You can then define a function for an output format like this:

    def tex(lines, contact_lines, *args):
        '''
        Returns the pre-processed Markdown output suitable for tex processing,
        as a string.

        lines -- a list of lines, without the contact lines
        contact_lines -- the extracted contact lines
        args -- any extra command-line arguments
        '''

And finally run it like this:

    python resume.py tex < resume.md

"""
import sys
import re

class Processor(object):
    handlers = {}

    def register(self, fn):
        self.handlers[fn.__name__] = fn
        return fn

    def process(self, format, lines, contact_lines, *args):
        try:
            handler = self.handlers[format]
        except KeyError:
            raise Exception("Unknown format: %s" % format)

        return handler(lines, contact_lines, *args)

processor = Processor()

@processor.register
def tex(lines, contact_lines, *args):
    # pandoc doesn't seem to support markdown inside latex blocks, so we're
    # just going to hardcode the two most common link formats for now so people
    # can put links in their contact info
    def replace_links(line):
        line = re.sub(r"<([^:]+@.+?)>", r"\href{mailto:\1}{\1}", line)
        line = re.sub(r"<(http.+?)>", r"\url{\1}", line)
        return re.sub(r"\[([^\]]+)\]\(([^\)]+)\)", r"\href{\2}{\1}", line)

    lines.insert(0, "\\begin{nospace}\\begin{flushright}\n" +
                    "\n\n".join(map(replace_links, contact_lines)) +
                    "\n\\end{flushright}\\end{nospace}\n")
    replace = {
        '~': '$\sim$',
    }

    escape = ['#']

    out = "".join(lines)
    for search in replace:
        out = re.sub(search, replace[search], out)

    for c in escape:
        out = re.sub(r'([^\\])\%s' % c, r'\1\%s' % c, out)

    return out

@processor.register
def html(lines, contact_lines, *args):
    untex = ['LaTeX']

    for word in untex:
        # yuck
        replace = lambda l: l.replace(r"\%s" % word, word)
        lines = map(replace, lines)
        contact_lines = map(replace, contact_lines)

    lines.insert(0, "

%s
\n" %
                         ("
" + "

".join(contact_lines) + "
"))
    lines.insert(1, "
")
    lines.append("
")

    return "".join(lines)

def main():
    try:
        format = sys.argv[1]
    except IndexError:
        raise Exception("No format specified")

    lines = sys.stdin.readlines()

    contact_lines = []
    for line in lines[3:]:
        lines.remove(line)
        parts = [x.strip() for x in line.split("•")]
        if parts == ['']:
            break

        contact_lines.extend(parts)

    print processor.process(format, lines, contact_lines, *sys.argv[1:])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
from cStringIO import StringIO
import os
from string import punctuation
from zipfile import ZipFile

from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.db import models
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.core.fields import FileField
from mezzanine.core.models import Orderable, RichText
from mezzanine.pages.models import Page
from mezzanine.utils.importing import import_dotted_path

# Set the directory where gallery images are uploaded to,
# either MEDIA_ROOT + 'galleries', or filebrowser's upload
# directory if being used.
GALLERIES_UPLOAD_DIR = "galleries"
if settings.PACKAGE_NAME_FILEBROWSER in settings.INSTALLED_APPS:
    fb_settings = "%s.settings" % settings.PACKAGE_NAME_FILEBROWSER
    try:
        GALLERIES_UPLOAD_DIR = import_dotted_path(fb_settings).DIRECTORY
    except ImportError:
        pass

class Gallery(Page, RichText):
    """
    Page bucket for gallery photos.
    """

    zip_import = models.FileField(upload_to="galleries", blank=True,
                    help_text=_("Upload a zip file containing images, and "
                                "they'll be imported into this gallery."))

    class Meta:
        verbose_name = _("Gallery")
        verbose_name_plural = _("Galleries")

    def save(self, delete_zip_import=True, *args, **kwargs):
        """
        If a zip file is uploaded, extract any images from it and add
        them to the gallery, before removing the zip file.
        """
        super(Gallery, self).save(*args, **kwargs)
        if self.zip_import:
            zip_file = ZipFile(self.zip_import)
            from PIL import Image
            for name in zip_file.namelist():
                data = zip_file.read(name)
                try:
                    image = Image.open(StringIO(data))
                    image.load()
                    image = Image.open(StringIO(data))
                    image.verify()
                except:
                    continue
                path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                    name.decode("utf-8"))
                try:
                    saved_path = default_storage.save(path, ContentFile(data))
                except UnicodeEncodeError:
                    from warnings import warn
                    warn("A file was saved that contains unicode "
                         "characters in its path, but somehow the current "
                         "locale does not support utf-8. You may need to set "
                         "'LC_ALL' to a correct value, eg: 'en_US.UTF-8'.")
                    path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                        unicode(name, errors="ignore"))
                    saved_path = default_storage.save(path, ContentFile(data))
                self.images.add(GalleryImage(file=saved_path))
            if delete_zip_import:
                zip_file.close()
                self.zip_import.delete(save=True)

class GalleryImage(Orderable):

    gallery = models.ForeignKey("Gallery", related_name="images")
    file = FileField(max_length=200, upload_to="galleries", format="Image")
    description = models.CharField(max_length=1000, blank=True)

    class Meta:
        verbose_name = _("Image")
        verbose_name_plural = _("Images")

    def __unicode__(self):
        return self.description

    def save(self, *args, **kwargs):
        """
        If no description is given when created, create one from the
        file name.
        """
        if not self.id and not self.description:
            name = self.file.path.split("/")[-1].rsplit(".", 1)[0]
            name = name.replace("'", "")
            name = "".join([c if c not in punctuation else " " for c in name])
            # str.title() doesn't deal with unicode very well.
            # http://bugs.python.org/issue6412
            name = "".join([s.upper() if i == 0 or name[i - 1] == " " else s
                            for i, s in enumerate(name)])
            self.description = name
        super(GalleryImage, self).save(*args, **kwargs)
<commit_msg>Use file.url instead of file.path to auto-generate descriptions for gallery image, as remote storage backends such as S3BotoStorage do not support the file.path method.<commit_after>
from cStringIO import StringIO
import os
from string import punctuation
from urllib import unquote
from zipfile import ZipFile

from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.db import models
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.core.fields import FileField
from mezzanine.core.models import Orderable, RichText
from mezzanine.pages.models import Page
from mezzanine.utils.importing import import_dotted_path

# Set the directory where gallery images are uploaded to,
# either MEDIA_ROOT + 'galleries', or filebrowser's upload
# directory if being used.
GALLERIES_UPLOAD_DIR = "galleries"
if settings.PACKAGE_NAME_FILEBROWSER in settings.INSTALLED_APPS:
    fb_settings = "%s.settings" % settings.PACKAGE_NAME_FILEBROWSER
    try:
        GALLERIES_UPLOAD_DIR = import_dotted_path(fb_settings).DIRECTORY
    except ImportError:
        pass

class Gallery(Page, RichText):
    """
    Page bucket for gallery photos.
    """

    zip_import = models.FileField(upload_to="galleries", blank=True,
                    help_text=_("Upload a zip file containing images, and "
                                "they'll be imported into this gallery."))

    class Meta:
        verbose_name = _("Gallery")
        verbose_name_plural = _("Galleries")

    def save(self, delete_zip_import=True, *args, **kwargs):
        """
        If a zip file is uploaded, extract any images from it and add
        them to the gallery, before removing the zip file.
        """
        super(Gallery, self).save(*args, **kwargs)
        if self.zip_import:
            zip_file = ZipFile(self.zip_import)
            from PIL import Image
            for name in zip_file.namelist():
                data = zip_file.read(name)
                try:
                    image = Image.open(StringIO(data))
                    image.load()
                    image = Image.open(StringIO(data))
                    image.verify()
                except:
                    continue
                path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                    name.decode("utf-8"))
                try:
                    saved_path = default_storage.save(path, ContentFile(data))
                except UnicodeEncodeError:
                    from warnings import warn
                    warn("A file was saved that contains unicode "
                         "characters in its path, but somehow the current "
                         "locale does not support utf-8. You may need to set "
                         "'LC_ALL' to a correct value, eg: 'en_US.UTF-8'.")
                    path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                        unicode(name, errors="ignore"))
                    saved_path = default_storage.save(path, ContentFile(data))
                self.images.add(GalleryImage(file=saved_path))
            if delete_zip_import:
                zip_file.close()
                self.zip_import.delete(save=True)

class GalleryImage(Orderable):

    gallery = models.ForeignKey("Gallery", related_name="images")
    file = FileField(max_length=200, upload_to="galleries", format="Image")
    description = models.CharField(max_length=1000, blank=True)

    class Meta:
        verbose_name = _("Image")
        verbose_name_plural = _("Images")

    def __unicode__(self):
        return self.description

    def save(self, *args, **kwargs):
        """
        If no description is given when created, create one from the
        file name.
        """
        if not self.id and not self.description:
            name = unquote(self.file.url).split("/")[-1].rsplit(".", 1)[0]
            name = name.replace("'", "")
            name = "".join([c if c not in punctuation else " " for c in name])
            # str.title() doesn't deal with unicode very well.
            # http://bugs.python.org/issue6412
            name = "".join([s.upper() if i == 0 or name[i - 1] == " " else s
                            for i, s in enumerate(name)])
            self.description = name
        super(GalleryImage, self).save(*args, **kwargs)
<|endoftext|>


<commit_before>'''
synbiochem (c) University of Manchester 2015

synbiochem is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
import itertools
from matplotlib.colors import LinearSegmentedColormap
import numpy
import pylab
import scipy.spatial
import sys
import tempfile
import urllib

from Bio.PDB.PDBParser import PDBParser

# KD Hydrophobicity, EIIP, Helix, Sheet, Turn
AMINO_ACID_PROPERTIES = {
    'A': [1.8, -0.0667, 32.9, -23.6, -41.6],
    'R': [-4.5, 0.2674, 0, -6.2, -5.1],
    'N': [-3.5, -0.2589, -24.8, -41.6, 44.5],
    'D': [-3.5, 0.4408, 5.8, -41.6, 37.8],
    'C': [2.5, 0.1933, -5.1, 6.8, 17.4],
    'Q': [-3.5, 0.1545, 11.3, 0, -2],
    'E': [-3.5, -0.2463, 36.5, -67.3, -30.1],
    'G': [-0.4, -0.2509, -46.2, -13.9, 44.5],
    'H': [-3.2, -0.1414, 11.3, -18.6, -5.1],
    'I': [4.5, -0.2794, -1, 45.1, -75.5],
    'L': [3.8, -0.2794, 26.2, 15.7, -52.8],
    'K': [-3.9, -0.0679, 19.1, -31.5, 1],
    'M': [1.9, 0.1899, 27.8, 1, -51.1],
    'F': [2.8, 0.26, 10.4, 20.7, -51.1],
    'P': [-1.6, -0.1665, -59.8, -47.8, 41.9],
    'S': [-0.8, 0.1933, -32.9, -6.2, 35.8],
    'T': [-0.7, 0.2572, -24.8, 28.5, -4.1],
    'W': [-0.9, 0.0331, 3, 21.5, -4.1],
    'Y': [-1.3, 0.0148, -31.5, 27, 13.1],
    'V': [4.2, -0.2469, -3, 49.5, -69.3]
}

def calc_proximity(pdb_id):
    '''Calculates residue proximities from PDB file.'''
    with tempfile.TemporaryFile() as pdb_file:
        opener = urllib.URLopener()
        opener.retrieve('http://www.rcsb.org/pdb/files/' + pdb_id + '.pdb',
                        pdb_file.name)
        parser = PDBParser()

        chains = [c for c in parser.get_structure(pdb_id,
                                                  pdb_file.name).get_chains()]

        coords = [residue.child_dict['CA'].get_coord()
                  for residue in chains[0] if 'CA' in residue.child_dict]

        return scipy.spatial.distance.cdist(coords, coords, 'euclidean')

def plot(values, plot_filename, plot_format, title, max_value=None):
    '''Plots 3d matrix values.'''
    fig = pylab.figure()
    sub_plot = fig.add_subplot(111)

    cmap = LinearSegmentedColormap.from_list(name='name', colors=['g', 'w'],
                                             N=10)

    cax = sub_plot.imshow(values, interpolation='nearest', cmap=cmap)
    cax.set_clim(0.0, max_value)
    sub_plot.set_title(title)

    # Add colorbar, make sure to specify tick locations to match desired tick
    # labels
    min_val = numpy.min(values)
    max_val = numpy.max(values)
    cbar = fig.colorbar(cax, ticks=[min_val, max_val])
    cbar.set_ticks([min_val, max_val])

    pylab.savefig(plot_filename, format=plot_format)

def export_html(values):
    print ','.join(['[' + ','.join([str(v) for v in value.tolist()]) + ']'
                    for value in values])

def main(argv):
    '''main method.'''
    pdb_id = argv[1]
    proximities = calc_proximity(pdb_id)

    input = [AMINO_ACID_PROPERTIES[amino_acid] for amino_acid in amino_acids]
    output = itertools.chain.from_iterable(proximities)

    # export_html(proximities)

    plot_format = 'png'
    plot_filename = pdb_id + '.' + plot_format
    plot(proximities, plot_filename, plot_format,
         pdb_id + ' proximity plot')

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Added deep learning parameter extraction code.<commit_after>'''
synbiochem (c) University of Manchester 2015

synbiochem is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
from itertools import chain
import math
from matplotlib.colors import LinearSegmentedColormap
import pylab
import random
import scipy.spatial
import sys
import tempfile
import urllib
import urllib2

from Bio import SeqUtils
from Bio.PDB.PDBParser import PDBParser
import numpy

# KD Hydrophobicity, EIIP, Helix, Sheet, Turn
AMINO_ACID_PROPERTIES = {
    'A': [1.8, -0.0667, 32.9, -23.6, -41.6],
    'R': [-4.5, 0.2674, 0, -6.2, -5.1],
    'N': [-3.5, -0.2589, -24.8, -41.6, 44.5],
    'D': [-3.5, 0.4408, 5.8, -41.6, 37.8],
    'C': [2.5, 0.1933, -5.1, 6.8, 17.4],
    'Q': [-3.5, 0.1545, 11.3, 0, -2],
    'E': [-3.5, -0.2463, 36.5, -67.3, -30.1],
    'G': [-0.4, -0.2509, -46.2, -13.9, 44.5],
    'H': [-3.2, -0.1414, 11.3, -18.6, -5.1],
    'I': [4.5, -0.2794, -1, 45.1, -75.5],
    'L': [3.8, -0.2794, 26.2, 15.7, -52.8],
    'K': [-3.9, -0.0679, 19.1, -31.5, 1],
    'M': [1.9, 0.1899, 27.8, 1, -51.1],
    'F': [2.8, 0.26, 10.4, 20.7, -51.1],
    'P': [-1.6, -0.1665, -59.8, -47.8, 41.9],
    'S': [-0.8, 0.1933, -32.9, -6.2, 35.8],
    'T': [-0.7, 0.2572, -24.8, 28.5, -4.1],
    'W': [-0.9, 0.0331, 3, 21.5, -4.1],
    'Y': [-1.3, 0.0148, -31.5, 27, 13.1],
    'V': [4.2, -0.2469, -3, 49.5, -69.3]
}

def get_pdb_ids(scop_id, max_ids=None):
    '''Returns all PDB ids for a given SCOP id.'''
    url = 'http://www.rcsb.org/pdb/results/results.do' + \
        '?startAt=0&resultsperpage=1000000&outformat=text&qrid=' + \
        scop_id

    ids = [line.split()[0] for line in urllib2.urlopen(url)]

    return ids if max_ids is None \
        else random.sample(ids, min(len(ids), max_ids))

def get_residues(pdb_id):
    '''Gets the residues in a PDB file.'''
    return [SeqUtils.seq1(''.join([residue.get_resname()
                                   for residue in chn
                                   if 'CA' in residue.child_dict]))
            for chn in get_structure(pdb_id).get_chains()]

def get_structure(pdb_id):
    '''Returns a PDB structure.'''
    with tempfile.TemporaryFile() as pdb_file:
        opener = urllib.URLopener()
        opener.retrieve('http://www.rcsb.org/pdb/files/' + pdb_id + '.pdb',
                        pdb_file.name)
        parser = PDBParser()
        return parser.get_structure(pdb_id, pdb_file.name)

def calc_proximities(pdb_id):
    '''Calculates residue proximities from PDB file.'''
    structure = get_structure(pdb_id)
    chains = [c for c in structure.get_chains()]

    coords = [[residue.child_dict['CA'].get_coord()
              for residue in chn
              if 'CA' in residue.child_dict]
              for chn in chains]

    return [scipy.spatial.distance.cdist(coord, coord, 'euclidean')
            for coord in coords]

def plot_proximities(pdb_id):
    '''Plots proximity plot(s).'''
    all_proximities = calc_proximities(pdb_id)

    plot_format = 'png'

    for idx, proximities in enumerate(all_proximities):
        name = pdb_id + '_' + str(idx+1)
        _plot(proximities, name + '.' + plot_format, plot_format,
              name + ' proximity plot')

def get_learning_data(pdb_id):
    '''Gets input/output for deep learning.'''
    all_proximities = calc_proximities(pdb_id)
    all_residues = get_residues(pdb_id)

    inpt = [list(chain.from_iterable([AMINO_ACID_PROPERTIES[amino_acid]
                                      if amino_acid in AMINO_ACID_PROPERTIES
                                      else [0.0, 0.0, 0.0]
                                      for amino_acid in residues]))
            for residues in all_residues]

    output = [list(chain.from_iterable(proximities))
              for proximities in all_proximities]

    return pdb_id, all_residues, inpt, output

def sample_learning_data(pdb_id, num_samples, nmer_len):
    pdb_id, all_residues, inpt, output = get_learning_data(pdb_id)

    results = []

    num_aa_props = len(AMINO_ACID_PROPERTIES['A'])

    for _ in range(num_samples):
        chn = int(random.random() * len(all_residues))
        start = int(random.random() * (len(all_residues[chn]) - nmer_len))
        results.append([pdb_id,
                        all_residues[chn][start:start+nmer_len],
                        inpt[chn][start * num_aa_props:(start + nmer_len) *
                                  num_aa_props]])

    return results

def _plot(values, plot_filename, plot_format, title, max_value=None):
    '''Plots 3d matrix values.'''
    fig = pylab.figure()
    sub_plot = fig.add_subplot(111)

    cmap = LinearSegmentedColormap.from_list(name='name', colors=['g', 'w'],
                                             N=10)

    cax = sub_plot.imshow(values, interpolation='nearest', cmap=cmap)
    cax.set_clim(0.0, max_value)
    sub_plot.set_title(title)

    # Add colorbar, make sure to specify tick locations to match desired tick
    # labels
    min_val = numpy.min(values)
    max_val = numpy.max(values)
    cbar = fig.colorbar(cax, ticks=[min_val, max_val])
    cbar.set_ticks([min_val, max_val])

    pylab.savefig(plot_filename, format=plot_format)

def main(argv):
    '''main method.'''
    for pdb_id in get_pdb_ids(argv[1], argv[2]):
        print '\t'.join([str(x) for x in sample_learning_data(pdb_id,
                                                              int(argv[3]),
                                                              int(argv[4]))])

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 matrix.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from twisted.internet import defer, reactor
from twisted.web.client import _AgentBase, _URI, readBody
from twisted.web.http_headers import Headers

from synapse.http.endpoint import matrix_endpoint
from synapse.util.async import sleep

from syutil.jsonutil import encode_canonical_json

from synapse.api.errors import CodeMessageException

import json
import logging
import urllib

logger = logging.getLogger(__name__)

# FIXME: SURELY these should be killed?!
_destination_mappings = {
    "red": "localhost:8080",
    "blue": "localhost:8081",
    "green": "localhost:8082",
}

class HttpClient(object):
    """ Interface for talking json over http
    """

    def put_json(self, destination, path, data):
        """ Sends the specifed json data using PUT

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            data (dict): A dict containing the data that will be used as
                the request body. This will be encoded as JSON.

        Returns:
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body. On a 4xx or 5xx error response a
            CodeMessageException is raised.
        """
        pass

    def get_json(self, destination, path, args=None):
        """ Get's some json from the given host homeserver and path

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            args (dict): A dictionary used to create query strings, defaults to
                None.
                **Note**: The value of each key is assumed to be an iterable
                and *not* a string.

        Returns:
            Deferred: Succeeds when we get *any* HTTP response.

            The result of the deferred is a tuple of `(code, response)`,
            where `response` is a dict representing the decoded JSON body.
        """
        pass

class MatrixHttpAgent(_AgentBase):

    def __init__(self, reactor, pool=None):
        _AgentBase.__init__(self, reactor, pool)

    def request(self, destination, endpoint, method, path, params, query,
                headers, body_producer):

        host = b""
        port = 0
        fragment = b""

        parsed_URI = _URI(b"http", destination, host, port, path, params,
                          query, fragment)

        # Set the connection pool key to be the destination.
        key = destination

        return self._requestWithEndpoint(key, endpoint, method, parsed_URI,
                                         headers, body_producer,
                                         parsed_URI.originForm)

class TwistedHttpClient(HttpClient):
    """ Wrapper around the twisted HTTP client api.

    Attributes:
        agent (twisted.web.client.Agent): The twisted Agent used to send the
            requests.
    """

    def __init__(self, hs):
        self.agent = MatrixHttpAgent(reactor)
        self.hs = hs

    @defer.inlineCallbacks
    def put_json(self, destination, path, data):
        if destination in _destination_mappings:
            destination = _destination_mappings[destination]

        response = yield self._create_request(
            destination.encode("ascii"),
            "PUT",
            path.encode("ascii"),
            producer=_JsonProducer(data),
            headers_dict={"Content-Type": ["application/json"]}
        )

        logger.debug("Getting resp body")
        body = yield readBody(response)
        logger.debug("Got resp body")

        defer.returnValue((response.code, body))

    @defer.inlineCallbacks
    def get_json(self, destination, path, args={}):
        if destination in _destination_mappings:
            destination = _destination_mappings[destination]

        logger.debug("get_json args: %s", args)
        query_bytes = urllib.urlencode(args, True)

        response = yield self._create_request(
            destination.encode("ascii"),
            "GET",
            path.encode("ascii"),
            query_bytes=query_bytes
        )

        body = yield readBody(response)

        defer.returnValue(json.loads(body))

    @defer.inlineCallbacks
    def _create_request(self, destination, method, path_bytes, param_bytes=b"",
                        query_bytes=b"", producer=None, headers_dict={}):
        """ Creates and sends a request to the given url
        """
        headers_dict[b"User-Agent"] = [b"Synapse"]
        headers_dict[b"Host"] = [destination]

        logger.debug("Sending request to %s: %s %s;%s?%s",
                     destination, method, path_bytes, param_bytes, query_bytes)

        logger.debug(
            "Types: %s",
            [
                type(destination), type(method), type(path_bytes),
                type(param_bytes),
                type(query_bytes)
            ]
        )

        retries_left = 5

        # TODO: setup and pass in an ssl_context to enable TLS
        endpoint = matrix_endpoint(
            reactor, destination, timeout=10,
            ssl_context_factory=self.hs.tls_context_factory
        )

        while True:
            try:
                response = yield self.agent.request(
                    destination,
                    endpoint,
                    method,
                    path_bytes,
                    param_bytes,
                    query_bytes,
                    Headers(headers_dict),
                    producer
                )

                logger.debug("Got response to %s", method)
                break
            except Exception as e:
                logger.exception("Got error in _create_request")
                _print_ex(e)

                if retries_left:
                    yield sleep(2 ** (5 - retries_left))
                    retries_left -= 1
                else:
                    raise

        if 200 <= response.code < 300:
            # We need to update the transactions table to say it was sent?
            pass
        else:
            # :'(
            # Update transactions table?
            logger.error(
                "Got response %d %s", response.code, response.phrase
            )
            raise CodeMessageException(
                response.code, response.phrase
            )

        defer.returnValue(response)

def _print_ex(e):
    if hasattr(e, "reasons") and e.reasons:
        for ex in e.reasons:
            _print_ex(ex)
    else:
        logger.exception(e)

class _JsonProducer(object):
    """ Used by the twisted http client to create the HTTP body from json
    """
    def __init__(self, jsn):
        self.body = encode_canonical_json(jsn)
        self.length = len(self.body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return defer.succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass
<commit_msg>hs: Updated synapse.http.client to handle DNSLookupErrors and bail immediately.<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 matrix.org
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from twisted.internet import defer, reactor
from twisted.internet.error import DNSLookupError
from twisted.web.client import _AgentBase, _URI, readBody
from twisted.web.http_headers import Headers

from synapse.http.endpoint import matrix_endpoint
from synapse.util.async import sleep

from syutil.jsonutil import encode_canonical_json

from synapse.api.errors import CodeMessageException, SynapseError

import json
import logging
import urllib

logger = logging.getLogger(__name__)

# FIXME: SURELY these should be killed?!
_destination_mappings = {
    "red": "localhost:8080",
    "blue": "localhost:8081",
    "green": "localhost:8082",
}

class HttpClient(object):
    """ Interface for talking json over http
    """

    def put_json(self, destination, path, data):
        """ Sends the specifed json data using PUT

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            data (dict): A dict containing the data that will be used as
                the request body. This will be encoded as JSON.

        Returns:
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body. On a 4xx or 5xx error response a
            CodeMessageException is raised.
        """
        pass

    def get_json(self, destination, path, args=None):
        """ Get's some json from the given host homeserver and path

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            args (dict): A dictionary used to create query strings, defaults to
                None.
                **Note**: The value of each key is assumed to be an iterable
                and *not* a string.

        Returns:
            Deferred: Succeeds when we get *any* HTTP response.

            The result of the deferred is a tuple of `(code, response)`,
            where `response` is a dict representing the decoded JSON body.
        """
        pass

class MatrixHttpAgent(_AgentBase):

    def __init__(self, reactor, pool=None):
        _AgentBase.__init__(self, reactor, pool)

    def request(self, destination, endpoint, method, path, params, query,
                headers, body_producer):

        host = b""
        port = 0
        fragment = b""

        parsed_URI = _URI(b"http", destination, host, port, path, params,
                          query, fragment)

        # Set the connection pool key to be the destination.
        key = destination

        return self._requestWithEndpoint(key, endpoint, method, parsed_URI,
                                         headers, body_producer,
                                         parsed_URI.originForm)

class TwistedHttpClient(HttpClient):
    """ Wrapper around the twisted HTTP client api.

    Attributes:
        agent (twisted.web.client.Agent): The twisted Agent used to send the
            requests.
    """

    def __init__(self, hs):
        self.agent = MatrixHttpAgent(reactor)
        self.hs = hs

    @defer.inlineCallbacks
    def put_json(self, destination, path, data):
        if destination in _destination_mappings:
            destination = _destination_mappings[destination]

        response = yield self._create_request(
            destination.encode("ascii"),
            "PUT",
            path.encode("ascii"),
            producer=_JsonProducer(data),
            headers_dict={"Content-Type": ["application/json"]}
        )

        logger.debug("Getting resp body")
        body = yield readBody(response)
        logger.debug("Got resp body")

        defer.returnValue((response.code, body))

    @defer.inlineCallbacks
    def get_json(self, destination, path, args={}):
        if destination in _destination_mappings:
            destination = _destination_mappings[destination]

        logger.debug("get_json args: %s", args)
        query_bytes = urllib.urlencode(args, True)

        response = yield self._create_request(
            destination.encode("ascii"),
            "GET",
            path.encode("ascii"),
            query_bytes=query_bytes
        )

        body = yield readBody(response)

        defer.returnValue(json.loads(body))

    @defer.inlineCallbacks
    def _create_request(self, destination, method, path_bytes, param_bytes=b"",
                        query_bytes=b"", producer=None, headers_dict={}):
        """ Creates and sends a request to the given url
        """
        headers_dict[b"User-Agent"] = [b"Synapse"]
        headers_dict[b"Host"] = [destination]

        logger.debug("Sending request to %s: %s %s;%s?%s",
                     destination, method, path_bytes, param_bytes, query_bytes)

        logger.debug(
            "Types: %s",
            [
                type(destination), type(method), type(path_bytes),
                type(param_bytes),
                type(query_bytes)
            ]
        )

        retries_left = 5

        # TODO: setup and pass in an ssl_context to enable TLS
        endpoint = matrix_endpoint(
            reactor, destination, timeout=10,
            ssl_context_factory=self.hs.tls_context_factory
        )

        while True:
            try:
                response = yield self.agent.request(
                    destination,
                    endpoint,
                    method,
                    path_bytes,
                    param_bytes,
                    query_bytes,
                    Headers(headers_dict),
                    producer
                )

                logger.debug("Got response to %s", method)
                break
            except DNSLookupError as dns:
                logger.warn("DNS Lookup failed to %s with %s", destination,
                            dns)
                raise SynapseError(400, "Domain specified not found.")
            except Exception as e:
                logger.exception("Got error in _create_request")
                _print_ex(e)

                if retries_left:
                    yield sleep(2 ** (5 - retries_left))
                    retries_left -= 1
                else:
                    raise

        if 200 <= response.code < 300:
            # We need to update the transactions table to say it was sent?
            pass
        else:
            # :'(
            # Update transactions table?
            logger.error(
                "Got response %d %s", response.code, response.phrase
            )
            raise CodeMessageException(
                response.code, response.phrase
            )

        defer.returnValue(response)

def _print_ex(e):
    if hasattr(e, "reasons") and e.reasons:
        for ex in e.reasons:
            _print_ex(ex)
    else:
        logger.exception(e)

class _JsonProducer(object):
    """ Used by the twisted http client to create the HTTP body from json
    """
    def __init__(self, jsn):
        self.body = encode_canonical_json(jsn)
        self.length = len(self.body)

    def startProducing(self, consumer):
        consumer.write(self.body)
        return defer.succeed(None)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        pass
<|endoftext|>