<commit_before># -*- coding: utf-8 -*-
__author__ = 'peter'

from handlers.avatar import AvatarRequestHandler
from backends import db
import momoko
from oauthlib.oauth2 import RequestValidator, LegacyApplicationServer, BackendApplicationServer
import logging
from datetime import datetime, timedelta
from tornado import web
from tornado import gen
from models import Box, Authorization
from cred import AvatarCredentailsChecker, BoxCredentialsChecker, BoxClientIdClientSecret, AvatarEmailPasswordBoxId

log = logging.getLogger(__name__)

class TardisRequestValidator(RequestValidator):
    def __init__(self, client, authorization, avatar):
        super(TardisRequestValidator, self).__init__()
        self.client=client
        self.authorization=authorization
        self.avatar=avatar

    def authenticate_client(self, request, *args, **kwargs):
        request.client=self.client
        if request.client:
            return True
        return False

    def get_default_scopes(self, client_id, request, *args, **kwargs):
        return None

    def get_original_scopes(self, refresh_token, request, *args, **kwargs):
        return None

    def is_within_original_scope(self, request_scopes, refresh_token, request, *args, **kwargs):
        return True

    def save_bearer_token(self, token, request, *args, **kwargs):
        authorization=Authorization(access_token=token['access_token'], refresh_token=token['refresh_token'],
                                    expires_at=datetime.utcnow() + timedelta(seconds=100), avatar=request.avatar,
                                    created_at=datetime.utcnow(), updated_at=datetime.utcnow())
        self.authorization.save()

    def validate_bearer_token(self, token, scopes, request):
        if datetime.utcnow() > self.authorization.expires_at:
            return False
        request.access_token = self.authorization.access_token
        request.user = self.authorization.avatar
        request.client = self.authorization.avatar.box
        return True

    def validate_grant_type(self, client_id, grant_type, client, request, *args, **kwargs):
        return grant_type in ['password', 'client_credentials']

    def validate_refresh_token(self, refresh_token, client, request, *args, **kwargs):
        if self.authorization and self.authorization.avatar.box.id==client.box_id:
            request.client = self.authorization.avatar.box
            request.avatar =self.authorization.avatar
            return True
        return False

    def validate_scopes(self, client_id, scopes, client, request, *args, **kwargs):
        return True

    def validate_user(self, username, password, client, request, *args, **kwargs):
        request.avatar=self.avatar
        if request.avatar:
            return True
        return False

class TokenHandler(AvatarRequestHandler):
    @gen.coroutine
    def post(self):
        checker = BoxCredentialsChecker()
        client=yield gen.Task(checker.requestBox, BoxClientIdClientSecret(self.get_argument('client_id', default=None), self.get_argument('client_secret', default=None)))
        authorization=None

        access_token=self.get_argument('access_token', default=None)
        if access_token:
            authorization=yield gen.Task(Authorization.select().where(Authorization.access_token==access_token).get)

        refresh_token=self.get_argument('refresh_token', default=None)
        if refresh_token:
            authorization=yield gen.Task(Authorization.select().where(Authorization.refresh_token_token==refresh_token).get)

        credentials=AvatarEmailPasswordBoxId(self.get_argument('username', default=None),
                                             self.get_argument('password', default=None),
                                             client.id)
        checker=AvatarCredentailsChecker()
        avatar=checker.requestAvatar(credentials)

        validator = TardisRequestValidator(client, authorization, avatar)

        if self.get_argument('grant_type', default=None)=='password':
            resourceOwnerPasswordCredentialsProvider=LegacyApplicationServer(validator)
            headers, body, status=resourceOwnerPasswordCredentialsProvider.create_token_response(self.request.uri,
                                                                                http_method=self.request.method,
                                                                                body=self.request.body,
                                                                                headers=self.request.headers)
        if self.get_argument('grant_type', default=None)=='client_credentials':
            clientCredentialsProvider=BackendApplicationServer(validator)
            headers, body, status=clientCredentialsProvider.create_token_response(self.request.uri,
                                                                   http_method=self.request.method,
                                                                   body=self.request.body,
                                                                   headers=self.request.headers)
        self.set_status(status)
        self.write(body)
        for k, v in headers.items():
            self.set_header(k,v)
        self.finish()<commit_msg>synchronous RequestValidator<commit_after># -*- coding: utf-8 -*-
__author__ = 'peter'

from handlers.avatar import AvatarRequestHandler
from backends import db
import momoko
from oauthlib.oauth2 import RequestValidator, LegacyApplicationServer, BackendApplicationServer
import logging
from datetime import datetime, timedelta
from tornado import web
from tornado import gen
from models import Box, Authorization
from cred import AvatarCredentailsChecker, BoxCredentialsChecker, BoxClientIdClientSecret, AvatarEmailPasswordBoxId

log = logging.getLogger(__name__)

class TardisRequestValidator(RequestValidator):
    def __init__(self, client, authorization, avatar):
        super(TardisRequestValidator, self).__init__()
        self.client=client
        self.authorization=authorization
        self.avatar=avatar

    def authenticate_client(self, request, *args, **kwargs):
        request.client=self.client
        if request.client:
            return True
        return False

    def get_default_scopes(self, client_id, request, *args, **kwargs):
        return None

    def get_original_scopes(self, refresh_token, request, *args, **kwargs):
        return None

    def is_within_original_scope(self, request_scopes, refresh_token, request, *args, **kwargs):
        return True

    def save_bearer_token(self, token, request, *args, **kwargs):
        authorization=Authorization(access_token=token['access_token'], refresh_token=token['refresh_token'],
                                    expires_at=datetime.utcnow() + timedelta(seconds=100), avatar=request.avatar,
                                    created_at=datetime.utcnow(), updated_at=datetime.utcnow())
        authorization.save()

    def validate_bearer_token(self, token, scopes, request):
        if datetime.utcnow() > self.authorization.expires_at:
            return False
        request.access_token = self.authorization.access_token
        request.user = self.authorization.avatar
        request.client = self.authorization.avatar.box
        return True

    def validate_grant_type(self, client_id, grant_type, client, request, *args, **kwargs):
        return grant_type in ['password', 'client_credentials']

    def validate_refresh_token(self, refresh_token, client, request, *args, **kwargs):
        if self.authorization and self.authorization.avatar.box.id==client.box_id:
            request.client = self.authorization.avatar.box
            request.avatar =self.authorization.avatar
            return True
        return False

    def validate_scopes(self, client_id, scopes, client, request, *args, **kwargs):
        return True

    def validate_user(self, username, password, client, request, *args, **kwargs):
        request.avatar=self.avatar
        if request.avatar:
            return True
        return False

class TokenHandler(AvatarRequestHandler):
    @gen.coroutine
    def post(self):
        checker = BoxCredentialsChecker()
        client=yield gen.Task(checker.requestBox, BoxClientIdClientSecret(self.get_argument('client_id', default=None), self.get_argument('client_secret', default=None)))
        authorization=None

        access_token=self.get_argument('access_token', default=None)
        if access_token:
            authorization=yield gen.Task(Authorization.select().where(Authorization.access_token==access_token).get)

        refresh_token=self.get_argument('refresh_token', default=None)
        if refresh_token:
            authorization=yield gen.Task(Authorization.select().where(Authorization.refresh_token_token==refresh_token).get)

        credentials=AvatarEmailPasswordBoxId(self.get_argument('username', default=None),
                                             self.get_argument('password', default=None),
                                             client.id)
        checker=AvatarCredentailsChecker()
        avatar=checker.requestAvatar(credentials)

        validator = TardisRequestValidator(client, authorization, avatar)

        if self.get_argument('grant_type', default=None)=='password':
            resourceOwnerPasswordCredentialsProvider=LegacyApplicationServer(validator)
            headers, body, status=resourceOwnerPasswordCredentialsProvider.create_token_response(self.request.uri,
                                                                                http_method=self.request.method,
                                                                                body=self.request.body,
                                                                                headers=self.request.headers)
        if self.get_argument('grant_type', default=None)=='client_credentials':
            clientCredentialsProvider=BackendApplicationServer(validator)
            headers, body, status=clientCredentialsProvider.create_token_response(self.request.uri,
                                                                   http_method=self.request.method,
                                                                   body=self.request.body,
                                                                   headers=self.request.headers)
        self.set_status(status)
        self.write(body)
        for k, v in headers.items():
            self.set_header(k,v)
        self.finish()<|endoftext|>


<commit_before># -*- coding: utf8 -*-

from Model import Model

class CharacterModel(Model):
    pass
<commit_msg>working methods of Model added<commit_after># -*- coding: utf8 -*-

from Model import Model

class CharacterModel(Model):
    _characterFields = {}

    #public
    def setSpecies(self, species):
        self._characterFields["id_species"] = str(species)

    def setGender(self, gender):
        self._characterFields["id_gender"] = str(gender)

    def setName(self, name):
        self._characterFields["name"] = name

    def getName(self):
        return self._characterFields["name"]

    def getPk(self):
        return self._characterFields["id_character"]

    def save(self):
        self.__setPk(Model.insert("character", self._characterFields));
        return True;

    @staticmethod
    def loadByNameAndIdPlayer(name, playerId):
        character = {};

        query = "\
            SELECT\
                id_character,\
                name,\
                id_species,\
                id_gender\
            FROM\
                `character`\
            WHERE\
                name = ?\
            LIMIT 1";

        character = Model.fetchOneRow(query, (name));

        if len(character) == 0:
            return None
        else:
            model = CharacterModel()
            model._setPk(character[0].atoi())
            model.setName(character[1])
            model.setSpecies(character[2].atoi())
            model.setGender(character[3].atoi())

            return model

<|endoftext|>


<commit_before>###########
# Parsers #
###########
import re
from sqlalchemy import Table, MetaData
from datetime import datetime

pgsql_qualifier = re.compile(r"^[\w\d_\.]+$")

def is_pg_qual(x):
    return pgsql_qualifier.search(x)

def quote(x):
    if not x.startswith('"') and '(' not in x:
        return '"' + x + '"'
    else:
        return x

def read_pgvalue(x):
    # TODO: Implement check for valid values
    if isinstance(x,str):
        return "'" + x + "'"
    if x is None:
        return 'null'
    return x

def read_operator(x, right):
    # TODO: Implement check for valid operators
    if isinstance(right, dict) and right['type'] == 'value' and ('value' not in right or right['value'] is None and x == '='):
        return 'is'
    return x

class ValidationError(Exception):
    def __init__(self, message, value):
        self.message = message
        self.value = value

def read_bool(s):
    if isinstance(s, bool):
        return s
    if s.lower() in ["true", "false"]:
        return s.lower() == "true"
    else:
        raise ValidationError("Invalid value in binary field", s)

def read_pgid(s):
    if is_pg_qual(s):
        return s
    raise ValidationError("Invalid identifier", s)

def set_meta_info(method, user, message=None):
    val_dict = {}
    val_dict['_user'] = user  # TODO: Add user handling
    val_dict['_submitted'] = datetime.now()
    val_dict['_autocheck'] = False
    val_dict['_humancheck'] = False
    val_dict['_type'] = method
    val_dict['_message'] = message
    return val_dict

def parse_insert(d, engine, context, message=None):
    table = Table(read_pgid(d['table']), MetaData(bind=engine), autoload=True,
                  schema=read_pgid(d['schema']))

    meta_cols = ['_message', '_user', '_submitted', '_autocheck',
                   '_humancheck', '_type']

    field_strings = []
    for field in d.get('fields',[]):
        assert ('type' in field and field['type'] == 'column'), 'Only pure column expressions are allowed in insert'

        field_strings.append(parse_expression(field))

    query = table.insert()

    if not 'method' in d:
        d['method'] = 'values'
    if d['method'] == 'default':
        query.values()
    elif d['method'] == 'values':
        if field_strings:
            assert(isinstance(d['values'],list))
            values = map(lambda x: zip(field_strings,x), d['values'])
        else:
            values = d['values']

        def clear_meta(vals):
            val_dict = vals
            # make sure meta fields are not compromised
            if context['user'].is_anonymous:
                username = 'Anonymous'
            else:
                username = context['user'].name
            val_dict.update(set_meta_info('insert', username, message))
            return val_dict

        values = list(map(clear_meta, values))

        query = query.values(values)
    elif d['method'] == 'query':
        raise NotImplementedError

    if 'returning' in d:
        query = query.returning(*map(parse_expression, d['returning']))

    return query

def parse_select(d):
    """
        Defintion of a select query according to
        http://www.postgresql.org/docs/9.3/static/sql-select.html

        not implemented:
            [ WITH [ RECURSIVE ] with_query [, ...] ]
            [ WINDOW window_name AS ( window_definition ) [, ...] ]
            [ FOR { UPDATE | NO KEY UPDATE | SHARE | KEY SHARE } [ OF table_name [, ...] ] [ NOWAIT ] [...] ]
    """
    s = 'SELECT'
    # [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ]
    if 'all' in d and read_bool(d['all']):
        s += ' ALL'
    elif 'distinct' in d:
        s += ' DISTINCT'
        if d['distinct']:
            s += ' (' + ', '.join(map(parse_expression, d)) + ') '

    L = []

    if 'fields' in d and d['fields']:
        for field in d['fields']:
            ss = parse_expression(field)
            if 'as' in field and '_count' not in d:
                ss += ' AS ' + read_pgid(field['as'])
            L.append(ss)
        s += ' ' + ', '.join(L)
    else:
        s += ' * '

    # [ FROM from_item [, ...] ]
    if 'from' in d:
        s += ' FROM ' + ', '.join(parse_from_item(f) for f in d['from'])

    # [ WHERE condition ]
    if 'where' in d:
        s += ' WHERE ' + parse_condition(d['where'])

    if 'group_by' in d:
        s += ' GROUP BY ' + ', '.join(
            parse_expression(f) for f in d['group_by'])

    if 'having' in d:
        s += ' HAVING ' + ', '.join(parse_condition(f) for f in d['having'])

    if 'select' in d:
        sel = d['select']
        if sel['type'].lower() in ['union', 'intersect', 'except']:
            s += ' ' + sel['type']
        else:
            raise ValidationError('UNION/INTERSECT/EXCEPT expected')
        if 'all' in sel and read_bool(sel['all']):
            s += ' ALL '
        elif 'distinct' in sel and read_bool(sel['distinct']):
            s += ' DISTINCT '
        s += parse_select(sel['select'])

    if 'order_by' in d:
        L = []
        for ob in d['order_by']:
            ss = ''
            ss += ' ORDER BY ' + parse_expression(ob)
            if 'ordering' in ob:
                if ob['ordering'] in ['asc', 'desc']:
                    ss += ' ' + ob['ordering']
                else:
                    ss += parse_operator(ob['ordering'])
            if 'nulls' in ob:
                ss += ' NULLS '
                if ob['nulls'].lower() in ['first', 'last']:
                    ss += ob['nulls']
                else:
                    raise ValidationError('Invalid NULLS option')
            L.append(ss)
        s += ', '.join(L)

    if 'limit' in d:
        s += ' LIMIT'
        if isinstance(d['limit'], str) and d['limit'].lower() == 'all':
            s += ' ALL'
        elif isinstance(d['limit'], int) or d['limit'].is_digit():
            s += ' ' + str(d['limit'])
        else:
            raise ValidationError('Invalid LIMIT (expected ALL or a digit)')

    if 'offset' in d and (
                isinstance(d['offset'], int) or d['offset'].is_digit()):
        s += ' OFFSET {} ROWS'.format(d['offset'])

    if 'fetch' in d and d['fetch'].is_digit():
        s += ' FETCH NEXT {} ROWS ONLY'.format(d['fetch'])
    return s

def parse_from_item(d):
    """
        Defintion of a from_item according to
        http://www.postgresql.org/docs/9.3/static/sql-select.html

        return: A from_item string with checked psql qualifiers.

        Not implemented:
            with_query_name [ [ AS ] alias [ ( column_alias [, ...] ) ] ]
            [ LATERAL ] function_name ( [ argument [, ...] ] ) [ AS ] alias [ ( column_alias [, ...] | column_definition [, ...] ) ]
            [ LATERAL ] function_name ( [ argument [, ...] ] ) AS ( column_definition [, ...] )
    """
    # TODO: If 'type' is not set assume just a table name is present
    if d['type'] == 'table':
        s = 'only ' if d.pop('only', False) else ''
        schema = read_pgid(d.pop('schema', ''))
        if schema:
            s += schema + "."
        s += read_pgid(d['table'])
        if 'star' in d and read_bool(d['star']):
            s += ' * '
        if 'alias' in d and read_bool(d['alias']):
            s += ' AS ' + read_pgid(d['alias'])
            s += ' (' + \
                 ', '.join(
                     map(read_pgid, d['alias'].pop('column_aliases', []))) + \
                 ')'

    elif d['type'] == 'select':
        s = 'LATERAL ' if read_bool(d['lateral']) else ''
        s += '(' + parse_select(d['select']) + ') '
        if 'alias' in d and read_bool(d['alias']):
            s += ' AS ' + read_pgid(d['alias'])
            s += ' (' + \
                 ', '.join(
                     map(read_pgid, d['alias'].pop('column_aliases', []))) + \
                 ')'

    elif d['type'] == 'join':
        s = parse_from_item(d['left'])
        if 'natural' in d and read_bool(d['natural']):
            s += ' NATURAL'
        if 'join_type' in d:
            if d['join_type'].lower().strip() in ['join', 'inner join', 'left join',
                                          'left outer join', 'right join',
                                          'right outer join', 'full join',
                                          'full inner join', 'full outer join', 'cross join']:
                s += ' ' + d['join_type']
            else:
                raise ValidationError('Invalid join type')
        else:
            s += ' JOIN '

        s += ' ' + parse_from_item(d['right'])

        if 'on' in d:
            s += ' ON ' + parse_condition(d['on'])
        elif 'using' in d:
            s += ' USING (' + ', '.join(map(read_pgid, d['using'])) + ')'
    return s

def parse_expression(d):
    # TODO: Implement
    if isinstance(d, dict):
        if d['type'] == 'column':
            name = quote(d['column'])
            if 'table' in d:
                name = quote(d['table']) + '.' + name
                if 'schema' in d:
                    name = quote(d['schema']) + '.' + name
            return name
        if d['type'] == 'grouping':
            return '(' + parse_expression(d['grouping']) + ')'
        if d['type'] == 'star':
            return ' * '
        if d['type'] == 'operator':
            return parse_operator(d)
        if d['type'] == 'operator_binary':
            return parse_operator(d)
        if d['type'] == 'operator_unary':
            return parse_operator_unary(d)
        if d['type'] == 'modifier_unary':
            return parse_modifier_unary(d)
        if d['type'] == 'function':
            return parse_function(d)
        if d['type'] == 'value':
            if 'value' in d:
                return read_pgvalue(d['value'])
            else:
                return 'null'
    if isinstance(d, list):
        return ', '.join(parse_expression(x) for x in d)
    if isinstance(d, str):
        return '\'' + d + '\''
    else:
        return d

def parse_condition(dl):
    # TODO: Implement
    import json
    print(json.dumps(dl, indent=2))
    if type(dl) == dict:
        dl = [dl]

    return " " + " AND ".join([parse_expression(d) for d in dl])

def parse_operator(d):
    if d['operator'] == 'as':
        return parse_expression(d['labeled']) + " AS " + d['label_name']
    else:
        return "%s %s %s" % (
            parse_expression(d['left']), read_operator(d['operator'], d['right']),
            parse_expression(d['right']))

def parse_operator_unary(d):
    return "%s %s" % (read_operator(d['operator'], d['operand']),
                      parse_expression(d['operand']))

def parse_modifier_unary(d):
    return "%s %s" % (parse_expression(d['operand']),
                      read_operator(d['operator'], d['operand']))

def parse_function(d):
    assert (read_pgid(d['function']))
    operand_struc = d['operands']
    if isinstance(operand_struc, list):
        operands = ', '.join(map(parse_expression, d['operands']))
    else:
        operands = parse_expression(operand_struc)

    return '{f}{ops}'.format(f=d['function'], ops=operands)

def cadd(d, key, string=None):
    if not string:
        string = key.upper() + ' '
    if d.pop(key, None):
        return string
    else:
        return ''

def parse_create_table(d):
    s = 'CREATE '
    if d.pop('global', None):
        s += 'GLOBAL '
    elif d.pop('local', None):
        s += 'LOCAL '

    s += (cadd(d, 'temp')
          + cadd(d, 'unlogged')
          + 'TABLE '
          + cadd(d, 'if_not_exists', 'IF NOT EXISTS ')
          + read_pgid(d['name']))

    fieldstrings = []

    for field in d.pop('fields', []):
        fs = ''
        if field['type'] == 'column':
            fs += read_pgid(field['name']) + ' '
            fs += read_pgid(field['data_type']) + ' '
            collate = field.pop('collate', None)
            if collate:
                fs += read_pgid(collate)
            fs += ', '.join([parse_column_constraint(cons)
                             for cons in field.pop('constraints', [])]) + ' '
        elif field['type'] == 'table_constraint':
            fs += parse_table_constraint(field)
        elif field['type'] == 'like':
            fs += 'LIKE '

def parse_column_constraint(d):
    raise NotImplementedError
    # TODO: Implement

def parse_table_constraint(d):
    raise NotImplementedError
    # TODO: Implement
<commit_msg>Fix condition handling<commit_after>###########
# Parsers #
###########
import re
from sqlalchemy import Table, MetaData
from datetime import datetime

pgsql_qualifier = re.compile(r"^[\w\d_\.]+$")

def is_pg_qual(x):
    return pgsql_qualifier.search(x)

def quote(x):
    if not x.startswith('"') and '(' not in x:
        return '"' + x + '"'
    else:
        return x

def read_pgvalue(x):
    # TODO: Implement check for valid values
    if isinstance(x,str):
        return "'" + x + "'"
    if x is None:
        return 'null'
    return x

def read_operator(x, right):
    # TODO: Implement check for valid operators
    if isinstance(right, dict) and right['type'] == 'value' and ('value' not in right or right['value'] is None and x == '='):
        return 'is'
    return x

class ValidationError(Exception):
    def __init__(self, message, value):
        self.message = message
        self.value = value

def read_bool(s):
    if isinstance(s, bool):
        return s
    if s.lower() in ["true", "false"]:
        return s.lower() == "true"
    else:
        raise ValidationError("Invalid value in binary field", s)

def read_pgid(s):
    if is_pg_qual(s):
        return s
    raise ValidationError("Invalid identifier", s)

def set_meta_info(method, user, message=None):
    val_dict = {}
    val_dict['_user'] = user  # TODO: Add user handling
    val_dict['_submitted'] = datetime.now()
    val_dict['_autocheck'] = False
    val_dict['_humancheck'] = False
    val_dict['_type'] = method
    val_dict['_message'] = message
    return val_dict

def parse_insert(d, engine, context, message=None):
    table = Table(read_pgid(d['table']), MetaData(bind=engine), autoload=True,
                  schema=read_pgid(d['schema']))

    meta_cols = ['_message', '_user', '_submitted', '_autocheck',
                   '_humancheck', '_type']

    field_strings = []
    for field in d.get('fields',[]):
        assert ('type' in field and field['type'] == 'column'), 'Only pure column expressions are allowed in insert'

        field_strings.append(parse_expression(field))

    query = table.insert()

    if not 'method' in d:
        d['method'] = 'values'
    if d['method'] == 'default':
        query.values()
    elif d['method'] == 'values':
        if field_strings:
            assert(isinstance(d['values'],list))
            values = map(lambda x: zip(field_strings,x), d['values'])
        else:
            values = d['values']

        def clear_meta(vals):
            val_dict = vals
            # make sure meta fields are not compromised
            if context['user'].is_anonymous:
                username = 'Anonymous'
            else:
                username = context['user'].name
            val_dict.update(set_meta_info('insert', username, message))
            return val_dict

        values = list(map(clear_meta, values))

        query = query.values(values)
    elif d['method'] == 'query':
        raise NotImplementedError

    if 'returning' in d:
        query = query.returning(*map(parse_expression, d['returning']))

    return query

def parse_select(d):
    """
        Defintion of a select query according to
        http://www.postgresql.org/docs/9.3/static/sql-select.html

        not implemented:
            [ WITH [ RECURSIVE ] with_query [, ...] ]
            [ WINDOW window_name AS ( window_definition ) [, ...] ]
            [ FOR { UPDATE | NO KEY UPDATE | SHARE | KEY SHARE } [ OF table_name [, ...] ] [ NOWAIT ] [...] ]
    """
    s = 'SELECT'
    # [ ALL | DISTINCT [ ON ( expression [, ...] ) ] ]
    if 'all' in d and read_bool(d['all']):
        s += ' ALL'
    elif 'distinct' in d:
        s += ' DISTINCT'
        if d['distinct']:
            s += ' (' + ', '.join(map(parse_expression, d)) + ') '

    L = []

    if 'fields' in d and d['fields']:
        for field in d['fields']:
            ss = parse_expression(field)
            if 'as' in field and '_count' not in d:
                ss += ' AS ' + read_pgid(field['as'])
            L.append(ss)
        s += ' ' + ', '.join(L)
    else:
        s += ' * '

    # [ FROM from_item [, ...] ]
    if 'from' in d:
        s += ' FROM ' + ', '.join(parse_from_item(f) for f in d['from'])

    # [ WHERE condition ]
    if 'where' in d:
        s += ' WHERE ' + parse_condition(d['where'])

    if 'group_by' in d:
        s += ' GROUP BY ' + ', '.join(
            parse_expression(f) for f in d['group_by'])

    if 'having' in d:
        s += ' HAVING ' + ', '.join(parse_condition(f) for f in d['having'])

    if 'select' in d:
        sel = d['select']
        if sel['type'].lower() in ['union', 'intersect', 'except']:
            s += ' ' + sel['type']
        else:
            raise ValidationError('UNION/INTERSECT/EXCEPT expected')
        if 'all' in sel and read_bool(sel['all']):
            s += ' ALL '
        elif 'distinct' in sel and read_bool(sel['distinct']):
            s += ' DISTINCT '
        s += parse_select(sel['select'])

    if 'order_by' in d:
        L = []
        for ob in d['order_by']:
            ss = ''
            ss += ' ORDER BY ' + parse_expression(ob)
            if 'ordering' in ob:
                if ob['ordering'] in ['asc', 'desc']:
                    ss += ' ' + ob['ordering']
                else:
                    ss += parse_operator(ob['ordering'])
            if 'nulls' in ob:
                ss += ' NULLS '
                if ob['nulls'].lower() in ['first', 'last']:
                    ss += ob['nulls']
                else:
                    raise ValidationError('Invalid NULLS option')
            L.append(ss)
        s += ', '.join(L)

    if 'limit' in d:
        s += ' LIMIT'
        if isinstance(d['limit'], str) and d['limit'].lower() == 'all':
            s += ' ALL'
        elif isinstance(d['limit'], int) or d['limit'].is_digit():
            s += ' ' + str(d['limit'])
        else:
            raise ValidationError('Invalid LIMIT (expected ALL or a digit)')

    if 'offset' in d and (
                isinstance(d['offset'], int) or d['offset'].is_digit()):
        s += ' OFFSET {} ROWS'.format(d['offset'])

    if 'fetch' in d and d['fetch'].is_digit():
        s += ' FETCH NEXT {} ROWS ONLY'.format(d['fetch'])
    return s

def parse_from_item(d):
    """
        Defintion of a from_item according to
        http://www.postgresql.org/docs/9.3/static/sql-select.html

        return: A from_item string with checked psql qualifiers.

        Not implemented:
            with_query_name [ [ AS ] alias [ ( column_alias [, ...] ) ] ]
            [ LATERAL ] function_name ( [ argument [, ...] ] ) [ AS ] alias [ ( column_alias [, ...] | column_definition [, ...] ) ]
            [ LATERAL ] function_name ( [ argument [, ...] ] ) AS ( column_definition [, ...] )
    """
    # TODO: If 'type' is not set assume just a table name is present
    if d['type'] == 'table':
        s = 'only ' if d.pop('only', False) else ''
        schema = read_pgid(d.pop('schema', ''))
        if schema:
            s += schema + "."
        s += read_pgid(d['table'])
        if 'star' in d and read_bool(d['star']):
            s += ' * '
        if 'alias' in d and read_bool(d['alias']):
            s += ' AS ' + read_pgid(d['alias'])
            s += ' (' + \
                 ', '.join(
                     map(read_pgid, d['alias'].pop('column_aliases', []))) + \
                 ')'

    elif d['type'] == 'select':
        s = 'LATERAL ' if read_bool(d['lateral']) else ''
        s += '(' + parse_select(d['select']) + ') '
        if 'alias' in d and read_bool(d['alias']):
            s += ' AS ' + read_pgid(d['alias'])
            s += ' (' + \
                 ', '.join(
                     map(read_pgid, d['alias'].pop('column_aliases', []))) + \
                 ')'

    elif d['type'] == 'join':
        s = parse_from_item(d['left'])
        if 'natural' in d and read_bool(d['natural']):
            s += ' NATURAL'
        if 'join_type' in d:
            if d['join_type'].lower().strip() in ['join', 'inner join', 'left join',
                                          'left outer join', 'right join',
                                          'right outer join', 'full join',
                                          'full inner join', 'full outer join', 'cross join']:
                s += ' ' + d['join_type']
            else:
                raise ValidationError('Invalid join type')
        else:
            s += ' JOIN '

        s += ' ' + parse_from_item(d['right'])

        if 'on' in d:
            s += ' ON ' + parse_condition(d['on'])
        elif 'using' in d:
            s += ' USING (' + ', '.join(map(read_pgid, d['using'])) + ')'
    return s

def parse_expression(d, separator=', '):
    # TODO: Implement
    if isinstance(d, dict):
        if d['type'] == 'column':
            name = quote(d['column'])
            if 'table' in d:
                name = quote(d['table']) + '.' + name
                if 'schema' in d:
                    name = quote(d['schema']) + '.' + name
            return name
        if d['type'] == 'grouping':
            return '(' + parse_expression(d['grouping']) + ')'
        if d['type'] == 'star':
            return ' * '
        if d['type'] == 'operator':
            return parse_operator(d)
        if d['type'] == 'operator_binary':
            return parse_operator(d)
        if d['type'] == 'operator_unary':
            return parse_operator_unary(d)
        if d['type'] == 'modifier_unary':
            return parse_modifier_unary(d)
        if d['type'] == 'function':
            return parse_function(d)
        if d['type'] == 'value':
            if 'value' in d:
                return read_pgvalue(d['value'])
            else:
                return 'null'
    if isinstance(d, list):
        return separator.join(parse_expression(x) for x in d)
    if isinstance(d, str):
        return '\'' + d + '\''
    else:
        return d

def parse_condition(dl):
    # TODO: Implement
    import json
    print(json.dumps(dl, indent=2))
    if type(dl) == dict:
        dl = [dl]

    return " " + " AND ".join([parse_expression(d, separator=' AND ') for d in dl])

def parse_operator(d):
    if d['operator'] == 'as':
        return parse_expression(d['labeled']) + " AS " + d['label_name']
    else:
        return "%s %s %s" % (
            parse_expression(d['left']), read_operator(d['operator'], d['right']),
            parse_expression(d['right']))

def parse_operator_unary(d):
    return "%s %s" % (read_operator(d['operator'], d['operand']),
                      parse_expression(d['operand']))

def parse_modifier_unary(d):
    return "%s %s" % (parse_expression(d['operand']),
                      read_operator(d['operator'], d['operand']))

def parse_function(d):
    assert (read_pgid(d['function']))
    operand_struc = d['operands']
    if isinstance(operand_struc, list):
        operands = ', '.join(map(parse_expression, d['operands']))
    else:
        operands = parse_expression(operand_struc)

    return '{f}{ops}'.format(f=d['function'], ops=operands)

def cadd(d, key, string=None):
    if not string:
        string = key.upper() + ' '
    if d.pop(key, None):
        return string
    else:
        return ''

def parse_create_table(d):
    s = 'CREATE '
    if d.pop('global', None):
        s += 'GLOBAL '
    elif d.pop('local', None):
        s += 'LOCAL '

    s += (cadd(d, 'temp')
          + cadd(d, 'unlogged')
          + 'TABLE '
          + cadd(d, 'if_not_exists', 'IF NOT EXISTS ')
          + read_pgid(d['name']))

    fieldstrings = []

    for field in d.pop('fields', []):
        fs = ''
        if field['type'] == 'column':
            fs += read_pgid(field['name']) + ' '
            fs += read_pgid(field['data_type']) + ' '
            collate = field.pop('collate', None)
            if collate:
                fs += read_pgid(collate)
            fs += ', '.join([parse_column_constraint(cons)
                             for cons in field.pop('constraints', [])]) + ' '
        elif field['type'] == 'table_constraint':
            fs += parse_table_constraint(field)
        elif field['type'] == 'like':
            fs += 'LIKE '

def parse_column_constraint(d):
    raise NotImplementedError
    # TODO: Implement

def parse_table_constraint(d):
    raise NotImplementedError
    # TODO: Implement
<|endoftext|>


<commit_before>import cgi
from wsgiref import simple_server

import falcon
import uuid
import pyrax

class VideoCollection(object):

    def __init__(self):
        pyrax.keyring_auth("rackervision")
        pyrax.queues.client_id = _generate_id()
        self.unprocessed_container = pyrax.cloudfiles.get_container("unprocessed")
        self.processed_container = pyrax.cloudfiles.get_container("processed")
        self.queue = pyrax.queues.get("unprocessed")

    def on_get(self, req, resp):
        # TODO(kgriffs): List videos
        pass

    def on_post(self, req, resp):
        form = cgi.FieldStorage(fp=req.stream, environ=req.env)
        title, video = form['title'].value, form['video'].file

        # if video_field.file:
        #     pass

        #data = video
        uuid = _generate_id()
        self.unprocessed_container.upload_file(video, uuid)
        self.queue.post_message(uuid, 1209599)

        # TODO(kgriffs): Stream into Swift
        # TODO(kgriffs): On successful stream, create video document in mongo

def _generate_id():
    return str(uuid.uuid1())

app = application = falcon.API()

app.add_route('/videos', VideoCollection())

if __name__ == '__main__' :
    httpd = simple_server.make_server('localhost', 8000, app)
    print 'Serving on localhost:8000'
    httpd.serve_forever()
<commit_msg>removed cruft<commit_after>import cgi
from wsgiref import simple_server

import falcon
import uuid
import pyrax

class VideoCollection(object):

    def __init__(self):
        pyrax.keyring_auth("rackervision")
        pyrax.queues.client_id = _generate_id()
        self.unprocessed_container = pyrax.cloudfiles.get_container("unprocessed")
        self.processed_container = pyrax.cloudfiles.get_container("processed")
        self.queue = pyrax.queues.get("unprocessed")

    def on_get(self, req, resp):
        # TODO(kgriffs): List videos
        pass

    def on_post(self, req, resp):
        form = cgi.FieldStorage(fp=req.stream, environ=req.env)
        title, video = form['title'].value, form['video'].file

        # if video_field.file:
        #     pass

        uuid = _generate_id()
        self.unprocessed_container.upload_file(video, uuid)
        self.queue.post_message(uuid, 1209599)

        # TODO(kgriffs): Stream into Swift
        # TODO(kgriffs): On successful stream, create video document in mongo

def _generate_id():
    return str(uuid.uuid1())

app = application = falcon.API()

app.add_route('/videos', VideoCollection())

if __name__ == '__main__' :
    httpd = simple_server.make_server('localhost', 8000, app)
    print 'Serving on localhost:8000'
    httpd.serve_forever()
<|endoftext|>


<commit_before># Copyright (C) 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

__author__ = 'MeanEYE.rcf <meaneye.rcf@gmail.com>'
__copyright__ = 'Copyright (c) 2010, Google Inc.'
__license__ = 'Apache License, Version 2.0'

__all__ = [
    'AlertDialog',
    'ProgressDialog',
    'SpinnerProgress',
    'BUTTON_LEFT',
    'BUTTON_RIGHT',
    'BUTTON_MIDDLE',
    'PROGRESS_SPINNER',
    'PROGRESS_HORIZONTAL',
    ]

# button position constants
BUTTON_LEFT = 0
BUTTON_RIGHT = 1
BUTTON_MIDDLE = 2

# progress dialog types
PROGRESS_SPINNER = 0
PROGRESS_HORIZONTAL = 1

class Dialog:
  """Base dialog

  This class is not meant for regular users. It provides
  commonly used methods to other classes.

  Do *NOT* use this class in your programs!

  """
  def __init__(self, AndroidProxy):
    self._uuid = None
    self._proxy = AndroidProxy

    self._title = None
    self._message = None

  def show(self):
    """Show dialog

    Note: Once dialog is displayed you
    can't change anything!

    """
    assert self._uuid is not None
    self._proxy.dialogShow(self._uuid)

  def dismiss(self):
    """Dismiss dialog"""
    assert self._uuid is not None

    self._proxy.dialogDismiss(self._uuid)
    self._uuid = None;

  def set_title(self, title):
    """Set dialog title"""
    self._title = title

  def set_message(self, message):
    """Set dialog message"""
    self._message = message

  def set_cancelable(self, cancelable=True):
    """Set dialog cancelable"""
    self._cancelable = cancelable

class AlertDialog(Dialog):
  """Alert dialog with additional options"""

  def __init__(self, AndroidProxy, title='', message='', cancelable=False):
    Dialog.__init__(self, AndroidProxy)

    self._title = title
    self._message = message
    self._cancelable = cancelable

  def set_button(self, text, position=BUTTON_LEFT):
    """Add button to alert dialog"""
    assert self._uuid is not None

    self._proxy.dialogSetButton(self._uuid, position, text)

  def get_response(self):
    """Wait for user response on dialog"""
    assert self._uuid is not None

    result = self._proxy.dialogGetResponse(self._uuid).result
    return result

  def show(self):
    """Show dialog"""

    # create alert dialog
    self._uuid = self._proxy.dialogCreateAlert(
                        self._title,
                        self._message,
                        self._cancelable
                      ).result

    # show newly created dialog
    Dialog.show(self)

class ProgressDialog(Dialog):
  """Progress dialog"""

  def __init__(self, AndroidProxy, type=PROGRESS_SPINNER, title='', message='', cancelable=False):
    Dialog.__init__(self, AndroidProxy)

    self._type = type
    self._title = title
    self._message = message
    self._cancelable = cancelable
    self._max = 100
    self._current = 0

  def set_max(self, max):
    """Set progress maximum value"""
    self._max = max

    if self._uuid is not None:
      self._proxy.dialogProgressSetMax(self._uuid, self._max)

  def set_current(self, current):
    """Set current dialog progress"""
    assert self._uuid is not None

    self._current = current
    self._proxy.dialogProgressSetCurrent(self._uuid, self._current)

  def show(self):
    """Show dialog"""

    method = {
        PROGRESS_SPINNER: self._proxy.dialogCreateSpinnerProgress,
        PROGRESS_HORIZONTAL: self._proxy.dialogCreateHorizontalProgress,
      }

    # create dialog
    self._uuid = method[self._type](
                    self._title,
                    self._message,
                    self._max,
                    self._cancelable
                  ).result

    # show newly created dialog
    Dialog.show(self)

class SpinnerProgress(ProgressDialog):
  """Spinner progress dialog class"""

  def __init__(self, AndroidProxy, title='', message='', max=100, cancelable=False):
    ProgressDialog.__init__(
                self,
                AndroidProxy,
                PROGRESS_SPINNER,
                title,
                message,
                cancelable
              )

    self.set_max(max)

class HorizontalProgress(ProgressDialog):
  """Horizontal progress dialog class"""

  def __init__(self, AndroidProxy, title='', message='', max=100, cancelable=False):
    ProgressDialog.__init__(
                self,
                AndroidProxy,
                PROGRESS_HORIZONTAL,
                title,
                message,
                cancelable
              )

    self.set_max(max)
<commit_msg>Refactor Python ui module.<commit_after># Copyright (C) 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

__author__ = 'MeanEYE.rcf <meaneye.rcf@gmail.com>'
__copyright__ = 'Copyright (c) 2010, Google Inc.'
__license__ = 'Apache License, Version 2.0'

__all__ = [
    'AlertDialog',
    'HorizontalProgressDialog',
    'SpinnerProgressDialog',
    'BUTTON_LEFT',
    'BUTTON_RIGHT',
    'BUTTON_MIDDLE',
    ]

# Button position constants.
BUTTON_LEFT = -1
BUTTON_MIDDLE = -2
BUTTON_RIGHT = -3

class _Dialog(object):
  """Base dialog."""

  def __init__(self, proxy):
    self._uuid = None
    self._proxy = proxy
    self._title = None
    self._message = None

  def show(self):
    """Show dialog."""
    assert self._uuid is not None
    self._proxy.dialogShow(self._uuid)

  def dismiss(self):
    """Dismiss dialog."""
    assert self._uuid is not None
    self._proxy.dialogDismiss(self._uuid)
    self._uuid = None;

  def set_title(self, title):
    """Set dialog title."""
    self._title = title

  def set_message(self, message):
    """Set dialog message."""
    self._message = message

  def set_cancelable(self, cancelable=True):
    """Set dialog cancelable."""
    self._cancelable = cancelable

class AlertDialog(_Dialog):
  """Alert dialog with additional options."""

  def __init__(self, proxy, title='', message='', cancelable=False):
    _Dialog.__init__(self, proxy)
    self._title = title
    self._message = message
    self._cancelable = cancelable

  def set_button(self, text, position=BUTTON_LEFT):
    """Add button to alert dialog."""
    assert self._uuid is not None
    self._proxy.dialogSetButton(self._uuid, position, text)

  def get_response(self):
    """Wait for user response to dialog."""
    assert self._uuid is not None
    return self._proxy.dialogGetResponse(self._uuid).result

  def show(self):
    """Show dialog."""
    self._uuid = self._proxy.dialogCreateAlert(self._title, self._message,
                                               self._cancelable).result
    _Dialog.show(self)

class _ProgressDialog(_Dialog):
  """Progress dialog."""

  def __init__(self, proxy, title='', message='', cancelable=False):
    _Dialog.__init__(self, proxy)
    self._type = type
    self._title = title
    self._message = message
    self._cancelable = cancelable
    self._max = 100
    self._current = 0

  def set_max(self, max):
    """Set progress maximum value."""
    self._max = max
    if self._uuid is not None:
      self._proxy.dialogProgressSetMax(self._uuid, self._max)

  def set_current(self, current):
    """Set current dialog progress."""
    assert self._uuid is not None
    self._current = current
    self._proxy.dialogProgressSetCurrent(self._uuid, self._current)

  def _show_spinner(self):
    """Show spinner progress dialog."""
    self._uuid = self._proxy.dialogCreateSpinnerProgress(
        self._title,self._message, self._max, self._cancelable).result
    _Dialog.show(self)

  def _show_horizontal(self):
    """Show horizontal progress dialog."""
    self._uuid = self._proxy.dialogCreateHorizontalProgress(
        self._title,self._message, self._max, self._cancelable).result
    _Dialog.show(self)

class SpinnerProgressDialog(_ProgressDialog):
  """Spinner progress dialog class."""

  def __init__(self, proxy, title='', message='', max=100, cancelable=False):
    _ProgressDialog.__init__(self, proxy, title, message, cancelable)
    self.set_max(max)

  def show(self):
    """Show dialog."""
    self._show_spinner()

class HorizontalProgressDialog(_ProgressDialog):
  """Horizontal progress dialog class."""

  def __init__(self, proxy, title='', message='', max=100, cancelable=False):
    _ProgressDialog.__init__(self, proxy, title, message, cancelable)
    self.set_max(max)

  def show(self):
    """Show dialog."""
    self._show_horizontal()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import sys
import os
import re
import getopt
import cherrypy
import json

from summpy import tools

class Summarizer(object):

    def __init__(self):
        self.summarizers = {}

    def get_summarizer(self, name):
        '''
        import summarizers on-demand
        '''
        if name in self.summarizers:
            pass
        elif name == 'lexrank':
            from summpy import lexrank
            self.summarizers[name] = lexrank.summarize
        elif name == 'mcp':
            from summpy import mcp_summ
            self.summarizers[name] = mcp_summ.summarize

        return self.summarizers[name]

    @cherrypy.expose
    def summarize(self, text=None, algo=u'lexrank', **summarizer_params):
        '''
        Args:
          text: text to be summarized
          algo: summarizaion algorithm
              - 'lexrank' (default) graph-based
              - 'clexrank' Continuous LexRank
              - 'mcp' select sentences in terms of maximum coverage problem

          summarizer_params examples:
            char_limit: summary length (the number of characters)
            sent_limit: (not supported with mcp)
              summary length (the number of sentences)
            imp_require: (lexrank only)
              cumulative LexRank score [0.0-1.0]
        '''
        try:  # TODO: generate more useful error message
            # fix parameter type
            for param, value in summarizer_params.items():
                if value == '':
                    del summarizer_params[param]
                    continue
                elif re.match(r'^\d*.\d+$', value):
                    value = float(value)
                elif re.match(r'^\d+$', value):
                    value = int(value)
                elif value == 'true':
                    value = True
                elif value == 'false':
                    value = False
                summarizer_params[param] = value

            if algo in ('lexrank', 'clexrank'):
                summarizer = self.get_summarizer('lexrank')
                if algo == 'clexrank':
                    summarizer_params['continuous'] = True
            elif algo == 'mcp':
                summarizer = self.get_summarizer('mcp')

            summary, debug_info = summarizer(text, **summarizer_params)

        except Exception, e:
            return json.dumps({'error': str(e)}, ensure_ascii=False, indent=2)
        else:
            res = json.dumps(
                tools.tree_encode({
                    'summary': summary, 'debug_info': debug_info
                }),
                ensure_ascii=False, indent=2
            )
            return res

if __name__ == '__main__':

    options, args = getopt.getopt(sys.argv[1:], 'h:p:')
    options = dict(options)
    host, port = options['-h'], int(options['-p'])

    cherrypy.config.update({
        'server.socket_host': host,
        'server.socket_port': port
    })

    conf = {
        '/': {
            'tools.staticdir.root': os.path.dirname(os.path.abspath(__file__))
        },
        '/summarize': {
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [
                ('Content-type', 'application/json')
            ]
        },
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': './public'
        }
    }
    cherrypy.quickstart(Summarizer(), '/', conf)
<commit_msg>add charset at header info of json response.<commit_after>#!/usr/bin/env python
# coding: utf-8

import sys
import os
import re
import getopt
import cherrypy
import json

from summpy import tools

class Summarizer(object):

    def __init__(self):
        self.summarizers = {}

    def get_summarizer(self, name):
        '''
        import summarizers on-demand
        '''
        if name in self.summarizers:
            pass
        elif name == 'lexrank':
            from summpy import lexrank
            self.summarizers[name] = lexrank.summarize
        elif name == 'mcp':
            from summpy import mcp_summ
            self.summarizers[name] = mcp_summ.summarize

        return self.summarizers[name]

    @cherrypy.expose
    def summarize(self, text=None, algo=u'lexrank', **summarizer_params):
        '''
        Args:
          text: text to be summarized
          algo: summarizaion algorithm
              - 'lexrank' (default) graph-based
              - 'clexrank' Continuous LexRank
              - 'mcp' select sentences in terms of maximum coverage problem

          summarizer_params examples:
            char_limit: summary length (the number of characters)
            sent_limit: (not supported with mcp)
              summary length (the number of sentences)
            imp_require: (lexrank only)
              cumulative LexRank score [0.0-1.0]
        '''
        try:  # TODO: generate more useful error message
            # fix parameter type
            for param, value in summarizer_params.items():
                if value == '':
                    del summarizer_params[param]
                    continue
                elif re.match(r'^\d*.\d+$', value):
                    value = float(value)
                elif re.match(r'^\d+$', value):
                    value = int(value)
                elif value == 'true':
                    value = True
                elif value == 'false':
                    value = False
                summarizer_params[param] = value

            if algo in ('lexrank', 'clexrank'):
                summarizer = self.get_summarizer('lexrank')
                if algo == 'clexrank':
                    summarizer_params['continuous'] = True
            elif algo == 'mcp':
                summarizer = self.get_summarizer('mcp')

            summary, debug_info = summarizer(text, **summarizer_params)

        except Exception, e:
            return json.dumps({'error': str(e)}, ensure_ascii=False, indent=2)
        else:
            res = json.dumps(
                tools.tree_encode({
                    'summary': summary, 'debug_info': debug_info
                }),
                ensure_ascii=False, indent=2
            )
            return res

if __name__ == '__main__':

    options, args = getopt.getopt(sys.argv[1:], 'h:p:')
    options = dict(options)
    host, port = options['-h'], int(options['-p'])

    cherrypy.config.update({
        'server.socket_host': host,
        'server.socket_port': port
    })

    conf = {
        '/': {
            'tools.staticdir.root': os.path.dirname(os.path.abspath(__file__))
        },
        '/summarize': {
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [
                ('Content-type', 'application/json; charset=utf-8')
            ]
        },
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': './public'
        }
    }
    cherrypy.quickstart(Summarizer(), '/', conf)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os
import argparse
import subprocess
import pysam
import datetime

from re import sub
from uuid import uuid4

def now():
    return str(datetime.datetime.now())

def bamtofastq(bam, outdir, samtofastq, threads=1):
    assert os.path.exists(samtofastq)
    assert bam.endswith('.bam')

    outfq = sub('bam$', 'fastq', outdir + '/' + os.path.basename(bam))

    cmd = ['java', '-XX:ParallelGCThreads=' + str(threads), '-Xmx4g', '-jar', samtofastq,
           'INPUT=' + bam, 'INTERLEAVE=true', 'VALIDATION_STRINGENCY=SILENT', 'FASTQ=' + outfq]
    sys.stderr.write("INFO\t" + now() + "\tconverting BAM " + bam + " to FASTQ\n")
    subprocess.call(cmd)

    assert os.path.exists(outfq) # conversion failed

    return outfq

def cramtofastq(cram, outdir, cramjar, ref, threads=1):
    assert os.path.exists(cramjar)
    assert cram.endswith('.cram')

    outfq = sub('cram$', 'fastq', outdir + '/' + os.path.basename(cram))

    cmd = ['java', '-XX:ParallelGCThreads=' + str(threads), '-Xmx4g', '-jar', cramjar,
           'fastq', '-I', cram, '-R', ref, '--enumerate', '--skip-md5-check']
    sys.stderr.write("INFO\t" + now() + "\tconverting CRAM " + cram + " to FASTQ\n")

    with open(outfq, 'w') as fq:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        for line in p.stdout:
            fq.write(line)

    assert os.path.exists(outfq) # conversion failed

    return outfq

def bwamem(fq, ref, outdir, threads=1, width=150, uid=None, sortmem='8G'):
    ''' FIXME: add parameters to commandline '''
    assert os.path.exists(ref + '.fai')

    fqroot = sub('fastq$', '', fq)
    if uid is not None:
        fqroot = uid

    fqroot   = outdir + '/' + os.path.basename(fqroot)

    if str(sortmem).rstrip('Gg') != str(sortmem):
        sortmem = int(str(sortmem).rstrip('Gg')) * 1000000000

    if str(sortmem).rstrip('Mm') != str(sortmem):
        sortmem = int(str(sortmem).rstrip('Gg')) * 1000000

    sortmem = sortmem/int(threads) # avoid PBS killing my jobs

    sam_out   = fqroot + 'capture.sam'
    bam_out   = fqroot + 'capture.bam'
    sort_out  = fqroot + 'capture.sort'

    sam_cmd  = ['bwa', 'mem', '-t', str(threads), '-M', ref, fq]
    bam_cmd  = ['samtools', 'view', '-bt', ref + '.fai', '-o', bam_out, sam_out]
    sort_cmd = ['samtools', 'sort', '-@', str(threads), '-m', str(sortmem), bam_out, sort_out]
    idx_cmd  = ['samtools', 'index', sort_out + '.bam']

    sys.stderr.write("INFO\t" + now() + "\trunning bwa-mem: " + ' '.join(sam_cmd) + "\n")

    with open(sam_out, 'w') as sam:
        p = subprocess.Popen(sam_cmd, stdout=subprocess.PIPE)
        for line in p.stdout:
            sam.write(line)

    sys.stderr.write("INFO\t" + now() + "\twriting " + sam_out + " to BAM...\n")
    subprocess.call(bam_cmd)
    os.remove(sam_out)

    sys.stdout.write("INFO\t" + now() + "\tsorting output: " + ' '.join(sort_cmd) + "\n")
    subprocess.call(sort_cmd)
    os.remove(bam_out)

    sys.stdout.write("INFO\t" + now() + "\tindexing: " + ' '.join(idx_cmd) + "\n")
    subprocess.call(idx_cmd)

    return sort_out + '.bam'

def getmatch(read):
    ''' return number of mismatches / aligned length of TE sub-read '''
    nm = [value for (tag, value) in read.tags if tag == 'NM'][0]
    return 1.0 - (float(nm)/float(read.alen))

def parsebam(inbam, fqout, minmatch=0.0):
    # pass 1, identify hardclipped reads
    hc = {}
    bam = pysam.Samfile(inbam, 'rb')
    for read in bam.fetch():
        if read.is_secondary:
            hc[read.qname] = True

    sys.stderr.write("INFO\t" + now() + "\tIdentified " + str(len(hc)) + " hardclipped reads, starting second pass\n")

    # pass 2, output reads
    found_hc = 0
    outcount = 0

    bam.close()
    with open(fqout, 'w') as fq:
        bam = pysam.Samfile(inbam, 'rb')
        for read in bam.fetch(until_eof=True):
            if getmatch(read) >= float(minmatch) and not read.is_secondary:
                fq.write(fqread(read))
                outcount += 1
            elif read.qname in hc:
                fq.write(fqread(read))
                found_hc += 1
                outcount += 1

    sys.stderr.write("INFO\t" + now() + "\toutput " + str(outcount) + " reads, " + str(found_hc) + " of them rescued from hardclips\n")

def fqread(read):
    salt = str(uuid4()).split('-')[0]
    return '\n'.join(('@' + read.qname + ':' + salt, seq, '+', qual)) + '\n'

def main(args):
    sys.stderr.write("INFO\t" + now() + "\tstarting " + sys.argv[0] + " called with args:\n" + ' '.join(sys.argv) + "\n")

    if not os.path.exists(args.outdir):
        try:
            os.mkdir(args.outdir)
        except:
            sys.stderr.write("ERROR\t" + now() + "\tcould not create output directory: " + args.outdir + "\n")
            sys.exit(1)

    for seq in args.seqs:
        sys.stderr.write("INFO\t" + now() + "\tprocessing " + seq + "\n")
        fastq = None
        temp  = False

        if seq.endswith('.bam'):
            fastq = bamtofastq(seq, args.outdir, args.samtofastq, threads=int(args.threads))
            temp  = True

        elif seq.endswith('.cram'):
            fastq = cramtofastq(seq, args.outdir, args.cramjar, threads=int(args.threads))
            temp  = True

        elif seq.endswith('.fastq') or seq.endswith('.fastq.gz'):
            fastq = seq

        else:
            sys.stderr.write("ERROR\t" + now() + "\tunrecognized file format (extension != .bam or .cram or .fastq\n")
            sys.exit(1)

        assert fastq.endswith('.fastq')

        sys.stderr.write("INFO\t" + now() + "\taligning fastq " + fastq + " to reference " + args.ref + "\n")
        bam = bwamem(fastq, args.ref, args.outdir, threads=args.threads, sortmem=args.sortmem)

        if not args.keepfastq and temp:
            os.remove(fastq)

        parsebam(bam, args.fqout, minmatch=float(args.minmatch))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='find relevant reads from sequence data')
    parser.add_argument(metavar='<input file bam/cram/fastq>', dest='seqs', nargs=1, help='input files')
    parser.add_argument('--samtofastq', default=None, help='path to picard SamToFastq.jar')
    parser.add_argument('--cramjar', default=None, help='path to cramtools .jar')
    parser.add_argument('-o', '--outdir', default='.', help='path to output directory')
    parser.add_argument('-f', '--fqout', required=True, help='output FASTQ file')
    parser.add_argument('-r', '--ref', required=True, help='reference fasta (needs bwa index and samtools faidx')
    parser.add_argument('-t', '--threads', default=1, help='threads for alignment (default = 1)')
    parser.add_argument('-m', '--minmatch', default=0.90, help='minimum percent match to output read (default=0.9)')
    parser.add_argument('--sortmem', default='8G', help='memory limit for sorting BAMs')
    parser.add_argument('--keepfastq', action='store_true', default=False, help='keep temporary fastq file if created (default = False)')
    args = parser.parse_args()
    main(args)
<commit_msg>update capture script<commit_after>#!/usr/bin/env python

import sys, os
import argparse
import subprocess
import pysam
import datetime
import gzip

from re import sub
from uuid import uuid4

def now():
    return str(datetime.datetime.now())

def bamtofastq(bam, outdir, samtofastq, threads=1):
    assert os.path.exists(samtofastq)
    assert bam.endswith('.bam')

    outfq = sub('bam$', 'fastq', outdir + '/' + os.path.basename(bam))

    cmd = ['java', '-XX:ParallelGCThreads=' + str(threads), '-Xmx4g', '-jar', samtofastq,
           'INPUT=' + bam, 'INTERLEAVE=true', 'VALIDATION_STRINGENCY=SILENT', 'FASTQ=' + outfq]
    sys.stderr.write("INFO\t" + now() + "\tconverting BAM " + bam + " to FASTQ\n")
    subprocess.call(cmd)

    assert os.path.exists(outfq) # conversion failed

    return outfq

def cramtofastq(cram, outdir, cramjar, ref, threads=1):
    assert os.path.exists(cramjar)
    assert cram.endswith('.cram')

    outfq = sub('cram$', 'fastq', outdir + '/' + os.path.basename(cram))

    cmd = ['java', '-XX:ParallelGCThreads=' + str(threads), '-Xmx4g', '-jar', cramjar,
           'fastq', '-I', cram, '-R', ref, '--enumerate', '--skip-md5-check']
    sys.stderr.write("INFO\t" + now() + "\tconverting CRAM " + cram + " to FASTQ\n")

    with open(outfq, 'w') as fq:
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        for line in p.stdout:
            fq.write(line)

    assert os.path.exists(outfq) # conversion failed

    return outfq

def bwamem(fq, ref, outdir, threads=1, width=150, uid=None):
    ''' FIXME: add parameters to commandline '''
    assert os.path.exists(ref + '.fai')

    fqroot = sub('fastq$', '', fq)
    if uid is not None:
        fqroot = uid

    fqroot   = outdir + '/' + os.path.basename(fqroot)

    sam_out   = fqroot + 'capture.sam'
    bam_out   = fqroot + 'capture.bam'
    sort_out  = fqroot + 'capture.sort'

    sam_cmd  = ['bwa', 'mem', '-t', str(threads), '-M', ref, fq]
    bam_cmd  = ['samtools', 'view', '-bt', ref + '.fai', '-o', bam_out, sam_out]

    sys.stderr.write("INFO\t" + now() + "\trunning bwa-mem: " + ' '.join(sam_cmd) + "\n")

    with open(sam_out, 'w') as sam:
        p = subprocess.Popen(sam_cmd, stdout=subprocess.PIPE)
        for line in p.stdout:
            sam.write(line)

    sys.stderr.write("INFO\t" + now() + "\twriting " + sam_out + " to BAM...\n")
    subprocess.call(bam_cmd)

    os.remove(sam_out)
    return bam_out

def getmatch(read):
    ''' return number of mismatches / aligned length of TE sub-read '''
    nm = [value for (tag, value) in read.tags if tag == 'NM'][0]
    return 1.0 - (float(nm)/float(read.alen))

def parsebam(inbam, fqout, minmatch=0.0):
    # pass 1, identify hardclipped reads
    hc = {}
    bam = pysam.Samfile(inbam, 'rb')
    for read in bam.fetch(until_eof=True):
        if read.is_secondary:
            hc[read.qname] = True

    sys.stderr.write("INFO\t" + now() + "\tIdentified " + str(len(hc)) + " hardclipped reads\n")

    # pass 2, output reads
    found_hc = 0
    outcount = 0
    bam.close()

    sys.stderr.write("INFO\t" + now() + "\toutputting to " + fqout + "\n")
    with gzip.open(fqout, 'wb') as fq:
        bam = pysam.Samfile(inbam, 'rb')
        for read in bam.fetch(until_eof=True):
            if read.qname in hc and not read.is_secondary:
                fq.write(fqread(read))
                found_hc += 1
                outcount += 1
            elif getmatch(read) >= float(minmatch) and not read.is_secondary:
                fq.write(fqread(read))
                outcount += 1

    sys.stderr.write("INFO\t" + now() + "\toutput " + str(outcount) + " reads, " + str(found_hc) + " of them rescued from hardclips\n")

def fqread(read):
    salt = str(uuid4()).split('-')[0]
    return '\n'.join(('@' + read.qname + ':' + salt, read.seq, '+', read.qual)) + '\n'

def main(args):
    sys.stderr.write("INFO\t" + now() + "\tstarting " + sys.argv[0] + " called with args:\n" + ' '.join(sys.argv) + "\n")

    if not os.path.exists(args.outdir):
        try:
            os.mkdir(args.outdir)
        except:
            sys.stderr.write("ERROR\t" + now() + "\tcould not create output directory: " + args.outdir + "\n")
            sys.exit(1)

    for seq in args.seqs:
        sys.stderr.write("INFO\t" + now() + "\tprocessing " + seq + "\n")
        fastq = None
        temp  = False

        if seq.endswith('.bam'):
            fastq = bamtofastq(seq, args.outdir, args.samtofastq, threads=int(args.threads))
            temp  = True

        elif seq.endswith('.cram'):
            assert args.genome is not None and os.path.exists(args.genome), "must specify a valid reference genome for CRAM mode\n"
            fastq = cramtofastq(seq, args.outdir, args.cramjar, args.genome, threads=int(args.threads))
            temp  = True

        elif seq.endswith('.fastq') or seq.endswith('.fastq.gz'):
            fastq = seq

        else:
            sys.stderr.write("ERROR\t" + now() + "\tunrecognized file format (extension != .bam or .cram or .fastq\n")
            sys.exit(1)

        assert fastq.endswith('.fastq')

        sys.stderr.write("INFO\t" + now() + "\taligning fastq " + fastq + " to TE reference " + args.ref + "\n")
        bam = bwamem(fastq, args.ref, args.outdir, threads=args.threads)

        if not args.keepfastq and temp:
            os.remove(fastq)

        fqout = args.outdir + '/' + os.path.basename(args.fqout)
        if not fqout.endswith('.gz'):
            fqout += '.gz'
        parsebam(bam, fqout, minmatch=float(args.minmatch))
        os.remove(bam)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='find relevant reads from sequence data')
    parser.add_argument(metavar='<input file bam/cram/fastq>', dest='seqs', nargs=1, help='input files')
    parser.add_argument('--samtofastq', default=None, help='path to picard SamToFastq.jar')
    parser.add_argument('--cramjar', default=None, help='path to cramtools .jar')
    parser.add_argument('-o', '--outdir', default='.', help='path to output directory')
    parser.add_argument('-f', '--fqout', required=True, help='output FASTQ file')
    parser.add_argument('-r', '--ref', required=True, help='TE reference fasta (needs bwa index and samtools faidx')
    parser.add_argument('-g', '--genome', default=None, help='genome reference fasta (e.g. GRCh37, needs bwa index and samtools faidx')
    parser.add_argument('-t', '--threads', default=1, help='threads for alignment (default = 1)')
    parser.add_argument('-m', '--minmatch', default=0.90, help='minimum percent match to output read (default=0.9)')
    parser.add_argument('--keepfastq', action='store_true', default=False, help='keep temporary fastq file if created (default = False)')
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>"""Convert Tensorflow SavedModel to TensorFlow.js web format.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
import os
import json
import numpy as np
import tensorflow as tf

from absl import flags

from tensorflow.core.protobuf import device_properties_pb2, rewriter_config_pb2
from tensorflow.python.grappler import cluster as gcluster
from tensorflow.python.grappler import tf_optimizer
from tensorflow.python.lib.io import file_io
from tensorflow.python.tools import freeze_graph

from google.protobuf import text_format

sys.path.append(
    os.path.join(
        os.path.dirname(__file__), '..',
        'node_modules/deeplearn-src/scripts/'))
import write_weights as tfc

flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('output_node_names', '',
                    'The names of the output nodes, comma separated.')
flags.DEFINE_string('output_graph', '', 'The name of the output graph file')
flags.DEFINE_string(
    'saved_model_tags', 'serve',
    'Tags of the MetaGraphDef to load, in comma separated string format.')

FLAGS = flags.FLAGS

def get_cluster():
    """ Grappler optimization configuration for GPU."""
    named_device = device_properties_pb2.NamedDevice()
    named_device.name = '/GPU:0'
    named_device.properties.type = 'GPU'
    named_device.properties.environment['architecture'] = '4'
    cluster = gcluster.Cluster(devices=[named_device])
    return cluster

def load_graph(graph_filename, output_node_names):
    """Loads GraphDef. Returns Python Graph object.

    Args:
      graph_filename: string file name for the frozen graph
    """
    with tf.gfile.Open(graph_filename, 'rb') as file:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(file.read())

    with tf.Graph().as_default() as graph:
        # Set name to empty to avoid using the default name 'import'.
        tf.import_graph_def(graph_def, name='')

    for node in output_node_names.split(','):
        graph.add_to_collection('train_op',
                                graph.get_operation_by_name(node.strip()))

    return graph

def validate(nodes):
    """Validate if the node's op is compatible with TensorFlow.js.

    Args:
      nodes: tf.NodeDef tensorflow NodeDef objects from GraphDef
    """
    ops = []
    op_list_path = 'src/operations/op_list/'
    dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
    for filename in os.listdir(os.path.join(dir_path, op_list_path)):
        with open(os.path.join(dir_path, op_list_path, filename)) as json_data:
            ops += json.load(json_data)

    names = set([x['tfOpName'] for x in ops])
    not_supported = set(
        [x.op for x in [x for x in nodes if x.op not in names]])
    return not_supported

def optimize_graph(graph, output_graph):
    """Takes a Python Graph object and optimizes the graph.

    Args:
      graph: tf.Graph tensorflow dataflow graph
    """
    rewriter_config = rewriter_config_pb2.RewriterConfig()
    rewriter_config.optimizers[:] = [
        'pruning', 'constfold', 'arithmetic', 'dependency', 'pruning',
        'constfold', 'arithmetic', 'dependency'
    ]
    meta_graph = tf.train.export_meta_graph(
        graph_def=graph.as_graph_def(), graph=graph)
    optimized_graph = tf_optimizer.OptimizeGraph(
        rewriter_config, meta_graph, cluster=get_cluster())

    extract_weights(graph, optimized_graph, output_graph)
    return optimize_graph

def extract_weights(graph, graph_def, output_graph):
    """Takes a Python GraphDef object and extract the weights.

    Args:
      graph: tf.Graph tensorflow dataflow graph
      graph_def: tf.GraphDef tensorflow GraphDef proto object, which represents
        the model topology
    """
    constants = [node for node in graph_def.node if node.op == 'Const']
    print('Writing weight file ' + output_graph + '...')
    const_manifest = []
    path = os.path.dirname(output_graph)

    with tf.Session(graph=graph) as sess:
        for const in constants:
            tensor = graph.get_tensor_by_name(const.name + ':0')
            value = tensor.eval(session=sess)
            if not isinstance(value, np.ndarray):
                value = np.array(value)

            const_manifest.append({'name': const.name, 'data': value})

            # Remove the binary array from tensor and save it to the external file.
            const.attr["value"].tensor.ClearField('tensor_content')

    tfc.write_weights([const_manifest], path)

    file_io.atomic_write_string_to_file(
        os.path.abspath(output_graph), graph_def.SerializeToString())

    file_io.atomic_write_string_to_file(
        os.path.abspath(output_graph + 'txt'),
        text_format.MessageToString(graph_def))

def convert(output_node_names, output_graph, saved_model_tags,
            saved_model_dir):
    """Freeze the SavedModel and check the model compatibility with Tensorflow.js.
    Optimize and convert the model to Tensorflow.js format, when the model passes
    the compatiblity check.

    Args
      output_node_names: string The names of the output nodes, comma separated.
      output_graph: string The name of the output graph file.
      saved_model_tags: string Tagset of the MetaGraphDef to load, in comma separated string format.
      saved_model_dir: string The saved model directory.

    """
    freeze_graph.freeze_graph(
        '',
        '',
        True,
        '',
        output_node_names,
        '',
        '',
        output_graph + '.frozen',
        True,
        '',
        saved_model_tags=saved_model_tags,
        input_saved_model_dir=saved_model_dir)
    graph = load_graph(output_graph + '.frozen', output_node_names)
    unsupported = validate(graph.as_graph_def().node)
    if unsupported:
        print('Unsupported Ops in the model\n' + ', '.join(unsupported))
    else:
        optimize_graph(graph, output_graph)

def main(_):
    convert(FLAGS.output_node_names, FLAGS.output_graph,
            FLAGS.saved_model_tags, FLAGS.saved_model_dir)

if __name__ == '__main__':
    FLAGS(sys.argv)
    tf.app.run(main)
<commit_msg>Frozen not found for fixing issue #41 (#43)<commit_after>"""Convert Tensorflow SavedModel to TensorFlow.js web format.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import sys
import os
import json
import numpy as np
import tensorflow as tf

from absl import flags

from tensorflow.core.protobuf import device_properties_pb2, rewriter_config_pb2
from tensorflow.python.grappler import cluster as gcluster
from tensorflow.python.grappler import tf_optimizer
from tensorflow.python.lib.io import file_io
from tensorflow.python.tools import freeze_graph

sys.path.append(
    os.path.join(
        os.path.dirname(__file__), '..',
        'node_modules/deeplearn-src/scripts/'))
import write_weights as tfc

flags.DEFINE_string('saved_model_dir', '', 'The saved model directory.')
flags.DEFINE_string('output_node_names', '',
                    'The names of the output nodes, comma separated.')
flags.DEFINE_string('output_graph', '', 'The name of the output graph file')
flags.DEFINE_string(
    'saved_model_tags', 'serve',
    'Tags of the MetaGraphDef to load, in comma separated string format.')

FLAGS = flags.FLAGS

def get_cluster():
    """ Grappler optimization configuration for GPU."""
    named_device = device_properties_pb2.NamedDevice()
    named_device.name = '/GPU:0'
    named_device.properties.type = 'GPU'
    named_device.properties.environment['architecture'] = '4'
    cluster = gcluster.Cluster(devices=[named_device])
    return cluster

def load_graph(graph_filename, output_node_names):
    """Loads GraphDef. Returns Python Graph object.

    Args:
      graph_filename: string file name for the frozen graph
    """
    with tf.gfile.Open(graph_filename, 'rb') as file:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(file.read())

    with tf.Graph().as_default() as graph:
        # Set name to empty to avoid using the default name 'import'.
        tf.import_graph_def(graph_def, name='')

    for node in output_node_names.split(','):
        graph.add_to_collection('train_op',
                                graph.get_operation_by_name(node.strip()))

    return graph

def validate(nodes):
    """Validate if the node's op is compatible with TensorFlow.js.

    Args:
      nodes: tf.NodeDef tensorflow NodeDef objects from GraphDef
    """
    ops = []
    op_list_path = 'src/operations/op_list/'
    dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
    for filename in os.listdir(os.path.join(dir_path, op_list_path)):
        if os.path.splitext(filename)[1] == '.json':
            with open(os.path.join(dir_path, op_list_path,
                                   filename)) as json_data:
                ops += json.load(json_data)

    names = set([x['tfOpName'] for x in ops])
    not_supported = set(
        [x.op for x in [x for x in nodes if x.op not in names]])
    return not_supported

def optimize_graph(graph, output_graph):
    """Takes a Python Graph object and optimizes the graph.

    Args:
      graph: tf.Graph tensorflow dataflow graph
    """
    rewriter_config = rewriter_config_pb2.RewriterConfig()
    rewriter_config.optimizers[:] = [
        'pruning', 'constfold', 'arithmetic', 'dependency', 'pruning',
        'constfold', 'arithmetic', 'dependency'
    ]
    meta_graph = tf.train.export_meta_graph(
        graph_def=graph.as_graph_def(), graph=graph)
    optimized_graph = tf_optimizer.OptimizeGraph(
        rewriter_config, meta_graph, cluster=get_cluster())

    extract_weights(graph, optimized_graph, output_graph)
    return optimize_graph

def extract_weights(graph, graph_def, output_graph):
    """Takes a Python GraphDef object and extract the weights.

    Args:
      graph: tf.Graph tensorflow dataflow graph
      graph_def: tf.GraphDef tensorflow GraphDef proto object, which represents
        the model topology
    """
    constants = [node for node in graph_def.node if node.op == 'Const']
    print('Writing weight file ' + output_graph + '...')
    const_manifest = []
    path = os.path.dirname(output_graph)

    with tf.Session(graph=graph) as sess:
        for const in constants:
            tensor = graph.get_tensor_by_name(const.name + ':0')
            value = tensor.eval(session=sess)
            if not isinstance(value, np.ndarray):
                value = np.array(value)

            const_manifest.append({'name': const.name, 'data': value})

            # Remove the binary array from tensor and save it to the external file.
            const.attr["value"].tensor.ClearField('tensor_content')

    tfc.write_weights([const_manifest], path)

    file_io.atomic_write_string_to_file(
        os.path.abspath(output_graph), graph_def.SerializeToString())

def convert(output_node_names, output_graph, saved_model_tags,
            saved_model_dir):
    """Freeze the SavedModel and check the model compatibility with Tensorflow.js.
    Optimize and convert the model to Tensorflow.js format, when the model passes
    the compatiblity check.

    Args
      output_node_names: string The names of the output nodes, comma separated.
      output_graph: string The name of the output graph file.
      saved_model_tags: string Tagset of the MetaGraphDef to load, in comma separated string format.
      saved_model_dir: string The saved model directory.

    """

    directory = os.path.dirname(output_graph)
    if not os.path.exists(directory):
        os.makedirs(directory)

    frozen_file = output_graph + '.frozen'
    freeze_graph.freeze_graph(
        '',
        '',
        True,
        '',
        output_node_names,
        '',
        '',
        frozen_file,
        True,
        '',
        saved_model_tags=saved_model_tags,
        input_saved_model_dir=saved_model_dir)
    graph = load_graph(output_graph + '.frozen', output_node_names)
    unsupported = validate(graph.as_graph_def().node)
    if unsupported:
        print('Unsupported Ops in the model\n' + ', '.join(unsupported))
    else:
        optimize_graph(graph, output_graph)

    # clean up the temp files
    if os.path.exists(frozen_file):
        os.remove(frozen_file)

def main(_):
    convert(FLAGS.output_node_names, FLAGS.output_graph,
            FLAGS.saved_model_tags, FLAGS.saved_model_dir)

if __name__ == '__main__':
    FLAGS(sys.argv)
    tf.app.run(main)
<|endoftext|>