<commit_before>#    Copyright (c) 2014 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from sqlalchemy import or_
from sqlalchemy.orm import attributes
from sqlalchemy import sql
from webob import exc

from muranoapi.db import models
from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _  # noqa
from muranoapi.openstack.common import log as logging

SEARCH_MAPPING = {'fqn': 'fully_qualified_name',
                  'name': 'name',
                  'created': 'created'
                  }

LOG = logging.getLogger(__name__)

def category_get_names():
    session = db_session.get_session()
    categories = []
    for row in session.query(models.Category.name).all():
        for name in row:
            categories.append(name)
    return categories

def _package_get(package_id, session):
    package = session.query(models.Package).get(package_id)
    if not package:
        msg = _("Package id '{0}' is not found").format(package_id)
        LOG.error(msg)
        raise exc.HTTPNotFound(msg)

    return package

def _authorize_package(package, context, allow_public=False):
    if context.is_admin:
        return

    if package.owner_id != context.tenant:
        if not allow_public:
            msg = _("Package '{0}' is not owned by "
                    "tenant '{1}'").format(package.id, context.tenant)
            LOG.error(msg)
            raise exc.HTTPForbidden(msg)
        if not package.is_public:
            msg = _("Package '{0}' is not public and not owned by "
                    "tenant '{1}' ").format(package.id, context.tenant)
            LOG.error(msg)
            raise exc.HTTPForbidden(msg)

def package_get(package_id, context):
    """
    Return package details
    :param package_id: ID of a package, string
    :returns: detailed information about package, dict
    """
    session = db_session.get_session()
    package = _package_get(package_id, session)
    _authorize_package(package, context, allow_public=True)
    return package

def _get_categories(category_names, session=None):
    """
    Return existing category objects or raise an exception
    :param category_names: name of categories to associate with package, list
    :returns: list of Category objects to associate with package, list
    """
    if session is None:
        session = db_session.get_session()
    categories = []
    for ctg_name in category_names:
        ctg_obj = session.query(models.Category).filter_by(
            name=ctg_name).first()
        if not ctg_obj:
            msg = _("Category '{name}' doesn't exist").format(name=ctg_name)
            LOG.error(msg)
            # it's not allowed to specify non-existent categories
            raise exc.HTTPBadRequest(msg)

        categories.append(ctg_obj)
    return categories

def _get_tags(tag_names, session=None):
    """
    Return existing tags object or create new ones
    :param tag_names: name of tags to associate with package, list
    :returns: list of Tag objects to associate with package, list
    """
    if session is None:
        session = db_session.get_session()
    tags = []
    for tag_name in tag_names:
        tag_obj = session.query(models.Tag).filter_by(name=tag_name).first()
        if tag_obj:
            tags.append(tag_obj)
        else:
            tag_record = models.Tag(name=tag_name)
            tags.append(tag_record)
    return tags

def _get_class_definitions(class_names, session=None):
    classes = []
    for name in class_names:
        class_record = models.Class(name=name)
        classes.append(class_record)
    return classes

def _do_replace(package, change):
    path = change['path'][0]
    value = change['value']
    calculate = {'categories': _get_categories,
                 'tags': _get_tags}
    if path in ('categories', 'tags'):
        existing_items = getattr(package, path)

        duplicates = list(set(i.name for i in existing_items) & set(value))
        unique_values = [x for x in value if x not in duplicates]
        items_to_replace = calculate[path](unique_values)

        # NOTE(efedorova): Replacing duplicate entities is not allowed,
        # so need to remove anything, but duplicates
        # and append everything but duplicates
        for item in list(existing_items):
            if item.name not in duplicates:
                existing_items.remove(item)

        existing_items.extend(items_to_replace)
    else:
        setattr(package, path, value)

    return package

def _do_add(package, change):
    # Only categories and tags support addition
    path = change['path'][0]
    value = change['value']

    calculate = {'categories': _get_categories,
                 'tags': _get_tags}
    items_to_add = calculate[path](value)
    for item in items_to_add:
        try:
            getattr(package, path).append(item)
        except AssertionError:
            msg = _('One of the specified {0} is already '
                    'associated with a package. Doing nothing.')
            LOG.warning(msg.format(path))
    return package

def _do_remove(package, change):
    # Only categories and tags support removal
    def find(seq, predicate):
        for elt in seq:
            if predicate(elt):
                return elt

    path = change['path'][0]
    values = change['value']

    current_values = getattr(package, path)
    for value in values:
        if value not in [i.name for i in current_values]:
            msg = _("Value '{0}' of property '{1}' "
                    "does not exist.").format(value, path)
            LOG.error(msg)
            raise exc.HTTPNotFound(msg)
        if path == 'categories' and len(current_values) == 1:
            msg = _("At least one category should be assigned to the package")
            LOG.error(msg)
            raise exc.HTTPBadRequest(msg)
        item_to_remove = find(current_values, lambda i: i.name == value)
        current_values.remove(item_to_remove)
    return package

def package_update(pkg_id, changes, context):
    """
    Update package information
    :param changes: parameters to update
    :returns: detailed information about new package, dict
    """

    operation_methods = {'add': _do_add,
                         'replace': _do_replace,
                         'remove': _do_remove}
    session = db_session.get_session()
    with session.begin():
        pkg = _package_get(pkg_id, session)
        _authorize_package(pkg, context)

        for change in changes:
            pkg = operation_methods[change['op']](pkg, change)
        session.add(pkg)
    return pkg

def package_search(filters, context):
    """
    Search packages with different filters
      * Admin is allowed to browse all the packages
      * Regular user is allowed to browse all packages belongs to user tenant
        and all other packages marked is_public.
        Also all packages should be enabled.
      * Use marker and limit for pagination:
        The typical pattern of limit and marker is to make an initial limited
        request and then to use the ID of the last package from the response
        as the marker parameter in a subsequent limited request.
    """

    def _validate_limit(value):
        if value is None:
            return
        try:
            value = int(value)
        except ValueError:
            msg = _("limit param must be an integer")
            LOG.error(msg)
            raise exc.HTTPBadRequest(explanation=msg)

        if value < 0:
            msg = _("limit param must be positive")
            LOG.error(msg)
            raise exc.HTTPBadRequest(explanation=msg)

        return value

    def get_pkg_attr(package_obj, search_attr_name):
        return getattr(package_obj, SEARCH_MAPPING[search_attr_name])

    limit = _validate_limit(filters.get('limit'))

    session = db_session.get_session()
    pkg = models.Package

    if 'owned' in filters.keys():
        query = session.query(pkg).filter(pkg.owner_id == context.tenant)
    elif context.is_admin:
        query = session.query(pkg)
    else:
        query = session.query(pkg).filter(or_((pkg.is_public & pkg.enabled),
                                              pkg.owner_id == context.tenant))
    if 'type' in filters.keys():
        query = query.filter(pkg.type == filters['type'].title())

    if 'category' in filters.keys():
        query = query.filter(pkg.categories.any(
            models.Category.name.in_(filters['category'])))
    if 'tag' in filters.keys():
        query = query.filter(pkg.tags.any(
            models.Tag.name.in_(filters['tag'])))
    if 'class_name' in filters.keys():
        query = query.filter(pkg.class_definitions.any(
            models.Class.name == filters['class_name']))
    if 'fqn' in filters.keys():
        query = query.filter(pkg.fully_qualified_name == filters['fqn'])

    if 'search' in filters.keys():
        fk_fields = {'categories': 'Category',
                     'tags': 'Tag',
                     'class_definitions': 'Class'}
        conditions = []

        for attr in dir(pkg):
            if attr.startswith('_'):
                continue
            if isinstance(getattr(pkg, attr),
                          attributes.InstrumentedAttribute):
                search_str = filters['search']
                for delim in ',;':
                    search_str = search_str.replace(delim, ' ')
                for key_word in search_str.split():
                    _word = '%{value}%'.format(value=key_word)
                    if attr in fk_fields.keys():
                        condition = getattr(pkg, attr).any(
                            getattr(models, fk_fields[attr]).name.like(_word))
                        conditions.append(condition)
                    else:
                        conditions.append(getattr(pkg, attr).like(_word))
        query = query.filter(or_(*conditions))

    sort_keys = filters.get('order_by', []) or ['created']
    for key in sort_keys:
        query = query.order_by(get_pkg_attr(pkg, key))

    marker = filters.get('marker')
    if marker is not None:
        # Note(efedorova): Copied from Glance
        #   Pagination works by requiring a unique sort_key, specified by sort_
        #   keys. (If sort_keys is not unique, then we risk looping through
        #   values.) We use the last row in the previous page as the 'marker'
        #   for pagination. So we must return values that follow the passed
        #   marker in the order. With a single-valued sort_key, this would
        #   be easy: sort_key > X. With a compound-values sort_key,
        #   (k1, k2, k3) we must do this to repeat the lexicographical
        #   ordering: (k1 > X1) or (k1 == X1 && k2 > X2) or
        #   (k1 == X1 && k2 == X2 && k3 > X3)

        model_marker = _package_get(marker, session)
        marker_values = []
        for sort_key in sort_keys:
            v = getattr(model_marker, sort_key)
            marker_values.append(v)

        # Build up an array of sort criteria as in the docstring
        criteria_list = []
        for i in range(len(sort_keys)):
            crit_attrs = []
            for j in range(i):
                model_attr = get_pkg_attr(pkg, sort_keys[j])
                crit_attrs.append((model_attr == marker_values[j]))

            model_attr = get_pkg_attr(pkg, sort_keys[i])
            crit_attrs.append((model_attr > marker_values[i]))

            criteria = sql.and_(*crit_attrs)
            criteria_list.append(criteria)
            f = sql.or_(*criteria_list)
            query = query.filter(f)

    if limit is not None:
        query = query.limit(limit)

    return query.all()

def package_upload(values, tenant_id):
    """
    Upload a package with new application
    :param values: parameters describing the new package
    :returns: detailed information about new package, dict
    """
    session = db_session.get_session()
    package = models.Package()

    composite_attr_to_func = {'categories': _get_categories,
                              'tags': _get_tags,
                              'class_definitions': _get_class_definitions}
    with session.begin():
        for attr, func in composite_attr_to_func.iteritems():
            if values.get(attr):
                result = func(values[attr], session)
                setattr(package, attr, result)
                del values[attr]

        package.update(values)
        package.owner_id = tenant_id
        package.save(session)
    return package

def package_delete(package_id):
    """
    Delete package information from the system ID of a package, string
    parameters to update
    """
    session = db_session.get_session()
    with session.begin():
        package = session.query(models.Package).get(package_id)

        session.delete(package)

def categories_list():
    session = db_session.get_session()
    return session.query(models.Category).all()
<commit_msg>Remove hardcoded foreign keys from db.catalog.api<commit_after>#    Copyright (c) 2014 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from sqlalchemy import inspection
from sqlalchemy import or_
from sqlalchemy.orm import attributes
from sqlalchemy import sql
from webob import exc

from muranoapi.db import models
from muranoapi.db import session as db_session
from muranoapi.openstack.common.gettextutils import _  # noqa
from muranoapi.openstack.common import log as logging

SEARCH_MAPPING = {'fqn': 'fully_qualified_name',
                  'name': 'name',
                  'created': 'created'
                  }

LOG = logging.getLogger(__name__)

def category_get_names():
    session = db_session.get_session()
    categories = []
    for row in session.query(models.Category.name).all():
        for name in row:
            categories.append(name)
    return categories

def _package_get(package_id, session):
    package = session.query(models.Package).get(package_id)
    if not package:
        msg = _("Package id '{0}' is not found").format(package_id)
        LOG.error(msg)
        raise exc.HTTPNotFound(msg)

    return package

def _authorize_package(package, context, allow_public=False):
    if context.is_admin:
        return

    if package.owner_id != context.tenant:
        if not allow_public:
            msg = _("Package '{0}' is not owned by "
                    "tenant '{1}'").format(package.id, context.tenant)
            LOG.error(msg)
            raise exc.HTTPForbidden(msg)
        if not package.is_public:
            msg = _("Package '{0}' is not public and not owned by "
                    "tenant '{1}' ").format(package.id, context.tenant)
            LOG.error(msg)
            raise exc.HTTPForbidden(msg)

def package_get(package_id, context):
    """
    Return package details
    :param package_id: ID of a package, string
    :returns: detailed information about package, dict
    """
    session = db_session.get_session()
    package = _package_get(package_id, session)
    _authorize_package(package, context, allow_public=True)
    return package

def _get_categories(category_names, session=None):
    """
    Return existing category objects or raise an exception
    :param category_names: name of categories to associate with package, list
    :returns: list of Category objects to associate with package, list
    """
    if session is None:
        session = db_session.get_session()
    categories = []
    for ctg_name in category_names:
        ctg_obj = session.query(models.Category).filter_by(
            name=ctg_name).first()
        if not ctg_obj:
            msg = _("Category '{name}' doesn't exist").format(name=ctg_name)
            LOG.error(msg)
            # it's not allowed to specify non-existent categories
            raise exc.HTTPBadRequest(msg)

        categories.append(ctg_obj)
    return categories

def _get_tags(tag_names, session=None):
    """
    Return existing tags object or create new ones
    :param tag_names: name of tags to associate with package, list
    :returns: list of Tag objects to associate with package, list
    """
    if session is None:
        session = db_session.get_session()
    tags = []
    for tag_name in tag_names:
        tag_obj = session.query(models.Tag).filter_by(name=tag_name).first()
        if tag_obj:
            tags.append(tag_obj)
        else:
            tag_record = models.Tag(name=tag_name)
            tags.append(tag_record)
    return tags

def _get_class_definitions(class_names, session=None):
    classes = []
    for name in class_names:
        class_record = models.Class(name=name)
        classes.append(class_record)
    return classes

def _do_replace(package, change):
    path = change['path'][0]
    value = change['value']
    calculate = {'categories': _get_categories,
                 'tags': _get_tags}
    if path in ('categories', 'tags'):
        existing_items = getattr(package, path)

        duplicates = list(set(i.name for i in existing_items) & set(value))
        unique_values = [x for x in value if x not in duplicates]
        items_to_replace = calculate[path](unique_values)

        # NOTE(efedorova): Replacing duplicate entities is not allowed,
        # so need to remove anything, but duplicates
        # and append everything but duplicates
        for item in list(existing_items):
            if item.name not in duplicates:
                existing_items.remove(item)

        existing_items.extend(items_to_replace)
    else:
        setattr(package, path, value)

    return package

def _do_add(package, change):
    # Only categories and tags support addition
    path = change['path'][0]
    value = change['value']

    calculate = {'categories': _get_categories,
                 'tags': _get_tags}
    items_to_add = calculate[path](value)
    for item in items_to_add:
        try:
            getattr(package, path).append(item)
        except AssertionError:
            msg = _('One of the specified {0} is already '
                    'associated with a package. Doing nothing.')
            LOG.warning(msg.format(path))
    return package

def _do_remove(package, change):
    # Only categories and tags support removal
    def find(seq, predicate):
        for elt in seq:
            if predicate(elt):
                return elt

    path = change['path'][0]
    values = change['value']

    current_values = getattr(package, path)
    for value in values:
        if value not in [i.name for i in current_values]:
            msg = _("Value '{0}' of property '{1}' "
                    "does not exist.").format(value, path)
            LOG.error(msg)
            raise exc.HTTPNotFound(msg)
        if path == 'categories' and len(current_values) == 1:
            msg = _("At least one category should be assigned to the package")
            LOG.error(msg)
            raise exc.HTTPBadRequest(msg)
        item_to_remove = find(current_values, lambda i: i.name == value)
        current_values.remove(item_to_remove)
    return package

def package_update(pkg_id, changes, context):
    """
    Update package information
    :param changes: parameters to update
    :returns: detailed information about new package, dict
    """

    operation_methods = {'add': _do_add,
                         'replace': _do_replace,
                         'remove': _do_remove}
    session = db_session.get_session()
    with session.begin():
        pkg = _package_get(pkg_id, session)
        _authorize_package(pkg, context)

        for change in changes:
            pkg = operation_methods[change['op']](pkg, change)
        session.add(pkg)
    return pkg

def package_search(filters, context):
    """
    Search packages with different filters
      * Admin is allowed to browse all the packages
      * Regular user is allowed to browse all packages belongs to user tenant
        and all other packages marked is_public.
        Also all packages should be enabled.
      * Use marker and limit for pagination:
        The typical pattern of limit and marker is to make an initial limited
        request and then to use the ID of the last package from the response
        as the marker parameter in a subsequent limited request.
    """

    def _validate_limit(value):
        if value is None:
            return
        try:
            value = int(value)
        except ValueError:
            msg = _("limit param must be an integer")
            LOG.error(msg)
            raise exc.HTTPBadRequest(explanation=msg)

        if value < 0:
            msg = _("limit param must be positive")
            LOG.error(msg)
            raise exc.HTTPBadRequest(explanation=msg)

        return value

    def get_pkg_attr(package_obj, search_attr_name):
        return getattr(package_obj, SEARCH_MAPPING[search_attr_name])

    limit = _validate_limit(filters.get('limit'))

    session = db_session.get_session()
    pkg = models.Package

    if 'owned' in filters.keys():
        query = session.query(pkg).filter(pkg.owner_id == context.tenant)
    elif context.is_admin:
        query = session.query(pkg)
    else:
        query = session.query(pkg).filter(or_((pkg.is_public & pkg.enabled),
                                              pkg.owner_id == context.tenant))
    if 'type' in filters.keys():
        query = query.filter(pkg.type == filters['type'].title())

    if 'category' in filters.keys():
        query = query.filter(pkg.categories.any(
            models.Category.name.in_(filters['category'])))
    if 'tag' in filters.keys():
        query = query.filter(pkg.tags.any(
            models.Tag.name.in_(filters['tag'])))
    if 'class_name' in filters.keys():
        query = query.filter(pkg.class_definitions.any(
            models.Class.name == filters['class_name']))
    if 'fqn' in filters.keys():
        query = query.filter(pkg.fully_qualified_name == filters['fqn'])

    if 'search' in filters.keys():
        conditions = []
        search_str = filters['search']
        for delimiter in ',;':
            search_str = search_str.replace(delimiter, ' ')
        key_words = ['%{0}%'.format(word) for word in search_str.split()]
        relationships = inspection.inspect(pkg).relationships

        searchable_attrs = set([
            'fully_qualified_name', 'author', 'name', 'description', 'tags',
            'categories', 'class_definitions'])
        searchable_pkg_attrs = set(dir(pkg)) & searchable_attrs

        for attr_name in searchable_pkg_attrs:
            attr = getattr(pkg, attr_name)
            if isinstance(attr, attributes.InstrumentedAttribute):
                if attr_name in relationships:
                    model = relationships[attr_name].mapper.class_
                    for key_word in key_words:
                        condition = attr.any(model.name.like(key_word))
                        conditions.append(condition)
                else:
                    for key_word in key_words:
                        conditions.append(attr.like(key_word))
        query = query.filter(or_(*conditions))

    sort_keys = filters.get('order_by', []) or ['created']
    for key in sort_keys:
        query = query.order_by(get_pkg_attr(pkg, key))

    marker = filters.get('marker')
    if marker is not None:
        # Note(efedorova): Copied from Glance
        #   Pagination works by requiring a unique sort_key, specified by sort_
        #   keys. (If sort_keys is not unique, then we risk looping through
        #   values.) We use the last row in the previous page as the 'marker'
        #   for pagination. So we must return values that follow the passed
        #   marker in the order. With a single-valued sort_key, this would
        #   be easy: sort_key > X. With a compound-values sort_key,
        #   (k1, k2, k3) we must do this to repeat the lexicographical
        #   ordering: (k1 > X1) or (k1 == X1 && k2 > X2) or
        #   (k1 == X1 && k2 == X2 && k3 > X3)

        model_marker = _package_get(marker, session)
        marker_values = []
        for sort_key in sort_keys:
            v = getattr(model_marker, sort_key)
            marker_values.append(v)

        # Build up an array of sort criteria as in the docstring
        criteria_list = []
        for i in range(len(sort_keys)):
            crit_attrs = []
            for j in range(i):
                model_attr = get_pkg_attr(pkg, sort_keys[j])
                crit_attrs.append((model_attr == marker_values[j]))

            model_attr = get_pkg_attr(pkg, sort_keys[i])
            crit_attrs.append((model_attr > marker_values[i]))

            criteria = sql.and_(*crit_attrs)
            criteria_list.append(criteria)
            f = sql.or_(*criteria_list)
            query = query.filter(f)

    if limit is not None:
        query = query.limit(limit)

    return query.all()

def package_upload(values, tenant_id):
    """
    Upload a package with new application
    :param values: parameters describing the new package
    :returns: detailed information about new package, dict
    """
    session = db_session.get_session()
    package = models.Package()

    composite_attr_to_func = {'categories': _get_categories,
                              'tags': _get_tags,
                              'class_definitions': _get_class_definitions}
    with session.begin():
        for attr, func in composite_attr_to_func.iteritems():
            if values.get(attr):
                result = func(values[attr], session)
                setattr(package, attr, result)
                del values[attr]

        package.update(values)
        package.owner_id = tenant_id
        package.save(session)
    return package

def package_delete(package_id):
    """
    Delete package information from the system ID of a package, string
    parameters to update
    """
    session = db_session.get_session()
    with session.begin():
        package = session.query(models.Package).get(package_id)

        session.delete(package)

def categories_list():
    session = db_session.get_session()
    return session.query(models.Category).all()
<|endoftext|>


<commit_before># encoding: utf-8

import argparse
import collections
import os.path

from nltk.corpus import comtrans
from nltk.translate import bleu_score
import numpy
import six

import chainer
from chainer import cuda
import chainer.functions as F
import chainer.links as L
from chainer import reporter
from chainer import training
from chainer.training import extensions

import europal

def sequence_embed(embed, xs):
    x_len = [len(x) for x in xs]
    x_section = numpy.cumsum(x_len[:-1])
    ex = embed(F.concat(xs, axis=0))
    exs = F.split_axis(ex, x_section, 0, force_tuple=True)
    return exs

class Seq2seq(chainer.Chain):

    def __init__(self, n_layers, n_source_vocab, n_target_vocab, n_units):
        super(Seq2seq, self).__init__(
            embed_x=L.EmbedID(n_source_vocab, n_units),
            embed_y=L.EmbedID(n_target_vocab, n_units),
            encoder=L.NStepLSTM(n_layers, n_units, n_units, 0.1),
            decoder=L.NStepLSTM(n_layers, n_units, n_units, 0.1),
            W=L.Linear(n_units, n_target_vocab),
        )
        self.n_layers = n_layers
        self.n_units = n_units

    def __call__(self, *inputs):
        xs = inputs[:len(inputs) // 2]
        ys = inputs[len(inputs) // 2:]

        xs = [x[::-1] for x in xs]

        eos = self.xp.zeros(1, 'i')
        ys_in = [F.concat([eos, y], axis=0) for y in ys]
        ys_out = [F.concat([y, eos], axis=0) for y in ys]

        exs = sequence_embed(self.embed_x, xs)
        eys = sequence_embed(self.embed_y, ys_in)

        batch = len(xs)
        # Initial hidden variable and cell variable
        zero = self.xp.zeros((self.n_layers, batch, self.n_units), 'f')
        hx, cx, _ = self.encoder(zero, zero, exs)
        _, _, os = self.decoder(hx, cx, eys)
        concat_os = F.concat(os, axis=0)
        concat_ys_out = F.concat(ys_out, axis=0)
        loss = F.softmax_cross_entropy(
            self.W(concat_os), concat_ys_out, normalize=False) \
            * concat_ys_out.shape[0] / batch

        reporter.report({'loss': loss.data}, self)
        perp = self.xp.exp(loss.data / concat_ys_out.shape[0] * batch)
        reporter.report({'perp': perp}, self)
        return loss

    def translate(self, xs, max_length=50):
        batch = len(xs)
        with chainer.no_backprop_mode():
            xs = [x[::-1] for x in xs]
            exs = sequence_embed(self.embed_x, xs)
            # Initial hidden variable and cell variable
            zero = self.xp.zeros((self.n_layers, batch, self.n_units), 'f')
            h, c, _ = self.encoder(zero, zero, exs, train=False)
            ys = self.xp.zeros(batch, 'i')
            result = []
            for i in range(max_length):
                eys = self.embed_y(ys)
                eys = chainer.functions.split_axis(
                    eys, batch, 0, force_tuple=True)
                h, c, ys = self.decoder(h, c, eys, train=False)
                cys = chainer.functions.concat(ys, axis=0)
                wy = self.W(cys)
                ys = self.xp.argmax(wy.data, axis=1).astype('i')
                result.append(ys)

        result = cuda.to_cpu(self.xp.stack(result).T)

        # Remove EOS taggs
        outs = []
        for y in result:
            inds = numpy.argwhere(y == 0)
            if len(inds) > 0:
                y = y[:inds[0, 0]]
            outs.append(y)
        return outs

def convert(batch, device):
    def to_device_batch(batch):
        if device is None:
            return batch
        elif device < 0:
            return [chainer.dataset.to_device(device, x) for x in batch]
        else:
            xp = cuda.cupy.get_array_module(*batch)
            concat = xp.concatenate(batch, axis=0)
            sections = numpy.cumsum([len(x) for x in batch[:-1]], dtype='i')
            concat_dev = chainer.dataset.to_device(device, concat)
            batch_dev = cuda.cupy.split(concat_dev, sections)
            return batch_dev

    return tuple(
        to_device_batch([x for x, _ in batch]) +
        to_device_batch([y for _, y in batch]))

class CalculateBleu(chainer.training.Extension):

    trigger = 1, 'epoch'

    def __init__(self, model, test_data, batch=100, device=-1):
        self.model = model
        self.test_data = test_data
        self.batch = batch
        self.device = device

    def __call__(self, trainer):
        with chainer.no_backprop_mode():
            references = []
            hypotheses = []
            for i in range(0, len(self.test_data), self.batch):
                sources, targets = zip(*self.test_data[i:i + self.batch])
                references.extend([[t.tolist()] for t in targets])

                sources = [
                    chainer.dataset.to_device(self.device, x) for x in sources]
                ys = [y.tolist() for y in self.model.translate(sources)]
                hypotheses.extend(ys)

        bleu = bleu_score.corpus_bleu(
            references, hypotheses,
            smoothing_function=bleu_score.SmoothingFunction().method1)
        print('BELU {}'.format(bleu))

def main():
    parser = argparse.ArgumentParser(description='Chainer example: seq2seq')
    parser.add_argument('--batchsize', '-b', type=int, default=64,
                        help='Number of images in each mini-batch')
    parser.add_argument('--epoch', '-e', type=int, default=20,
                        help='Number of sweeps over the dataset to train')
    parser.add_argument('--gpu', '-g', type=int, default=-1,
                        help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--resume', '-r', default='',
                        help='Resume the training from snapshot')
    parser.add_argument('--unit', '-u', type=int, default=1024,
                        help='Number of units')
    parser.add_argument('--input', '-i', type=str, default='wmt',
                        help='Input directory')
    parser.add_argument('--out', '-o', default='result',
                        help='Directory to output the result')
    args = parser.parse_args()

    if False:
        sentences = comtrans.aligned_sents('alignment-en-fr.txt')
        source_ids = collections.defaultdict(lambda: len(source_ids))
        target_ids = collections.defaultdict(lambda: len(target_ids))
        target_ids['eos']
        data = []
        for sentence in sentences:
            source = numpy.array([source_ids[w] for w in sentence.words], 'i')
            target = numpy.array([target_ids[w] for w in sentence.mots], 'i')
            data.append((source, target))
        print('Source vocabulary: %d' % len(source_ids))
        print('Target vocabulary: %d' % len(target_ids))

        test_data = data[:len(data) / 10]
        train_data = data[len(data) / 10:]
    else:
        # Check file
        en_path = os.path.join(args.input, 'giga-fren.release2.fixed.en')
        source_vocab = ['<eos>', '<unk>'] + europal.count_words(en_path)
        source_data = europal.make_dataset(en_path, source_vocab)
        fr_path = os.path.join(args.input, 'giga-fren.release2.fixed.fr')
        target_vocab = ['<eos>', '<unk>'] + europal.count_words(fr_path)
        target_data = europal.make_dataset(fr_path, target_vocab)
        assert len(source_data) == len(target_data)
        print('Original training data size: %d' % len(source_data))
        train_data = [(s, t)
                      for s, t in six.moves.zip(source_data, target_data)
                      if 0 < len(s) < 50 and 0 < len(t) < 50]
        print('Filtered training data size: %d' % len(train_data))

        en_path = os.path.join(args.input, 'dev', 'newstest2013.en')
        source_data = europal.make_dataset(en_path, source_vocab)
        fr_path = os.path.join(args.input, 'dev', 'newstest2013.fr')
        target_data = europal.make_dataset(fr_path, target_vocab)
        assert len(source_data) == len(target_data)
        test_data = [(s, t) for s, t in six.moves.zip(source_data, target_data)
                     if 0 < len(s) and 0 < len(t)]

        source_ids = {word: index for index, word in enumerate(source_vocab)}
        target_ids = {word: index for index, word in enumerate(target_vocab)}

    target_words = {i: w for w, i in target_ids.items()}
    source_words = {i: w for w, i in source_ids.items()}

    model = Seq2seq(3, len(source_ids), len(target_ids), args.unit)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu(args.gpu)

    optimizer = chainer.optimizers.Adam()
    optimizer.setup(model)

    train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize)
    updater = training.StandardUpdater(
        train_iter, optimizer, converter=convert, device=args.gpu)
    trainer = training.Trainer(updater, (args.epoch, 'epoch'))
    trainer.extend(extensions.LogReport(trigger=(200, 'iteration')),
                   trigger=(200, 'iteration'))
    trainer.extend(extensions.PrintReport(
        ['epoch', 'iteration', 'main/loss', 'validation/main/loss',
         'main/perp', 'validation/main/perp', 'elapsed_time']),
        trigger=(200, 'iteration'))

    def translate_one(source, target):
        words = europal.split_sentence(source)
        print('# source : ' + ' '.join(words))
        x = model.xp.array(
            [source_ids.get(w, 1) for w in words], 'i')
        ys = model.translate([x])[0]
        words = [target_words[y] for y in ys]
        print('#  result : ' + ' '.join(words))
        print('#  expect : ' + target)

    @chainer.training.make_extension(trigger=(200, 'iteration'))
    def translate(trainer):
        translate_one(
            'Who are we ?',
            'Qui sommes-nous?')
        translate_one(
            'And it often costs over a hundred dollars ' +
            'to obtain the required identity card .',
            'Or, il en coûte souvent plus de cent dollars ' +
            'pour obtenir la carte d\'identité requise.')

        source, target = test_data[numpy.random.choice(len(test_data))]
        source = ' '.join([source_words[i] for i in source])
        target = ' '.join([target_words[i] for i in target])
        translate_one(source, target)

    trainer.extend(translate, trigger=(200, 'iteration'))
    trainer.extend(CalculateBleu(model, test_data, device=args.gpu),
                   trigger=(10000, 'iteration'))
    print('start training')
    trainer.run()

if __name__ == '__main__':
    main()
<commit_msg>Fix typo<commit_after># encoding: utf-8

import argparse
import collections
import os.path

from nltk.corpus import comtrans
from nltk.translate import bleu_score
import numpy
import six

import chainer
from chainer import cuda
import chainer.functions as F
import chainer.links as L
from chainer import reporter
from chainer import training
from chainer.training import extensions

import europal

def sequence_embed(embed, xs):
    x_len = [len(x) for x in xs]
    x_section = numpy.cumsum(x_len[:-1])
    ex = embed(F.concat(xs, axis=0))
    exs = F.split_axis(ex, x_section, 0, force_tuple=True)
    return exs

class Seq2seq(chainer.Chain):

    def __init__(self, n_layers, n_source_vocab, n_target_vocab, n_units):
        super(Seq2seq, self).__init__(
            embed_x=L.EmbedID(n_source_vocab, n_units),
            embed_y=L.EmbedID(n_target_vocab, n_units),
            encoder=L.NStepLSTM(n_layers, n_units, n_units, 0.1),
            decoder=L.NStepLSTM(n_layers, n_units, n_units, 0.1),
            W=L.Linear(n_units, n_target_vocab),
        )
        self.n_layers = n_layers
        self.n_units = n_units

    def __call__(self, *inputs):
        xs = inputs[:len(inputs) // 2]
        ys = inputs[len(inputs) // 2:]

        xs = [x[::-1] for x in xs]

        eos = self.xp.zeros(1, 'i')
        ys_in = [F.concat([eos, y], axis=0) for y in ys]
        ys_out = [F.concat([y, eos], axis=0) for y in ys]

        exs = sequence_embed(self.embed_x, xs)
        eys = sequence_embed(self.embed_y, ys_in)

        batch = len(xs)
        # Initial hidden variable and cell variable
        zero = self.xp.zeros((self.n_layers, batch, self.n_units), 'f')
        hx, cx, _ = self.encoder(zero, zero, exs)
        _, _, os = self.decoder(hx, cx, eys)
        concat_os = F.concat(os, axis=0)
        concat_ys_out = F.concat(ys_out, axis=0)
        loss = F.softmax_cross_entropy(
            self.W(concat_os), concat_ys_out, normalize=False) \
            * concat_ys_out.shape[0] / batch

        reporter.report({'loss': loss.data}, self)
        perp = self.xp.exp(loss.data / concat_ys_out.shape[0] * batch)
        reporter.report({'perp': perp}, self)
        return loss

    def translate(self, xs, max_length=50):
        batch = len(xs)
        with chainer.no_backprop_mode():
            xs = [x[::-1] for x in xs]
            exs = sequence_embed(self.embed_x, xs)
            # Initial hidden variable and cell variable
            zero = self.xp.zeros((self.n_layers, batch, self.n_units), 'f')
            h, c, _ = self.encoder(zero, zero, exs, train=False)
            ys = self.xp.zeros(batch, 'i')
            result = []
            for i in range(max_length):
                eys = self.embed_y(ys)
                eys = chainer.functions.split_axis(
                    eys, batch, 0, force_tuple=True)
                h, c, ys = self.decoder(h, c, eys, train=False)
                cys = chainer.functions.concat(ys, axis=0)
                wy = self.W(cys)
                ys = self.xp.argmax(wy.data, axis=1).astype('i')
                result.append(ys)

        result = cuda.to_cpu(self.xp.stack(result).T)

        # Remove EOS taggs
        outs = []
        for y in result:
            inds = numpy.argwhere(y == 0)
            if len(inds) > 0:
                y = y[:inds[0, 0]]
            outs.append(y)
        return outs

def convert(batch, device):
    def to_device_batch(batch):
        if device is None:
            return batch
        elif device < 0:
            return [chainer.dataset.to_device(device, x) for x in batch]
        else:
            xp = cuda.cupy.get_array_module(*batch)
            concat = xp.concatenate(batch, axis=0)
            sections = numpy.cumsum([len(x) for x in batch[:-1]], dtype='i')
            concat_dev = chainer.dataset.to_device(device, concat)
            batch_dev = cuda.cupy.split(concat_dev, sections)
            return batch_dev

    return tuple(
        to_device_batch([x for x, _ in batch]) +
        to_device_batch([y for _, y in batch]))

class CalculateBleu(chainer.training.Extension):

    trigger = 1, 'epoch'

    def __init__(self, model, test_data, batch=100, device=-1):
        self.model = model
        self.test_data = test_data
        self.batch = batch
        self.device = device

    def __call__(self, trainer):
        with chainer.no_backprop_mode():
            references = []
            hypotheses = []
            for i in range(0, len(self.test_data), self.batch):
                sources, targets = zip(*self.test_data[i:i + self.batch])
                references.extend([[t.tolist()] for t in targets])

                sources = [
                    chainer.dataset.to_device(self.device, x) for x in sources]
                ys = [y.tolist() for y in self.model.translate(sources)]
                hypotheses.extend(ys)

        bleu = bleu_score.corpus_bleu(
            references, hypotheses,
            smoothing_function=bleu_score.SmoothingFunction().method1)
        print('BLEU {}'.format(bleu))

def main():
    parser = argparse.ArgumentParser(description='Chainer example: seq2seq')
    parser.add_argument('--batchsize', '-b', type=int, default=64,
                        help='Number of images in each mini-batch')
    parser.add_argument('--epoch', '-e', type=int, default=20,
                        help='Number of sweeps over the dataset to train')
    parser.add_argument('--gpu', '-g', type=int, default=-1,
                        help='GPU ID (negative value indicates CPU)')
    parser.add_argument('--resume', '-r', default='',
                        help='Resume the training from snapshot')
    parser.add_argument('--unit', '-u', type=int, default=1024,
                        help='Number of units')
    parser.add_argument('--input', '-i', type=str, default='wmt',
                        help='Input directory')
    parser.add_argument('--out', '-o', default='result',
                        help='Directory to output the result')
    args = parser.parse_args()

    if False:
        sentences = comtrans.aligned_sents('alignment-en-fr.txt')
        source_ids = collections.defaultdict(lambda: len(source_ids))
        target_ids = collections.defaultdict(lambda: len(target_ids))
        target_ids['eos']
        data = []
        for sentence in sentences:
            source = numpy.array([source_ids[w] for w in sentence.words], 'i')
            target = numpy.array([target_ids[w] for w in sentence.mots], 'i')
            data.append((source, target))
        print('Source vocabulary: %d' % len(source_ids))
        print('Target vocabulary: %d' % len(target_ids))

        test_data = data[:len(data) / 10]
        train_data = data[len(data) / 10:]
    else:
        # Check file
        en_path = os.path.join(args.input, 'giga-fren.release2.fixed.en')
        source_vocab = ['<eos>', '<unk>'] + europal.count_words(en_path)
        source_data = europal.make_dataset(en_path, source_vocab)
        fr_path = os.path.join(args.input, 'giga-fren.release2.fixed.fr')
        target_vocab = ['<eos>', '<unk>'] + europal.count_words(fr_path)
        target_data = europal.make_dataset(fr_path, target_vocab)
        assert len(source_data) == len(target_data)
        print('Original training data size: %d' % len(source_data))
        train_data = [(s, t)
                      for s, t in six.moves.zip(source_data, target_data)
                      if 0 < len(s) < 50 and 0 < len(t) < 50]
        print('Filtered training data size: %d' % len(train_data))

        en_path = os.path.join(args.input, 'dev', 'newstest2013.en')
        source_data = europal.make_dataset(en_path, source_vocab)
        fr_path = os.path.join(args.input, 'dev', 'newstest2013.fr')
        target_data = europal.make_dataset(fr_path, target_vocab)
        assert len(source_data) == len(target_data)
        test_data = [(s, t) for s, t in six.moves.zip(source_data, target_data)
                     if 0 < len(s) and 0 < len(t)]

        source_ids = {word: index for index, word in enumerate(source_vocab)}
        target_ids = {word: index for index, word in enumerate(target_vocab)}

    target_words = {i: w for w, i in target_ids.items()}
    source_words = {i: w for w, i in source_ids.items()}

    model = Seq2seq(3, len(source_ids), len(target_ids), args.unit)
    if args.gpu >= 0:
        chainer.cuda.get_device(args.gpu).use()
        model.to_gpu(args.gpu)

    optimizer = chainer.optimizers.Adam()
    optimizer.setup(model)

    train_iter = chainer.iterators.SerialIterator(train_data, args.batchsize)
    updater = training.StandardUpdater(
        train_iter, optimizer, converter=convert, device=args.gpu)
    trainer = training.Trainer(updater, (args.epoch, 'epoch'))
    trainer.extend(extensions.LogReport(trigger=(200, 'iteration')),
                   trigger=(200, 'iteration'))
    trainer.extend(extensions.PrintReport(
        ['epoch', 'iteration', 'main/loss', 'validation/main/loss',
         'main/perp', 'validation/main/perp', 'elapsed_time']),
        trigger=(200, 'iteration'))

    def translate_one(source, target):
        words = europal.split_sentence(source)
        print('# source : ' + ' '.join(words))
        x = model.xp.array(
            [source_ids.get(w, 1) for w in words], 'i')
        ys = model.translate([x])[0]
        words = [target_words[y] for y in ys]
        print('#  result : ' + ' '.join(words))
        print('#  expect : ' + target)

    @chainer.training.make_extension(trigger=(200, 'iteration'))
    def translate(trainer):
        translate_one(
            'Who are we ?',
            'Qui sommes-nous?')
        translate_one(
            'And it often costs over a hundred dollars ' +
            'to obtain the required identity card .',
            'Or, il en coûte souvent plus de cent dollars ' +
            'pour obtenir la carte d\'identité requise.')

        source, target = test_data[numpy.random.choice(len(test_data))]
        source = ' '.join([source_words[i] for i in source])
        target = ' '.join([target_words[i] for i in target])
        translate_one(source, target)

    trainer.extend(translate, trigger=(200, 'iteration'))
    trainer.extend(CalculateBleu(model, test_data, device=args.gpu),
                   trigger=(10000, 'iteration'))
    print('start training')
    trainer.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# Cuckoo Sandbox - Automated Malware Analysis
# Copyright (C) 2010-2011  Claudio "nex" Guarnieri (nex@cuckoobox.org)
# http://www.cuckoobox.org
#
# This file is part of Cuckoo.
#
# Cuckoo is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Cuckoo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses/.

import os
import sys
import string
import random
import logging
from shutil import *
from ctypes import sizeof, byref, c_int, c_ulong, wintypes
import ctypes

sys.path.append("\\\\VBOXSVR\\setup\\lib\\")

import cuckoo.defines
from cuckoo.paths import *

# The following function was taken from PyBox:
# http://code.google.com/p/pyboxed
########################################################################
# Copyright (c) 2010
# Felix S. Leder <leder<at>cs<dot>uni-bonn<dot>de>
# Daniel Plohmann <plohmann<at>cs<dot>uni-bonn<dot>de>
# All rights reserved.
########################################################################
def grant_debug_privilege(pid = 0):
    log = logging.getLogger("Inject.GrantDebugPrivilege")
    """ grant SeDebugPrivilege to own process
    @param pid: Process id to set permissions of (or 0 if current)
    @type pid: int
    @return: True if operation was successful,
              False otherwise
    """
    cuckoo.defines.ADVAPI32.OpenProcessToken.argtypes = (wintypes.HANDLE,
                                                         wintypes.DWORD,
                                                         ctypes.POINTER(wintypes.HANDLE))

    cuckoo.defines.ADVAPI32.LookupPrivilegeValueW.argtypes = (wintypes.LPWSTR,
                                                              wintypes.LPWSTR,
                                                              ctypes.POINTER(cuckoo.defines.LUID))
    cuckoo.defines.ADVAPI32.AdjustTokenPrivileges.argtypes = (wintypes.HANDLE,
                                                              wintypes.BOOL,
                                                              ctypes.POINTER(cuckoo.defines.TOKEN_PRIVILEGES),
                                                              wintypes.DWORD,
                                                              ctypes.POINTER(cuckoo.defines.TOKEN_PRIVILEGES),
                                                              ctypes.POINTER(wintypes.DWORD))

    h_process = None
    if pid == 0:
        h_process = cuckoo.defines.KERNEL32.GetCurrentProcess()
    else:
        h_process = cuckoo.defines.KERNEL32.OpenProcess(cuckoo.defines.PROCESS_ALL_ACCESS,
                                                        False,
                                                        pid)

    if not h_process:
        log.error("Unable to open process handle (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    # obtain token to process
    h_current_token = wintypes.HANDLE()
    if not cuckoo.defines.ADVAPI32.OpenProcessToken(h_process,
                                                    cuckoo.defines.TOKEN_ALL_ACCESS,
                                                    h_current_token):
        log.error("Unable to open process token (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    # look up current privilege value
    se_original_luid = cuckoo.defines.LUID()
    if not cuckoo.defines.ADVAPI32.LookupPrivilegeValueW(None,
                                                         "SeDebugPrivilege",
                                                         se_original_luid):
        log.error("Unable to lookup current privilege (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    luid_attributes = cuckoo.defines.LUID_AND_ATTRIBUTES()
    luid_attributes.Luid = se_original_luid
    luid_attributes.Attributes = cuckoo.defines.SE_PRIVILEGE_ENABLED
    token_privs = cuckoo.defines.TOKEN_PRIVILEGES()
    token_privs.PrivilegeCount = 1;
    token_privs.Privileges = luid_attributes;

    if not cuckoo.defines.ADVAPI32.AdjustTokenPrivileges(h_current_token,
                                                         False,
                                                         token_privs,
                                                         0,
                                                         None,
                                                         None):
        log.error("Unable to adjust token privileges (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    cuckoo.defines.KERNEL32.CloseHandle(h_current_token)
    cuckoo.defines.KERNEL32.CloseHandle(h_process)

    log.info("Successfully granted debug privileges on Cuckoo process.")

    return True

def randomize_dll(dll_path):
    log = logging.getLogger("Inject.RandomizeDll")

    new_dll_name = "".join(random.choice(string.ascii_letters) for x in range(6))
    new_dll_path = os.path.join(CUCKOO_DLL_FOLDER, "%s.dll" % new_dll_name)

    try:
        copy(dll_path, new_dll_path)
        return new_dll_path
    except (IOError, os.error), why:
        log.error("Something went wrong while randomzing DLL to path \"%s\": %s"
                  % (new_dll_path, why))
        return dll_path

def cuckoo_inject(pid, dll_path):
    log = logging.getLogger("Inject.Inject")

    if not os.path.exists(dll_path):
        log.error("DLL does not exist at path \"%s\"." % dll_path)
        return False

    dll_path = randomize_dll(dll_path)

    # If target process is current,
    if pid == os.getpid():
        return False

    grant_debug_privilege()

    h_process = cuckoo.defines.KERNEL32.OpenProcess(cuckoo.defines.PROCESS_ALL_ACCESS,
                                                    False,
                                                    int(pid))

    if not h_process:
        log.error("Unable to obtain handle on process with PID %d (GLE=%d). " \
                  "Abort."
                  % (pid, cuckoo.defines.KERNEL32.GetLastError()))
        return False

    ll_param = cuckoo.defines.KERNEL32.VirtualAllocEx(h_process,
                                                      0,
                                                      len(dll_path),
                                                      cuckoo.defines.MEM_RESERVE | \
                                                      cuckoo.defines.MEM_COMMIT,
                                                      cuckoo.defines.PAGE_READWRITE)

    bytes_written = c_int(0)

    if not cuckoo.defines.KERNEL32.WriteProcessMemory(h_process,
                                                      ll_param,
                                                      dll_path,
                                                      len(dll_path),
                                                      byref(bytes_written)):
        log.error("Unable to write on memory of process with PID %d (GLE=%d)." \
                  "" % (pid, cuckoo.defines.KERNEL32.GetLastError()))
        return False

    lib_addr = cuckoo.defines.KERNEL32.GetProcAddress(cuckoo.defines.KERNEL32.GetModuleHandleA("kernel32.dll"),
                                                      "LoadLibraryA")

    new_thread_id = c_ulong(0)

    if not cuckoo.defines.KERNEL32.CreateRemoteThread(h_process,
                                                      None,
                                                      0,
                                                      lib_addr,
                                                      ll_param,
                                                      0,
                                                      byref(new_thread_id)):
        log.error("Unable to create a remote thread in process with PID %d " \
                  "(GLE=%d)." % (pid, cuckoo.defines.GetLastError()))
        return False

    log.debug("Process with PID %d successfully injected with DLL at path " \
              "\"%s\"." % (pid, dll_path))

    return True
<commit_msg>Added log entry to attempt on injecting cuckoo analyzer process<commit_after>#!/usr/bin/python
# Cuckoo Sandbox - Automated Malware Analysis
# Copyright (C) 2010-2011  Claudio "nex" Guarnieri (nex@cuckoobox.org)
# http://www.cuckoobox.org
#
# This file is part of Cuckoo.
#
# Cuckoo is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Cuckoo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses/.

import os
import sys
import string
import random
import logging
from shutil import *
from ctypes import sizeof, byref, c_int, c_ulong, wintypes
import ctypes

sys.path.append("\\\\VBOXSVR\\setup\\lib\\")

import cuckoo.defines
from cuckoo.paths import *

# The following function was taken from PyBox:
# http://code.google.com/p/pyboxed
########################################################################
# Copyright (c) 2010
# Felix S. Leder <leder<at>cs<dot>uni-bonn<dot>de>
# Daniel Plohmann <plohmann<at>cs<dot>uni-bonn<dot>de>
# All rights reserved.
########################################################################
def grant_debug_privilege(pid = 0):
    log = logging.getLogger("Inject.GrantDebugPrivilege")
    """ grant SeDebugPrivilege to own process
    @param pid: Process id to set permissions of (or 0 if current)
    @type pid: int
    @return: True if operation was successful,
              False otherwise
    """
    cuckoo.defines.ADVAPI32.OpenProcessToken.argtypes = (wintypes.HANDLE,
                                                         wintypes.DWORD,
                                                         ctypes.POINTER(wintypes.HANDLE))

    cuckoo.defines.ADVAPI32.LookupPrivilegeValueW.argtypes = (wintypes.LPWSTR,
                                                              wintypes.LPWSTR,
                                                              ctypes.POINTER(cuckoo.defines.LUID))
    cuckoo.defines.ADVAPI32.AdjustTokenPrivileges.argtypes = (wintypes.HANDLE,
                                                              wintypes.BOOL,
                                                              ctypes.POINTER(cuckoo.defines.TOKEN_PRIVILEGES),
                                                              wintypes.DWORD,
                                                              ctypes.POINTER(cuckoo.defines.TOKEN_PRIVILEGES),
                                                              ctypes.POINTER(wintypes.DWORD))

    h_process = None
    if pid == 0:
        h_process = cuckoo.defines.KERNEL32.GetCurrentProcess()
    else:
        h_process = cuckoo.defines.KERNEL32.OpenProcess(cuckoo.defines.PROCESS_ALL_ACCESS,
                                                        False,
                                                        pid)

    if not h_process:
        log.error("Unable to open process handle (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    # obtain token to process
    h_current_token = wintypes.HANDLE()
    if not cuckoo.defines.ADVAPI32.OpenProcessToken(h_process,
                                                    cuckoo.defines.TOKEN_ALL_ACCESS,
                                                    h_current_token):
        log.error("Unable to open process token (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    # look up current privilege value
    se_original_luid = cuckoo.defines.LUID()
    if not cuckoo.defines.ADVAPI32.LookupPrivilegeValueW(None,
                                                         "SeDebugPrivilege",
                                                         se_original_luid):
        log.error("Unable to lookup current privilege (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    luid_attributes = cuckoo.defines.LUID_AND_ATTRIBUTES()
    luid_attributes.Luid = se_original_luid
    luid_attributes.Attributes = cuckoo.defines.SE_PRIVILEGE_ENABLED
    token_privs = cuckoo.defines.TOKEN_PRIVILEGES()
    token_privs.PrivilegeCount = 1;
    token_privs.Privileges = luid_attributes;

    if not cuckoo.defines.ADVAPI32.AdjustTokenPrivileges(h_current_token,
                                                         False,
                                                         token_privs,
                                                         0,
                                                         None,
                                                         None):
        log.error("Unable to adjust token privileges (GLE=%d)."
                  % cuckoo.defines.KERNEL32.GetLastError())
        return False

    cuckoo.defines.KERNEL32.CloseHandle(h_current_token)
    cuckoo.defines.KERNEL32.CloseHandle(h_process)

    log.info("Successfully granted debug privileges on Cuckoo process.")

    return True

def randomize_dll(dll_path):
    log = logging.getLogger("Inject.RandomizeDll")

    new_dll_name = "".join(random.choice(string.ascii_letters) for x in range(6))
    new_dll_path = os.path.join(CUCKOO_DLL_FOLDER, "%s.dll" % new_dll_name)

    try:
        copy(dll_path, new_dll_path)
        return new_dll_path
    except (IOError, os.error), why:
        log.error("Something went wrong while randomzing DLL to path \"%s\": %s"
                  % (new_dll_path, why))
        return dll_path

def cuckoo_inject(pid, dll_path):
    log = logging.getLogger("Inject.Inject")

    if not os.path.exists(dll_path):
        log.error("DLL does not exist at path \"%s\"." % dll_path)
        return False

    dll_path = randomize_dll(dll_path)

    # If target process is current, obviously abort.
    if pid == os.getpid():
        log.warning("The process to be injected is Cuckoo! Abort.")
        return False

    grant_debug_privilege()

    h_process = cuckoo.defines.KERNEL32.OpenProcess(cuckoo.defines.PROCESS_ALL_ACCESS,
                                                    False,
                                                    int(pid))

    if not h_process:
        log.error("Unable to obtain handle on process with PID %d (GLE=%d). " \
                  "Abort."
                  % (pid, cuckoo.defines.KERNEL32.GetLastError()))
        return False

    ll_param = cuckoo.defines.KERNEL32.VirtualAllocEx(h_process,
                                                      0,
                                                      len(dll_path),
                                                      cuckoo.defines.MEM_RESERVE | \
                                                      cuckoo.defines.MEM_COMMIT,
                                                      cuckoo.defines.PAGE_READWRITE)

    bytes_written = c_int(0)

    if not cuckoo.defines.KERNEL32.WriteProcessMemory(h_process,
                                                      ll_param,
                                                      dll_path,
                                                      len(dll_path),
                                                      byref(bytes_written)):
        log.error("Unable to write on memory of process with PID %d (GLE=%d)." \
                  "" % (pid, cuckoo.defines.KERNEL32.GetLastError()))
        return False

    lib_addr = cuckoo.defines.KERNEL32.GetProcAddress(cuckoo.defines.KERNEL32.GetModuleHandleA("kernel32.dll"),
                                                      "LoadLibraryA")

    new_thread_id = c_ulong(0)

    if not cuckoo.defines.KERNEL32.CreateRemoteThread(h_process,
                                                      None,
                                                      0,
                                                      lib_addr,
                                                      ll_param,
                                                      0,
                                                      byref(new_thread_id)):
        log.error("Unable to create a remote thread in process with PID %d " \
                  "(GLE=%d)." % (pid, cuckoo.defines.GetLastError()))
        return False

    log.debug("Process with PID %d successfully injected with DLL at path " \
              "\"%s\"." % (pid, dll_path))

    return True
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#

import os
import sys

from inspect import ismethod
from time import sleep
from OSEncryptionState import *

class EncryptBlockDeviceState(OSEncryptionState):
    def __init__(self, context):
        super(EncryptBlockDeviceState, self).__init__('EncryptBlockDeviceState', context)

    def should_enter(self):
        self.context.logger.log("Verifying if machine should enter encrypt_block_device state")

        if not super(EncryptBlockDeviceState, self).should_enter():
            return False

        self.context.logger.log("Performing enter checks for encrypt_block_device state")

        return True

    def enter(self):
        if not self.should_enter():
            return

        self.context.logger.log("Entering encrypt_block_device state")

        self._find_bek_and_execute_action('_dump_passphrase')
        self._find_bek_and_execute_action('_luks_format')
        self._find_bek_and_execute_action('_luks_open')

        self.command_executor.Execute('dd if=/dev/sda2 of=/dev/mapper/osencrypt bs=52428800', True)

    def should_exit(self):
        self.context.logger.log("Verifying if machine should exit encrypt_block_device state")

        return super(EncryptBlockDeviceState, self).should_exit()

    def _luks_format(self, bek_path):
        self.command_executor.Execute('mkdir /boot/luks', True)
        self.command_executor.Execute('dd if=/dev/zero of=/boot/luks/osluksheader bs=33554432 count=1', True)
        self.command_executor.Execute('cryptsetup luksFormat --header /boot/luks/osluksheader -d {0} /dev/sda2 -q'.format(bek_path),
                                      raise_exception_on_failure=True)

    def _luks_open(self, bek_path):
        self.command_executor.Execute('cryptsetup luksOpen --header /boot/luks/osluksheader /dev/sda2 osencrypt -d {0}'.format(bek_path),
                                      raise_exception_on_failure=True)

    def _dump_passphrase(self, bek_path):
        proc_comm = ProcessCommunicator()

        self.command_executor.Execute(command_to_execute="od -c {0}".format(bek_path),
                                      raise_exception_on_failure=True,
                                      communicator=proc_comm)
        self.context.logger.log("Passphrase:")
        self.context.logger.log(proc_comm.stdout)

    def _find_bek_and_execute_action(self, callback_method_name):
        callback_method = getattr(self, callback_method_name)
        if not ismethod(callback_method):
            raise Exception("{0} is not a method".format(callback_method_name))

        bek_path = self.bek_util.get_bek_passphrase_file(self.encryption_config)
        callback_method(bek_path)
<commit_msg>Add exit check for EncryptBlockDevice state<commit_after>#!/usr/bin/env python
#
# VM Backup extension
#
# Copyright 2015 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Requires Python 2.7+
#

import os
import sys

from inspect import ismethod
from time import sleep
from OSEncryptionState import *

class EncryptBlockDeviceState(OSEncryptionState):
    def __init__(self, context):
        super(EncryptBlockDeviceState, self).__init__('EncryptBlockDeviceState', context)

    def should_enter(self):
        self.context.logger.log("Verifying if machine should enter encrypt_block_device state")

        if not super(EncryptBlockDeviceState, self).should_enter():
            return False

        self.context.logger.log("Performing enter checks for encrypt_block_device state")

        return True

    def enter(self):
        if not self.should_enter():
            return

        self.context.logger.log("Entering encrypt_block_device state")

        self._find_bek_and_execute_action('_dump_passphrase')
        self._find_bek_and_execute_action('_luks_format')
        self._find_bek_and_execute_action('_luks_open')

        self.command_executor.Execute('dd if=/dev/sda2 of=/dev/mapper/osencrypt bs=52428800', True)

    def should_exit(self):
        self.context.logger.log("Verifying if machine should exit encrypt_block_device state")

        self.command_executor.Execute('mount /dev/mapper/osencrypt /oldroot', True)
        self.command_executor.Execute('umount /oldroot', True)

        return super(EncryptBlockDeviceState, self).should_exit()

    def _luks_format(self, bek_path):
        self.command_executor.Execute('mkdir /boot/luks', True)
        self.command_executor.Execute('dd if=/dev/zero of=/boot/luks/osluksheader bs=33554432 count=1', True)
        self.command_executor.Execute('cryptsetup luksFormat --header /boot/luks/osluksheader -d {0} /dev/sda2 -q'.format(bek_path),
                                      raise_exception_on_failure=True)

    def _luks_open(self, bek_path):
        self.command_executor.Execute('cryptsetup luksOpen --header /boot/luks/osluksheader /dev/sda2 osencrypt -d {0}'.format(bek_path),
                                      raise_exception_on_failure=True)

    def _dump_passphrase(self, bek_path):
        proc_comm = ProcessCommunicator()

        self.command_executor.Execute(command_to_execute="od -c {0}".format(bek_path),
                                      raise_exception_on_failure=True,
                                      communicator=proc_comm)
        self.context.logger.log("Passphrase:")
        self.context.logger.log(proc_comm.stdout)

    def _find_bek_and_execute_action(self, callback_method_name):
        callback_method = getattr(self, callback_method_name)
        if not ismethod(callback_method):
            raise Exception("{0} is not a method".format(callback_method_name))

        bek_path = self.bek_util.get_bek_passphrase_file(self.encryption_config)
        callback_method(bek_path)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# transcoder is the main interface for several transcoders
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jan 2, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import baidu_uck
import burify
from administration.config import hparser
from image_processor import image_helper
import os
import simplr
import threading

from administration.config import NEWS_TEMPLATE
from administration.config import NEWS_TEMPLATE_ARABIC
from administration.config import TRANSCODED_LOCAL_DIR
from administration.config import TRANSCODED_PUBLIC_DIR
from administration.config import TRANSCODING_BTN_AR
from administration.config import TRANSCODING_BTN_EN
from administration.config import TRANSCODING_BTN_IND
from administration.config import TRANSCODING_BTN_JA
from administration.config import TRANSCODING_BTN_PT
from administration.config import TRANSCODING_BTN_TH

# create a local dir for transcoded content if dir does not exist
if not os.path.exists(TRANSCODED_LOCAL_DIR):
    os.mkdir(TRANSCODED_LOCAL_DIR)

class TranscoderAPI(threading.Thread):
    """
    call a transcoder
    """
    def __init__(self, url="this should not exist", transcoder="simplr", language=None):
        threading.Thread.__init__(self)
        self.transcoder = transcoder
        self.url = url
        self.language = language
        self.result = None

    def run(self):
        if self.language:
            self.result = eval(self.transcoder).convert(self.url, self.language)
        else:
            self.result = eval(self.transcoder).convert(self.url)

def _save(data, path):
    """
    save the file on local disk and return web and local path
    """
    if not data or not path:
        return None

    local_path = '%s%s.html' % (TRANSCODED_LOCAL_DIR, path)
    web_path = '%s%s.html' % (TRANSCODED_PUBLIC_DIR, path)

    f = open(local_path, 'w')
    f.write(hparser.unescape(data))
    f.close()
    return web_path, local_path

def _compose(language, title, content):
    """
    combine content with a template
    """
    if not content or not language or not title:
        raise Exception("ERROR: Method not well formed!")

    transcode_button = {'en': TRANSCODING_BTN_EN, 'ja': TRANSCODING_BTN_JA, 'th': TRANSCODING_BTN_TH, 'pt':
                        TRANSCODING_BTN_PT, 'ind': TRANSCODING_BTN_IND, 'en-rIN': TRANSCODING_BTN_EN, 'ar': TRANSCODING_BTN_AR}

    # f reads the template
    f = None
    if language == 'ar':
        f = open(NEWS_TEMPLATE_ARABIC, 'r')
    else:
        f = open(NEWS_TEMPLATE, 'r')
    # a template is found
    if f:
        template = str(f.read())
        f.close()
        return template % (title, title, content, transcode_button[language])
    else:
        raise Exception("ERROR: Cannot find a template!")

def _combine(content, images):
    """
    combine results from transcoders
    """
    if not content or not images:
        return content, images

    # for now, if there are more than one image, take only one of them
    biggest = image_helper.find_biggest_image(images)
    IMAGE_TAG = ''
    image = IMAGE_TAG % (
        biggest['url'], str(biggest['width']), str(biggest['height']))
    return "%s %s" % (image, content), images

# TODO: add http string checkers
def _transcode(url, transcoders, language=None):
    """
    organize different transcoders
    """
    if not url or not transcoders:
        raise Exception("ERROR: Method not well formed!")

    threads = {}
    for transcoder in transcoders:
        if transcoder == 'simplr':
            transcoding_request = TranscoderAPI(url, transcoder, language)
        else:
            transcoding_request = TranscoderAPI(url, transcoder)
        # thread could be found via transcoder name
        threads[transcoder] = transcoding_request
        transcoding_request.start()
        # 10 second to wait UCK server
        transcoding_request.join(10 * 1000)

    # after a while ... put data in the proper variables
    uck_content = simplr_content = burify_content = None
    uck_images = simplr_images = burify_images = None
    simplr_title = burify_title = None

    if 'baidu_uck' in transcoders and 'baidu_uck' in threads:
        if threads['baidu_uck'].result:
            uck_content, uck_images = threads['baidu_uck'].result
    if 'simplr' in transcoders and 'simplr' in threads:
        if threads['simplr'].result:
            simplr_title, simplr_content, simplr_images = threads['simplr'].result
    if 'burify' in transcoders and 'burify' in threads:
        if threads['burify'].result:
            burify_title, burify_content, burify_images = threads['burify'].result

    # use different combinations to create a news page with pictures
    if 'simplr' in transcoders or 'burify' in transcoders:
        if 'simplr' in transcoders and simplr_content:
            # if simplr found any image
            if simplr_images:
                return simplr_title, simplr_content, simplr_images
            elif uck_images:  # add images from uck
                new_content, new_images = _combine(simplr_content, uck_images)
                return simplr_title, new_content, new_images
            else:  # no image at all
                return simplr_title, simplr_content, simplr_images
        elif 'burify' in transcoders and burify_content:
            # if burify found any image
            if burify_images:
                return burify_title, burify_content, burify_images
            elif uck_images:  # add images from uck
                new_content, new_images = _combine(burify_content, uck_images)
                return burify_title, new_content, new_images
            else:  # no image at all
                return burify_title, burify_content, burify_images
    # only uck
    if uck_content:
        return "", uck_content, uck_images
    else:
        raise Exception("ERROR: UCK failed!")

def _organize_transcoders(transcoder="chengdujin"):
    """
    get data from different transcoders
    chengdujin: simplr.py
    readability: burify.py
    uck: baidu_uck.py
    """
    transcoders = []
    if transcoder == 'chengdujin':
        transcoders.append("simplr")
        transcoders.append("baidu_uck")
    elif transcoder == 'readability':
        transcoders.append("burify")
        transcoders.append("baidu_uck")
    else:
        transcoders.append("baidu_uck")
    return transcoders

def _preprocess(link):
    """
    get the real address out
    """
    last_http_index = link.rfind('http')
    return link[last_http_index:].strip()

def convert(language="en", title=None, link=None, transcoder="chengdujin", relative_path=None):
    """
    select a transcoder
    send the link
    gather the data
    combine them with the template
    generate paths
    return news and images
    """
    if not language or not title or not link or not relative_path:
        raise Exception('ERROR: Method not well formed!')

    link = _preprocess(link)
    transcoders = _organize_transcoders(transcoder)
    title_new, content, images = _transcode(link, transcoders, language)
    # in case uck cannot find a proper title
    if title_new:
        title = title_new

    if content:
        # embed content in template
        news = _compose(language, title, content)
        # create web/local path
        web_path, local_path = _save(news, relative_path)
        return web_path, local_path, images
    else:
        raise Exception("ERROR: Transcoder %s failed for %s" %
                        (transcoder, link))
<commit_msg>simplr needs language to get a short title<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# transcoder is the main interface for several transcoders
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jan 2, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import baidu_uck
import burify
from administration.config import hparser
from image_processor import image_helper
import os
import simplr
import threading

from administration.config import NEWS_TEMPLATE
from administration.config import NEWS_TEMPLATE_ARABIC
from administration.config import TRANSCODED_LOCAL_DIR
from administration.config import TRANSCODED_PUBLIC_DIR
from administration.config import TRANSCODING_BTN_AR
from administration.config import TRANSCODING_BTN_EN
from administration.config import TRANSCODING_BTN_IND
from administration.config import TRANSCODING_BTN_JA
from administration.config import TRANSCODING_BTN_PT
from administration.config import TRANSCODING_BTN_TH

# create a local dir for transcoded content if dir does not exist
if not os.path.exists(TRANSCODED_LOCAL_DIR):
    os.mkdir(TRANSCODED_LOCAL_DIR)

class TranscoderAPI(threading.Thread):
    """
    call a transcoder
    """
    def __init__(self, url="this should not exist", transcoder="simplr", language=None):
        threading.Thread.__init__(self)
        self.transcoder = transcoder
        self.url = url
        self.language = language
        self.result = None

    def run(self):
        if self.transcoder == 'simplr':
            self.result = eval(self.transcoder).convert(self.url, self.language)
        else:
            self.result = eval(self.transcoder).convert(self.url)

def _save(data, path):
    """
    save the file on local disk and return web and local path
    """
    if not data or not path:
        return None

    local_path = '%s%s.html' % (TRANSCODED_LOCAL_DIR, path)
    web_path = '%s%s.html' % (TRANSCODED_PUBLIC_DIR, path)

    f = open(local_path, 'w')
    f.write(hparser.unescape(data))
    f.close()
    return web_path, local_path

def _compose(language, title, content):
    """
    combine content with a template
    """
    if not content or not language or not title:
        raise Exception("ERROR: Method not well formed!")

    transcode_button = {'en': TRANSCODING_BTN_EN, 'ja': TRANSCODING_BTN_JA, 'th': TRANSCODING_BTN_TH, 'pt':
                        TRANSCODING_BTN_PT, 'ind': TRANSCODING_BTN_IND, 'en-rIN': TRANSCODING_BTN_EN, 'ar': TRANSCODING_BTN_AR}

    # f reads the template
    f = None
    if language == 'ar':
        f = open(NEWS_TEMPLATE_ARABIC, 'r')
    else:
        f = open(NEWS_TEMPLATE, 'r')
    # a template is found
    if f:
        template = str(f.read())
        f.close()
        return template % (title, title, content, transcode_button[language])
    else:
        raise Exception("ERROR: Cannot find a template!")

def _combine(content, images):
    """
    combine results from transcoders
    """
    if not content or not images:
        return content, images

    # for now, if there are more than one image, take only one of them
    biggest = image_helper.find_biggest_image(images)
    IMAGE_TAG = ''
    image = IMAGE_TAG % (
        biggest['url'], str(biggest['width']), str(biggest['height']))
    return "%s %s" % (image, content), images

# TODO: add http string checkers
def _transcode(url, transcoders, language=None):
    """
    organize different transcoders
    """
    if not url or not transcoders:
        raise Exception("ERROR: Method not well formed!")

    threads = {}
    for transcoder in transcoders:
        if transcoder == 'simplr':
            transcoding_request = TranscoderAPI(url, transcoder, language)
        else:
            transcoding_request = TranscoderAPI(url, transcoder)
        # thread could be found via transcoder name
        threads[transcoder] = transcoding_request
        transcoding_request.start()
        # 10 second to wait UCK server
        transcoding_request.join(10 * 1000)

    # after a while ... put data in the proper variables
    uck_content = simplr_content = burify_content = None
    uck_images = simplr_images = burify_images = None
    simplr_title = burify_title = None

    if 'baidu_uck' in transcoders and 'baidu_uck' in threads:
        if threads['baidu_uck'].result:
            uck_content, uck_images = threads['baidu_uck'].result
    if 'simplr' in transcoders and 'simplr' in threads:
        if threads['simplr'].result:
            simplr_title, simplr_content, simplr_images = threads['simplr'].result
    if 'burify' in transcoders and 'burify' in threads:
        if threads['burify'].result:
            burify_title, burify_content, burify_images = threads['burify'].result

    # use different combinations to create a news page with pictures
    if 'simplr' in transcoders or 'burify' in transcoders:
        if 'simplr' in transcoders and simplr_content:
            # if simplr found any image
            if simplr_images:
                return simplr_title, simplr_content, simplr_images
            elif uck_images:  # add images from uck
                new_content, new_images = _combine(simplr_content, uck_images)
                return simplr_title, new_content, new_images
            else:  # no image at all
                return simplr_title, simplr_content, simplr_images
        elif 'burify' in transcoders and burify_content:
            # if burify found any image
            if burify_images:
                return burify_title, burify_content, burify_images
            elif uck_images:  # add images from uck
                new_content, new_images = _combine(burify_content, uck_images)
                return burify_title, new_content, new_images
            else:  # no image at all
                return burify_title, burify_content, burify_images
    # only uck
    if uck_content:
        return "", uck_content, uck_images
    else:
        raise Exception("ERROR: UCK failed!")

def _organize_transcoders(transcoder="chengdujin"):
    """
    get data from different transcoders
    chengdujin: simplr.py
    readability: burify.py
    uck: baidu_uck.py
    """
    transcoders = []
    if transcoder == 'chengdujin':
        transcoders.append("simplr")
        transcoders.append("baidu_uck")
    elif transcoder == 'readability':
        transcoders.append("burify")
        transcoders.append("baidu_uck")
    else:
        transcoders.append("baidu_uck")
    return transcoders

def _preprocess(link):
    """
    get the real address out
    """
    last_http_index = link.rfind('http')
    return link[last_http_index:].strip()

def convert(language="en", title=None, link=None, transcoder="chengdujin", relative_path=None):
    """
    select a transcoder
    send the link
    gather the data
    combine them with the template
    generate paths
    return news and images
    """
    if not language or not title or not link or not relative_path:
        raise Exception('ERROR: Method not well formed!')

    link = _preprocess(link)
    transcoders = _organize_transcoders(transcoder)
    title_new, content, images = _transcode(link, transcoders, language)
    # in case uck cannot find a proper title
    if title_new:
        title = title_new

    if content:
        # embed content in template
        news = _compose(language, title, content)
        # create web/local path
        web_path, local_path = _save(news, relative_path)
        return web_path, local_path, images
    else:
        raise Exception("ERROR: Transcoder %s failed for %s" %
                        (transcoder, link))
<|endoftext|>