<commit_before>"""
A set of tools which allow specifying a model consisting of sheets
organized in levels, and projections connecting these sheets. The
sheets have an attribute matchconditions allowing to specify which
other (incoming) sheets a sheet should connect to.

Instances of the RegisteredMethod decorator are offered for setting
parameters/matchconditions for a sheet within a level, as well as
parameters for projections.
"""

from collections import OrderedDict
import itertools
from functools import wraps

import param

import topo
from topo import sheet,transferfn,pattern,projection
from topo.base.sheet import Sheet

from dataviews.collector import AttrTree

class Specification(object):
    @property
    def spec_type(self):
        return self._spec_type

    def __init__(self, spec_type, parameters=None):
        self._spec_type = spec_type
        self.parameters = {}
        for param_name, default_value in spec_type.params().items():
            self.parameters[param_name]=default_value.default
        if parameters is not None: self.parameters.update(parameters)

class SheetSpec(Specification):
    """
    SheetSpec acts as a template for sheet objects. It is also
    possible to resolve the actual sheet object after it has been
    instantiated.

    The following attributes can be accessed:

    :'properties': Dictionary specifying the properties of the
    sheet. There must be a value given for the key 'level'.
    :'spec_type': Subclass of topo.base.sheet.Sheet.(read-only)
    :'parameters': Dictionary specifying which parameters should be
    passed to the sheet object specified with spec_type. Keys are the
    parameter names, values the parameter values.
    :'matchconditions': Dictionary specifying the matchconditions of
    the sheet. This may be used to determine which other sheets this
    sheet should connect to.
    """

    def __init__(self, spec_type, properties, properties_order=None,
                 parameters=None, matchconditions=None):
        """
        Initialize a SheetSpec object. All arguments but parameters
        are just passed to the internal attributes. For parameters,
        additional key-value pairs with all possible parameters for
        the sheet type specified with sheet_type are added. This
        allows a lookup which parameters can be set.
        """
        super(SheetSpec,self).__init__(spec_type, parameters)

        if properties_order:
            properties_ordered = OrderedDict()
            for k in properties_order:
                if k in properties:
                    properties_ordered[k] = properties[k]

            self.properties = properties_ordered
        else:
            self.properties = properties

        self.matchconditions = matchconditions
        if self.matchconditions is None: self.matchconditions={}

    def __call__(self):
        """
        Instantiate the sheet and register it in topo.sim.
        """
        topo.sim[str(self)]=self.spec_type(**self.parameters)

    def resolve(self):
        """
        Returns the actual sheet object after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __str__(self):
        """
        Returns a string representation combined from the properties values.
        This might be used as name for the actual sheet object.
        """
        name=''
        for prop in self.properties.itervalues():
            name+=prop

        return name

class ProjectionSpec(Specification):
    """
    ProjectionSpec acts as a template for projections. It is also
    possible to resolve the actual projection after it has been
    instantiated.

    The following attributes can be accessed:

    :'src': SheetSpec of the source sheet
    :'dest': SheetSpec of the destionation sheet
    :'spec_type': Subclass of topo.base.projection.Projection
    :'match_name': Name of the matchcondition which has been used to
    set this projection up.  This might be used to set the parameters
    of the ProjectionSpec
    :'parameters': Dictionary specifying which parameters should be
    passed to the projection specified with connection_type. Keys are
    the parameter names, values the parameter values.
    """

    def __init__(self, spec_type, src, dest, match_name=None,
                 parameters=None, properties=None):
        """
        Initialize a ProjectionSpec object. All arguments but
        parameters are just passed to the internal attributes. For
        parameters, additional key-value pairs with all possible
        parameters for the projection type specified with
        connection_type are added. This allows a lookup which
        parameters can be set. Furthermore, parameters['src'] and
        parameters['dest'] are removed, as these must be set with the
        'src' and 'dest' arguments instead.
        """
        super(ProjectionSpec, self).__init__(spec_type, parameters)

        self.src = src
        self.dest = dest
        self.match_name = match_name
        self.properties = properties
        # These parameters are directly passed into topo.sim.connect()!
        del self.parameters['src']
        del self.parameters['dest']

    def __call__(self):
        """
        Instantiate the projection and register it in topo.sim.
        """
        topo.sim.connect(str(self.src),str(self.dest),self.spec_type,**self.parameters)

    def resolve(self):
        """
        Returns the actual projection after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self.dest)+'.'+self.parameters['name'])

class RegisteredMethod(object):
    def __init__(self):
        self.registry = {}
        self.types = {}
        self.options = {}

    def __call__(self, key, obj_type=None, option=None):
        def decorator(f):
            @wraps(f)
            def inner(*args, **kwargs):
                return f(*args, **kwargs)

            self.registry[key] = inner
            if obj_type is not None:
                self.types[key] = obj_type
            if option is not None:
                self.options[key] = option

            return inner
        return decorator

class Model(param.Parameterized):
    """
    The available setup options are:

        :'training_patterns': fills the training_patterns AttrTree
        with pattern generator instances. The path is the name of the
        input sheet. Usually calls PatternCoordinator to do this.
        :'setup_sheets': determines the amount of sheets, their types
        and names sets sheet parameters according to the registered
        methods in level sets sheet matchconditions according to the
        registered methods in matchconditions
        :'projections': determines which connections should be present
        between the sheets according to the matchconditions of
        SheetSpec objects, using projection_spec to specify the
        connection type and sets their parameters according to the
        registered methods in projection_spec

    The available instantiate options are:

        :'sheets': instantiates all sheets and registers them in
        topo.sim
        :'projections': instantiates all projections and registers
        them in topo.sim
    """

    available_setup_options = ['training_patterns','sheets','projections','analysis']

    available_instantiate_options = ['sheets','projections']

    __abstract = True

    level = RegisteredMethod()
    matchconditions = RegisteredMethod()
    projection_spec = RegisteredMethod()

    def __init__(self, setup_options=True, **params):
        super(Model,self).__init__(**params)

        self.training_patterns = AttrTree()
        self.sheets = AttrTree()
        self.projections = AttrTree()

        self.setup(setup_options)

    def setup(self,setup_options):
        """
        This method can be used to setup certain aspects of the
        submodel.  If setup_options=True, all setup methods are
        called.  setup_options can also be a list, whereas all list
        items of available_setup_options are accepted.
        """

        if setup_options==True:
            setup_options = self.available_setup_options

        if 'training_patterns' in setup_options:
            self._setup_training_patterns()
        if 'sheets' in setup_options:
            self._setup_sheets()
            self._set_sheet_parameters()
            self._set_sheet_matchconditions()
        if 'projections' in setup_options:
            self._setup_projections()
            self._set_projection_parameters()
        if 'analysis' in setup_options:
            self._setup_analysis()

    def _setup_training_patterns(self):
        """
        Adds new PatternGenerators to self.training_patterns, with
        the name of the input sheet where the training patterns should
        be installed as path.
        """
        raise NotImplementedError

    def _setup_sheets(self):
        """
        Adds new SheetSpec items to self.sheets.
        The AttrTree path is named after the sheet name.
        """
        raise NotImplementedError

    def _setup_projections(self):
        # Loop through all possible combinations of SheetSpec objects in self.sheets
        # If the src_sheet fulfills all criteria specified in dest_sheet.matchconditions,
        # create a new ProjectionSpec object and add this item to self.projections
        self.projections = AttrTree()

        for src_sheet, dest_sheet in itertools.product(self.sheets.path_items.values(),
                                                       self.sheets.path_items.values()):
            for matchname, matchconditions in dest_sheet.matchconditions.items():
                is_match=True
                if matchconditions is None:
                    continue
                for incoming_key, incoming_value in matchconditions.items():
                    if incoming_key in src_sheet.properties and \
                        str(src_sheet.properties[incoming_key]) not in str(incoming_value):
                            is_match=False
                            break

                if is_match:
                    options = self.projection_spec.options.get(matchname, {})
                    options.update({'matchname':[matchname]})
                    combinations = [i for i in options.values()]
                    for combination_prod in itertools.product(*combinations):
                        properties={}
                        for idx, option in enumerate(options.keys()):
                            properties.update({option:combination_prod[idx]})
                        appendix=''
                        for key, value in properties.items():
                            if key is not 'matchname':
                                appendix+=key
                                appendix+=str(value)
                        self.projections.set_path(str(dest_sheet)+'.'+str(src_sheet)+'.'+matchname+appendix,
                                                  ProjectionSpec(self.projection_spec.types[matchname],
                                                                 src_sheet, dest_sheet, matchname,
                                                                 properties=properties))

    def _set_sheet_parameters(self):
        for sheet_item in self.sheets.path_items.values():
            if(callable(self.level.registry[sheet_item.properties['level']])):
                sheet_item.parameters.update(self.level.registry[sheet_item.properties['level']]
                                             (self,sheet_item.properties))
            else:
                sheet_item.parameters.update(self.level.registry[sheet_item.properties['level']])

    def _set_sheet_matchconditions(self):
        for sheet_item in self.sheets.path_items.values():
            matchcondition = self.matchconditions.registry.get(sheet_item.properties['level'], False)
            if(callable(matchcondition)):
                sheet_item.matchconditions.update(self.matchconditions.registry[sheet_item.properties['level']]
                                                  (self,sheet_item.properties))
            elif matchcondition:
                sheet_item.matchconditions.update(matchcondition)

    def _set_projection_parameters(self):
        for proj in self.projections.path_items.values():
            proj.parameters.update(self.projection_spec.registry[proj.match_name](self,proj))

    def _setup_analysis(self):
        """
        Set up appropriate defaults for analysis functions in topo.analysis.featureresponses
        """
        pass

    def __call__(self,instantiate_options=True):
        """
        Instantiates all sheets / projections in self.sheets /
        self.projections and registers them in topo.sim If
        instantiate_options=True, all items are initialised
        instantiate_options can also be a list, whereas all list items
        of available_instantiate_options are accepted.
        """
        if instantiate_options==True:
            instantiate_options=self.available_instantiate_options

        if 'sheets' in instantiate_options:
            self.message('Sheets:\n')
            for sheet_item in self.sheets.path_items.itervalues():
                self.message(sheet_item)
                sheet_item()
            self.message('\n\n')

        if 'projections' in instantiate_options:
            self.message('Connections:\n')
            #Ugly hack to take the order to initialize sheets into account
            #As soon as weight initialization is done with time_dependent=True,
            #this can be simplified to:
            #for proj in self.projections.path_items.itervalues():
            connection_order=['Afferent','AfferentCenter','AfferentSurround',
                              'LateralGC','AfferentV1On','AfferentV1Off',
                              'LateralV1Excitatory','LateralV1Inhibitory']
            for proj in sorted(self.projections.path_items.itervalues(),
                               key=lambda projection: connection_order.index(projection.match_name)):
                self.message('Connect ' + str(proj.src) + ' with ' + str(proj.dest) + \
                             ' (Match name: ' + proj.match_name + \
                             ', connection name: ' + str(proj.parameters['name']) + ')')
                proj()
<commit_msg>Simplified ordering of properties for naming purposes<commit_after>"""
A set of tools which allow specifying a model consisting of sheets
organized in levels, and projections connecting these sheets. The
sheets have an attribute matchconditions allowing to specify which
other (incoming) sheets a sheet should connect to.

Instances of the RegisteredMethod decorator are offered for setting
parameters/matchconditions for a sheet within a level, as well as
parameters for projections.
"""

from collections import OrderedDict
import itertools
from functools import wraps

import param

import topo
from topo import sheet,transferfn,pattern,projection
from topo.base.sheet import Sheet

from dataviews.collector import AttrTree

class Specification(object):
    @property
    def spec_type(self):
        return self._spec_type

    def __init__(self, spec_type, parameters=None):
        self._spec_type = spec_type
        self.parameters = {}
        for param_name, default_value in spec_type.params().items():
            self.parameters[param_name]=default_value.default
        if parameters is not None: self.parameters.update(parameters)

class SheetSpec(Specification):
    """
    SheetSpec acts as a template for sheet objects. It is also
    possible to resolve the actual sheet object after it has been
    instantiated.

    The following attributes can be accessed:

    :'properties': Dictionary specifying the properties of the
    sheet. There must be a value given for the key 'level'.
    :'spec_type': Subclass of topo.base.sheet.Sheet.(read-only)
    :'parameters': Dictionary specifying which parameters should be
    passed to the sheet object specified with spec_type. Keys are the
    parameter names, values the parameter values.
    :'matchconditions': Dictionary specifying the matchconditions of
    the sheet. This may be used to determine which other sheets this
    sheet should connect to.
    """

    def __init__(self, spec_type, properties, name_ordering=None,
                 parameters=None, matchconditions=None):
        """
        Initialize a SheetSpec object. All arguments but parameters
        are just passed to the internal attributes. For parameters,
        additional key-value pairs with all possible parameters for
        the sheet type specified with sheet_type are added. This
        allows a lookup which parameters can be set.
        """
        super(SheetSpec,self).__init__(spec_type, parameters)

        if name_ordering:
            properties = [(k, properties[k]) for k in name_ordering]
        self.properties = OrderedDict(properties)

        self.matchconditions = matchconditions
        if self.matchconditions is None: self.matchconditions={}

    def __call__(self):
        """
        Instantiate the sheet and register it in topo.sim.
        """
        topo.sim[str(self)]=self.spec_type(**self.parameters)

    def resolve(self):
        """
        Returns the actual sheet object after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self))

    def __str__(self):
        """
        Returns a string representation combined from the properties values.
        This might be used as name for the actual sheet object.
        """
        name=''
        for prop in self.properties.itervalues():
            name+=prop

        return name

class ProjectionSpec(Specification):
    """
    ProjectionSpec acts as a template for projections. It is also
    possible to resolve the actual projection after it has been
    instantiated.

    The following attributes can be accessed:

    :'src': SheetSpec of the source sheet
    :'dest': SheetSpec of the destionation sheet
    :'spec_type': Subclass of topo.base.projection.Projection
    :'match_name': Name of the matchcondition which has been used to
    set this projection up.  This might be used to set the parameters
    of the ProjectionSpec
    :'parameters': Dictionary specifying which parameters should be
    passed to the projection specified with connection_type. Keys are
    the parameter names, values the parameter values.
    """

    def __init__(self, spec_type, src, dest, match_name=None,
                 parameters=None, properties=None):
        """
        Initialize a ProjectionSpec object. All arguments but
        parameters are just passed to the internal attributes. For
        parameters, additional key-value pairs with all possible
        parameters for the projection type specified with
        connection_type are added. This allows a lookup which
        parameters can be set. Furthermore, parameters['src'] and
        parameters['dest'] are removed, as these must be set with the
        'src' and 'dest' arguments instead.
        """
        super(ProjectionSpec, self).__init__(spec_type, parameters)

        self.src = src
        self.dest = dest
        self.match_name = match_name
        self.properties = properties
        # These parameters are directly passed into topo.sim.connect()!
        del self.parameters['src']
        del self.parameters['dest']

    def __call__(self):
        """
        Instantiate the projection and register it in topo.sim.
        """
        topo.sim.connect(str(self.src),str(self.dest),self.spec_type,**self.parameters)

    def resolve(self):
        """
        Returns the actual projection after it has been instantiated.
        """
        from topo import sim
        return eval('sim.'+str(self.dest)+'.'+self.parameters['name'])

class RegisteredMethod(object):
    def __init__(self):
        self.registry = {}
        self.types = {}
        self.options = {}

    def __call__(self, key, obj_type=None, option=None):
        def decorator(f):
            @wraps(f)
            def inner(*args, **kwargs):
                return f(*args, **kwargs)

            self.registry[key] = inner
            if obj_type is not None:
                self.types[key] = obj_type
            if option is not None:
                self.options[key] = option

            return inner
        return decorator

class Model(param.Parameterized):
    """
    The available setup options are:

        :'training_patterns': fills the training_patterns AttrTree
        with pattern generator instances. The path is the name of the
        input sheet. Usually calls PatternCoordinator to do this.
        :'setup_sheets': determines the amount of sheets, their types
        and names sets sheet parameters according to the registered
        methods in level sets sheet matchconditions according to the
        registered methods in matchconditions
        :'projections': determines which connections should be present
        between the sheets according to the matchconditions of
        SheetSpec objects, using projection_spec to specify the
        connection type and sets their parameters according to the
        registered methods in projection_spec

    The available instantiate options are:

        :'sheets': instantiates all sheets and registers them in
        topo.sim
        :'projections': instantiates all projections and registers
        them in topo.sim
    """

    available_setup_options = ['training_patterns','sheets','projections','analysis']

    available_instantiate_options = ['sheets','projections']

    __abstract = True

    level = RegisteredMethod()
    matchconditions = RegisteredMethod()
    projection_spec = RegisteredMethod()

    def __init__(self, setup_options=True, **params):
        super(Model,self).__init__(**params)

        self.training_patterns = AttrTree()
        self.sheets = AttrTree()
        self.projections = AttrTree()

        self.setup(setup_options)

    def setup(self,setup_options):
        """
        This method can be used to setup certain aspects of the
        submodel.  If setup_options=True, all setup methods are
        called.  setup_options can also be a list, whereas all list
        items of available_setup_options are accepted.
        """

        if setup_options==True:
            setup_options = self.available_setup_options

        if 'training_patterns' in setup_options:
            self._setup_training_patterns()
        if 'sheets' in setup_options:
            self._setup_sheets()
            self._set_sheet_parameters()
            self._set_sheet_matchconditions()
        if 'projections' in setup_options:
            self._setup_projections()
            self._set_projection_parameters()
        if 'analysis' in setup_options:
            self._setup_analysis()

    def _setup_training_patterns(self):
        """
        Adds new PatternGenerators to self.training_patterns, with
        the name of the input sheet where the training patterns should
        be installed as path.
        """
        raise NotImplementedError

    def _setup_sheets(self):
        """
        Adds new SheetSpec items to self.sheets.
        The AttrTree path is named after the sheet name.
        """
        raise NotImplementedError

    def _setup_projections(self):
        # Loop through all possible combinations of SheetSpec objects in self.sheets
        # If the src_sheet fulfills all criteria specified in dest_sheet.matchconditions,
        # create a new ProjectionSpec object and add this item to self.projections
        self.projections = AttrTree()

        for src_sheet, dest_sheet in itertools.product(self.sheets.path_items.values(),
                                                       self.sheets.path_items.values()):
            for matchname, matchconditions in dest_sheet.matchconditions.items():
                is_match=True
                if matchconditions is None:
                    continue
                for incoming_key, incoming_value in matchconditions.items():
                    if incoming_key in src_sheet.properties and \
                        str(src_sheet.properties[incoming_key]) not in str(incoming_value):
                            is_match=False
                            break

                if is_match:
                    options = self.projection_spec.options.get(matchname, {})
                    options.update({'matchname':[matchname]})
                    combinations = [i for i in options.values()]
                    for combination_prod in itertools.product(*combinations):
                        properties={}
                        for idx, option in enumerate(options.keys()):
                            properties.update({option:combination_prod[idx]})
                        appendix=''
                        for key, value in properties.items():
                            if key is not 'matchname':
                                appendix+=key
                                appendix+=str(value)
                        self.projections.set_path(str(dest_sheet)+'.'+str(src_sheet)+'.'+matchname+appendix,
                                                  ProjectionSpec(self.projection_spec.types[matchname],
                                                                 src_sheet, dest_sheet, matchname,
                                                                 properties=properties))

    def _set_sheet_parameters(self):
        for sheet_item in self.sheets.path_items.values():
            if(callable(self.level.registry[sheet_item.properties['level']])):
                sheet_item.parameters.update(self.level.registry[sheet_item.properties['level']]
                                             (self,sheet_item.properties))
            else:
                sheet_item.parameters.update(self.level.registry[sheet_item.properties['level']])

    def _set_sheet_matchconditions(self):
        for sheet_item in self.sheets.path_items.values():
            matchcondition = self.matchconditions.registry.get(sheet_item.properties['level'], False)
            if(callable(matchcondition)):
                sheet_item.matchconditions.update(self.matchconditions.registry[sheet_item.properties['level']]
                                                  (self,sheet_item.properties))
            elif matchcondition:
                sheet_item.matchconditions.update(matchcondition)

    def _set_projection_parameters(self):
        for proj in self.projections.path_items.values():
            proj.parameters.update(self.projection_spec.registry[proj.match_name](self,proj))

    def _setup_analysis(self):
        """
        Set up appropriate defaults for analysis functions in topo.analysis.featureresponses
        """
        pass

    def __call__(self,instantiate_options=True):
        """
        Instantiates all sheets / projections in self.sheets /
        self.projections and registers them in topo.sim If
        instantiate_options=True, all items are initialised
        instantiate_options can also be a list, whereas all list items
        of available_instantiate_options are accepted.
        """
        if instantiate_options==True:
            instantiate_options=self.available_instantiate_options

        if 'sheets' in instantiate_options:
            self.message('Sheets:\n')
            for sheet_item in self.sheets.path_items.itervalues():
                self.message(sheet_item)
                sheet_item()
            self.message('\n\n')

        if 'projections' in instantiate_options:
            self.message('Connections:\n')
            #Ugly hack to take the order to initialize sheets into account
            #As soon as weight initialization is done with time_dependent=True,
            #this can be simplified to:
            #for proj in self.projections.path_items.itervalues():
            connection_order=['Afferent','AfferentCenter','AfferentSurround',
                              'LateralGC','AfferentV1On','AfferentV1Off',
                              'LateralV1Excitatory','LateralV1Inhibitory']
            for proj in sorted(self.projections.path_items.itervalues(),
                               key=lambda projection: connection_order.index(projection.match_name)):
                self.message('Connect ' + str(proj.src) + ' with ' + str(proj.dest) + \
                             ' (Match name: ' + proj.match_name + \
                             ', connection name: ' + str(proj.parameters['name']) + ')')
                proj()
<|endoftext|>


<commit_before>import os

from .. import data
from six.moves import urllib
from bs4 import UnicodeDammit

class TREC(data.ZipDataset):

    url_base = 'http://cogcomp.org/Data/QA/QC/'
    train_filename = 'train_5500.label'
    test_filename = 'TREC_10.label'
    dirname = 'trec'

    def __init__(self, path, text_field, label_field,
                 fine_grained=False, **kwargs)
        """Create an SST dataset instance given a path and fields.

        Arguments:
            path: Path to the data file.
            text_field: The field that will be used for text data.
            label_field: The field that will be used for label data.
            fine_grained: Whether to use the fine-grained (50-class) version of TREC
                or the coarse grained (6-class) version.
            Remaining keyword arguments: Passed to the constructor of
                data.Dataset.
        """
        fields = [('text', text_field), ('label', label_field)]
        examples = []

        def get_label_str(label):
                return label.split(':')[0] if not fine_grained else label
        label_field.preprocessing = data.Pipeline(get_label_str)

        for line in open(os.path.expanduser(path), 'rb'):
            line = UnicodeDammit(line).unicode_markup.split()
            label, text = line[0], ' '.join(line[1:])
            examples.append(data.Example.fromlist([text, label], fields))

        super(TREC, self).__init__(examples, fields, **kwargs)

    @classmethod
    def download(cls, root):
        path = os.path.join(root, cls.dirname)
        if not os.path.isdir(path):
            for fn in [cls.train_filename, cls.test_filename]:
                fpath = os.path.join(path, fn)
                if not os.path.isfile(fpath):
                    os.makedirs(os.path.dirname(fpath), exist_ok=True)
                    print('downloading {}'.format(fn))
                    urllib.request.urlretrieve(os.path.join(cls.url_base, fn), fpath)
        return os.path.join(path, '')

    @classmethod
    def splits(cls, text_field, label_field, root='.',
               train=train_filename, test=test_filename, **kwargs):
        """Create dataset objects for splits of the TREC dataset.

        Arguments:
            text_field: The field that will be used for the sentence.
            label_field: The field that will be used for label data.
            root: The root directory that the dataset's zip archive will be
                expanded into; therefore the directory in whose trees
                subdirectory the data files will be stored.
            train: The filename of the train data. Default: 'train_5500.label'.
            test: The filename of the test data, or None to not load the test
                set. Default: 'TREC_10.label'.
            Remaining keyword arguments: Passed to the splits method of
                Dataset.
        """
        path = cls.download(root)

        train_data = None if train is None else cls(
            path + train, text_field, label_field, **kwargs)
        test_data = None if test is None else cls(
            path + test, text_field, label_field, **kwargs)
        return tuple(d for d in (train_data, test_data)
                     if d is not None)

    @classmethod
    def iters(cls, batch_size=32, device=0, root='.', wv_dir='.',
              wv_type=None, wv_dim='300d', **kwargs):
        """Creater iterator objects for splits of the TREC dataset.

        Arguments:
            batch_size: Batch_size
            device: Device to create batches on. Use - 1 for CPU and None for
                the currently active GPU device.
            root: The root directory that the dataset's zip archive will be
                expanded into; therefore the directory in whose trees
                subdirectory the data files will be stored.
            wv_dir, wv_type, wv_dim: Passed to the Vocab constructor for the
                text field. The word vectors are accessible as
                train.dataset.fields['text'].vocab.vectors.
            Remaining keyword arguments: Passed to the splits method.
        """
        TEXT = data.Field()
        LABEL = data.Field(sequential=False)

        train, test = cls.splits(TEXT, LABEL, root=root, **kwargs)

        TEXT.build_vocab(train, wv_dir=wv_dir, wv_type=wv_type, wv_dim=wv_dim)
        LABEL.build_vocab(train)

        return data.BucketIterator.splits(
            (train, test), batch_size=batch_size, device=device)
<commit_msg>typo in docstring<commit_after>import os

from .. import data
from six.moves import urllib
from bs4 import UnicodeDammit

class TREC(data.ZipDataset):

    url_base = 'http://cogcomp.org/Data/QA/QC/'
    train_filename = 'train_5500.label'
    test_filename = 'TREC_10.label'
    dirname = 'trec'

    def __init__(self, path, text_field, label_field,
                 fine_grained=False, **kwargs):
        """Create an TREC dataset instance given a path and fields.

        Arguments:
            path: Path to the data file.
            text_field: The field that will be used for text data.
            label_field: The field that will be used for label data.
            fine_grained: Whether to use the fine-grained (50-class) version of TREC
                or the coarse grained (6-class) version.
            Remaining keyword arguments: Passed to the constructor of
                data.Dataset.
        """
        fields = [('text', text_field), ('label', label_field)]
        examples = []

        def get_label_str(label):
                return label.split(':')[0] if not fine_grained else label
        label_field.preprocessing = data.Pipeline(get_label_str)

        for line in open(os.path.expanduser(path), 'rb'):
            line = UnicodeDammit(line).unicode_markup.split()
            label, text = line[0], ' '.join(line[1:])
            examples.append(data.Example.fromlist([text, label], fields))

        super(TREC, self).__init__(examples, fields, **kwargs)

    @classmethod
    def download(cls, root):
        path = os.path.join(root, cls.dirname)
        if not os.path.isdir(path):
            for fn in [cls.train_filename, cls.test_filename]:
                fpath = os.path.join(path, fn)
                if not os.path.isfile(fpath):
                    os.makedirs(os.path.dirname(fpath), exist_ok=True)
                    print('downloading {}'.format(fn))
                    urllib.request.urlretrieve(os.path.join(cls.url_base, fn), fpath)
        return os.path.join(path, '')

    @classmethod
    def splits(cls, text_field, label_field, root='.',
               train=train_filename, test=test_filename, **kwargs):
        """Create dataset objects for splits of the TREC dataset.

        Arguments:
            text_field: The field that will be used for the sentence.
            label_field: The field that will be used for label data.
            root: The root directory that the dataset's zip archive will be
                expanded into; therefore the directory in whose trees
                subdirectory the data files will be stored.
            train: The filename of the train data. Default: 'train_5500.label'.
            test: The filename of the test data, or None to not load the test
                set. Default: 'TREC_10.label'.
            Remaining keyword arguments: Passed to the splits method of
                Dataset.
        """
        path = cls.download(root)

        train_data = None if train is None else cls(
            path + train, text_field, label_field, **kwargs)
        test_data = None if test is None else cls(
            path + test, text_field, label_field, **kwargs)
        return tuple(d for d in (train_data, test_data)
                     if d is not None)

    @classmethod
    def iters(cls, batch_size=32, device=0, root='.', wv_dir='.',
              wv_type=None, wv_dim='300d', **kwargs):
        """Creater iterator objects for splits of the TREC dataset.

        Arguments:
            batch_size: Batch_size
            device: Device to create batches on. Use - 1 for CPU and None for
                the currently active GPU device.
            root: The root directory that the dataset's zip archive will be
                expanded into; therefore the directory in whose trees
                subdirectory the data files will be stored.
            wv_dir, wv_type, wv_dim: Passed to the Vocab constructor for the
                text field. The word vectors are accessible as
                train.dataset.fields['text'].vocab.vectors.
            Remaining keyword arguments: Passed to the splits method.
        """
        TEXT = data.Field()
        LABEL = data.Field(sequential=False)

        train, test = cls.splits(TEXT, LABEL, root=root, **kwargs)

        TEXT.build_vocab(train, wv_dir=wv_dir, wv_type=wv_type, wv_dim=wv_dim)
        LABEL.build_vocab(train)

        return data.BucketIterator.splits(
            (train, test), batch_size=batch_size, device=device)
<|endoftext|>


<commit_before># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commit sha for bazelbuild/rules_docker."""

version = "1f1e68e64e9cd1556846734d938e7dc00f47d5fa"
<commit_msg>Project import generated by Copybara. (#641)<commit_after># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Commit sha for bazelbuild/rules_docker."""

version = "b3a59276bc2f6e66bd3a34b5960978481c147fee"
<|endoftext|>


<commit_before>def eigenDecompose(bed, outFile=None):
	import eigenDecompose
	return eigenDecompose.eigenDecompose(bed, outFile)

def findRelated(bed, outFile=None, cutoff=0.05):
	import findRelated
	return findRelated.findRelated(bed, outFile, cutoff)

def calcH2(pheno, prev, eigen, keepArr=None, covar=None, numRemovePCs=10, h2coeff=0.875, lowtail=False):
	import calc_h2
	return calc_h2.calc_h2(pheno, prev, eigen, keepArr, covar, numRemovePCs, h2coeff, lowtail)

def probit(bed, pheno, h2, prev, eigen, outFile=None, keepArr=None, covar=None, thresholds=None, nofail=False,
				numSkipTopPCs=0, mineig=1e-3, hess=False, recenter=True, maxFixedIters=100, epsilon=1e-3, treatFixedAsRandom=False):
	import probit
	return probit.probit(bed, pheno, h2, prev, eigen, outFile, keepArr, covar, thresholds, nofail,
				numSkipTopPCs, mineig, hess, recenter, maxFixedIters, epsilon, treatFixedAsRandom=treatFixedAsRandom)

def leapGwas(bedSim, bedTest, pheno, h2, outFile=None, eigenFile=None, covar=None):
	import leap_gwas
	return leap_gwas.gwas(bedSim, bedTest, pheno, h2, outFile, eigenFile, covar)

	<commit_msg>Added option to use a custom kinship matrix<commit_after>def eigenDecompose(bed, kinshipFile=None, outFile=None):
	import eigenDecompose
	return eigenDecompose.eigenDecompose(bed, kinshipFile, outFile)

def findRelated(bed, outFile=None, cutoff=0.05, kinshipFile=None):
	import findRelated
	return findRelated.findRelated(bed, outFile, cutoff, kinshipFile)

def calcH2(pheno, prev, eigen, keepArr=None, covar=None, numRemovePCs=10, lowtail=False):
	import calc_h2
	return calc_h2.calc_h2(pheno, prev, eigen, keepArr, covar, numRemovePCs, lowtail)

def probit(bed, pheno, h2, prev, eigen, outFile=None, keepArr=None, covar=None, thresholds=None, nofail=False,
				numSkipTopPCs=0, mineig=1e-3, hess=False, recenter=True, maxFixedIters=100, epsilon=1e-3, treatFixedAsRandom=False):
	import probit
	return probit.probit(bed, pheno, h2, prev, eigen, outFile, keepArr, covar, thresholds, nofail,
				numSkipTopPCs, mineig, hess, recenter, maxFixedIters, epsilon, treatFixedAsRandom=treatFixedAsRandom)

def leapGwas(bedSim, bedTest, pheno, h2, outFile=None, eigenFile=None, covar=None):
	import leap_gwas
	return leap_gwas.gwas(bedSim, bedTest, pheno, h2, outFile, eigenFile, covar)

	<|endoftext|>


<commit_before>from typing import Optional

from ..base_bot import BaseBot
from ..toolbox import add_torrent_from_url, get_registered_torrents, remove_torrent
from ..utils import get_torrent_from_url, RPCObjectsRegistry

try:
    import telegram
    from telegram import (
        ReplyKeyboardMarkup, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton, Bot, Update
    )
    from telegram.ext import (
        Filters, Updater, ConversationHandler, CommandHandler, MessageHandler, RegexHandler, CallbackQueryHandler
    )

except ImportError:
    telegram = None

class TelegramBot(BaseBot):

    START, URL, PATH = 1, 2, 3

    alias: str = 'telegram'
    url: str = 'https://api.telegram.org/bot'

    def __init__(self, token: str, allowed_users: str = None):
        """
        :param token: str Telegram's bot token
        :param allowed_users: comma-joined list of users allowed to add new torrents.

        """
        if not telegram:
            self.log_error('You have not installed python-telegram-bot library.')
            return

        self.token = token
        self.allowed_users = allowed_users or ''
        self.handler_kwargs = {}
        self.updater = Updater(token=self.token)
        self.dispatcher = self.updater.dispatcher

        if self.allowed_users:
            self.handler_kwargs = {'filters': Filters.user(username=self.allowed_users.split(','))}

        super().__init__()

    def test_configuration(self) -> bool:
        return telegram and bool(self.updater)

    def run(self):
        if self.test_configuration():
            self.add_handlers()
            self.updater.start_polling()
            self.updater.idle()

    def add_handlers(self):
        path_handler_regex = r'^/(?!(cancel|start|add|list|remove|help)(?!/)).+|\.'

        kwargs = self.handler_kwargs

        conv_handler = ConversationHandler(
            entry_points=[CommandHandler('start', self.command_start, **kwargs)],

            states={
                self.URL: [RegexHandler('http[s]?://', self.handle_process_url, pass_user_data=True)],
                self.PATH: [RegexHandler(path_handler_regex, self.handle_ask_download_path, pass_user_data=True)],
            },

            fallbacks=[CommandHandler('cancel', self.cancel_handler)],
            allow_reentry=True
        )

        add_handler = self.dispatcher.add_handler

        add_handler(conv_handler)
        add_handler(CallbackQueryHandler(self.handle_callbacks))
        add_handler(CommandHandler('add', self.command_add_torrent, **kwargs))
        add_handler(CommandHandler('list', self.command_list_torrents, **kwargs))
        add_handler(CommandHandler('remove', self.command_remove_torrents, **kwargs))
        add_handler(CommandHandler('help', self.command_help, **kwargs))

    def handle_callbacks(self, bot: 'Bot', update: 'Update'):
        """Handler to process all callbacks from buttons"""

        handlers = {
            'add_torrent': self.handle_ask_url,
            'list_torrents': self.command_list_torrents,
            'delete_torrent': self.command_remove_torrents
        }
        handler = handlers.get(update.callback_query.data)

        if handler:
            handler(bot, update.callback_query)

        elif update.callback_query.data.startswith('hash:'):
            self.handle_remove_torrents(bot, update)

    def handle_ask_url(self, bot: 'Bot', update: 'Update'):

        update.message.reply_text(
            text="Give me a URL and I'll do the rest.",
            reply_markup=ReplyKeyboardRemove()
        )
        return self.URL

    def handle_process_url(self, bot: 'Bot', update: 'Update', user_data: Optional[dict]):

        torrent_url = update.message.text
        torrent_data = get_torrent_from_url(torrent_url)

        if torrent_data is None:
            update.message.reply_text(
                f'Unable to add torrent from `{torrent_url}`',
                reply_markup=ReplyKeyboardRemove())

            return ConversationHandler.END

        else:

            user_data['url'] = torrent_url
            download_dirs = set()

            for rpc_alias, rpc in RPCObjectsRegistry.get().items():

                if not rpc.enabled:
                    continue

                torrents = rpc.method_get_torrents()

                for torrent in torrents:
                    download_dirs.add(torrent['download_to'])

            choices = [[directory] for directory in download_dirs]

            update.message.reply_text(
                'Where to download data? Send absolute path or "."',
                reply_markup=ReplyKeyboardMarkup(choices, one_time_keyboard=True))

            return self.PATH

    def handle_ask_download_path(self, bot: 'Bot', update: 'Update', user_data: Optional[dict]):

        try:
            torrent_url = user_data['url']

            if not torrent_url:
                update.message.reply_text('Something wrong. Try again')

        except KeyError:
            update.message.reply_text('Something wrong. Try again')

        else:

            path = update.message.text
            if path == '.':
                path = None

            torrents_count = len(get_registered_torrents())

            try:
                add_torrent_from_url(torrent_url, download_to=path)

            except Exception as e:

                self.log_error(f'Unable to add torrent: {e}')

                update.message.reply_text(
                    'Error was occurred during registering torrent.',
                    reply_markup=ReplyKeyboardRemove())

            if len(get_registered_torrents()) > torrents_count:

                update.message.reply_text(
                    f'Torrent from `{torrent_url}` was added',
                    reply_markup=ReplyKeyboardRemove())

            else:
                update.message.reply_text(
                    'Unable to add torrent.',
                    reply_markup=ReplyKeyboardRemove())

        return ConversationHandler.END

    def cancel_handler(self, bot: 'Bot', update: 'Update'):

        update.message.reply_text(
            'Bye! I hope to see you again.',
            reply_markup=ReplyKeyboardRemove())

        return ConversationHandler.END

    def handle_remove_torrents(self, bot: 'Bot', update: 'Update'):
        """
        Handler for torrent remove action.
        data is colon-joined string which is contains:
        1. keyword `hash` - command prefix
        2. torrent HASH from configured RPC
        3. 0|1 - optional attribute (with_data) responsible for data removal from RPC
        For example 'hash:1234567890' or 'hash:1234567890:0'
        """
        data = update.callback_query.data
        split_data = data.split(':')[1:]
        torrent_hash = split_data.pop(0)

        if not split_data:
            # with_data attribute was not set yet, ask user

            update.callback_query.message.reply_text(
                'Do you want to delete data?',
                reply_markup=InlineKeyboardMarkup([[
                    InlineKeyboardButton(text='Yes', callback_data=data + ':1'),
                    InlineKeyboardButton(text='No', callback_data=data + ':0'),
                ]])
            )

        else:

            torrent_data = get_registered_torrents().get(torrent_hash)

            if torrent_data:
                remove_torrent(torrent_hash, with_data=bool(int(split_data[0])))
                update.callback_query.message.reply_text(
                    f"Torrent `{torrent_data['name']}` was removed")

            else:
                update.callback_query.message.reply_text(
                    'Torrent not found. Try one more time with /remove')

        return

    def command_start(self, bot: 'Bot', update: 'Update'):
        """Start dialog handler"""

        kb = InlineKeyboardMarkup([[
            InlineKeyboardButton(text='Add torrent', callback_data='add_torrent'),
            InlineKeyboardButton(text='List torrents', callback_data='list_torrents'),
            InlineKeyboardButton(text='Delete torrent', callback_data='delete_torrent'),
        ]])
        bot.send_message(update.message.chat_id, 'What do you want to do?', reply_markup=kb)

        return self.URL

    def command_add_torrent(self, bot: 'Bot', update: 'Update'):
        """Stand-alone handler to add torrent"""

        torrent_url = update.message.text.lstrip('/add ')

        if not torrent_url:

            update.message.reply_text(
                'Please provide link to the tracker page. '
                'For example: \n/add https://rutracker.org/forum/viewtopic.php?t=1')

            return

        torrents_count = len(get_registered_torrents())
        chat_id = update.message.chat_id

        try:
            add_torrent_from_url(torrent_url)

        except Exception as e:

            self.log_error(f'Unable to register the torrent: {e}')
            bot.send_message(chat_id, text='Unable to register the torrent due to an error.')

        if len(get_registered_torrents()) > torrents_count:
            bot.send_message(chat_id, text='The torrent is successfully registered.')

        else:
            bot.send_message(chat_id, text='Unable to register the torrent.')

    def command_list_torrents(self, bot: 'Bot', update: 'Update'):
        """Command to list all monitored torrents"""

        torrents = []

        for idx, torrent in enumerate(get_registered_torrents().values(), 1):
            if torrent.get('url'):
                torrents.append(f"{idx}. {torrent['name']}\n{torrent['url']}")

            else:
                torrents.append(f"{idx}. {torrent['name']}")

        if torrents:
            update.message.reply_text('\n\n'.join(torrents))

        else:
            update.message.reply_text('No torrents yet.')

    def command_remove_torrents(self, bot: 'Bot', update: 'Update'):
        """Command to remove torrent"""

        buttons = []

        for torrent in get_registered_torrents().values():
            buttons.append([
                InlineKeyboardButton(
                    text=torrent['name'],
                    callback_data=f"hash:{torrent['hash']}"
                )
            ])

        bot.send_message(
            update.message.chat_id,
            text='Which torrent do you want to remove?',
            reply_markup=InlineKeyboardMarkup(buttons))

    def command_help(self, bot: 'Bot', update: 'Update'):
        """Command for help"""

        helptext = (
            'Available commands:\n'
            '/start: Interactive wizard for torrent management.\n'
            '/add <URL>: Quick way to add a new torrent.\n'
            '/list: Display all registered torrents.\n'
            '/remove: Remove torrent.\n'
            '/cancel: Cancel current operation.'
        )

        update.message.reply_text(helptext)
<commit_msg>Add mock-friendliness for test.<commit_after>from typing import Optional

from ..base_bot import BaseBot
from ..toolbox import add_torrent_from_url, get_registered_torrents, remove_torrent
from ..utils import get_torrent_from_url, RPCObjectsRegistry

try:
    import telegram
    from telegram import (
        ReplyKeyboardMarkup, ReplyKeyboardRemove, InlineKeyboardMarkup, InlineKeyboardButton, Bot, Update
    )
    from telegram.ext import (
        Filters, Updater, ConversationHandler, CommandHandler, MessageHandler, RegexHandler, CallbackQueryHandler
    )

except ImportError:
    telegram = None

class TelegramBot(BaseBot):

    START, URL, PATH = 1, 2, 3

    alias: str = 'telegram'
    url: str = 'https://api.telegram.org/bot'

    def __init__(self, token: str, allowed_users: str = None):
        """
        :param token: str Telegram's bot token
        :param allowed_users: comma-joined list of users allowed to add new torrents.

        """
        self.token = token
        self.allowed_users = allowed_users or ''

        if not telegram:
            self.log_error('You have not installed python-telegram-bot library.')
            return

        self.handler_kwargs = {}
        self.updater = Updater(token=self.token)
        self.dispatcher = self.updater.dispatcher

        if self.allowed_users:
            self.handler_kwargs = {'filters': Filters.user(username=self.allowed_users.split(','))}

        super().__init__()

    def test_configuration(self) -> bool:
        return telegram and bool(self.updater)

    def run(self):
        if self.test_configuration():
            self.add_handlers()
            self.updater.start_polling()
            self.updater.idle()

    def add_handlers(self):
        path_handler_regex = r'^/(?!(cancel|start|add|list|remove|help)(?!/)).+|\.'

        kwargs = self.handler_kwargs

        conv_handler = ConversationHandler(
            entry_points=[CommandHandler('start', self.command_start, **kwargs)],

            states={
                self.URL: [RegexHandler('http[s]?://', self.handle_process_url, pass_user_data=True)],
                self.PATH: [RegexHandler(path_handler_regex, self.handle_ask_download_path, pass_user_data=True)],
            },

            fallbacks=[CommandHandler('cancel', self.cancel_handler)],
            allow_reentry=True
        )

        add_handler = self.dispatcher.add_handler

        add_handler(conv_handler)
        add_handler(CallbackQueryHandler(self.handle_callbacks))
        add_handler(CommandHandler('add', self.command_add_torrent, **kwargs))
        add_handler(CommandHandler('list', self.command_list_torrents, **kwargs))
        add_handler(CommandHandler('remove', self.command_remove_torrents, **kwargs))
        add_handler(CommandHandler('help', self.command_help, **kwargs))

    def handle_callbacks(self, bot: 'Bot', update: 'Update'):
        """Handler to process all callbacks from buttons"""

        handlers = {
            'add_torrent': self.handle_ask_url,
            'list_torrents': self.command_list_torrents,
            'delete_torrent': self.command_remove_torrents
        }
        handler = handlers.get(update.callback_query.data)

        if handler:
            handler(bot, update.callback_query)

        elif update.callback_query.data.startswith('hash:'):
            self.handle_remove_torrents(bot, update)

    def handle_ask_url(self, bot: 'Bot', update: 'Update'):

        update.message.reply_text(
            text="Give me a URL and I'll do the rest.",
            reply_markup=ReplyKeyboardRemove()
        )
        return self.URL

    def handle_process_url(self, bot: 'Bot', update: 'Update', user_data: Optional[dict]):

        torrent_url = update.message.text
        torrent_data = get_torrent_from_url(torrent_url)

        if torrent_data is None:
            update.message.reply_text(
                f'Unable to add torrent from `{torrent_url}`',
                reply_markup=ReplyKeyboardRemove())

            return ConversationHandler.END

        else:

            user_data['url'] = torrent_url
            download_dirs = set()

            for rpc_alias, rpc in RPCObjectsRegistry.get().items():

                if not rpc.enabled:
                    continue

                torrents = rpc.method_get_torrents()

                for torrent in torrents:
                    download_dirs.add(torrent['download_to'])

            choices = [[directory] for directory in download_dirs]

            update.message.reply_text(
                'Where to download data? Send absolute path or "."',
                reply_markup=ReplyKeyboardMarkup(choices, one_time_keyboard=True))

            return self.PATH

    def handle_ask_download_path(self, bot: 'Bot', update: 'Update', user_data: Optional[dict]):

        try:
            torrent_url = user_data['url']

            if not torrent_url:
                update.message.reply_text('Something wrong. Try again')

        except KeyError:
            update.message.reply_text('Something wrong. Try again')

        else:

            path = update.message.text
            if path == '.':
                path = None

            torrents_count = len(get_registered_torrents())

            try:
                add_torrent_from_url(torrent_url, download_to=path)

            except Exception as e:

                self.log_error(f'Unable to add torrent: {e}')

                update.message.reply_text(
                    'Error was occurred during registering torrent.',
                    reply_markup=ReplyKeyboardRemove())

            if len(get_registered_torrents()) > torrents_count:

                update.message.reply_text(
                    f'Torrent from `{torrent_url}` was added',
                    reply_markup=ReplyKeyboardRemove())

            else:
                update.message.reply_text(
                    'Unable to add torrent.',
                    reply_markup=ReplyKeyboardRemove())

        return ConversationHandler.END

    def cancel_handler(self, bot: 'Bot', update: 'Update'):

        update.message.reply_text(
            'Bye! I hope to see you again.',
            reply_markup=ReplyKeyboardRemove())

        return ConversationHandler.END

    def handle_remove_torrents(self, bot: 'Bot', update: 'Update'):
        """
        Handler for torrent remove action.
        data is colon-joined string which is contains:
        1. keyword `hash` - command prefix
        2. torrent HASH from configured RPC
        3. 0|1 - optional attribute (with_data) responsible for data removal from RPC
        For example 'hash:1234567890' or 'hash:1234567890:0'
        """
        data = update.callback_query.data
        split_data = data.split(':')[1:]
        torrent_hash = split_data.pop(0)

        if not split_data:
            # with_data attribute was not set yet, ask user

            update.callback_query.message.reply_text(
                'Do you want to delete data?',
                reply_markup=InlineKeyboardMarkup([[
                    InlineKeyboardButton(text='Yes', callback_data=data + ':1'),
                    InlineKeyboardButton(text='No', callback_data=data + ':0'),
                ]])
            )

        else:

            torrent_data = get_registered_torrents().get(torrent_hash)

            if torrent_data:
                remove_torrent(torrent_hash, with_data=bool(int(split_data[0])))
                update.callback_query.message.reply_text(
                    f"Torrent `{torrent_data['name']}` was removed")

            else:
                update.callback_query.message.reply_text(
                    'Torrent not found. Try one more time with /remove')

        return

    def command_start(self, bot: 'Bot', update: 'Update'):
        """Start dialog handler"""

        kb = InlineKeyboardMarkup([[
            InlineKeyboardButton(text='Add torrent', callback_data='add_torrent'),
            InlineKeyboardButton(text='List torrents', callback_data='list_torrents'),
            InlineKeyboardButton(text='Delete torrent', callback_data='delete_torrent'),
        ]])
        bot.send_message(update.message.chat_id, 'What do you want to do?', reply_markup=kb)

        return self.URL

    def command_add_torrent(self, bot: 'Bot', update: 'Update'):
        """Stand-alone handler to add torrent"""

        torrent_url = update.message.text.lstrip('/add ')

        if not torrent_url:

            update.message.reply_text(
                'Please provide link to the tracker page. '
                'For example: \n/add https://rutracker.org/forum/viewtopic.php?t=1')

            return

        torrents_count = len(get_registered_torrents())
        chat_id = update.message.chat_id

        try:
            add_torrent_from_url(torrent_url)

        except Exception as e:

            self.log_error(f'Unable to register the torrent: {e}')
            bot.send_message(chat_id, text='Unable to register the torrent due to an error.')

        if len(get_registered_torrents()) > torrents_count:
            bot.send_message(chat_id, text='The torrent is successfully registered.')

        else:
            bot.send_message(chat_id, text='Unable to register the torrent.')

    def command_list_torrents(self, bot: 'Bot', update: 'Update'):
        """Command to list all monitored torrents"""

        torrents = []

        for idx, torrent in enumerate(get_registered_torrents().values(), 1):
            if torrent.get('url'):
                torrents.append(f"{idx}. {torrent['name']}\n{torrent['url']}")

            else:
                torrents.append(f"{idx}. {torrent['name']}")

        if torrents:
            update.message.reply_text('\n\n'.join(torrents))

        else:
            update.message.reply_text('No torrents yet.')

    def command_remove_torrents(self, bot: 'Bot', update: 'Update'):
        """Command to remove torrent"""

        buttons = []

        for torrent in get_registered_torrents().values():
            buttons.append([
                InlineKeyboardButton(
                    text=torrent['name'],
                    callback_data=f"hash:{torrent['hash']}"
                )
            ])

        bot.send_message(
            update.message.chat_id,
            text='Which torrent do you want to remove?',
            reply_markup=InlineKeyboardMarkup(buttons))

    def command_help(self, bot: 'Bot', update: 'Update'):
        """Command for help"""

        helptext = (
            'Available commands:\n'
            '/start: Interactive wizard for torrent management.\n'
            '/add <URL>: Quick way to add a new torrent.\n'
            '/list: Display all registered torrents.\n'
            '/remove: Remove torrent.\n'
            '/cancel: Cancel current operation.'
        )

        update.message.reply_text(helptext)
<|endoftext|>


<commit_before># Copyright 2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This code is not currently exposed publically. It is an example of how to
# stream from a aws log using the FilterLogEvents API.

import time

from touchdown.core import plan
from touchdown.core.datetime import as_seconds
from touchdown.aws import common
from touchdown.aws.logs import LogGroup

class Plan(common.SimplePlan, plan.Plan):

    name = "tail"
    resource = LogGroup
    service_name = "logs"

    def tail(self, start, end, follow):
        kwargs = {
            'logGroupName': self.resource.name,
        }
        if start:
            kwargs['startTime'] = as_seconds(start)
        if end:
            kwargs['endTime'] = as_seconds(end)

        def pull(kwargs, previous_events):
            seen = set()
            filters = {}
            filters.update(kwargs)
            results = self.client.filter_log_events(**filters)
            while True:
                for event in results.get('events', []):
                    seen.add(event['eventId'])
                    if event['eventId'] in previous_events:
                        continue
                    print(u"[{logStreamName}] {message}".format(**event))
                    kwargs['startTime'] = event['timestamp']
                if 'nextToken' not in results:
                    break
                filters['nextToken'] = results['nextToken']
                results = self.client.filter_log_events(**filters)
            return seen

        seen = pull(kwargs, set())
        while follow:
            seen = pull(kwargs, seen)
            time.sleep(2)
<commit_msg>Fix log escaping<commit_after># Copyright 2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This code is not currently exposed publically. It is an example of how to
# stream from a aws log using the FilterLogEvents API.

import time

from touchdown.core import plan
from touchdown.core.datetime import as_seconds
from touchdown.aws import common
from touchdown.aws.logs import LogGroup

class Plan(common.SimplePlan, plan.Plan):

    name = "tail"
    resource = LogGroup
    service_name = "logs"

    def tail(self, start, end, follow):
        kwargs = {
            'logGroupName': self.resource.name,
        }
        if start:
            kwargs['startTime'] = as_seconds(start)
        if end:
            kwargs['endTime'] = as_seconds(end)

        def pull(kwargs, previous_events):
            seen = set()
            filters = {}
            filters.update(kwargs)
            results = self.client.filter_log_events(**filters)
            while True:
                for event in results.get('events', []):
                    seen.add(event['eventId'])
                    if event['eventId'] in previous_events:
                        continue
                    print("[{logStreamName}] {message}".format(**{
                        "logStreamName": event.get('logStreamName', ''),
                        "message": event['message'].encode("utf-8", "ignore"),
                    }))
                    kwargs['startTime'] = event['timestamp']
                if 'nextToken' not in results:
                    break
                filters['nextToken'] = results['nextToken']
                results = self.client.filter_log_events(**filters)
            return seen

        seen = pull(kwargs, set())
        while follow:
            seen = pull(kwargs, seen)
            time.sleep(2)
<|endoftext|>


<commit_before># Copyright 2014-2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import hashlib

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan
from touchdown.core import argument

from .bucket import Bucket
from ..common import SimpleDescribe, SimpleApply

class Folder(Resource):

    resource_name = "folder"

    name = argument.String()
    source = argument.String()

    acl = argument.String(
        default="private",
        choices=["private", "public-read", "public-read-write", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control"],
        field="ACL",
    )

    bucket = argument.Resource(Bucket, field="Bucket")

class Describe(SimpleDescribe, Plan):

    resource = Folder
    service_name = 's3'
    key = 'Name'

    def get_folder_contents(self):
        paginator = self.client.get_paginator("list_objects")
        pages = paginator.paginate(
            Bucket=self.resource.bucket.name,
            Prefix=self.resource.name
        )

        for page in pages:
            for key in page.get('Contents', []):
                if key["Key"].startswith(self.resource.name):
                    yield key["Key"][len(self.resource.name):].lstrip("/"), {
                        "LastModified": key['LastModified'],
                        "ETag": key['ETag'],
                        "Md5": key['ETag'].strip('"'),
                        "Size": key['Size'],
                    }

    def describe_object(self):
        return {
            'Name': self.resource.name,
            'Arn': 's3:aws:s3:::{}{}'.format(
                self.resource.bucket.name,
                self.resource.source,
            ),
        }

class Apply(SimpleApply, Describe):

    create_action = "put_object"
    create_response = "not-that-useful"

    def update_object(self):
        remote = {}
        local = {}

        base = self.resource.source
        for root, dirs, files in os.walk(base):
            for f in files:
                path = os.path.join(root, f)
                with open(path) as fp:
                    h = hashlib.md5(fp.read())
                local[os.path.relpath(path, base)] = {
                    "Md5": h.hexdigest(),
                }

        if self.runner.get_plan(self.resource.bucket).resource_id:
            remote = {k: v for k, v in self.get_folder_contents()}

        for path in local:
            if not path in remote:
                yield self.generic_action(
                    "Add {}".format(path),
                    self.client.put_object,
                    Key=os.path.join(self.resource.name, path),
                    Body=open(os.path.join(base, path)).read(),
                    ACL=self.resource.acl,
                    Bucket=self.resource.bucket.name,
                )
            elif local[path]['Md5'] != remote[path]['Md5']:
                yield self.generic_action(
                    "Update {}".format(path),
                    self.client.put_object,
                    Key=os.path.join(self.resource.name, path),
                    Body=open(os.path.join(base, path)).read(),
                    ACL=self.resource.acl,
                    Bucket=self.resource.bucket.name,
                )

        for path in remote:
            if not path in local:
                yield self.generic_action(
                    "Remove {}".format(path),
                    self.client.delete_object,
                    Bucket=self.resource.bucket.name,
                    Key=os.path.join(self.resource.name, path),
                )
<commit_msg>Add content type guessing and set default cache control header<commit_after># Copyright 2014-2015 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import mimetypes
import hashlib

from touchdown.core.resource import Resource
from touchdown.core.plan import Plan
from touchdown.core import argument

from .bucket import Bucket
from ..common import SimpleDescribe, SimpleApply

class Folder(Resource):

    resource_name = "folder"

    name = argument.String()
    source = argument.String()

    acl = argument.String(
        default="private",
        choices=["private", "public-read", "public-read-write", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control"],
        field="ACL",
    )

    bucket = argument.Resource(Bucket, field="Bucket")

class Describe(SimpleDescribe, Plan):

    resource = Folder
    service_name = 's3'
    key = 'Name'

    def get_folder_contents(self):
        paginator = self.client.get_paginator("list_objects")
        pages = paginator.paginate(
            Bucket=self.resource.bucket.name,
            Prefix=self.resource.name
        )

        for page in pages:
            for key in page.get('Contents', []):
                if key["Key"].startswith(self.resource.name):
                    yield key["Key"][len(self.resource.name):].lstrip("/"), {
                        "LastModified": key['LastModified'],
                        "ETag": key['ETag'],
                        "Md5": key['ETag'].strip('"'),
                        "Size": key['Size'],
                    }

    def describe_object(self):
        return {
            'Name': self.resource.name,
            'Arn': 's3:aws:s3:::{}{}'.format(
                self.resource.bucket.name,
                self.resource.source,
            ),
        }

class Apply(SimpleApply, Describe):

    create_action = "put_object"
    create_response = "not-that-useful"

    def update_object(self):
        remote = {}
        local = {}

        base = self.resource.source
        for root, dirs, files in os.walk(base):
            for f in files:
                path = os.path.join(root, f)
                with open(path) as fp:
                    h = hashlib.md5(fp.read())
                local[os.path.relpath(path, base)] = {
                    "Md5": h.hexdigest(),
                }

        if self.runner.get_plan(self.resource.bucket).resource_id:
            remote = {k: v for k, v in self.get_folder_contents()}

        for path in local:
            contenttype = mimetypes.guess_type(path)[0]
            if not path in remote:
                yield self.generic_action(
                    "Add {} ({})".format(path, contenttype),
                    self.client.put_object,
                    Key=os.path.join(self.resource.name, path),
                    Body=open(os.path.join(base, path)).read(),
                    ACL=self.resource.acl,
                    Bucket=self.resource.bucket.name,
                    CacheControl="max-age=0",
                    ContentType=contenttype,
                )
            elif local[path]['Md5'] != remote[path]['Md5']:
                yield self.generic_action(
                    "Update {} ({})".format(path, contenttype),
                    self.client.put_object,
                    Key=os.path.join(self.resource.name, path),
                    Body=open(os.path.join(base, path)).read(),
                    ACL=self.resource.acl,
                    Bucket=self.resource.bucket.name,
                    CacheControl="max-age=0",
                    ContentType=contenttype,
                )

        for path in remote:
            if not path in local:
                yield self.generic_action(
                    "Remove {}".format(path),
                    self.client.delete_object,
                    Bucket=self.resource.bucket.name,
                    Key=os.path.join(self.resource.name, path),
                )
<|endoftext|>


<commit_before>#!/usr/bin/env python
import yaml
import urllib2
import tempfile
import zipfile
import re
import subprocess, os, sys
import azure.mgmt.network
import requests
import time
import json

from jinja2 import Template
from subprocess import call
from subprocess import Popen, PIPE
from azure.mgmt.common import SubscriptionCloudCredentials

from Utils.WAAgentUtil import waagent
import Utils.HandlerUtil as Util

def authorizedPost(url, token):
    req = urllib2.Request(url)
    req.add_header("Authorization", "Token {0}".format(token))
    req.add_header("Accept", "application/json")
    req.data = ''

    res = urllib2.urlopen(req)
    return res

def get_token_from_client_credentials(endpoint, client_id, client_secret):
    payload = {
        'grant_type': 'client_credentials',
        'client_id': client_id,
        'client_secret': client_secret,
        'resource': 'https://management.core.windows.net/',
    }
    response = requests.post(endpoint, data=payload).json()
    return response['access_token']

def get_job_ip_address(username, password, host, deployment_name, job_name):
    url = "https://{0}:25555/deployments/{1}/vms?format=full".format(host, deployment_name)
    response = requests.get(url, auth=(username, password), verify=False)
    id = response.json()['id']

    result_url = "https://{0}:25555/tasks/{1}/output?type=result".format(host, id)
    response = requests.get(result_url, auth=(username, password), verify=False)

    while not response.content:
        time.sleep(1)
        response = requests.get(result_url, auth=(username, password), verify=False)

    for vm in response.content.split("\n"):
        if vm:
            vm_dict = json.loads(vm)
            if vm_dict['job_name'] == job_name:
                return vm_dict['ips'][0]

    return null

# Get settings from CustomScriptForLinux extension configurations
waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout')
hutil =  Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script")
hutil.do_parse_context("enable")
settings = hutil.get_public_settings()

username = settings["username"]
home_dir = os.path.join("/home", username)
install_log = os.path.join(home_dir, "install.log")

# Unbuffer output to install_log
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

tee = subprocess.Popen(["tee", "-a", install_log], stdin=subprocess.PIPE)
os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
os.dup2(tee.stdin.fileno(), sys.stderr.fileno())

os.environ["HOME"] = home_dir

# deploy director
os.environ["BOSH_INIT_LOG_LEVEL"] = 'INFO'
os.environ["BOSH_INIT_LOG_PATH"] = './bosh-init-debug.log'

# do we have a targeted bosh?
p = Popen(['bosh', 'target'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()

# assume if no bosh is target, there is no director
if "Target not set" in err:

    res = None

    while res != 0:
        res = call("bosh-init deploy ./bosh/bosh.yml", shell=True)

    p = Popen(['bosh', 'target', '10.0.0.4'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
    output, err = p.communicate(b"admin\r\nadmin\r\n")
    rc = p.returncode

# change owner for bosh.yaml
call("chown -R {0} {1}/.bosh_config".format(username, home_dir), shell=True)

# load manifests in to memory
f = open('./manifests/index.yml')
manifests = yaml.safe_load(f)
f.close()

# run bosh status and get the id back to inject in to manifests
bosh_uuid = subprocess.Popen(["bosh", "status", "--uuid"], stdout=subprocess.PIPE).communicate()[0]
print "Director uuid is {0}".format(bosh_uuid)

# set the director id on the manifests
for m in manifests['manifests']:
    with open ("{0}/manifests/{1}".format(home_dir, m['file']), 'r+') as f:

        contents = f.read()
        template = Template(contents)
        contents = template.render(DIRECTOR_UUID=bosh_uuid)

        f.seek(0)
        f.write(contents)
        f.truncate()
        f.close()

pivnetAPIToken = settings["pivnet-api-token"]

eula_urls = ["https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance".format(m['release-name'], m['release-number'])
    for m in manifests['manifests']]

release_urls = ["https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download".format(m['release-name'], m['release-number'], m['file-number'])
    for m in manifests['manifests']]

stemcell_urls = [m['stemcell'] for m in manifests['manifests']]

# accept eula for each product
for url in eula_urls:
    res = authorizedPost(url, pivnetAPIToken)
    code = res.getcode()

# releases
is_release_file = re.compile("^releases\/.+")
call("mkdir -p /tmp/releases", shell=True)

print "Processing releases."

for url in release_urls:

    print "Downloading {0}.".format(url)

    res = authorizedPost(url, pivnetAPIToken)
    code = res.getcode()

    length = int(res.headers["Content-Length"])

    # content-length
    if code is 200:

        total = 0
        pcent = 0.0
        CHUNK = 16 * 1024

        with tempfile.TemporaryFile() as temp:
            while True:
                chunk = res.read(CHUNK)
                total += CHUNK
                pcent = (float(total) / float(length)) * 100

                sys.stdout.write("Download progress: %.2f%% (%.2fM)\r" % (pcent, total / 1000000.0) )
                sys.stdout.flush()

                if not chunk: break
                temp.write(chunk)

            print "Download complete."

            z = zipfile.ZipFile(temp)
            for name in z.namelist():

                # is this a release?
                if is_release_file.match(name):

                    release_filename = "/tmp/{0}".format(name)

                    print "Unpacking {0}.".format(name)
                    z.extract(name, "/tmp")

                    # upload the file with bosh
                    print "Uploading release {0} to BOSH director.".format(name)
                    call("bosh upload release {0}".format(release_filename), shell=True)

                    os.unlink(release_filename)

            z.close()
            temp.close()

# stemcells
print "Processing stemcells."

for url in stemcell_urls:
    print "Processing stemcell {0}".format(url)
    call("bosh upload stemcell {0}".format(url), shell=True)

# deploy!
for m in manifests['manifests']:
    call("bosh deployment {0}/manifests/{1}".format(home_dir, m['file']), shell=True)
    call("bosh -n deploy", shell=True)

# cf specific configuration (configure security groups for haproxy)
bosh_address = settings("bosh-ip")
deployment_name = "router-and-lb"
subscription_id = settings('SUBSCRIPTION-ID')
tenant = settings('TENANT-ID')
endpoint = "https://login.microsoftonline.com/{0}/oauth2/token".format(tenant)
client_token = settings('CLIENT-ID')
client_secret = settings('CLIENT-SECRET')

ip = get_job_ip_address("admin", "admin", bosh_address, deployment_name, "haproxy")

token = get_token_from_client_credentials(endpoint, client_token, client_secret)
creds = SubscriptionCloudCredentials(subscription_id, token)

network_client = azure.mgmt.network.NetworkResourceProviderClient(creds)
rules_client = azure.mgmt.network.networkresourceprovider.SecurityRuleOperations(network_client)

ha_proxy_address = "10.0.16.200"

rule = azure.mgmt.network.SecurityRule(
    description="",
    protocol="*",
    source_port_range="*",
    destination_port_range="80",
    source_address_prefix="*",
    destination_address_prefix=ha_proxy_address,
    access="Allow",
    priority=1100,
    direction="InBound"
)

rules_client.create_or_update("cf", settings('NSG-NAME-FOR-CF'), "http_inbound", rule)

rule = azure.mgmt.network.SecurityRule(
    description="",
    protocol="*",
    source_port_range="*",
    destination_port_range="443",
    source_address_prefix="*",
    destination_address_prefix=ha_proxy_address,
    access="Allow",
    priority=1200,
    direction="InBound"
)

rules_client.create_or_update("cf", "CFSecurityGroup", "http_inbound", rule)
<commit_msg>Fix routine calls to dict!<commit_after>#!/usr/bin/env python
import yaml
import urllib2
import tempfile
import zipfile
import re
import subprocess, os, sys
import azure.mgmt.network
import requests
import time
import json

from jinja2 import Template
from subprocess import call
from subprocess import Popen, PIPE
from azure.mgmt.common import SubscriptionCloudCredentials

from Utils.WAAgentUtil import waagent
import Utils.HandlerUtil as Util

def authorizedPost(url, token):
    req = urllib2.Request(url)
    req.add_header("Authorization", "Token {0}".format(token))
    req.add_header("Accept", "application/json")
    req.data = ''

    res = urllib2.urlopen(req)
    return res

def get_token_from_client_credentials(endpoint, client_id, client_secret):
    payload = {
        'grant_type': 'client_credentials',
        'client_id': client_id,
        'client_secret': client_secret,
        'resource': 'https://management.core.windows.net/',
    }
    response = requests.post(endpoint, data=payload).json()
    return response['access_token']

def get_job_ip_address(username, password, host, deployment_name, job_name):
    url = "https://{0}:25555/deployments/{1}/vms?format=full".format(host, deployment_name)
    response = requests.get(url, auth=(username, password), verify=False)
    id = response.json()['id']

    result_url = "https://{0}:25555/tasks/{1}/output?type=result".format(host, id)
    response = requests.get(result_url, auth=(username, password), verify=False)

    while not response.content:
        time.sleep(1)
        response = requests.get(result_url, auth=(username, password), verify=False)

    for vm in response.content.split("\n"):
        if vm:
            vm_dict = json.loads(vm)
            if vm_dict['job_name'] == job_name:
                return vm_dict['ips'][0]

    return null

# Get settings from CustomScriptForLinux extension configurations
waagent.LoggerInit('/var/log/waagent.log', '/dev/stdout')
hutil =  Util.HandlerUtility(waagent.Log, waagent.Error, "bosh-deploy-script")
hutil.do_parse_context("enable")
settings = hutil.get_public_settings()

username = settings["username"]
home_dir = os.path.join("/home", username)
install_log = os.path.join(home_dir, "install.log")

# Unbuffer output to install_log
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

tee = subprocess.Popen(["tee", "-a", install_log], stdin=subprocess.PIPE)
os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
os.dup2(tee.stdin.fileno(), sys.stderr.fileno())

os.environ["HOME"] = home_dir

# deploy director
os.environ["BOSH_INIT_LOG_LEVEL"] = 'INFO'
os.environ["BOSH_INIT_LOG_PATH"] = './bosh-init-debug.log'

# do we have a targeted bosh?
p = Popen(['bosh', 'target'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()

# assume if no bosh is target, there is no director
if "Target not set" in err:

    res = None

    while res != 0:
        res = call("bosh-init deploy ./bosh/bosh.yml", shell=True)

    p = Popen(['bosh', 'target', '10.0.0.4'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
    output, err = p.communicate(b"admin\r\nadmin\r\n")
    rc = p.returncode

# change owner for bosh.yaml
call("chown -R {0} {1}/.bosh_config".format(username, home_dir), shell=True)

# load manifests in to memory
f = open('./manifests/index.yml')
manifests = yaml.safe_load(f)
f.close()

# run bosh status and get the id back to inject in to manifests
bosh_uuid = subprocess.Popen(["bosh", "status", "--uuid"], stdout=subprocess.PIPE).communicate()[0]
print "Director uuid is {0}".format(bosh_uuid)

# set the director id on the manifests
for m in manifests['manifests']:
    with open ("{0}/manifests/{1}".format(home_dir, m['file']), 'r+') as f:

        contents = f.read()
        template = Template(contents)
        contents = template.render(DIRECTOR_UUID=bosh_uuid)

        f.seek(0)
        f.write(contents)
        f.truncate()
        f.close()

pivnetAPIToken = settings["pivnet-api-token"]

eula_urls = ["https://network.pivotal.io/api/v2/products/{0}/releases/{1}/eula_acceptance".format(m['release-name'], m['release-number'])
    for m in manifests['manifests']]

release_urls = ["https://network.pivotal.io/api/v2/products/{0}/releases/{1}/product_files/{2}/download".format(m['release-name'], m['release-number'], m['file-number'])
    for m in manifests['manifests']]

stemcell_urls = [m['stemcell'] for m in manifests['manifests']]

# accept eula for each product
for url in eula_urls:
    res = authorizedPost(url, pivnetAPIToken)
    code = res.getcode()

# releases
is_release_file = re.compile("^releases\/.+")
call("mkdir -p /tmp/releases", shell=True)

print "Processing releases."

for url in release_urls:

    print "Downloading {0}.".format(url)

    res = authorizedPost(url, pivnetAPIToken)
    code = res.getcode()

    length = int(res.headers["Content-Length"])

    # content-length
    if code is 200:

        total = 0
        pcent = 0.0
        CHUNK = 16 * 1024

        with tempfile.TemporaryFile() as temp:
            while True:
                chunk = res.read(CHUNK)
                total += CHUNK
                pcent = (float(total) / float(length)) * 100

                sys.stdout.write("Download progress: %.2f%% (%.2fM)\r" % (pcent, total / 1000000.0) )
                sys.stdout.flush()

                if not chunk: break
                temp.write(chunk)

            print "Download complete."

            z = zipfile.ZipFile(temp)
            for name in z.namelist():

                # is this a release?
                if is_release_file.match(name):

                    release_filename = "/tmp/{0}".format(name)

                    print "Unpacking {0}.".format(name)
                    z.extract(name, "/tmp")

                    # upload the file with bosh
                    print "Uploading release {0} to BOSH director.".format(name)
                    call("bosh upload release {0}".format(release_filename), shell=True)

                    os.unlink(release_filename)

            z.close()
            temp.close()

# stemcells
print "Processing stemcells."

for url in stemcell_urls:
    print "Processing stemcell {0}".format(url)
    call("bosh upload stemcell {0}".format(url), shell=True)

# deploy!
for m in manifests['manifests']:
    call("bosh deployment {0}/manifests/{1}".format(home_dir, m['file']), shell=True)
    call("bosh -n deploy", shell=True)

# cf specific configuration (configure security groups for haproxy)
bosh_address = "10.0.0.4"
deployment_name = "router-and-lb"
subscription_id = settings['SUBSCRIPTION-ID']
tenant = settings['TENANT-ID']
endpoint = "https://login.microsoftonline.com/{0}/oauth2/token".format(tenant)
client_token = settings['CLIENT-ID']
client_secret = settings['CLIENT-SECRET']

ip = get_job_ip_address("admin", "admin", bosh_address, deployment_name, "haproxy")

token = get_token_from_client_credentials(endpoint, client_token, client_secret)
creds = SubscriptionCloudCredentials(subscription_id, token)

network_client = azure.mgmt.network.NetworkResourceProviderClient(creds)
rules_client = azure.mgmt.network.networkresourceprovider.SecurityRuleOperations(network_client)

ha_proxy_address = "10.0.16.200"

rule = azure.mgmt.network.SecurityRule(
    description="",
    protocol="*",
    source_port_range="*",
    destination_port_range="80",
    source_address_prefix="*",
    destination_address_prefix=ha_proxy_address,
    access="Allow",
    priority=1100,
    direction="InBound"
)

rules_client.create_or_update("cf", settings['NSG-NAME-FOR-CF'], "http_inbound", rule)

rule = azure.mgmt.network.SecurityRule(
    description="",
    protocol="*",
    source_port_range="*",
    destination_port_range="443",
    source_address_prefix="*",
    destination_address_prefix=ha_proxy_address,
    access="Allow",
    priority=1200,
    direction="InBound"
)

rules_client.create_or_update("cf", "CFSecurityGroup", "http_inbound", rule)
<|endoftext|>


<commit_before>from hdl_toolkit.hdlObjects.types.defs import INT, BOOL, STR, BIT
from hdl_toolkit.hdlObjects.types.bits import Bits
from hdl_toolkit.hdlObjects.types.typeCast import toHVal

# create hdl integer value (for example integer value in vhdl)
hInt = lambda val: INT.fromPy(val)

# create hdl boolean value (for example boolean value in vhdl)
hBool = lambda val: BOOL.fromPy(val)

# create hdl string value (for example string value in vhdl)
hStr = lambda val: STR.fromPy(val)

# create hdl bit value (for example STD_LOGIC value in vhdl)
hBit = lambda val: BIT.fromPy(val)

def mkRange(width):
    """Make hdl range (for example 1 downto 0 in vhdl)
       @return: (width -1, 0)
    """
    to = toHVal(width)
    to = to - 1
    return to._downto(0)

def vecT(width, signed=None):
    """Make contrained vector type for example std_logic_vector(width-1 downto 0) in vhdl"""
    return Bits(widthConstr=mkRange(width), signed=signed, forceVector=True)

def vec(val, width):
    """create hdl vector value"""
    assert val < 2 ** width
    return vecT(width).fromPy(val)

def hRange(upper, lower):
    upper = toHVal(upper)
    return upper._downto(lower)
<commit_msg>fix: vec - support for width specified by parameter<commit_after>from hdl_toolkit.hdlObjects.types.defs import INT, BOOL, STR, BIT
from hdl_toolkit.hdlObjects.types.bits import Bits
from hdl_toolkit.hdlObjects.types.typeCast import toHVal
from hdl_toolkit.synthetisator.param import evalParam

# create hdl integer value (for example integer value in vhdl)
hInt = lambda val: INT.fromPy(val)

# create hdl boolean value (for example boolean value in vhdl)
hBool = lambda val: BOOL.fromPy(val)

# create hdl string value (for example string value in vhdl)
hStr = lambda val: STR.fromPy(val)

# create hdl bit value (for example STD_LOGIC value in vhdl)
hBit = lambda val: BIT.fromPy(val)

def mkRange(width):
    """Make hdl range (for example 1 downto 0 in vhdl)
       @return: (width -1, 0)
    """
    to = toHVal(width)
    to = to - 1
    return to._downto(0)

def vecT(width, signed=None):
    """Make contrained vector type for example std_logic_vector(width-1 downto 0) in vhdl"""
    return Bits(widthConstr=mkRange(width), signed=signed, forceVector=True)

def vec(val, width):
    """create hdl vector value"""
    assert val < evalParam(hInt(2) ** width).val
    return vecT(width).fromPy(val)

def hRange(upper, lower):
    upper = toHVal(upper)
    return upper._downto(lower)
<|endoftext|>