<commit_before># -*- coding: utf-8 -*-
"""Config utility functions for vcspull.

vcspull.config
~~~~~~~~~~~~~~

A lot of these items are todo.

"""

from __future__ import (
    absolute_import, division, print_function, with_statement, unicode_literals
)

import os
import fnmatch
import glob
import logging

import kaptan

from . import exc
from .util import update_dict
from ._compat import string_types

log = logging.getLogger(__name__)

def in_dir(
    config_dir=os.path.expanduser('~/.vcspull'),
    extensions=['.yml', '.yaml', '.json']
):
    """Return a list of configs in ``config_dir``.

    :param config_dir: directory to search
    :type config_dir: string
    :param extensions: filetypes to check (e.g. ``['.yaml', '.json']``).
    :type extensions: list
    :rtype: list

    """
    configs = []

    for filename in os.listdir(config_dir):
        if is_config_file(filename, extensions) and \
           not filename.startswith('.'):
            configs.append(filename)

    return configs

def expand_config(config):
    """Return expanded configuration.

    end-user configuration permit inline configuration shortcuts, expand to
    identical format for parsing.

    :param config: the repo config in :py:class:`dict` format.
    :type config: dict
    :rtype: dict

    """
    for directory, repos in config.items():
        for repo, repo_data in repos.items():

            '''
            repo_name: http://myrepo.com/repo.git

            to

            repo_name: { url: 'http://myrepo.com/repo.git' }

            also assures the repo is a :py:class:`dict`.
            '''

            if isinstance(repo_data, string_types):
                config[directory][repo] = {'url': repo_data}

            '''
            ``shell_command_after``: if str, turn to list.
            '''
            if 'shell_command_after' in repo_data:
                if isinstance(repo_data['shell_command_after'], string_types):
                    repo_data['shell_command_after'] = [
                        repo_data['shell_command_after']
                    ]

    config = dict(
        (os.path.expandvars(directory), repo_data)
        for directory, repo_data in config.items()
    )

    config = dict(
        (os.path.expanduser(directory), repo_data) for
        directory, repo_data in config.items()
    )

    return config

def is_config_file(filename, extensions=['.yml', '.yaml', '.json']):
    """Return True if file has a valid config file type.

    :param filename: filename to check (e.g. ``mysession.json``).
    :type filename: string
    :param extensions: filetypes to check (e.g. ``['.yaml', '.json']``).
    :type extensions: list or string
    :rtype: bool

    """

    extensions = [extensions] if isinstance(
        extensions, string_types) else extensions
    return any(filename.endswith(e) for e in extensions)

def find_home_config_files(filetype=['json', 'yaml']):
    """Return configs of ``.vcspull.{yaml,json}`` in user's home directory.

    """

    configs = []

    yaml_config = os.path.expanduser('~/.vcspull.yaml')
    has_yaml_config = os.path.exists(yaml_config)
    json_config = os.path.expanduser('~/.vcspull.json')
    has_json_config = os.path.exists(json_config)

    if not has_yaml_config and not has_json_config:
        log.debug(
            'No config file found. Create a .vcspull.yaml or .vcspull.json'
            ' in your $HOME directory. http://vcspull.rtfd.org for a'
            ' quickstart.'
        )
    else:
        if sum(filter(None, [has_json_config, has_yaml_config])) > int(1):
            raise exc.MultipleRootConfigs(
                'multiple configs found in home directory use only one.'
                ' .yaml, .json.'
            )
        if has_yaml_config:
            configs.append(yaml_config)
        if has_json_config:
            configs.append(json_config)

    return configs

def find_config_files(
    path=['~/.vcspull'],
    match=['*'],
    filetype=['json', 'yaml'],
    include_home=False
):
    """Return repos from a directory and match. Not recursive.

    :param path: list of paths to search
    :type path: list
    :param match: list of globs to search against
    :type match: list
    :param filetype: list of filetypes to search against
    :type filetype: list
    :param include_home: Include home configuration files
    :type include_home: bool
    :raises:
        - LoadConfigRepoConflict: There are two configs that have same path
          and name with different repo urls.
        - NoConfigsFound: No configs found in home directory or ~/.vcspull
          directory.
    :returns: list of absolute paths to config files.
    :rtype: list

    """

    configs = []

    if include_home is True:
        configs.extend(find_home_config_files())

    if isinstance(path, list):
        for p in path:
            configs.extend(find_config_files(p, match, filetype))
            return configs
    else:
        path = os.path.expanduser(path)
        if isinstance(match, list):
            for m in match:
                configs.extend(find_config_files(path, m, filetype))
        else:
            if isinstance(filetype, list):
                for f in filetype:
                    configs.extend(find_config_files(path, match, f))
            else:
                match = os.path.join(path, match)
                match += ".{filetype}".format(filetype=filetype)

                configs = glob.glob(match)

    return configs

def load_configs(configs):
    """Return repos from a directory and fnmatch. Not recursive.

    :param configs: paths to config file
    :type path: list
    :returns: expanded config dict item
    :rtype: iter(dict)

    """

    configdict = {}

    # rewrite this...
    for config in configs:
        fName, fExt = os.path.splitext(config)
        conf = kaptan.Kaptan(handler=fExt.lstrip('.'))
        conf.import_config(config)

        newconfigdict = expand_config(conf.export('dict'))

        if configdict:
            for path in newconfigdict:
                if path in configdict:
                    for repo_name in newconfigdict[path]:
                        if repo_name in configdict[path]:
                            if newconfigdict[path][repo_name] != configdict[path][repo_name]:
                                print('same path + repo for %s' % repo_name)
                                if newconfigdict[path][repo_name]['url'] != configdict[path][repo_name]['url']:
                                    msg = (
                                        'same path + repo, different vcs (%s)\n'
                                        '%s\n%s' %
                                        (
                                            repo_name,
                                            configdict[path][repo_name]['url'],
                                            newconfigdict[path][repo_name]['url']
                                        )
                                    )
                                    raise Exception(msg)
        configdict = update_dict(configdict, newconfigdict)
        # configdict.update(conf.export('dict'))

    # if configs load and validate_schema, then load.
    # configs = [config for config in configs if validate_schema(config)]

    configdict = expand_config(configdict)

    return configdict

def expand_config(config):
    """Return expanded configuration.

    end-user configuration permit inline configuration shortcuts, expand to
    identical format for parsing.

    :param config: the repo config in :py:class:`dict` format.
    :type config: dict
    :rtype: dict

    """
    for directory, repos in config.items():
        for repo, repo_data in repos.items():

            '''
            repo_name: http://myrepo.com/repo.git

            to

            repo_name: { url: 'http://myrepo.com/repo.git' }

            also assures the repo is a :py:class:`dict`.
            '''

            if isinstance(repo_data, string_types):
                config[directory][repo] = {'url': repo_data}

            '''
            ``shell_command_after``: if str, turn to list.
            '''
            if 'shell_command_after' in repo_data:
                if isinstance(repo_data['shell_command_after'], string_types):
                    repo_data['shell_command_after'] = [
                        repo_data['shell_command_after']
                    ]

    config = dict(
        (os.path.expandvars(directory), repo_data) for
        directory, repo_data in config.items()
    )

    config = dict(
        (os.path.expanduser(directory), repo_data) for
        directory, repo_data in config.items()
    )

    return config

def flatten_config(config):
    """Return config flattened into a list of dictionary.

    :param config: expanded config
    :type config: dict
    :rtype: list of dict
    """

    repos = []
    for repocwd, reponode in config.items():
        for reponame, repo in reponode.items():
            repo['name'] = reponame
            repo['cwd'] = repocwd
            repos.append(repo)
    return repos

<commit_msg>pep257 and remove duplicate function in config<commit_after># -*- coding: utf-8 -*-
"""Config utility functions for vcspull.

vcspull.config
~~~~~~~~~~~~~~

A lot of these items are todo.

"""

from __future__ import (
    absolute_import, division, print_function, with_statement, unicode_literals
)

import os
import glob
import logging

import kaptan

from . import exc
from .util import update_dict
from ._compat import string_types

log = logging.getLogger(__name__)

def in_dir(
    config_dir=os.path.expanduser('~/.vcspull'),
    extensions=['.yml', '.yaml', '.json']
):
    """Return a list of configs in ``config_dir``.

    :param config_dir: directory to search
    :type config_dir: string
    :param extensions: filetypes to check (e.g. ``['.yaml', '.json']``).
    :type extensions: list
    :rtype: list

    """
    configs = []

    for filename in os.listdir(config_dir):
        if is_config_file(filename, extensions) and \
           not filename.startswith('.'):
            configs.append(filename)

    return configs

def expand_config(config):
    """Return expanded configuration.

    end-user configuration permit inline configuration shortcuts, expand to
    identical format for parsing.

    :param config: the repo config in :py:class:`dict` format.
    :type config: dict
    :rtype: dict

    """
    for directory, repos in config.items():
        for repo, repo_data in repos.items():

            '''
            repo_name: http://myrepo.com/repo.git

            to

            repo_name: { url: 'http://myrepo.com/repo.git' }

            also assures the repo is a :py:class:`dict`.
            '''

            if isinstance(repo_data, string_types):
                config[directory][repo] = {'url': repo_data}

            '''
            ``shell_command_after``: if str, turn to list.
            '''
            if 'shell_command_after' in repo_data:
                if isinstance(repo_data['shell_command_after'], string_types):
                    repo_data['shell_command_after'] = [
                        repo_data['shell_command_after']
                    ]

    config = dict(
        (os.path.expandvars(directory), repo_data)
        for directory, repo_data in config.items()
    )

    config = dict(
        (os.path.expanduser(directory), repo_data) for
        directory, repo_data in config.items()
    )

    return config

def is_config_file(filename, extensions=['.yml', '.yaml', '.json']):
    """Return True if file has a valid config file type.

    :param filename: filename to check (e.g. ``mysession.json``).
    :type filename: string
    :param extensions: filetypes to check (e.g. ``['.yaml', '.json']``).
    :type extensions: list or string
    :rtype: bool

    """
    extensions = [extensions] if isinstance(
        extensions, string_types) else extensions
    return any(filename.endswith(e) for e in extensions)

def find_home_config_files(filetype=['json', 'yaml']):
    """Return configs of ``.vcspull.{yaml,json}`` in user's home directory."""
    configs = []

    yaml_config = os.path.expanduser('~/.vcspull.yaml')
    has_yaml_config = os.path.exists(yaml_config)
    json_config = os.path.expanduser('~/.vcspull.json')
    has_json_config = os.path.exists(json_config)

    if not has_yaml_config and not has_json_config:
        log.debug(
            'No config file found. Create a .vcspull.yaml or .vcspull.json'
            ' in your $HOME directory. http://vcspull.rtfd.org for a'
            ' quickstart.'
        )
    else:
        if sum(filter(None, [has_json_config, has_yaml_config])) > int(1):
            raise exc.MultipleRootConfigs(
                'multiple configs found in home directory use only one.'
                ' .yaml, .json.'
            )
        if has_yaml_config:
            configs.append(yaml_config)
        if has_json_config:
            configs.append(json_config)

    return configs

def find_config_files(
    path=['~/.vcspull'],
    match=['*'],
    filetype=['json', 'yaml'],
    include_home=False
):
    """Return repos from a directory and match. Not recursive.

    :param path: list of paths to search
    :type path: list
    :param match: list of globs to search against
    :type match: list
    :param filetype: list of filetypes to search against
    :type filetype: list
    :param include_home: Include home configuration files
    :type include_home: bool
    :raises:
        - LoadConfigRepoConflict: There are two configs that have same path
          and name with different repo urls.
        - NoConfigsFound: No configs found in home directory or ~/.vcspull
          directory.
    :returns: list of absolute paths to config files.
    :rtype: list

    """
    configs = []

    if include_home is True:
        configs.extend(find_home_config_files())

    if isinstance(path, list):
        for p in path:
            configs.extend(find_config_files(p, match, filetype))
            return configs
    else:
        path = os.path.expanduser(path)
        if isinstance(match, list):
            for m in match:
                configs.extend(find_config_files(path, m, filetype))
        else:
            if isinstance(filetype, list):
                for f in filetype:
                    configs.extend(find_config_files(path, match, f))
            else:
                match = os.path.join(path, match)
                match += ".{filetype}".format(filetype=filetype)

                configs = glob.glob(match)

    return configs

def load_configs(configs):
    """Return repos from a directory and fnmatch. Not recursive.

    :param configs: paths to config file
    :type path: list
    :returns: expanded config dict item
    :rtype: iter(dict)

    """
    configdict = {}

    # rewrite this...
    for config in configs:
        fName, fExt = os.path.splitext(config)
        conf = kaptan.Kaptan(handler=fExt.lstrip('.'))
        conf.import_config(config)

        newconfigdict = expand_config(conf.export('dict'))

        if configdict:
            for path in newconfigdict:
                if path in configdict:
                    for repo_name in newconfigdict[path]:
                        if repo_name in configdict[path]:
                            if newconfigdict[path][repo_name] != configdict[path][repo_name]:
                                print('same path + repo for %s' % repo_name)
                                if newconfigdict[path][repo_name]['url'] != configdict[path][repo_name]['url']:
                                    msg = (
                                        'same path + repo, diffvcs (%s)\n'
                                        '%s\n%s' %
                                        (
                                            repo_name,
                                            configdict[path][repo_name]['url'],
                                            newconfigdict[path][
                                                repo_name]['url']
                                        )
                                    )
                                    raise Exception(msg)
        configdict = update_dict(configdict, newconfigdict)
        # configdict.update(conf.export('dict'))

    # if configs load and validate_schema, then load.
    # configs = [config for config in configs if validate_schema(config)]

    configdict = expand_config(configdict)

    return configdict

def flatten_config(config):
    """Return config flattened into a list of dictionary.

    :param config: expanded config
    :type config: dict
    :rtype: list of dict
    """
    repos = []
    for repocwd, reponode in config.items():
        for reponame, repo in reponode.items():
            repo['name'] = reponame
            repo['cwd'] = repocwd
            repos.append(repo)
    return repos
<|endoftext|>


<commit_before>__version__ = (0, 1, 0)
<commit_msg>Version bump.<commit_after>__version__ = (0, 1, 1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Runs the server for backend application `verejne`."""
import argparse
import db_old
import simplejson as json
from paste import httpserver
import os
import sys
import webapp2

from api_GetInfos import get_GetInfos
from state import Entities
from utils import yaml_load

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/db')))
from db import DatabaseConnection

class MyServer(webapp2.RequestHandler):
    """Abstract class defining a server hook.

    All individual hooks inherit from this class and implement the
    method `get` performing the actual logic of the computation.
    """

    def get(self):
        raise NotImplementedError('Must implement method `get`.')

    def returnJSON(self, j):
        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(json.dumps(j, separators=(',',':')))

class GetAddresses(MyServer):
    def get(self):
        # Parse URL parameters:
        try:
            lat1 = float(self.request.GET['lat1'])
            lat2 = float(self.request.GET['lat2'])
            lng1 = float(self.request.GET['lng1'])
            lng2 = float(self.request.GET['lng2'])
            restrictToSlovakia = bool(
                self.request.GET.get('restrictToSlovakia', False))
            level = int(self.request.GET.get('level', 3))
        except:
            self.abort(400, detail='Could not parse parameters')

        # Find addresses within the specified rectangle:
        q = """
            SELECT
              id AS address_id,
              lat,
              lng,
              address_flags.trade_with_government,
              address_flags.political_entity,
              address_flags.contact_with_politics
            FROM address
            LEFT JOIN address_flags
              ON address_flags.address_id=address.id
            WHERE  '(%s, %s), (%s, %s)'::box @> point(lat, lng)
            LIMIT 1500;
            """
        q_data = (lat1, lng1, lat2, lng2)
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

class GetEntitiesAtAddressId(MyServer):
    def get(self):
        # Parse address_id from URL
        try:
            address_id = int(self.request.GET['address_id'])
        except:
            self.abort(400, detail="Could not parse parameter 'address_id' as an integer")

        # Find entities with the given address_id in the database
        q = """
            SELECT id, name FROM entities
            WHERE  address_id=%s;
            """
        q_data = [address_id]
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

class GetEntities(MyServer):
    def get(self):
        try:
            lat1 = float(self.request.GET["lat1"])
            lat2 = float(self.request.GET["lat2"])
            lng1 = float(self.request.GET["lng1"])
            lng2 = float(self.request.GET["lng2"])
            restrictToSlovakia = bool(
                    self.request.GET.get("restrictToSlovakia", False))
            level = int(self.request.GET.get("level", 3))
        except:
            self.abort(400, detail="Could not parse parameters")

        if (level == 0) and ((lat2 - lat1) + (lng2 - lng1) > 2):
            self.abort(400, detail="Requested area is too large. If you want to download data for your own use, please contact us on Facebook.")

        entities = webapp2.get_app().registry['entities']
        self.response.headers['Content-Type'] = 'application/json'
        entities.getEntities(
                self.response, lat1, lng1, lat2, lng2, level, restrictToSlovakia)

class GetRelated(MyServer):
    def get(self):
        try:
            eid = int(self.request.GET["eid"])
        except:
            self.abort(400, detail="Could not parse parameter 'eid' as an integer")

        if eid < 0:
            self.abort(400, detail="Provided 'eid' is not an entity id")

        entities = webapp2.get_app().registry['entities']
        self.returnJSON(entities.getRelated(eid))

# Read info from all info tables and returns all related entities for the given eid
class GetInfo(MyServer):
    def get(self):
        try:
            eid = int(self.request.GET["eid"])
        except:
            self.abort(400, detail="Could not parse parameter 'eid' as an integer")

        # Copy total sum of contracts if present
        result = {}
        entities = webapp2.get_app().registry['entities']
        if eid in entities.eid_to_index:
            entity = entities.entities[entities.eid_to_index[eid]]
            result["total_contracts"] = entity.contracts

        # load info data from individual tables
        data_sources = webapp2.get_app().registry['data_sources']
        for table in data_sources:
            print "Processing table", table
            columns = ",".join(data_sources[table])
            sql = "SELECT DISTINCT " + columns + " FROM entities"
            if (table != "entities"):
                sql += " JOIN " + table + " ON " + "entities.id=" + table + ".id"
            sql += " WHERE entities.eid=%s"
            with db_old.getCursor() as cur:
                cur = db_old.execute(cur, sql, [eid])
                current = []
                for row in cur:
                    # TODO: is this needed?
                    r = {key: row[key] for key in row}
                    current.append(r)
                result[table] = current

        result["related"] = entities.getRelated(eid)
        self.returnJSON(result)

class GetInfos(MyServer):
    """ Given a list of eIDs, return a dictionary indexed by eIDs containing
        information from production tables about the respective entities.
        If an eID is not found, the corresponding information is an empty dict.
    """

    def get(self):
        # Parse eIDs
        try:
            eIDs = [int(x) for x in (self.request.GET['eids'].split(','))]
        except:
            self.abort(400, detail='Could not parse parameter "eids" as a list of integers')

        # Limit the number of eIDs in a single query
        max_count = 50
        if len(eIDs) > max_count:
            self.abort(400, detail='Too many eIDs requested at once (maximum %d)' % (max_count))

        # Obtain pointer to the database connection object
        db = webapp2.get_app().registry['db']

        # Carry out the API logic
        result = get_GetInfos(db, eIDs)

        # Return the result as JSON
        self.returnJSON(result)

# Search entity by name
class SearchEntityByName(MyServer):
    def get(self):
        try:
            text = self.request.GET["name"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        q = """
            SELECT DISTINCT id AS eid FROM entities_search
            WHERE search_vector @@ plainto_tsquery(unaccent(%s))
            LIMIT 20
            """

        q_data = [text]
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

# Search entity by name
class SearchEntity(MyServer):
    def get(self):
        try:
            text = self.request.GET["text"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        with db_old.getCursor() as cur:
            sql = "SELECT DISTINCT eid AS eid FROM entities " + \
                  "WHERE to_tsvector('unaccent', entity_name) @@ plainto_tsquery('unaccent', %s) " + \
                  "LIMIT 20"
            cur = db_old.execute(cur, sql, [text])
            result = []
            for row in cur:
                try:
                    result.append({"eid": row["eid"]})
                except:
                    pass
            self.returnJSON(result)

class SearchEntityByNameAndAddress(MyServer):
    """ Server hook allowing to search for entities by name and address
        (given in text format). Returns a list of dictionaries, with each
        dictionary of the form {'eid': 123456}. """
    def get(self):
        # Parse input
        try:
            firstname = self.request.GET["firstname"].encode("utf8")
            surname = self.request.GET["surname"].encode("utf8")
            address = self.request.GET["address"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        # Carry-out the logic
        q = """
            SELECT DISTINCT
                eid AS eid
            FROM
                entities
            WHERE
                to_tsvector('unaccent', entity_name) @@ plainto_tsquery('unaccent', %s)
                AND
                to_tsvector('unaccent', address) @@ plainto_tsquery('unaccent', %s)
            LIMIT 20;
            """
        with db_old.getCursor() as cur:
            cur = db_old.execute(cur, q, (firstname + ' ' + surname, address))
            result = []
            for row in cur:
                try:
                    result.append({"eid": row["eid"]})
                except:
                    pass
            self.returnJSON(result)

# For given ico, find the corresponding eid and redirect to to its url.
# If no matching entity found redirect to default
class IcoRedirect(MyServer):
    def getEidForIco(self, ico):
        def getForTable(table):
            sql = "SELECT eid FROM " + table + \
                  " JOIN entities ON entities.id = " + table + ".id" + \
                  " WHERE ico = %s" + \
                  " LIMIT 1"
            with db_old.getCursor() as cur:
                cur = db_old.execute(cur, sql, [ico])
                row = cur.fetchone()
                if row is None: return None
                return row["eid"]

        eid = None
        eid = getForTable("new_orsr_data")
        if eid is None: eid = getForTable("firmy_data")
        if eid is None: eid = getForTable("orsresd_data")
        return eid

    def get(self):
        try:
            ico = int(self.request.GET["ico"])
        except:
            self.abort(400, detail="Could not parse parameter 'ico' as an integer")

        eid = self.getEidForIco(ico)
        print("LOG: icoredirect " + str(ico) + " " + str(eid))
        entities = webapp2.get_app().registry['entities']
        if (eid is None) or (not eid in entities.eid_to_index):
            return self.redirect("/")
        entity = entities.entities[entities.eid_to_index[eid]]
        return self.redirect("/?zobraz&%.7f&%.7f&%d" % (entity.lat, entity.lng, eid))

# Setup of the webapp2 WSGI application
app = webapp2.WSGIApplication([
    ('/getAddresses', GetAddresses),
    ('/getEntitiesAtAddressId', GetEntitiesAtAddressId),
    ('/getEntities', GetEntities),
    ('/getInfo', GetInfo),
    ('/getInfos', GetInfos),
    ('/getRelated', GetRelated),
    ('/ico', IcoRedirect),
    ('/searchEntity', SearchEntity),
    ('/searchEntityByName', SearchEntityByName),
    ('/searchEntityByNameAndAddress', SearchEntityByNameAndAddress),
], debug=False)

def initialise_app(serving_directory, disable_old_database=False):
    """Precomputes values shared across requests to this app.

    The registry property is intended for storing these precomputed
    values, so as to avoid global variables.
    """

    # Connect to the database:
    db = DatabaseConnection(path_config='db_config.yaml')
    schema = db.get_latest_schema('prod_')
    db.execute('SET search_path to ' + schema + ',public;')
    app.registry['db'] = db

    # For faster unit testing:
    if disable_old_database:
        return

    # Load (old) data sources:
    data_sources = yaml_load('datasources.yaml')
    app.registry['data_sources'] = data_sources

    # Load (old) entities:
    entities = Entities()
    entities.loadFromDirectory(serving_directory)
    app.registry['entities'] = entities

def main(args_dict):
    initialise_app(args_dict['serving_directory'])

    host, port = args_dict['listen'].split(':')
    httpserver.serve(
        app,
        host=host,
        port=port,
        request_queue_size=128,
        use_threadpool=True,
        threadpool_workers=32,
    )

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--listen',
        default='127.0.0.1:8080',
        help='host:port to listen on')
    parser.add_argument(
        '--serving_directory',
        default='/data/www/verejne.digital/serving/prod/',
        help='Directory with serving data')
    args_dict = vars(parser.parse_args())
    main(args_dict)
<commit_msg>fix oliver solga search<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Runs the server for backend application `verejne`."""
import argparse
import db_old
import simplejson as json
from paste import httpserver
import os
import sys
import webapp2

from api_GetInfos import get_GetInfos
from state import Entities
from utils import yaml_load

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../data/db')))
from db import DatabaseConnection

class MyServer(webapp2.RequestHandler):
    """Abstract class defining a server hook.

    All individual hooks inherit from this class and implement the
    method `get` performing the actual logic of the computation.
    """

    def get(self):
        raise NotImplementedError('Must implement method `get`.')

    def returnJSON(self, j):
        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(json.dumps(j, separators=(',',':')))

class GetAddresses(MyServer):
    def get(self):
        # Parse URL parameters:
        try:
            lat1 = float(self.request.GET['lat1'])
            lat2 = float(self.request.GET['lat2'])
            lng1 = float(self.request.GET['lng1'])
            lng2 = float(self.request.GET['lng2'])
            restrictToSlovakia = bool(
                self.request.GET.get('restrictToSlovakia', False))
            level = int(self.request.GET.get('level', 3))
        except:
            self.abort(400, detail='Could not parse parameters')

        # Find addresses within the specified rectangle:
        q = """
            SELECT
              id AS address_id,
              lat,
              lng,
              address_flags.trade_with_government,
              address_flags.political_entity,
              address_flags.contact_with_politics
            FROM address
            LEFT JOIN address_flags
              ON address_flags.address_id=address.id
            WHERE  '(%s, %s), (%s, %s)'::box @> point(lat, lng)
            LIMIT 1500;
            """
        q_data = (lat1, lng1, lat2, lng2)
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

class GetEntitiesAtAddressId(MyServer):
    def get(self):
        # Parse address_id from URL
        try:
            address_id = int(self.request.GET['address_id'])
        except:
            self.abort(400, detail="Could not parse parameter 'address_id' as an integer")

        # Find entities with the given address_id in the database
        q = """
            SELECT id, name FROM entities
            WHERE  address_id=%s;
            """
        q_data = [address_id]
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

class GetEntities(MyServer):
    def get(self):
        try:
            lat1 = float(self.request.GET["lat1"])
            lat2 = float(self.request.GET["lat2"])
            lng1 = float(self.request.GET["lng1"])
            lng2 = float(self.request.GET["lng2"])
            restrictToSlovakia = bool(
                    self.request.GET.get("restrictToSlovakia", False))
            level = int(self.request.GET.get("level", 3))
        except:
            self.abort(400, detail="Could not parse parameters")

        if (level == 0) and ((lat2 - lat1) + (lng2 - lng1) > 2):
            self.abort(400, detail="Requested area is too large. If you want to download data for your own use, please contact us on Facebook.")

        entities = webapp2.get_app().registry['entities']
        self.response.headers['Content-Type'] = 'application/json'
        entities.getEntities(
                self.response, lat1, lng1, lat2, lng2, level, restrictToSlovakia)

class GetRelated(MyServer):
    def get(self):
        try:
            eid = int(self.request.GET["eid"])
        except:
            self.abort(400, detail="Could not parse parameter 'eid' as an integer")

        if eid < 0:
            self.abort(400, detail="Provided 'eid' is not an entity id")

        entities = webapp2.get_app().registry['entities']
        self.returnJSON(entities.getRelated(eid))

# Read info from all info tables and returns all related entities for the given eid
class GetInfo(MyServer):
    def get(self):
        try:
            eid = int(self.request.GET["eid"])
        except:
            self.abort(400, detail="Could not parse parameter 'eid' as an integer")

        # Copy total sum of contracts if present
        result = {}
        entities = webapp2.get_app().registry['entities']
        if eid in entities.eid_to_index:
            entity = entities.entities[entities.eid_to_index[eid]]
            result["total_contracts"] = entity.contracts

        # load info data from individual tables
        data_sources = webapp2.get_app().registry['data_sources']
        for table in data_sources:
            print "Processing table", table
            columns = ",".join(data_sources[table])
            sql = "SELECT DISTINCT " + columns + " FROM entities"
            if (table != "entities"):
                sql += " JOIN " + table + " ON " + "entities.id=" + table + ".id"
            sql += " WHERE entities.eid=%s"
            with db_old.getCursor() as cur:
                cur = db_old.execute(cur, sql, [eid])
                current = []
                for row in cur:
                    # TODO: is this needed?
                    r = {key: row[key] for key in row}
                    current.append(r)
                result[table] = current

        result["related"] = entities.getRelated(eid)
        self.returnJSON(result)

class GetInfos(MyServer):
    """ Given a list of eIDs, return a dictionary indexed by eIDs containing
        information from production tables about the respective entities.
        If an eID is not found, the corresponding information is an empty dict.
    """

    def get(self):
        # Parse eIDs
        try:
            eIDs = [int(x) for x in (self.request.GET['eids'].split(','))]
        except:
            self.abort(400, detail='Could not parse parameter "eids" as a list of integers')

        # Limit the number of eIDs in a single query
        max_count = 50
        if len(eIDs) > max_count:
            self.abort(400, detail='Too many eIDs requested at once (maximum %d)' % (max_count))

        # Obtain pointer to the database connection object
        db = webapp2.get_app().registry['db']

        # Carry out the API logic
        result = get_GetInfos(db, eIDs)

        # Return the result as JSON
        self.returnJSON(result)

# Search entity by name
class SearchEntityByName(MyServer):
    def get(self):
        try:
            text = self.request.GET["name"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        q = """
            SELECT DISTINCT id AS eid FROM entities_search
            WHERE search_vector @@ plainto_tsquery('simple', unaccent(%s))
            LIMIT 20
            """

        q_data = [text]
        response = webapp2.get_app().registry['db'].query(q, q_data)
        self.returnJSON(response)

# Search entity by name
class SearchEntity(MyServer):
    def get(self):
        try:
            text = self.request.GET["text"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        with db_old.getCursor() as cur:
            sql = "SELECT DISTINCT eid AS eid FROM entities " + \
                  "WHERE to_tsvector('unaccent', entity_name) @@ plainto_tsquery('unaccent', %s) " + \
                  "LIMIT 20"
            cur = db_old.execute(cur, sql, [text])
            result = []
            for row in cur:
                try:
                    result.append({"eid": row["eid"]})
                except:
                    pass
            self.returnJSON(result)

class SearchEntityByNameAndAddress(MyServer):
    """ Server hook allowing to search for entities by name and address
        (given in text format). Returns a list of dictionaries, with each
        dictionary of the form {'eid': 123456}. """
    def get(self):
        # Parse input
        try:
            firstname = self.request.GET["firstname"].encode("utf8")
            surname = self.request.GET["surname"].encode("utf8")
            address = self.request.GET["address"].encode("utf8")
        except:
            self.abort(400, detail="Unable to parse input text")

        # Carry-out the logic
        q = """
            SELECT DISTINCT
                eid AS eid
            FROM
                entities
            WHERE
                to_tsvector('unaccent', entity_name) @@ plainto_tsquery('unaccent', %s)
                AND
                to_tsvector('unaccent', address) @@ plainto_tsquery('unaccent', %s)
            LIMIT 20;
            """
        with db_old.getCursor() as cur:
            cur = db_old.execute(cur, q, (firstname + ' ' + surname, address))
            result = []
            for row in cur:
                try:
                    result.append({"eid": row["eid"]})
                except:
                    pass
            self.returnJSON(result)

# For given ico, find the corresponding eid and redirect to to its url.
# If no matching entity found redirect to default
class IcoRedirect(MyServer):
    def getEidForIco(self, ico):
        def getForTable(table):
            sql = "SELECT eid FROM " + table + \
                  " JOIN entities ON entities.id = " + table + ".id" + \
                  " WHERE ico = %s" + \
                  " LIMIT 1"
            with db_old.getCursor() as cur:
                cur = db_old.execute(cur, sql, [ico])
                row = cur.fetchone()
                if row is None: return None
                return row["eid"]

        eid = None
        eid = getForTable("new_orsr_data")
        if eid is None: eid = getForTable("firmy_data")
        if eid is None: eid = getForTable("orsresd_data")
        return eid

    def get(self):
        try:
            ico = int(self.request.GET["ico"])
        except:
            self.abort(400, detail="Could not parse parameter 'ico' as an integer")

        eid = self.getEidForIco(ico)
        print("LOG: icoredirect " + str(ico) + " " + str(eid))
        entities = webapp2.get_app().registry['entities']
        if (eid is None) or (not eid in entities.eid_to_index):
            return self.redirect("/")
        entity = entities.entities[entities.eid_to_index[eid]]
        return self.redirect("/?zobraz&%.7f&%.7f&%d" % (entity.lat, entity.lng, eid))

# Setup of the webapp2 WSGI application
app = webapp2.WSGIApplication([
    ('/getAddresses', GetAddresses),
    ('/getEntitiesAtAddressId', GetEntitiesAtAddressId),
    ('/getEntities', GetEntities),
    ('/getInfo', GetInfo),
    ('/getInfos', GetInfos),
    ('/getRelated', GetRelated),
    ('/ico', IcoRedirect),
    ('/searchEntity', SearchEntity),
    ('/searchEntityByName', SearchEntityByName),
    ('/searchEntityByNameAndAddress', SearchEntityByNameAndAddress),
], debug=False)

def initialise_app(serving_directory, disable_old_database=False):
    """Precomputes values shared across requests to this app.

    The registry property is intended for storing these precomputed
    values, so as to avoid global variables.
    """

    # Connect to the database:
    db = DatabaseConnection(path_config='db_config.yaml')
    schema = db.get_latest_schema('prod_')
    db.execute('SET search_path to ' + schema + ',public;')
    app.registry['db'] = db

    # For faster unit testing:
    if disable_old_database:
        return

    # Load (old) data sources:
    data_sources = yaml_load('datasources.yaml')
    app.registry['data_sources'] = data_sources

    # Load (old) entities:
    entities = Entities()
    entities.loadFromDirectory(serving_directory)
    app.registry['entities'] = entities

def main(args_dict):
    initialise_app(args_dict['serving_directory'])

    host, port = args_dict['listen'].split(':')
    httpserver.serve(
        app,
        host=host,
        port=port,
        request_queue_size=128,
        use_threadpool=True,
        threadpool_workers=32,
    )

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--listen',
        default='127.0.0.1:8080',
        help='host:port to listen on')
    parser.add_argument(
        '--serving_directory',
        default='/data/www/verejne.digital/serving/prod/',
        help='Directory with serving data')
    args_dict = vars(parser.parse_args())
    main(args_dict)
<|endoftext|>


<commit_before>from basic import *

class DotsParameter(Node):
    CODE = "210000"
    LABEL = "DotsParameter"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], DotsParameter.CODE)
        return DotsParameter(jsn['pos'])

class StructUnionName(Node):
    CODE = "60900"
    LABEL = "StructUnionName"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], StructUnionName.CODE)
        assert not jsn['children']
        return StructUnionName(jsn['pos'], jsn['label'])

    def __init__(self, pos, name):
        super().__init__(pos)
        self.__name = name

class TypeName(Node):
    CODE = "61000"
    LABEL = "TypeName"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], TypeName.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        assert isinstance(children[0], GenericString)
        return TypeName(jsn['pos'], children[0])

    def __init__(self, pos, name):
        super().__init__(pos)
        self.__name = name

class Pointer(Node):
    CODE = "60200"
    LABEL = "Pointer"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Pointer.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) <= 1
        typ = children[0] if children else None
        assert (typ is None) or isinstance(typ, FullType)
        return Pointer(jsn['pos'], typ)

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

# What is this?
class Si(Node):
    CODE = "80100"
    LABEL = "Si"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Si.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return Si(jsn['pos'], children[0])

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

class Void(Node):
    CODE = "70001"
    LABEL = "Void"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Void.CODE)
        return Void(jsn['pos'])

class CChar(Node):
    CODE = "80001"
    LABEL = "CChar"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], CChar.CODE)
        return CChar(jsn['pos'])

# And this?
class CInt(Node):
    CODE = "100003"
    LABEL = "CInt"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], CInt.CODE)
        return CInt(jsn['pos'])

class IntType(Node):
    CODE = "70100"
    LABEL = "IntType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], IntType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return IntType(jsn['pos'], children[0])

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

# TODO: Exactly what is this implementing?
class BaseType(Node):
    CODE = "60100"
    LABEL = "BaseType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], BaseType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return BaseType(jsn['pos'], children[0])

    def __init__(self, pos, base):
        super().__init__(pos)
        self.__base = base

class TypeQualifier(Node):
    CODE = "50000"
    LABEL = "TypeQualifier"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], TypeQualifier.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1 and isinstance(children[0], GenericString)
        return TypeQualifier(jsn['pos'], children[0])

    def __init__(self, pos, qualifier):
        super().__init__(pos)
        self.__qualifier = qualifier

# Provides a full type definition
class FullType(Node):
    CODE = "40000"
    LABEL = "FullType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], FullType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) <= 3

        qualifier = children.pop(0)
        if isinstance(children[0], TypeName):
            name = children.pop(0)
        else:
            name = None

        # Optional - may be Pointer or BaseType
        base_type = children.pop(0) if children else None

        assert isinstance(qualifier, TypeQualifier)
        assert name is None or isinstance(name, TypeName)
        assert base_type is None or isinstance(base_type, BaseType) or isinstance(base_type, Pointer)
        return FullType(jsn['pos'], qualifier, name, base_type)

    def __init__(self, pos, qualifier, name, base_type):
        super().__init__(pos)
        self.__qualifier = qualifier
        self.__name = name
        self.__base_type = base_type

class Storage(Node):
    CODE = "340000"
    LABEL = "Storage"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Storage.CODE)
        return Storage(jsn['pos'],\
                       GenericString.from_json(jsn['children'][0]))

    def __init__(self, pos, label):
        super().__init__(pos)
        self.__label = label
<commit_msg>Updated types<commit_after>from basic import *

class DotsParameter(Node):
    CODE = "210000"
    LABEL = "DotsParameter"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], DotsParameter.CODE)
        return DotsParameter(jsn['pos'])

class StructUnionName(Node):
    CODE = "60900"
    LABEL = "StructUnionName"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], StructUnionName.CODE)
        assert not jsn['children']
        return StructUnionName(jsn['pos'], jsn['label'])

    def __init__(self, pos, name):
        super().__init__(pos)
        self.__name = name

class TypeName(Node):
    CODE = "61000"
    LABEL = "TypeName"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], TypeName.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        assert isinstance(children[0], GenericString)
        return TypeName(jsn['pos'], children[0])

    def __init__(self, pos, name):
        super().__init__(pos)
        self.__name = name

class Pointer(Node):
    CODE = "60200"
    LABEL = "Pointer"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Pointer.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) <= 1
        typ = children[0] if children else None
        assert (typ is None) or isinstance(typ, FullType)
        return Pointer(jsn['pos'], typ)

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

# What is this?
class Si(Node):
    CODE = "80100"
    LABEL = "Si"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Si.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return Si(jsn['pos'], children[0])

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

class Void(Node):
    CODE = "70001"
    LABEL = "Void"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Void.CODE)
        return Void(jsn['pos'])

class CChar(Node):
    CODE = "80001"
    LABEL = "CChar"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], CChar.CODE)
        return CChar(jsn['pos'])

# And this?
class CInt(Node):
    CODE = "100003"
    LABEL = "CInt"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], CInt.CODE)
        return CInt(jsn['pos'])

class IntType(Node):
    CODE = "70100"
    LABEL = "IntType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], IntType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return IntType(jsn['pos'], children[0])

    def __init__(self, pos, typ):
        super().__init__(pos)
        self.__typ = typ

# TODO: Exactly what is this implementing?
class BaseType(Node):
    CODE = "60100"
    LABEL = "BaseType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], BaseType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1
        return BaseType(jsn['pos'], children[0])

    def __init__(self, pos, base):
        super().__init__(pos)
        self.__base = base

class TypeQualifier(Node):
    CODE = "50000"
    LABEL = "TypeQualifier"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], TypeQualifier.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) == 1 and isinstance(children[0], GenericString)
        return TypeQualifier(jsn['pos'], children[0])

    def __init__(self, pos, qualifier):
        super().__init__(pos)
        self.__qualifier = qualifier

# Provides a full type definition
class FullType(Node):
    CODE = "40000"
    LABEL = "FullType"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], FullType.CODE)
        children = [Node.from_json(c) for c in jsn['children']]
        assert len(children) <= 3

        qualifier = children.pop(0)
        if isinstance(children[0], TypeName):
            name = children.pop(0)
        else:
            name = None

        # Optional
        base_type = children.pop(0) if children else None

        assert isinstance(qualifier, TypeQualifier)
        assert name is None or isinstance(name, TypeName)
        return FullType(jsn['pos'], qualifier, name, base_type)

    def __init__(self, pos, qualifier, name, base_type):
        super().__init__(pos)
        self.__qualifier = qualifier
        self.__name = name
        self.__base_type = base_type

class Storage(Node):
    CODE = "340000"
    LABEL = "Storage"

    @staticmethod
    def from_json(jsn):
        Node.check_code(jsn['type'], Storage.CODE)
        return Storage(jsn['pos'],\
                       GenericString.from_json(jsn['children'][0]))

    def __init__(self, pos, label):
        super().__init__(pos)
        self.__label = label
<|endoftext|>


<commit_before>"""
Helper file to generate package version numbers on the fly

Wouldn't it be great if every time you bundled a python package, it was tagged
with a unique version number that reflected the current git ref? It's almost
like that's a feature that should be available without stupid hacks.

But it's not. So here's how the stupid hacks work.

When you run ``python setup.py``, if you are running it from inside of a git
repository this script with generate a unique version number and embed it in an
auto-generated file in your package. By default the file is named
'_version.py', and you should add it to your ``.gitignore``. Since this is a
python file and it's in your package, it will get bundled up and distributed
with your package. Then during the installation process, this script will
recognize that it is not longer being run from within a git repository and find
the version number from the file it generated earlier.

"""
import locale
import os
import re

import subprocess

# This version regex was constructed from the regex-ish description here:
# http://www.python.org/dev/peps/pep-0440/#public-version-identifiers
VERSION_SCHEME = re.compile(r'^\d+(\.\d+)+'
                            r'((a|b|c|rc)\d+)?'
                            r'(\.post\d+)?'
                            r'(\.dev\d+)?$')
GIT_DESCRIBE = ('git', 'describe')
GIT_DESCRIBE_ARGS = ('--tags', '--dirty', '--abbrev=40', '--long')

def find_package(path):
    """
    Find the directory that contains the python package in a repository

    Parameters
    ----------
    path : str
        The path to the repository

    Returns
    -------
    package_dir : str
        The name of the directory that contains the python package

    Raises
    ------
    error : :class:`IOError`
        If a single package cannot be found

    """
    dirname = os.path.basename(path)
    if os.path.isdir(os.path.join(path, dirname)):
        return dirname
    candidates = []
    for filename in os.listdir(path):
        init = os.path.join(path, filename, '__init__.py')
        if os.path.exists(init):
            candidates.append(filename)
    if len(candidates) == 1:
        return candidates[0]
    elif len(candidates) == 0:
        raise IOError("No package found in repo '%s'" % path)
    else:
        raise IOError("Multiple possible packages found in repo '%s'! "
                      "Please specify one." % path)

def parse_constants(filename):
    """ Parse python constants from a file """
    if not os.path.exists(filename):
        return {
            'source_label': 'NOTAG',
            'version': '0.0.0',
        }
    constants = {}
    with open(filename, 'r') as infile:
        for line in infile:
            components = line.split('=')
            if len(components) <= 1:
                continue
            key = components[0].strip(' _')
            value = '='.join(components[1:]).strip().strip('\'\"')
            constants[key] = value
    return constants

def write_constants(filename, **constants):
    """ Write python constants to a file """
    with open(filename, 'w') as outfile:
        outfile.write('""" This file is auto-generated during the '
                      'package-building process """%s' % os.linesep)
        for key, value in constants.items():
            outfile.write("__%s__ = '%s'%s" % (key, value, os.linesep))

def git_describe(describe_args):
    """
    Pull the version information from git

    Parameters
    ----------
    describe_args : list
        Arguments for ``describe_cmd`` to be passed to subprocess

    Returns
    -------
    data : dict
        Dictionary of repo data. The fields are listed below

    tag : str
        The git tag for this version
    description : str
        The output of ``git describe``
    is_dev : bool
        True if is_dirty or if addl_commits > 0
    is_dirty : bool
        True if the git repo is dirty
    addl_commits : int
        The number of additional commits on top of the tag ref
    ref : str
        The ref for the current commit
    dirty_suffix : str
        The string that would denote that the working copy is dirty

    Raises
    ------
    error : :class:`subprocess.CalledProcessError`
        If there is an error running ``git describe``

    """
    encoding = locale.getdefaultlocale()[1] or 'utf-8'
    proc = subprocess.Popen(GIT_DESCRIBE + describe_args,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    output, _ = proc.communicate()
    description = output.decode(encoding).strip()
    if proc.returncode != 0:
        print("Error parsing git revision! Make sure that you have tagged a "
              "commit, and that the tag matches the 'tag_match' argument")
        print(output)
        raise ValueError("Could not describe git version")

    components = description.split('-')
    # trim off the dirty suffix
    dirty_suffix = '-dirty'
    is_dirty = False
    for arg in describe_args:
        if arg.startswith('--dirty='):
            dirty_suffix = arg.split('=')[1]
            break
    if dirty_suffix.startswith('-') and components[-1] == dirty_suffix[1:]:
        components = components[:-1]
        is_dirty = True
    elif components[-1].endswith(dirty_suffix):
        components[-1] = components[-1][:-len(dirty_suffix)]
        is_dirty = True

    ref = components[-1][1:]
    addl_commits = int(components[-2])
    tag = '-'.join(components[:-2])
    return {
        'tag': tag,
        'description': description,
        'is_dirty': is_dirty,
        'is_dev': is_dirty or addl_commits > 0,
        'addl_commits': addl_commits,
        'ref': ref,
        'dirty_suffix': dirty_suffix,
    }

def git_version(package=None,
                tag_match='[0-9]*',
                version_mod='_version.py',
                post_process=None,
                source_url_format=None,
                source_url_on_dev=False,
                pep440=False):
    """
    Generate the version from the git revision, or retrieve it from the
    auto-generated module

    Parameters
    ----------
    package : str, optional
        The name of the directory that contains the package's code. If not
        specified, it will be inferred.
    tag_match : str, optional
        Match only tags with this format (default '[0-9]*'). Note that this
        uses glob matching, not PCRE.
    version_mod : str, optional
        The name of the file to write the version into (default '_version.py')
    post_process : callable, optional
        A function that accepts the output of :meth:`.git_describe` and
        optionally mutates it. This can be used to convert custom tags into
        version numbers (ex. 'v0.1' => '0.1') (default None)
    source_url_format : str, optional
        A format string for the url. (ex.
        'https://github.com/pypa/pip/archive/%(version)s.zip') (default None)
    source_url_on_dev : bool, optional
        Attach a source_url even on developer builds (default False)
    pep440 : bool, optional
        If true, create a PEP 440 compatible version number (default False)

    Returns
    -------
    data : dict
        Dictionary of version data. The fields are listed below.

    version : str
        The unique version of this package
    source_label : str
        VERSION-DISTANCE-gHASH
    source_url : str, optional
        A string containing a full URL where the source for this specific
        version of the distribution can be downloaded

    """
    here = os.path.abspath(os.path.dirname(__file__))

    if package is None:
        package = find_package(here)

    version_file_path = os.path.join(here, package, version_mod)

    if not os.path.isdir(os.path.join(here, '.git')):
        return parse_constants(version_file_path)

    describe_args = GIT_DESCRIBE_ARGS
    if tag_match is not None:
        describe_args += ('--match=%s' % tag_match,)
    version_data = git_describe(describe_args)
    if post_process is not None:
        post_process(version_data)
    if version_data['is_dev']:
        if pep440:
            version = (version_data['tag'] +
                       ".post0.dev%(addl_commits)d" % version_data)
        else:
            version = "{tag}-{addl_commits}-g{ref:<.7}".format(**version_data)
            if version_data['is_dirty']:
                version += version_data['dirty_suffix']
    else:
        version = version_data['tag']

    data = {
        'source_label': version_data['description'],
        'version': version,
    }

    if source_url_format is not None and \
            (source_url_on_dev or not version_data['is_dev']):
        data['source_url'] = source_url_format % data
    write_constants(version_file_path, **data)

    if pep440 and not VERSION_SCHEME.match(data['version']):
        raise Exception("Package version '%(version)s' "
                        "is not a valid PEP 440 version string!" % data)

    return data
<commit_msg>Updating the comments in version_helper.py<commit_after>"""
Helper file to generate package version numbers on the fly

You're probably already using git tags to tag releases of your project. If you
aren't, you really should. Wouldn't it be great if your python package
automatically updated its version number based on the most recent git tag? You
know, so you don't have to do it manually all the time? It's almost like
that's a feature that should be available without stupid hacks.

But it's not. So here's how the stupid hacks work.

When you run ``python setup.py``, if you are running it from inside of a git
repository this script with generate a unique version number and embed it in an
auto-generated file in your package. By default the file is named
'_version.py', and you should add it to your ``.gitignore``. Since this is a
python file and it's in your package, it will get bundled up and distributed
with your package. Then during the installation process, this script will
recognize that it is no longer being run from within a git repository and find
the version number from the file it generated earlier.

To use version_helper, first tag a release commit in your repository::

    git tag 0.1.0

Then add the following to your setup.py file::

    from setuptools import setup
    from version_helper import git_version

    setup(
        name='mypackage',
        ...
        **git_version())

You're done! That's it! To view the auto-generated version number of your
package, run::

    python setup.py -V

"""
import locale
import os
import re

import subprocess

# This version regex was constructed from the regex-ish description here:
# http://www.python.org/dev/peps/pep-0440/#public-version-identifiers
VERSION_SCHEME = re.compile(r'^\d+(\.\d+)+'
                            r'((a|b|c|rc)\d+)?'
                            r'(\.post\d+)?'
                            r'(\.dev\d+)?$')
GIT_DESCRIBE = ('git', 'describe')
GIT_DESCRIBE_ARGS = ('--tags', '--dirty', '--abbrev=40', '--long')

def find_package(path):
    """
    Find the directory that contains the python package in a repository

    Parameters
    ----------
    path : str
        The path to the repository

    Returns
    -------
    package_dir : str
        The name of the directory that contains the python package

    Raises
    ------
    error : :class:`IOError`
        If a single package cannot be found

    """
    dirname = os.path.basename(path)
    if os.path.isdir(os.path.join(path, dirname)):
        return dirname
    candidates = []
    for filename in os.listdir(path):
        init = os.path.join(path, filename, '__init__.py')
        if os.path.exists(init):
            candidates.append(filename)
    if len(candidates) == 1:
        return candidates[0]
    elif len(candidates) == 0:
        raise IOError("No package found in repo '%s'" % path)
    else:
        raise IOError("Multiple possible packages found in repo '%s'! "
                      "Please specify one." % path)

def parse_constants(filename):
    """ Parse python constants from a file """
    if not os.path.exists(filename):
        return {
            'source_label': 'NOTAG',
            'version': 'unknown',
        }
    constants = {}
    with open(filename, 'r') as infile:
        for line in infile:
            components = line.split('=')
            if len(components) <= 1:
                continue
            key = components[0].strip(' _')
            value = '='.join(components[1:]).strip().strip('\'\"')
            constants[key] = value
    return constants

def write_constants(filename, **constants):
    """ Write python constants to a file """
    with open(filename, 'w') as outfile:
        outfile.write('""" This file is auto-generated during the '
                      'package-building process """%s' % os.linesep)
        for key, value in constants.items():
            outfile.write("__%s__ = '%s'%s" % (key, value, os.linesep))

def git_describe(describe_args):
    """
    Pull the version information from git

    Parameters
    ----------
    describe_args : list
        Arguments for ``describe_cmd`` to be passed to subprocess

    Returns
    -------
    data : dict
        Dictionary of repo data. The fields are listed below

    tag : str
        The git tag for this version
    description : str
        The output of ``git describe``
    is_dev : bool
        True if is_dirty or if addl_commits > 0
    is_dirty : bool
        True if the git repo is dirty
    addl_commits : int
        The number of additional commits on top of the tag ref
    ref : str
        The ref for the current commit
    dirty_suffix : str
        The string that would denote that the working copy is dirty

    Raises
    ------
    error : :class:`subprocess.CalledProcessError`
        If there is an error running ``git describe``

    """
    encoding = locale.getdefaultlocale()[1] or 'utf-8'
    proc = subprocess.Popen(GIT_DESCRIBE + describe_args,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    output, _ = proc.communicate()
    description = output.decode(encoding).strip()
    if proc.returncode != 0:
        print("Error parsing git revision! Make sure that you have tagged a "
              "commit, and that the tag matches the 'tag_match' argument")
        print(output)
        raise ValueError("Could not describe git version")

    components = description.split('-')
    # trim off the dirty suffix
    dirty_suffix = '-dirty'
    is_dirty = False
    for arg in describe_args:
        if arg.startswith('--dirty='):
            dirty_suffix = arg.split('=')[1]
            break
    if dirty_suffix.startswith('-') and components[-1] == dirty_suffix[1:]:
        components = components[:-1]
        is_dirty = True
    elif components[-1].endswith(dirty_suffix):
        components[-1] = components[-1][:-len(dirty_suffix)]
        is_dirty = True

    ref = components[-1][1:]
    addl_commits = int(components[-2])
    tag = '-'.join(components[:-2])
    return {
        'tag': tag,
        'description': description,
        'is_dirty': is_dirty,
        'is_dev': is_dirty or addl_commits > 0,
        'addl_commits': addl_commits,
        'ref': ref,
        'dirty_suffix': dirty_suffix,
    }

def git_version(package=None,
                tag_match='[0-9]*',
                version_mod='_version.py',
                post_process=None,
                source_url_format=None,
                source_url_on_dev=False,
                pep440=False):
    """
    Generate the version from the git revision, or retrieve it from the
    auto-generated module

    Parameters
    ----------
    package : str, optional
        The name of the directory that contains the package's code. If not
        specified, it will be inferred.
    tag_match : str, optional
        Match only tags with this format (default '[0-9]*'). Note that this
        uses glob matching, not PCRE.
    version_mod : str, optional
        The name of the file to write the version into (default '_version.py')
    post_process : callable, optional
        A function that accepts the output of :meth:`.git_describe` and
        optionally mutates it. This can be used to convert custom tags into
        version numbers (ex. 'v0.1' => '0.1') (default None)
    source_url_format : str, optional
        A format string for the url. (ex.
        'https://github.com/pypa/pip/archive/%(version)s.zip') (default None)
    source_url_on_dev : bool, optional
        Attach a source_url even on developer builds (default False)
    pep440 : bool, optional
        If true, create a PEP 440 compatible version number (default False)

    Returns
    -------
    data : dict
        Dictionary of version data. The fields are listed below.

    version : str
        The unique version of this package
    source_label : str
        VERSION-DISTANCE-gHASH
    source_url : str, optional
        A string containing a full URL where the source for this specific
        version of the distribution can be downloaded

    """
    here = os.path.abspath(os.path.dirname(__file__))

    if package is None:
        package = find_package(here)

    version_file_path = os.path.join(here, package, version_mod)

    if not os.path.isdir(os.path.join(here, '.git')):
        return parse_constants(version_file_path)

    describe_args = GIT_DESCRIBE_ARGS
    if tag_match is not None:
        describe_args += ('--match=%s' % tag_match,)
    version_data = git_describe(describe_args)
    if post_process is not None:
        post_process(version_data)
    if version_data['is_dev']:
        if pep440:
            version = (version_data['tag'] +
                       ".post0.dev%(addl_commits)d" % version_data)
        else:
            version = "{tag}-{addl_commits}-g{ref:<.7}".format(**version_data)
            if version_data['is_dirty']:
                version += version_data['dirty_suffix']
    else:
        version = version_data['tag']

    data = {
        'source_label': version_data['description'],
        'version': version,
    }

    if source_url_format is not None and \
            (source_url_on_dev or not version_data['is_dev']):
        data['source_url'] = source_url_format % data
    write_constants(version_file_path, **data)

    if pep440 and not VERSION_SCHEME.match(data['version']):
        raise Exception("Package version '%(version)s' "
                        "is not a valid PEP 440 version string!" % data)

    return data
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Xm
#
# The xm library is hard to find and requires this special code.
#
# Author P G Jones - 11/07/2012 <p.g.jones@qmul.ac.uk> : First revision
# Author P G Jones - 22/09/2012 <p.g.jones@qmul.ac.uk> : Major refactor of snoing.
####################################################################################################
import systempackage
import system
import os

class Xm(systempackage.SystemPackage):
    """ Package for the Open Motif/Xm library."""
    def __init__(self, system):
        super(Xm, self).__init__("Xm", system, "Install Xm-dev (OpenMotif) on this system.")
    def check_state(self):
        """ Check the Xm state, slightly more involved on macs."""
        if self._system.get_os_type() == system.System.Mac:
            if os.path.exists("/sw/include/Xm"):
                flags = [ "-I%s" % "/sw/include", "-L%s" % "/sw/lib" ]
            elif os.path.exists("/usr/OpenMotif"):
                flags = [ "-I%s" % "/usr/OpenMotif/include", "-L%s" % "/usr/OpenMotif/lib" ]
            self._installed = self._system._test_compile(["Xm/Xm.h"], flags)
        else:
            self._installed = self._system.test_library("Xm", ["Xm/Xm.h"])
<commit_msg>Initialising unitialised variable.<commit_after>#!/usr/bin/env python
#
# Xm
#
# The xm library is hard to find and requires this special code.
#
# Author P G Jones - 11/07/2012 <p.g.jones@qmul.ac.uk> : First revision
# Author P G Jones - 22/09/2012 <p.g.jones@qmul.ac.uk> : Major refactor of snoing.
####################################################################################################
import systempackage
import system
import os

class Xm(systempackage.SystemPackage):
    """ Package for the Open Motif/Xm library."""
    def __init__(self, system):
        super(Xm, self).__init__("Xm", system, "Install Xm-dev (OpenMotif) on this system.")
    def check_state(self):
        """ Check the Xm state, slightly more involved on macs."""
        if self._system.get_os_type() == system.System.Mac:
            flags = []
            if os.path.exists("/sw/include/Xm"):
                flags = [ "-I%s" % "/sw/include", "-L%s" % "/sw/lib" ]
            elif os.path.exists("/usr/OpenMotif"):
                flags = [ "-I%s" % "/usr/OpenMotif/include", "-L%s" % "/usr/OpenMotif/lib" ]
            self._installed = self._system._test_compile(["Xm/Xm.h"], flags)
        else:
            self._installed = self._system.test_library("Xm", ["Xm/Xm.h"])
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys

import networkModel
import location.datacentre as datacentre
import location.location as location

class instanceCostNetwork (networkModel.networkModel):
    """ A generic virtual class which models the behaviour of a
    a networkModel e.g costs and locations"""

    def __init__(self):
        """Initialisation"""
        self.betweenCost= 0.0   #Cost per MB between data centres
        self.fullStream= 1.0    #Bandwidth of full_stream in MB
        self.compressedStream=0.2    #Bandwidth of compressed_stream
        self.MBPerInstance=10000 #MB per instance

    def parseJSON(self,js,fName):
        """ parse the JSON in file fName which
        is generic to any stream model"""

        try:
            netFName= js.pop("file")
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain "file" with filename of network description in', \
                    fName
            raise ValueError
        try:
            self.betweenCost= float(js.pop("between_cost_dollar_GB"))

        except:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain "between_cost_dollar_GB" -- cost of transfer', \
                ' between data centres as float in',fName
            raise ValueError
        try:
            self.fullStream= float(js.pop("full_stream_MBs"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain cost per MB/s as float "full_stream_MBs"'
            raise ValueError
        try:
            self.compressedStream= float(js.pop("compressed_stream_MBs"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain compressed stream rate MB/s as float', \
                '"compressed_stream_MBs"'
            raise ValueError
        try:
            self.MBPerInstance= int(js.pop("MB_per_instance"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain nnumber of users per instance as int', \
                '"users_per_instance"'
            raise ValueError
        self.parseInstCostFile(netFName,fName)

    def parseInstCostFile(self,netFName,fName):
        """ Read the network file that contains locations and prices"""
        try:
            f= open(netFName)
        except IOError:
            print >> sys.stderr, "Unable to open network_model file", \
                netFName,"defined in JSON",fName
            sys.exit()
        self.nodes=[]
        while True:
            origline= f.readline()
            if origline == '':
                break
            #Skip blank lines and comments
            if origline[0] == '\n' or origline[0] == '#' or origline[0] == '%':
                continue
            l=origline.split('%')[0].split('#')[0] # strip comments
            parts= l.split(",")
            if len(parts) != 4 and len(parts) != 5:
                print >> sys.stderr,"Lines in instanceCostNetwork file", \
                    netFName,' are expected to be 5 tuple floats or ', \
                    ' 6 tuple with zone string as third arg', \
                    'specified in JSON',fName
                print >> sys.stderr,'Line',origline
                raise ValueError
            try:
                loc= (float(parts[0]),float(parts[1]))
                if len(parts) == 6:
                    zone= parts[2]
                    incost= float(parts[3])
                    outcost= float(parts[4])
                    instcost= float(parts[5])
                else:
                    zone=None
                    incost= float(parts[2])
                    outcost= float(parts[3])
                    instcost= float(parts[4])
            except:
                print >> sys.stderr,"Lines in instanceCostNetwork file", \
                    netFName,' are expect to have loc then in, out and ',\
                    'instcost in ',fName
                print >> sys.stderr,'Line',origline
                raise ValueError
            if loc[0] < -90 or loc[0] > 90 or loc[1] < -180 or \
                loc[1] > 180:
                print >> sys.stderr,"Lat, long must be in ranges", \
                    '[-90,90],[-180,180] respectively in',netFName, \
                    'referenced from JSON',fName
                raise ValueError
            if incost < 0 or outcost < 0 or instcost < 0:
                print >> sys.stderr,'Specified costs must be +ve'\
                     'in',netFName, 'referenced from JSON',fName
                print >> sys.stderr,'line',origline
                print >> sys.stderr,'parts',parts
                raise ValueError
            self.nodes.append(datacentre.datacentre( \
                location.location(loc[0],loc[1],zone), \
                incost,outcost,instcost))
        f.close()

    def locations(self):
        """ return a list of locations where this network has nodes"""
        return self.nodes

    def getCostTuple(self, session, timedelta, routeMod):
        """ Return a 4-list of costs for a session following a given
        route model -- tuple is host->net net->net net->host cpu"""

        costs= session.getCosts()
        if costs == None:
            host2net= 0.0
            net2net=0.0
            net2host=0.0
            # part stream is what is normally sent on this route between this
            #user pair only -- averaged
            # 1/noUsers of time it is full stream
            # (noUsers-1)/noUsers of time it is compressed Stream
            noUsers= session.noUsers()
            if noUsers <= 1:
                costs= [0.0,0.0,0.0,0.0]
                session.setCosts(costs)
                return costs
            partStream= self.fullStream/noUsers+self.compressedStream* \
                (noUsers-1)/noUsers
            # Data is only sent once between a pair of routers
            # and once between a user and a router
            rdict={}  # Dictionary of links between routers
            indict={}  # Dictionary of links from server to router.
            cpuCost= 0.0
            if routeMod.symmetric():
                factor=2
            else:
                factor=1
            for r in session.getRoutes():
				# If we never saw r[0]--r[1] add inbound cost
                if not indict.has_key((r[0],r[1])):
                    indict[(r[0],r[1])]= True
                    host2net+= r[1].inCost*self.fullStream
                    cpuCost+= r[1].instCost*self.fullStream/self.MBPerInstance
                # Add route out bound cost
                net2host+= r[-2].outCost*partStream
                cpuCost+= r[-2].instCost*partStream/self.MBPerInstance
                # If routing is symmetric this route counts for
                # both directions -- so also add return costs
                # in same way
                if routeMod.symmetric():
                    net2host+= r[1].outCost*partStream
                    cpuCost+= r[1].instCost*partStream/self.MBPerInstance
                    if not indict.has_key((r[-1],r[-2])):
                        indict[(r[-1],r[-2])]= True
                        host2net+= r[-2].inCost*self.fullStream
                        cpuCost+= r[-2].instCost*self.fullStream/self.MBPerInstance
                if len(r) > 3:
                    for i in range(1,len(r)-2):
                        if not rdict.has_key((r[0],r[i],r[i+1])):
                            rdict[(r[0],r[i],r[i+1])]= True
                            net2net+=self.betweenCost*partStream
                            cpuCost+= r[i].instCost*partStream/self.MBPerInstance
                            if routeMod.symmetric():
                                rdict[(r[-1],r[i+1],r[i])]= True
                                net2net+=self.betweenCost*partStream
                                cpuCost+= r[i+1].instCost*partStream/self.MBPerInstance
            #1000 factor because data transfer in MB/s costs in GB
            # No CPU costs in this setting
            costs=[host2net/1000,net2net/1000,net2host/1000,cpuCost/1000]
            session.setCosts(costs)
        outcosts= list(costs)
        for i in range(len(outcosts)):
                outcosts[i]*= timedelta
        return outcosts

<commit_msg>Bug fix to length in fields of instance cost network<commit_after>#!/usr/bin/env python

import sys

import networkModel
import location.datacentre as datacentre
import location.location as location

class instanceCostNetwork (networkModel.networkModel):
    """ A generic virtual class which models the behaviour of a
    a networkModel e.g costs and locations"""

    def __init__(self):
        """Initialisation"""
        self.betweenCost= 0.0   #Cost per MB between data centres
        self.fullStream= 1.0    #Bandwidth of full_stream in MB
        self.compressedStream=0.2    #Bandwidth of compressed_stream
        self.MBPerInstance=10000 #MB per instance

    def parseJSON(self,js,fName):
        """ parse the JSON in file fName which
        is generic to any stream model"""

        try:
            netFName= js.pop("file")
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain "file" with filename of network description in', \
                    fName
            raise ValueError
        try:
            self.betweenCost= float(js.pop("between_cost_dollar_GB"))

        except:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain "between_cost_dollar_GB" -- cost of transfer', \
                ' between data centres as float in',fName
            raise ValueError
        try:
            self.fullStream= float(js.pop("full_stream_MBs"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain cost per MB/s as float "full_stream_MBs"'
            raise ValueError
        try:
            self.compressedStream= float(js.pop("compressed_stream_MBs"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain compressed stream rate MB/s as float', \
                '"compressed_stream_MBs"'
            raise ValueError
        try:
            self.MBPerInstance= int(js.pop("MB_per_instance"))
        except ValueError:
            print >> sys.stderr,"JSON for instanceCostNetwork must",\
                'contain nnumber of users per instance as int', \
                '"users_per_instance"'
            raise ValueError
        self.parseInstCostFile(netFName,fName)

    def parseInstCostFile(self,netFName,fName):
        """ Read the network file that contains locations and prices"""
        try:
            f= open(netFName)
        except IOError:
            print >> sys.stderr, "Unable to open network_model file", \
                netFName,"defined in JSON",fName
            sys.exit()
        self.nodes=[]
        while True:
            origline= f.readline()
            if origline == '':
                break
            #Skip blank lines and comments
            if origline[0] == '\n' or origline[0] == '#' or origline[0] == '%':
                continue
            l=origline.split('%')[0].split('#')[0] # strip comments
            parts= l.split(",")
            if len(parts) != 5 and len(parts) != 6:
                print >> sys.stderr,"Lines in instanceCostNetwork file", \
                    netFName,' are expected to be 5 tuple floats or ', \
                    ' 6 tuple with zone string as third arg', \
                    'specified in JSON',fName
                print >> sys.stderr,'Line',origline
                raise ValueError
            try:
                loc= (float(parts[0]),float(parts[1]))
                if len(parts) == 6:
                    zone= parts[2]
                    incost= float(parts[3])
                    outcost= float(parts[4])
                    instcost= float(parts[5])
                else:
                    zone=None
                    incost= float(parts[2])
                    outcost= float(parts[3])
                    instcost= float(parts[4])
            except:
                print >> sys.stderr,"Lines in instanceCostNetwork file", \
                    netFName,' are expect to have loc then in, out and ',\
                    'instcost in ',fName
                print >> sys.stderr,'Line',origline
                raise ValueError
            if loc[0] < -90 or loc[0] > 90 or loc[1] < -180 or \
                loc[1] > 180:
                print >> sys.stderr,"Lat, long must be in ranges", \
                    '[-90,90],[-180,180] respectively in',netFName, \
                    'referenced from JSON',fName
                raise ValueError
            if incost < 0 or outcost < 0 or instcost < 0:
                print >> sys.stderr,'Specified costs must be +ve'\
                     'in',netFName, 'referenced from JSON',fName
                print >> sys.stderr,'line',origline
                print >> sys.stderr,'parts',parts
                raise ValueError
            self.nodes.append(datacentre.datacentre( \
                location.location(loc[0],loc[1],zone), \
                incost,outcost,instcost))
        f.close()

    def locations(self):
        """ return a list of locations where this network has nodes"""
        return self.nodes

    def getCostTuple(self, session, timedelta, routeMod):
        """ Return a 4-list of costs for a session following a given
        route model -- tuple is host->net net->net net->host cpu"""

        costs= session.getCosts()
        if costs == None:
            host2net= 0.0
            net2net=0.0
            net2host=0.0
            # part stream is what is normally sent on this route between this
            #user pair only -- averaged
            # 1/noUsers of time it is full stream
            # (noUsers-1)/noUsers of time it is compressed Stream
            noUsers= session.noUsers()
            if noUsers <= 1:
                costs= [0.0,0.0,0.0,0.0]
                session.setCosts(costs)
                return costs
            partStream= self.fullStream/noUsers+self.compressedStream* \
                (noUsers-1)/noUsers
            # Data is only sent once between a pair of routers
            # and once between a user and a router
            rdict={}  # Dictionary of links between routers
            indict={}  # Dictionary of links from server to router.
            cpuCost= 0.0
            if routeMod.symmetric():
                factor=2
            else:
                factor=1
            for r in session.getRoutes():
				# If we never saw r[0]--r[1] add inbound cost
                if not indict.has_key((r[0],r[1])):
                    indict[(r[0],r[1])]= True
                    host2net+= r[1].inCost*self.fullStream
                    cpuCost+= r[1].instCost*self.fullStream/self.MBPerInstance
                # Add route out bound cost
                net2host+= r[-2].outCost*partStream
                cpuCost+= r[-2].instCost*partStream/self.MBPerInstance
                # If routing is symmetric this route counts for
                # both directions -- so also add return costs
                # in same way
                if routeMod.symmetric():
                    net2host+= r[1].outCost*partStream
                    cpuCost+= r[1].instCost*partStream/self.MBPerInstance
                    if not indict.has_key((r[-1],r[-2])):
                        indict[(r[-1],r[-2])]= True
                        host2net+= r[-2].inCost*self.fullStream
                        cpuCost+= r[-2].instCost*self.fullStream/self.MBPerInstance
                if len(r) > 3:
                    for i in range(1,len(r)-2):
                        if not rdict.has_key((r[0],r[i],r[i+1])):
                            rdict[(r[0],r[i],r[i+1])]= True
                            net2net+=self.betweenCost*partStream
                            cpuCost+= r[i].instCost*partStream/self.MBPerInstance
                            if routeMod.symmetric():
                                rdict[(r[-1],r[i+1],r[i])]= True
                                net2net+=self.betweenCost*partStream
                                cpuCost+= r[i+1].instCost*partStream/self.MBPerInstance
            #1000 factor because data transfer in MB/s costs in GB
            # No CPU costs in this setting
            costs=[host2net/1000,net2net/1000,net2host/1000,cpuCost/1000]
            session.setCosts(costs)
        outcosts= list(costs)
        for i in range(len(outcosts)):
                outcosts[i]*= timedelta
        return outcosts

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

from swid_generator.generators.utils import create_temp_folder
from swid_generator.command_manager import CommandManager
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class PacmanEnvironment(CommonEnvironment):
    """
    Environment class for distributions using pacman as package manager (used
    by Arch Linux).

    """
    executable = 'pacman'

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """

        command_args_packages = [cls.executable, '-Q', '--color', 'never']
        packages_output = CommandManager.run_command_check_output(command_args_packages)
        lines = filter(None, packages_output.rstrip().split('\n'))
        result = []
        for line in lines:
            split_line = line.split()
            assert len(split_line) == 2, repr(split_line)
            info = PackageInfo()
            info.package = split_line[0]
            info.version = split_line[1]
            result.append(info)
        return result

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Caching could be implemented by using `Ql` without any package name.

        Args:
            package_name (str):
                The package name as string (e.g. ``cowsay``).

        Returns:
            List of ``FileInfo`` instances.

        """
        command_args_files = [cls.executable, '-Ql', package_info.package]
        files_output = CommandManager.run_command_check_output(command_args_files)
        lines = filter(None, files_output.rstrip().split('\n'))
        result = []
        for line in lines:
            split_line = line.split(' ', 1)
            assert len(split_line) == 2, repr(split_line)
            file_path = split_line[1]
            if cls._is_file(file_path):
                file_info = FileInfo(file_path)
                # With the assumption that files in the '/etc'-Folders are mostly Configuration-Files
                if "/etc" in file_path:
                    file_info.mutable = True
                result.append(file_info)
        return result

    @classmethod
    def get_files_from_packagefile(cls, file_fullpathname):

        command_args_files = [cls.executable, '-Qlp', file_fullpathname]
        files_output = CommandManager.run_command_check_output(command_args_files)

        lines = files_output.split('\n')
        lines = filter(lambda l: len(l) > 0, lines)

        save_options = create_temp_folder(file_fullpathname)

        all_files = []

        command_args_extract_package = ['tar', '-xf',
                                        save_options['absolute_package_path']]

        CommandManager.run_command(command_args_extract_package,
                                   working_directory=save_options['save_location'])

        for line in lines:
            path = line.split(' ')[1]
            temporary_path = save_options['save_location'] + path
            if cls._is_file(temporary_path):
                file_info = FileInfo(path, actual_path=False)
                file_info.set_actual_path(temporary_path)
                if 'etc' in path:
                    file_info.mutable = True
                all_files.append(file_info)

        return all_files

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        command_args_packageinfo = [cls.executable, '--query', '--file', file_path]
        package_info_output = CommandManager.run_command_check_output(command_args_packageinfo)
        line_split = package_info_output.split(' ')

        package_info = PackageInfo()
        package_info.package = line_split[0]
        package_info.version = line_split[1].rstrip()

        return package_info
<commit_msg>Required Packages for PacmanEnvironment defined<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

from swid_generator.generators.utils import create_temp_folder
from swid_generator.command_manager import CommandManager
from .common import CommonEnvironment
from ..package_info import PackageInfo, FileInfo

class PacmanEnvironment(CommonEnvironment):
    """
    Environment class for distributions using pacman as package manager (used
    by Arch Linux).

    """
    executable = 'pacman'

    required_packages_package_file_method = [
        "tar"
    ]

    required_packages_sign_method = [
        "xmlsec1"
    ]

    @classmethod
    def get_package_list(cls):
        """
        Get list of installed packages.

        Returns:
            List of ``PackageInfo`` instances.

        """

        command_args_packages = [cls.executable, '-Q', '--color', 'never']
        packages_output = CommandManager.run_command_check_output(command_args_packages)
        lines = filter(None, packages_output.rstrip().split('\n'))
        result = []
        for line in lines:
            split_line = line.split()
            assert len(split_line) == 2, repr(split_line)
            info = PackageInfo()
            info.package = split_line[0]
            info.version = split_line[1]
            result.append(info)
        return result

    @classmethod
    def get_files_for_package(cls, package_info):
        """
        Get list of files related to the specified package.

        Caching could be implemented by using `Ql` without any package name.

        Args:
            package_name (str):
                The package name as string (e.g. ``cowsay``).

        Returns:
            List of ``FileInfo`` instances.

        """
        command_args_files = [cls.executable, '-Ql', package_info.package]
        files_output = CommandManager.run_command_check_output(command_args_files)
        lines = filter(None, files_output.rstrip().split('\n'))
        result = []
        for line in lines:
            split_line = line.split(' ', 1)
            assert len(split_line) == 2, repr(split_line)
            file_path = split_line[1]
            if cls._is_file(file_path):
                file_info = FileInfo(file_path)
                # With the assumption that files in the '/etc'-Folders are mostly Configuration-Files
                if "/etc" in file_path:
                    file_info.mutable = True
                result.append(file_info)
        return result

    @classmethod
    def get_files_from_packagefile(cls, file_fullpathname):

        command_args_files = [cls.executable, '-Qlp', file_fullpathname]
        files_output = CommandManager.run_command_check_output(command_args_files)

        lines = files_output.split('\n')
        lines = filter(lambda l: len(l) > 0, lines)

        save_options = create_temp_folder(file_fullpathname)

        all_files = []

        command_args_extract_package = ['tar', '-xf',
                                        save_options['absolute_package_path']]

        CommandManager.run_command(command_args_extract_package,
                                   working_directory=save_options['save_location'])

        for line in lines:
            path = line.split(' ')[1]
            temporary_path = save_options['save_location'] + path
            if cls._is_file(temporary_path):
                file_info = FileInfo(path, actual_path=False)
                file_info.set_actual_path(temporary_path)
                if 'etc' in path:
                    file_info.mutable = True
                all_files.append(file_info)

        return all_files

    @classmethod
    def get_packageinfo_from_packagefile(cls, file_path):
        command_args_packageinfo = [cls.executable, '--query', '--file', file_path]
        package_info_output = CommandManager.run_command_check_output(command_args_packageinfo)
        line_split = package_info_output.split(' ')

        package_info = PackageInfo()
        package_info.package = line_split[0]
        package_info.version = line_split[1].rstrip()

        return package_info
<|endoftext|>