<commit_before>#! /usr/bin/python3
# vim: set expandtab tabstop=4 shiftwidth=4 :
"""Module with functions wrapping urllib"""

import http.client
import urllib.request
import urllib.parse
import json
import shutil
import gzip
from bs4 import BeautifulSoup
import inspect
import logging
import time

def log(string):
    """Dirty logging function."""
    # TODO: https://docs.python.org/2/library/logging.html#logrecord-attributes
    # we do not need to retrieve the function name manually
    logging.debug(inspect.stack()[1][3] + " " + string)

def convert_iri_to_plain_ascii_uri(uri):
    """Convert IRI to plain ASCII URL
    Based on http://stackoverflow.com/questions/4389572/how-to-fetch-a-non-ascii-url-with-python-urlopen."""
    lis = list(urllib.parse.urlsplit(uri))
    lis[2] = urllib.parse.quote(lis[2])
    url = urllib.parse.urlunsplit(lis)
    if False and url != uri:
        print(uri, '->', url)
    return url

def urlopen_wrapper(url):
    """Wrapper around urllib.request.urlopen (user-agent, etc).

    url is a string
    Returns a byte object."""
    log('(url : %s)' % url)
    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/11.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30'
    try:
        response = urllib.request.urlopen(
            urllib.request.Request(
                url,
                headers={'User-Agent': user_agent, 'Accept': '*/*'}))
        if response.info().get('Content-Encoding') == 'gzip':
            return gzip.GzipFile(fileobj=response)
        return response
    except (urllib.error.HTTPError, urllib.error.URLError) as e:
        print("Exception %s for url %s" % (e, url))
        raise

def urljoin_wrapper(base, url):
    """Wrapper around urllib.parse.urljoin.
    Construct a full ("absolute") URL by combining a "base URL" (base) with
    another URL (url)."""
    return urllib.parse.urljoin(base, url)

def get_content(url):
    """Get content at url.

    url is a string
    Returns a string"""
    log('(url : %s)' % url)
    try:
        return urlopen_wrapper(url).read()
    except http.client.IncompleteRead as e:
        print("Incomplete read of %s (%d more bytes expected)" % (url, e.expected))
        return e.partial

def extensions_are_equivalent(ext1, ext2):
    """Return whether file extensions can be considered as equivalent."""
    synonyms = [{'jpg', 'jpeg'}]
    ext1, ext2 = ext1.lower(), ext2.lower()
    return ext1 == ext2 or any((ext1 in s and ext2 in s) for s in synonyms)

def add_extension_to_filename_if_needed(ext, filename):
    """Given an extension and a filename, add the extension to the filename
    if the filename does not already have this extension (or an extension
    considered to be equivalent."""
    filename_ext = filename.split('.')[-1]
    if extensions_are_equivalent(ext, filename_ext):
        return filename
    else:
        return filename + '.' + ext

def get_file_at_url(url, path):
    """Save content at url in path on file system.
    In theory, this could have been achieved with urlretrieve but it seems
    to be about to get deprecated and adding a user-agent seems to be quite
    awkward.

    url is a string
    path is a string corresponding to the file location
    Returns the path if the file is retrieved properly, None otherwise."""
    log('(url : %s, path : %s)' % (url, path))
    try:
        with urlopen_wrapper(url) as response:
            content_type = response.info().get('Content-Type', '').split('/')
            assert 1 <= len(content_type) <= 2
            if len(content_type) == 2:
                data = content_type[1].split(';')
                path = add_extension_to_filename_if_needed(data[0], path)
            with open(path, 'wb') as out_file:
                shutil.copyfileobj(response, out_file)
                return path
    except (urllib.error.HTTPError, urllib.error.URLError):
        return None

def get_filename_from_url(url):
    """Get filename from url

    url is a string
    Returns a string corresponding to the name of the file."""
    return urllib.parse.unquote(url).split('/')[-1]

def load_json_at_url(url):
    """Get content at url as JSON and return it."""
    return json.loads(get_content(url).decode())

def get_soup_at_url(url, detect_meta=False, detect_rel=False, save_in_file=False):
    """Get content at url as BeautifulSoup.

    url is a string
    detect_meta is a hacky flag used to detect comics using similar plugin to
        be able to reuse code at some point
    detect_rel is a hacky flag to detect next/first comics automatically
    save_in_file is a hacky flag to save content in temp file for debugging
        purposes
    Returns a BeautifulSoup object."""
    content = get_content(url)
    soup = BeautifulSoup(content, "html.parser")
    if detect_meta:
        for meta_val in ['generator', 'ComicPress', 'Comic-Easel']:
            meta = soup.find('meta', attrs={'name': meta_val})
            if meta is not None:
                print(meta)
    if detect_rel:
        for tag in ['a', 'link']:
            next_ = soup.find(tag, rel='next')
            if next_ is not None:
                print(next_)
    if save_in_file:
        time_ms = time.time() * 1000
        prefix = 'get_soup_at_url_' + str(time_ms) + '_'
        with open(prefix + 'raw', 'wb') as f:
            f.write(content)
        with open(prefix + 'content', 'wb') as f:
            f.write(soup.encode('utf-8'))
    return soup
<commit_msg>Fix error when e.expected is not an int (?)<commit_after>#! /usr/bin/python3
# vim: set expandtab tabstop=4 shiftwidth=4 :
"""Module with functions wrapping urllib"""

import http.client
import urllib.request
import urllib.parse
import json
import shutil
import gzip
from bs4 import BeautifulSoup
import inspect
import logging
import time

def log(string):
    """Dirty logging function."""
    # TODO: https://docs.python.org/2/library/logging.html#logrecord-attributes
    # we do not need to retrieve the function name manually
    logging.debug(inspect.stack()[1][3] + " " + string)

def convert_iri_to_plain_ascii_uri(uri):
    """Convert IRI to plain ASCII URL
    Based on http://stackoverflow.com/questions/4389572/how-to-fetch-a-non-ascii-url-with-python-urlopen."""
    lis = list(urllib.parse.urlsplit(uri))
    lis[2] = urllib.parse.quote(lis[2])
    url = urllib.parse.urlunsplit(lis)
    if False and url != uri:
        print(uri, '->', url)
    return url

def urlopen_wrapper(url):
    """Wrapper around urllib.request.urlopen (user-agent, etc).

    url is a string
    Returns a byte object."""
    log('(url : %s)' % url)
    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/11.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30'
    try:
        response = urllib.request.urlopen(
            urllib.request.Request(
                url,
                headers={'User-Agent': user_agent, 'Accept': '*/*'}))
        if response.info().get('Content-Encoding') == 'gzip':
            return gzip.GzipFile(fileobj=response)
        return response
    except (urllib.error.HTTPError, urllib.error.URLError) as e:
        print("Exception %s for url %s" % (e, url))
        raise

def urljoin_wrapper(base, url):
    """Wrapper around urllib.parse.urljoin.
    Construct a full ("absolute") URL by combining a "base URL" (base) with
    another URL (url)."""
    return urllib.parse.urljoin(base, url)

def get_content(url):
    """Get content at url.

    url is a string
    Returns a string"""
    log('(url : %s)' % url)
    try:
        return urlopen_wrapper(url).read()
    except http.client.IncompleteRead as e:
        print("%s for %s" % (e, url))
        return e.partial

def extensions_are_equivalent(ext1, ext2):
    """Return whether file extensions can be considered as equivalent."""
    synonyms = [{'jpg', 'jpeg'}]
    ext1, ext2 = ext1.lower(), ext2.lower()
    return ext1 == ext2 or any((ext1 in s and ext2 in s) for s in synonyms)

def add_extension_to_filename_if_needed(ext, filename):
    """Given an extension and a filename, add the extension to the filename
    if the filename does not already have this extension (or an extension
    considered to be equivalent."""
    filename_ext = filename.split('.')[-1]
    if extensions_are_equivalent(ext, filename_ext):
        return filename
    else:
        return filename + '.' + ext

def get_file_at_url(url, path):
    """Save content at url in path on file system.
    In theory, this could have been achieved with urlretrieve but it seems
    to be about to get deprecated and adding a user-agent seems to be quite
    awkward.

    url is a string
    path is a string corresponding to the file location
    Returns the path if the file is retrieved properly, None otherwise."""
    log('(url : %s, path : %s)' % (url, path))
    try:
        with urlopen_wrapper(url) as response:
            content_type = response.info().get('Content-Type', '').split('/')
            assert 1 <= len(content_type) <= 2
            if len(content_type) == 2:
                data = content_type[1].split(';')
                path = add_extension_to_filename_if_needed(data[0], path)
            with open(path, 'wb') as out_file:
                shutil.copyfileobj(response, out_file)
                return path
    except (urllib.error.HTTPError, urllib.error.URLError):
        return None

def get_filename_from_url(url):
    """Get filename from url

    url is a string
    Returns a string corresponding to the name of the file."""
    return urllib.parse.unquote(url).split('/')[-1]

def load_json_at_url(url):
    """Get content at url as JSON and return it."""
    return json.loads(get_content(url).decode())

def get_soup_at_url(url, detect_meta=False, detect_rel=False, save_in_file=False):
    """Get content at url as BeautifulSoup.

    url is a string
    detect_meta is a hacky flag used to detect comics using similar plugin to
        be able to reuse code at some point
    detect_rel is a hacky flag to detect next/first comics automatically
    save_in_file is a hacky flag to save content in temp file for debugging
        purposes
    Returns a BeautifulSoup object."""
    content = get_content(url)
    soup = BeautifulSoup(content, "html.parser")
    if detect_meta:
        for meta_val in ['generator', 'ComicPress', 'Comic-Easel']:
            meta = soup.find('meta', attrs={'name': meta_val})
            if meta is not None:
                print(meta)
    if detect_rel:
        for tag in ['a', 'link']:
            next_ = soup.find(tag, rel='next')
            if next_ is not None:
                print(next_)
    if save_in_file:
        time_ms = time.time() * 1000
        prefix = 'get_soup_at_url_' + str(time_ms) + '_'
        with open(prefix + 'raw', 'wb') as f:
            f.write(content)
        with open(prefix + 'content', 'wb') as f:
            f.write(soup.encode('utf-8'))
    return soup
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from datetime import datetime
from django.db import models
from util.models import BaseModel
from physical.models import Instance, Environment, Volume

class BackupGroup(BaseModel):

    def __unicode__(self):
        return "Backup from {}".format(self.created_at)

class BackupInfo(BaseModel):

    SNAPSHOPT = 1
    TYPE_CHOICES = (
        (SNAPSHOPT, 'Snapshot'),
    )

    RUNNING = 1
    SUCCESS = 2
    ERROR = 3
    WARNING = 4
    STATUS_CHOICES = (
        (RUNNING, 'Running'),
        (SUCCESS, 'Success'),
        (ERROR, 'Error'),
        (WARNING, 'Warning'),
    )

    class Meta:
        abstract = True

    start_at = models.DateTimeField(null=False, blank=False)
    end_at = models.DateTimeField(null=True, blank=True)
    purge_at = models.DateTimeField(null=True, blank=True)
    type = models.IntegerField(choices=TYPE_CHOICES, null=False, blank=False)
    status = models.IntegerField(
        choices=STATUS_CHOICES, null=False, blank=False
    )
    instance = models.ForeignKey(
        Instance, related_name="backup_instance",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )
    database_name = models.CharField(
        max_length=100, null=True, blank=True, db_index=True
    )
    size = models.BigIntegerField(null=True, blank=True)
    environment = models.ForeignKey(
        Environment, related_name="backup_environment",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )
    error = models.CharField(max_length=400, null=True, blank=True)
    is_automatic = models.BooleanField(
        default=True, help_text='Backup required by DBaaS routine'
    )
    group = models.ForeignKey(
        BackupGroup, related_name='backups', null=True, blank=True
    )
    volume = models.ForeignKey(
        Volume, related_name="backups",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )

    def __unicode__(self):
        return "{} from {} started at {}".format(
            self.type, self.database_name, self.start_at
        )

    @property
    def was_successful(self):
        return self.status == Snapshot.SUCCESS

    @property
    def has_warning(self):
        return self.status == Snapshot.WARNING

    @property
    def was_error(self):
        return self.status == Snapshot.ERROR

    def set_error(self, msg):
        self.status = Snapshot.ERROR
        self.error = msg
        self.size = 0
        self.end_at = datetime.now()
        self.purge_at = datetime.now()
        self.save()

    @classmethod
    def create(cls, instance, group, volume, environment=None):
        if environment is None:
            environment = instance.databaseinfra.databases.first().environment

        snapshot = cls()
        snapshot.start_at = datetime.now()
        snapshot.type = Snapshot.SNAPSHOPT
        snapshot.status = Snapshot.RUNNING
        snapshot.instance = instance
        snapshot.environment = environment
        snapshot.group = group
        snapshot.database_name = instance.databaseinfra.databases.first().name
        snapshot.volume = volume
        snapshot.save()
        return snapshot

class Snapshot(BackupInfo):
    snapshopt_id = models.CharField(
        verbose_name="Snapshot ID", max_length=100, null=True, blank=True
    )
    snapshot_name = models.CharField(
        verbose_name="Snapshot Name", max_length=200, null=True, blank=True
    )
    volume_path = models.CharField(
        verbose_name="Volume Path", max_length=255, null=True, blank=True
    )

    def __unicode__(self):
        return "Snapshot from {} started at {}".format(
            self.database_name, self.start_at
        )

    def done(self, response):
        self.snapshopt_id = response['identifier']
        self.snapshot_name = response['description']
        self.volume_path = response['volume_path']
        self.size = response['size']
<commit_msg>Fix default environment on snapshot create<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
from django.db import models
from util.models import BaseModel
from physical.models import Instance, Environment, Volume

class BackupGroup(BaseModel):

    def __unicode__(self):
        return "Backup from {}".format(self.created_at)

class BackupInfo(BaseModel):

    SNAPSHOPT = 1
    TYPE_CHOICES = (
        (SNAPSHOPT, 'Snapshot'),
    )

    RUNNING = 1
    SUCCESS = 2
    ERROR = 3
    WARNING = 4
    STATUS_CHOICES = (
        (RUNNING, 'Running'),
        (SUCCESS, 'Success'),
        (ERROR, 'Error'),
        (WARNING, 'Warning'),
    )

    class Meta:
        abstract = True

    start_at = models.DateTimeField(null=False, blank=False)
    end_at = models.DateTimeField(null=True, blank=True)
    purge_at = models.DateTimeField(null=True, blank=True)
    type = models.IntegerField(choices=TYPE_CHOICES, null=False, blank=False)
    status = models.IntegerField(
        choices=STATUS_CHOICES, null=False, blank=False
    )
    instance = models.ForeignKey(
        Instance, related_name="backup_instance",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )
    database_name = models.CharField(
        max_length=100, null=True, blank=True, db_index=True
    )
    size = models.BigIntegerField(null=True, blank=True)
    environment = models.ForeignKey(
        Environment, related_name="backup_environment",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )
    error = models.CharField(max_length=400, null=True, blank=True)
    is_automatic = models.BooleanField(
        default=True, help_text='Backup required by DBaaS routine'
    )
    group = models.ForeignKey(
        BackupGroup, related_name='backups', null=True, blank=True
    )
    volume = models.ForeignKey(
        Volume, related_name="backups",
        unique=False, null=True, blank=True, on_delete=models.SET_NULL
    )

    def __unicode__(self):
        return "{} from {} started at {}".format(
            self.type, self.database_name, self.start_at
        )

    @property
    def was_successful(self):
        return self.status == Snapshot.SUCCESS

    @property
    def has_warning(self):
        return self.status == Snapshot.WARNING

    @property
    def was_error(self):
        return self.status == Snapshot.ERROR

    def set_error(self, msg):
        self.status = Snapshot.ERROR
        self.error = msg
        self.size = 0
        self.end_at = datetime.now()
        self.purge_at = datetime.now()
        self.save()

    @classmethod
    def create(cls, instance, group, volume, environment=None):
        if environment is None:
            environment = instance.databaseinfra.environment

        snapshot = cls()
        snapshot.start_at = datetime.now()
        snapshot.type = Snapshot.SNAPSHOPT
        snapshot.status = Snapshot.RUNNING
        snapshot.instance = instance
        snapshot.environment = environment
        snapshot.group = group
        snapshot.database_name = instance.databaseinfra.databases.first().name
        snapshot.volume = volume
        snapshot.save()
        return snapshot

class Snapshot(BackupInfo):
    snapshopt_id = models.CharField(
        verbose_name="Snapshot ID", max_length=100, null=True, blank=True
    )
    snapshot_name = models.CharField(
        verbose_name="Snapshot Name", max_length=200, null=True, blank=True
    )
    volume_path = models.CharField(
        verbose_name="Volume Path", max_length=255, null=True, blank=True
    )

    def __unicode__(self):
        return "Snapshot from {} started at {}".format(
            self.database_name, self.start_at
        )

    def done(self, response):
        self.snapshopt_id = response['identifier']
        self.snapshot_name = response['description']
        self.volume_path = response['volume_path']
        self.size = response['size']
<|endoftext|>


<commit_before>"""
Dataset loader class.
"""

class Loader:
    """ HDF5 data loading class """

    def __init__(self, name):
        """
        Initialize class.
        """

    def get(self, field_name, id):
        """
        Retrieve the 'i'th' data from the field 'field_name' into a list.

        Parameters:
        -----------
            - field_name: field name identifier [Type=String]
		    - id: index number of the field. If it is a list, returns the data for all the value indexes of that list [Type=long/List]
        Returns:
        --------
            returns a list filled with data.
        """
        pass

    def object_id(self, id, is_value=False):
        """
        Retrieve the data of all fields of an object: It works as calling :get() for each field individually and grouping them into a list.

        Parameters:
        -----------
            - id: index number of the object list. If it is a list, returns the data for all the value indexes of that list [Type=long/List]
            - is_value: if False, outputs a list of indexes. If True, it outputs the values instead of the indexes. [Type=Boolean, default=False]

        Returns:
        --------
            Returns a list of indexes (or values if is_value=True)
        """
        pass

    def size(self, field_name=None):
        """
        Returns the size of the elements of a field_name.

        Parameters:
        -----------
            - field_name: field name identifier [Type=String]

        Returns:
        --------
            returns the number of elements of a field
            (if empty it returns the size of the object list).
        """
        pass

    def list(self):
        """
        Lists all field names (order w.r.t. the object list position).
        """
        pass

class DataLoader:
    """ Dataset loader """

    def __init__(self, name):
        """
        Initialize class.
        """
        self.data = Loader(name)
        self.train = self.data['train']
        self.val = self.data['val']
        self.test = self.data['test']

<commit_msg>initial work for the loader class<commit_after>"""
Dataset loader class.
"""

import storage

class ManagerHDF5:
    """ HDF5 data loading class """

    def __init__(self, handler_file, group_name):
        """Initialize class.

        Parameters
        ----------
        cache_path : str
            Path to the metadata cache file on disk.

        Returns
        -------
            None
        """
        self.data = handler_file.storage[group_name]

    def get(self, field_name, id):
        """
        Retrieve the 'i'th' data from the field 'field_name' into a list.

        Parameters
        ----------
            - field_name: field name identifier [Type=String]
		    - id: index number of the field. If it is a list, returns the data for all the value indexes of that list [Type=long/List]
        Returns:
        --------
            returns a list filled with data.
        """
        pass
        # >>> fiquei aqui <<<<

    def object_id(self, id, is_value=False):
        """
        Retrieve the data of all fields of an object: It works as calling :get() for each field individually and grouping them into a list.

        Parameters:
        -----------
            - id: index number of the object list. If it is a list, returns the data for all the value indexes of that list [Type=long/List]
            - is_value: if False, outputs a list of indexes. If True, it outputs the values instead of the indexes. [Type=Boolean, default=False]

        Returns:
        --------
            Returns a list of indexes (or values if is_value=True)
        """
        pass

    def size(self, field_name=None):
        """
        Returns the size of the elements of a field_name.

        Parameters:
        -----------
            - field_name: field name identifier [Type=String]

        Returns:
        --------
            returns the number of elements of a field
            (if empty it returns the size of the object list).
        """
        pass

    def list(self):
        """
        Lists all field names (order w.r.t. the object list position).
        """
        pass

class Loader:
    """ Dataset loader """

    def __init__(self, name, category, task, data_path, cache_path):
        """Initialize class.

        Parameters
        ----------
        name : str
            Name of the dataset.
        category : str
            Category of the dataset (e.g. image processing, natural language processing)
        task : str
            Name of the task.
        data_path : str
            Path to the dataset's data on disk.
        cache_path : str
            Path to the metadata cache file on disk.

        Returns
        -------
            None
        """
        # store info
        self.name = name
        self.category = category
        self.task = task
        self.data_path = data_path
        self.cache_path = cache_path

        # load the cache file
        self.file = storage.StorageHDF5()

        # make links for all groups (train/val/test) for easier access
        self.add_group_links()

    def add_group_links(self):
        """
        Adds links for the groups for easier access to data.
        """
        for name in self.file.storage.keys():
            setattr(self, name, ManagerHDF5(self.file, name))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""recipe cooking"""

import json
import yaml
from orderedattrdict import AttrDict
from orderedattrdict.yamlutils import AttrDictYAMLLoader

from . ingredient import *
from .. import config
from . procedure import *
from .. str import format_float_digits

import logging

# supported procedures, import from procedure.py
supported_procs = {
    'translate_column': translate_column,
    'translate_header': translate_header,
    'identity': identity,
    'merge': merge,
    'run_op': run_op,
    'filter_row': filter_row,
    'align': align,
    'filter_item': filter_item,
    'groupby': groupby,
    'accumulate': accumulate,
    'copy': copy,
    'add_concepts': add_concepts
}

def _loadfile(f):
    """load json/yaml file, into AttrDict"""
    if re.match('.*\.json', f):
        res = json.load(open(f), object_pairs_hook=AttrDict)
    else:
        res = yaml.load(open(f), Loader=AttrDictYAMLLoader)

    return res

# functions for reading/running recipe
def build_recipe(recipe_file, to_disk=False):
    """build a complete recipe file if there are includes in
    recipe file, if no includes found than return the file as is.
    """
    recipe = _loadfile(recipe_file)

    # the base dir of recipe file. for building paths for dictionary_dir and
    # sub recipe paths.
    base_dir = os.path.dirname(recipe_file)

    # the dictionary dir to retrieve translation dictionaries
    try:
        dict_dir = recipe['config']['dictionary_dir']
        config.DICT_PATH = dict_dir
    except KeyError:
        dict_dir = config.DICT_PATH

    # expand all files in the options
    if 'cooking' in recipe.keys():
        for p in ['concepts', 'datapoints', 'entities']:
            if p not in recipe['cooking'].keys():
                continue
            for i, procedure in enumerate(recipe['cooking'][p]):
                try:
                    opt_dict = procedure['options']['dictionary']
                except KeyError:
                    continue
                if isinstance(opt_dict, str):
                    # if the option dict is str, then it should be a filename
                    if dict_dir is None:
                        raise KeyError("dictionary_dir not found in config!")
                    if os.path.isabs(dict_dir):
                        path = os.path.join(dict_dir, opt_dict)
                    else:
                        path = os.path.join(base_dir, dict_dir, opt_dict)

                    recipe['cooking'][p][i]['options']['dictionary'] = _loadfile(path)

    if 'include' not in recipe.keys():
        return recipe
    else:  # append sub-recipe entities into main recipe
        recipe_dir = recipe['config']['recipes_dir']

        sub_recipes = []
        for i in recipe['include']:
            # TODO: maybe add support to expand user home and env vars
            if os.path.isabs(recipe_dir):
                path = os.path.join(recipe_dir, i)
            else:
                path = os.path.join(base_dir, recipe_dir, i)
            sub_recipes.append(build_recipe(path))

        for rcp in sub_recipes:
            # appending ingredients
            if 'ingredients' in recipe.keys():
                # ingredients = [*recipe['ingredients'], *rcp['ingredients']]  # not supportted by Python < 3.5
                ingredients = []
                [ingredients.append(ing) for ing in recipe['ingredients']]
                [ingredients.append(ing) for ing in rcp['ingredients']]
                # drop duplicated ingredients.
                rcp_dict_tmp = {}
                for v in ingredients:
                    if v['id'] not in rcp_dict_tmp.keys():
                        rcp_dict_tmp[v['id']] = v
                    else:
                        # raise error when ingredients with same ID have different contents.
                        if v != rcp_dict_tmp[v['id']]:
                            raise ValueError("Different content with same ingredient id detected: " + v['id'])
                recipe['ingredients'] = list(rcp_dict_tmp.values())
            else:
                recipe['ingredients'] = rcp['ingredients']

            # appending cooking procedures
            if 'cooking' not in rcp.keys():
                continue
            for p in ['datapoints', 'entities', 'concepts']:
                if p not in rcp['cooking'].keys():
                    continue
                if 'cooking' in recipe.keys():
                    if p in recipe['cooking'].keys():
                        # NOTE: the included cooking procedures should be placed in front of
                        # the origin ones.
                        # recipe['cooking'][p] = [*rcp['cooking'][p], *recipe['cooking'][p]]  # not supportted by Python < 3.5
                        new_procs = []
                        [new_procs.append(proc) for proc in rcp['cooking'][p]]
                        [new_procs.append(proc) for proc in recipe['cooking'][p]]
                        recipe['cooking'][p] = new_procs
                    else:
                        recipe['cooking'][p] = rcp['cooking'][p]
                else:
                    recipe['cooking'] = {}
                    recipe['cooking'][p] = rcp['cooking'][p]

        if to_disk:
            yaml.dump(recipe, open('recipe_dump.yaml', 'w'))

        return recipe

def update_recipe_last_update(recipe, outdir):
    pass

def check_dataset_availability(recipe):
    """check availability of all datasets required by the recipe.

    raise error if some dataset not available. otherwise do nothing.
    """
    ddf_dir = recipe.config.ddf_dir

    datasets = set()
    for ingred in recipe.ingredients:
        datasets.add(ingred.dataset)

    not_exists = []
    for d in datasets:
        if not os.path.exists(os.path.join(ddf_dir, d)):
            not_exists.append(d)

    if len(not_exists) > 0:
        logging.critical("not enough datasets! please checkout following datasets:\n{}\n"\
                            .format('\n'.join(not_exists)))
        raise ValueError('not enough datasets')
    return

def run_recipe(recipe):
    """run the recipe.

    returns a dictionary. keys are `concepts`, `entities` and `datapoints`,
    and values are ingredients return by the procedures
    """
    try:
        config.DDF_SEARCH_PATH = recipe['config']['ddf_dir']
    except KeyError:
        if not config.DDF_SEARCH_PATH:
            raise ValueError("no ddf_dir configured, please check your recipe")
    logging.info('path for searching DDF: ' + str(config.DDF_SEARCH_PATH))

    check_dataset_availability(recipe)

    # load ingredients
    ings = [Ingredient.from_dict(i) for i in recipe['ingredients']]
    ings_dict = dict([[i.ingred_id, i] for i in ings])

    # cooking
    funcs = supported_procs
    dishes = {}

    for k, pceds in recipe['cooking'].items():

        print("running "+k)
        dishes[k] = list()

        for p in pceds:
            func = p['procedure']

            if func not in funcs.keys() and func != 'serve':
                raise NotImplementedError("Not supported: " + func)

            ingredient = [ings_dict[i] for i in p['ingredients']]

            if func == 'serve':
                for i in ingredient:
                    dishes[k].append(i)
                continue

            # change the 'base'/'source_ingredients' option to actual ingredient
            # TODO: find better way to handle the ingredients in options
            if 'options' in p.keys():
                options = p['options']
                if 'base' in options.keys():
                    options['base'] = ings_dict[options['base']]
                if 'source_ingredients' in options.keys():
                    options['source_ingredients'] = [ings_dict[x] for x in options['source_ingredients']]

            if 'result' in p.keys():
                result = p['result']
                if 'options' in p.keys():
                    out = funcs[func](*ingredient, result=result, **options)
                else:
                    out = funcs[func](*ingredient, result=result)
            else:
                if 'options' in p.keys():
                    out = funcs[func](*ingredient, **options)
                else:
                    out = funcs[func](*ingredient)
                result = out.ingred_id

            if result in ings_dict.keys():
                logging.warning("overwriting existing ingredient: " + result)
            ings_dict[result] = out

        # if there is no seving procedures, use the last output Ingredient object as final result.
        if len(dishes[k]) == 0:
            logger.warning('serving last procedure output for {}: {}'.format(k, out.ingred_id))
            dishes[k].append(out)
    return dishes

def dish_to_csv(dishes, outpath):
    for t, ds in dishes.items():
        for dish in ds:
            all_data = dish.get_data()
            if isinstance(all_data, dict):
                for k, df in all_data.items():
                    if re.match('ddf--.*.csv', k):
                        path = os.path.join(outpath, k)
                    else:
                        if t == 'datapoints':
                            by = dish.key_to_list()
                            path = os.path.join(outpath, 'ddf--{}--{}--by--{}.csv'.format(t, k, '--'.join(by)))
                        elif t == 'concepts':
                            path = os.path.join(outpath, 'ddf--{}.csv'.format(t))
                        elif t == 'entities':
                            domain = dish.key[0]
                            if k == domain:
                                path = os.path.join(outpath, 'ddf--{}--{}.csv'.format(t, k))
                            else:
                                path = os.path.join(outpath, 'ddf--{}--{}--{}.csv'.format(t, domain, k))
                        else:
                            raise ValueError('Not a correct collection: ' + t)

                    if t == 'datapoints':
                        df = df.set_index(by)
                        if not np.issubdtype(df[k].dtype, np.number):
                            try:
                                df[k] = df[k].astype(float)
                                # TODO: make floating precision an option
                                df[k] = df[k].map(lambda x: format_float_digits(x, 5))
                            except ValueError:
                                logging.warning("data not numeric: " + k)
                        else:
                            df[k] = df[k].map(lambda x: format_float_digits(x, 5))
                        df[[k]].to_csv(path, encoding='utf8')
                    else:
                        df.to_csv(path, index=False, encoding='utf8')
            else:
                path = os.path.join(outpath, 'ddf--{}.csv'.format(t))
                all_data.to_csv(path, index=False, encoding='utf8')
<commit_msg>remove unused logic<commit_after># -*- coding: utf-8 -*-
"""recipe cooking"""

import json
import yaml
from orderedattrdict import AttrDict
from orderedattrdict.yamlutils import AttrDictYAMLLoader

from . ingredient import *
from .. import config
from . procedure import *
from .. str import format_float_digits

import logging

# supported procedures, import from procedure.py
supported_procs = {
    'translate_column': translate_column,
    'translate_header': translate_header,
    'identity': identity,
    'merge': merge,
    'run_op': run_op,
    'filter_row': filter_row,
    'align': align,
    'filter_item': filter_item,
    'groupby': groupby,
    'accumulate': accumulate,
    'copy': copy,
    'add_concepts': add_concepts
}

def _loadfile(f):
    """load json/yaml file, into AttrDict"""
    if re.match('.*\.json', f):
        res = json.load(open(f), object_pairs_hook=AttrDict)
    else:
        res = yaml.load(open(f), Loader=AttrDictYAMLLoader)

    return res

# functions for reading/running recipe
def build_recipe(recipe_file, to_disk=False):
    """build a complete recipe file if there are includes in
    recipe file, if no includes found than return the file as is.
    """
    recipe = _loadfile(recipe_file)

    # the base dir of recipe file. for building paths for dictionary_dir and
    # sub recipe paths.
    base_dir = os.path.dirname(recipe_file)

    # the dictionary dir to retrieve translation dictionaries
    try:
        dict_dir = recipe['config']['dictionary_dir']
        config.DICT_PATH = dict_dir
    except KeyError:
        dict_dir = config.DICT_PATH

    # expand all files in the options
    if 'cooking' in recipe.keys():
        for p in ['concepts', 'datapoints', 'entities']:
            if p not in recipe['cooking'].keys():
                continue
            for i, procedure in enumerate(recipe['cooking'][p]):
                try:
                    opt_dict = procedure['options']['dictionary']
                except KeyError:
                    continue
                if isinstance(opt_dict, str):
                    # if the option dict is str, then it should be a filename
                    if dict_dir is None:
                        raise KeyError("dictionary_dir not found in config!")
                    if os.path.isabs(dict_dir):
                        path = os.path.join(dict_dir, opt_dict)
                    else:
                        path = os.path.join(base_dir, dict_dir, opt_dict)

                    recipe['cooking'][p][i]['options']['dictionary'] = _loadfile(path)

    if 'include' not in recipe.keys():
        return recipe
    else:  # append sub-recipe entities into main recipe
        recipe_dir = recipe['config']['recipes_dir']

        sub_recipes = []
        for i in recipe['include']:
            # TODO: maybe add support to expand user home and env vars
            if os.path.isabs(recipe_dir):
                path = os.path.join(recipe_dir, i)
            else:
                path = os.path.join(base_dir, recipe_dir, i)
            sub_recipes.append(build_recipe(path))

        for rcp in sub_recipes:
            # appending ingredients
            if 'ingredients' in recipe.keys():
                # ingredients = [*recipe['ingredients'], *rcp['ingredients']]  # not supportted by Python < 3.5
                ingredients = []
                [ingredients.append(ing) for ing in recipe['ingredients']]
                [ingredients.append(ing) for ing in rcp['ingredients']]
                # drop duplicated ingredients.
                rcp_dict_tmp = {}
                for v in ingredients:
                    if v['id'] not in rcp_dict_tmp.keys():
                        rcp_dict_tmp[v['id']] = v
                    else:
                        # raise error when ingredients with same ID have different contents.
                        if v != rcp_dict_tmp[v['id']]:
                            raise ValueError("Different content with same ingredient id detected: " + v['id'])
                recipe['ingredients'] = list(rcp_dict_tmp.values())
            else:
                recipe['ingredients'] = rcp['ingredients']

            # appending cooking procedures
            if 'cooking' not in rcp.keys():
                continue
            for p in ['datapoints', 'entities', 'concepts']:
                if p not in rcp['cooking'].keys():
                    continue
                if 'cooking' in recipe.keys():
                    if p in recipe['cooking'].keys():
                        # NOTE: the included cooking procedures should be placed in front of
                        # the origin ones.
                        # recipe['cooking'][p] = [*rcp['cooking'][p], *recipe['cooking'][p]]  # not supportted by Python < 3.5
                        new_procs = []
                        [new_procs.append(proc) for proc in rcp['cooking'][p]]
                        [new_procs.append(proc) for proc in recipe['cooking'][p]]
                        recipe['cooking'][p] = new_procs
                    else:
                        recipe['cooking'][p] = rcp['cooking'][p]
                else:
                    recipe['cooking'] = {}
                    recipe['cooking'][p] = rcp['cooking'][p]

        if to_disk:
            yaml.dump(recipe, open('recipe_dump.yaml', 'w'))

        return recipe

def update_recipe_last_update(recipe, outdir):
    pass

def check_dataset_availability(recipe):
    """check availability of all datasets required by the recipe.

    raise error if some dataset not available. otherwise do nothing.
    """
    ddf_dir = recipe.config.ddf_dir

    datasets = set()
    for ingred in recipe.ingredients:
        datasets.add(ingred.dataset)

    not_exists = []
    for d in datasets:
        if not os.path.exists(os.path.join(ddf_dir, d)):
            not_exists.append(d)

    if len(not_exists) > 0:
        logging.critical("not enough datasets! please checkout following datasets:\n{}\n"\
                            .format('\n'.join(not_exists)))
        raise ValueError('not enough datasets')
    return

def run_recipe(recipe):
    """run the recipe.

    returns a dictionary. keys are `concepts`, `entities` and `datapoints`,
    and values are ingredients return by the procedures
    """
    try:
        config.DDF_SEARCH_PATH = recipe['config']['ddf_dir']
    except KeyError:
        if not config.DDF_SEARCH_PATH:
            raise ValueError("no ddf_dir configured, please check your recipe")
    logging.info('path for searching DDF: ' + str(config.DDF_SEARCH_PATH))

    check_dataset_availability(recipe)

    # load ingredients
    ings = [Ingredient.from_dict(i) for i in recipe['ingredients']]
    ings_dict = dict([[i.ingred_id, i] for i in ings])

    # cooking
    funcs = supported_procs
    dishes = {}

    for k, pceds in recipe['cooking'].items():

        print("running "+k)
        dishes[k] = list()

        for p in pceds:
            func = p['procedure']

            if func not in funcs.keys() and func != 'serve':
                raise NotImplementedError("Not supported: " + func)

            ingredient = [ings_dict[i] for i in p['ingredients']]

            if func == 'serve':
                for i in ingredient:
                    dishes[k].append(i)
                continue

            # change the 'base'/'source_ingredients' option to actual ingredient
            # TODO: find better way to handle the ingredients in options
            if 'options' in p.keys():
                options = p['options']
                if 'base' in options.keys():
                    options['base'] = ings_dict[options['base']]
                if 'source_ingredients' in options.keys():
                    options['source_ingredients'] = [ings_dict[x] for x in options['source_ingredients']]

            if 'result' in p.keys():
                result = p['result']
                if 'options' in p.keys():
                    out = funcs[func](*ingredient, result=result, **options)
                else:
                    out = funcs[func](*ingredient, result=result)
            else:
                if 'options' in p.keys():
                    out = funcs[func](*ingredient, **options)
                else:
                    out = funcs[func](*ingredient)
                result = out.ingred_id

            if result in ings_dict.keys():
                logging.warning("overwriting existing ingredient: " + result)
            ings_dict[result] = out

        # if there is no seving procedures, use the last output Ingredient object as final result.
        if len(dishes[k]) == 0:
            logger.warning('serving last procedure output for {}: {}'.format(k, out.ingred_id))
            dishes[k].append(out)
    return dishes

def dish_to_csv(dishes, outpath):
    for t, ds in dishes.items():
        for dish in ds:
            all_data = dish.get_data()
            if isinstance(all_data, dict):
                for k, df in all_data.items():
                    if t == 'datapoints':
                        by = dish.key_to_list()
                        path = os.path.join(outpath, 'ddf--{}--{}--by--{}.csv'.format(t, k, '--'.join(by)))
                    elif t == 'concepts':
                        path = os.path.join(outpath, 'ddf--{}.csv'.format(t))
                    elif t == 'entities':
                        domain = dish.key[0]
                        if k == domain:
                            path = os.path.join(outpath, 'ddf--{}--{}.csv'.format(t, k))
                        else:
                            path = os.path.join(outpath, 'ddf--{}--{}--{}.csv'.format(t, domain, k))
                    else:
                        raise ValueError('Not a correct collection: ' + t)

                    if t == 'datapoints':
                        df = df.set_index(by)
                        if not np.issubdtype(df[k].dtype, np.number):
                            try:
                                df[k] = df[k].astype(float)
                                # TODO: make floating precision an option
                                df[k] = df[k].map(lambda x: format_float_digits(x, 5))
                            except ValueError:
                                logging.warning("data not numeric: " + k)
                        else:
                            df[k] = df[k].map(lambda x: format_float_digits(x, 5))
                        df[[k]].to_csv(path, encoding='utf8')
                    else:
                        df.to_csv(path, index=False, encoding='utf8')
            else:
                path = os.path.join(outpath, 'ddf--{}.csv'.format(t))
                all_data.to_csv(path, index=False, encoding='utf8')
<|endoftext|>


<commit_before># file ddisearch/geo/utils.py
#
# Copyright 2014 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import re
from django.conf import settings
from ddisearch.geo.models import Location, GeonamesCountry, GeonamesContinent
from ddisearch.geo.geonames import GeonamesClient, GeonamesException
from ddisearch.geo.altnames import alternate_names

logger = logging.getLogger(__name__)

class CodebookGeocoder(object):
    '''Utility class to geocode the geographical coverage terms in
    a :class:`~dddisearch.ddi.models.CodeBook` in order to
    enable browsing records by geography and plotting locations on a map.
    Geographical information is stored in the database using
    :class:`~ddisearch.geo.models.Location`, and the corresponding geonames
    id is added to the xml.
    '''

    name_paren_re = re.compile('^(?P<name>[A-Z][a-zA-Z ]+) \((?P<restriction>[A-Za-z ]+)\)$')

    def __init__(self):
        # initialize the geocoder for reuse
        self.geonames = GeonamesClient(username=settings.GEONAMES_USERNAME)

        # save the continents for easy lookup
        continents = GeonamesContinent.objects.all()
        self.continents = dict([(c.name, c.geonames_id) for c in continents])

    def code_locations(self, cb):
        '''Code the geographic coverage locations in a codebook'''

        # in theory this could give us some help, but is often not set
        if cb.geo_unit:
            logger.debug('geogUnit = %s', '; '.join(cb.geo_unit))

        # check if geographic coverage includes "global" (indicates not US-only)
        is_global = any([geog.val.lower() == 'global' for geog in cb.geo_coverage])
        includes_us = any([geog.val == 'United States' for geog in cb.geo_coverage])

        # check if there are at least three US states in this record,
        # to help determine if we should assume US
        current_us_states = [geog.val for geog in cb.geo_coverage if geog.val in us_states]

        # If not global and includes U.S. and at least one state
        # OR if not global and includes more than three states,
        # restrict geocoded results to U.S. locations

        # set a flag for whether assume US is true for *entire* document
        if (not is_global and includes_us and current_us_states) \
          or (not is_global and len(current_us_states) >= 3):
            doc_assume_US = True
        else:
            doc_assume_US = False

        # loop through geographical coverage terms and look them up,
        # setting geonames id on the geogCover element
        for geog in cb.geo_coverage:

            # reset assume US flag to whatever the document assumption about US is,
            # since it could potentially change for individual locations with
            # U.S. states mentioned
            assume_US = doc_assume_US

            # skip coverage of global - nothing to code
            if geog.val == 'Global':
                continue

            logger.info('geogCover %s', geog.val)

            # special case: first check if we have a continent
            # (geonames not looking these up so well; for some reason
            # 'Europe' is geocoded as 'Minsk' and 'Africa' as 'Camayenne')
            if geog.val in self.continents:
                geog.id = 'geonames:%d' % self.continents[geog.val]

                # make sure continent is present in Locations db table
                if Location.objects.filter(name=geog.val, feature_code='CONT').count() == 0:
                    loc = self.geonames.geocode(name_equals=geog.val, feature_code='CONT')
                    dbloc = self.location_from_geoname(loc)

                continue

            geo_options = {}

            # clean names that are formatted this way:
            #   Portland (Maine)
            #   Hiroshima (prefecture)
            #   New York (state)
            # if (###) is a state, use it to filter; otherwise ignore
            match = self.name_paren_re.match(geog.val)
            if match:
                matchinfo = match.groupdict()
                geogname = matchinfo['name']
                restriction = matchinfo['restriction']
                # if the parenthetical text matches a U.S. state, use that in the lookup
                if restriction in us_states:
                    assume_US = True

                # sometimes used to clarify ambiguous names, e.g. New York (state)
                elif restriction == 'state' and assume_US:
                    geo_options['feature_code'] = 'ADM1'
                    # NOTE: this is the feature code for states in U.S.
                    # may need to generalize more to handle other countries
            else:
                geogname = geog.val
                restriction = None

            # if this place name is in our list of items with known alternate names,
            # lookup as if it were the other name
            if geogname in alternate_names:
                geogname = alternate_names[geogname]
            elif geog.val in alternate_names:
                geogname = alternate_names[geog.val]

            # first check if the name is in the country list
            # (except for assume US, to avoid coding Georgia, US as Republic of Georgia)
            dbloc = None
            if not assume_US:
                dbloc = self.lookup_country(geogname)

            # next check in the db, in case we've already looked this place up
            # NOTE: skipping this even though it will require more geocoding,
            # because adding duplicate logic here to restrict by location,
            # admin code, etc. seems problematic

            # if we still don't have a location, use the geocoder
            if dbloc is None:
                # use US-biased geocoder if we have hit a threshold where
                # we want to assume US (with certain exceptions)
                # NOTE: there may well be other locations like Puerto Rico!

                if assume_US and geogname != 'Puerto Rico':
                    geo_options['country'] = 'US'

                if assume_US and geogname in us_states and not restriction == 'state':
                    # FIXME: what is this doing?
                    geo_options['admin_code1'] = us_states[geogname]
                elif restriction is not None and restriction in us_states:
                    # if we have a restriction that matches a U.S. state
                    # (e.g. "Portland (Maine"), pass that along to the geocoder
                    geo_options['admin_code1'] = us_states[restriction]

                # special case: if record includes U.S. and only one state,
                # and current place is neither of those, look within the state first
                elif includes_us and len(current_us_states) == 1 and \
                  geogname != 'United States' and geogname not in current_us_states:
                    geo_options['admin_code1'] = us_states[current_us_states[0]]

                loc = self.lookup_name(geogname, geo_options)
                # if no location was found, warn and skip
                if not loc:
                    logger.warn('No geonames result found for %s', geogname)
                    continue

                logger.debug('geonames result: %s', unicode(loc))
                logger.debug(loc.raw)

                # check if geonames id is already in the db
                db_locations = Location.objects.filter(geonames_id=loc.raw['geonameId'])
                if db_locations.count():
                    dbloc = db_locations[0]
                else:
                    # if not, create new db location from geonames lookup
                    dbloc = self.location_from_geoname(loc)

            # set geonames id in the xml
            geog.id = 'geonames:%d' % dbloc.geonames_id
            logger.info('setting geonames id to %s (%s, %s, %s)',
                        geog.id, dbloc.name, dbloc.country_code,
                         dbloc.continent_code)

    def lookup_country(self, geogname):
        # lookup a name to see if matches one of our known countries
        country = GeonamesCountry.objects.filter(name=geogname).first()
        if country:
            try:
                loc = self.geonames.get_by_id(country.geonames_id)
                logger.debug('Found a country match for %s, using geonames %s',
                             geogname, country.geonames_id)
                dbloc = self.location_from_geoname(loc)
                return dbloc
            except GeonamesException:
                # possible to get an error if geonames id no longer exists
                pass

    def lookup_location(self, geogname, assume_US=False):
        # lookup a name to see if it matches a location in the database
        country_filter = {}
        # if conditions are met where we want to assume US, restrict to
        # country code US except for Puerto Rico
        if assume_US and geogname != 'Puerto Rico':
            country_filter['country_code'] = 'US'
        db_locations = Location.objects.filter(name=geogname, **country_filter)
        # if we *don't* have reason to assume US and we get Georgia,
        # look for non-US match
        # NOTE: possibly redundant with new lookup_country logic?
        if not assume_US and geogname == 'Georgia':
            # NOTE: we could use similar logic to give a country bias to geocoder
            db_locations = db_locations.exclude(country_code='US')

        # NOTE: historic German and Italian states are handled
        # via alternate names list, and coded at country level

        # If there is one and only one match, use it; otherwise defer to geocoder
        # to determine which place to use
        if db_locations.count() == 1:
            # store db location so we can put geonames id into the xml
            dbloc = db_locations[0]

            logger.debug('Found a location match for %s, using %s',
                geogname, dbloc)

            return dbloc

    def lookup_name(self, geogname, geo_options):
        # attempt to geocode the geographic name

        # first try for exact match, with feature class of A (country, state, region)
        loc = self.geonames.geocode(name_equals=geogname, feature_class='A',
                                    **geo_options)

        # if that doesn't work, try with exact name and broader feature class (P = city, village)
        if not loc:
            loc = self.geonames.geocode(name_equals=geogname, feature_class=['A', 'P'],
                                        **geo_options)

        # if that doesn't work, try with looser name match but smaller feature class
        if not loc:
            loc = self.geonames.geocode(name=geogname, feature_class='A',
                                        **geo_options)

        # if that still fails, try a looser name match
        if not loc:
            loc = self.geonames.geocode(name=geogname, feature_class=['A', 'P'],
                                        **geo_options)

        # could return none if no match
        return loc

    def location_from_geoname(self, loc):
        '''Create and return a :class:`ddisearch.geo.models.Location`
        based on a location as returned by
        :class:`geopy.geocoders.GeoNames`'''

        # check if requested location is already in the db by geoname id
        # if already present, do nothing
        dbloc = Location.objects.filter(geonames_id=loc.raw['geonameId'])
        if dbloc:
            return dbloc[0]

        # determine continent code based on country
        country_code = loc.raw.get('countryCode', None)

        # special case: for historic places without country code in geonames
        # where we have added local country codes, pull from country db
        # OR where we have overridden country code
        # (i.e. to differentiate Serbia and Montenegro from Serbia)
        geonames_id = int(loc.raw['geonameId'])
        # USSR, Yugoslavia, Serbia and Montenegro
        if geonames_id in [8354411, 8505035, 8505033]:
            country = GeonamesCountry.objects.get(geonames_id=geonames_id)
            country_code = country.code

        continent_code = None
        if country_code is not None:
            try:
                c = GeonamesCountry.objects.get(code=country_code)
                continent_code = c.continent
            except:
                pass
        # special case for continents
        elif loc.raw['fcode'] == 'CONT':
            try:
                c = GeonamesContinent.objects.get(geonames_id=loc.raw['geonameId'])
                continent_code = c.code
            except:
                pass

        # get state code if there is one
        state_code = None
        admin_code = loc.raw.get('adminCode1', None)
        if admin_code and admin_code != '00':
            state_code = admin_code

        dbloc = Location(name=loc.raw['name'],
                         geonames_id=loc.raw['geonameId'],
                         latitude=loc.latitude,
                         longitude=loc.longitude,
                         country_code=country_code,
                         feature_code=loc.raw['fcode'],
                         continent_code=continent_code,
                         state_code=state_code)
        dbloc.save()

        # for cities with state code, make sure the state location is in db
        # so that browse page will not 404
        # TODO: other codes that belong here?
        # FIXME: do we really want ISL ? or probably not?
        # - perhaps restrict via allowed feature codes on initial geocode request
        if dbloc.feature_code in ['PPLA', 'PPLA2', 'ISL', 'ADM2'] and dbloc.state_code:
            if not Location.objects.filter(country_code=dbloc.country_code,
                                       state_code=dbloc.state_code,
                                       feature_code='ADM1').count():
                state = self.geonames.geocode(country=dbloc.country_code,
                                              feature_code='ADM1',
                                              admin_code1=dbloc.state_code)

                if state:
                    self.location_from_geoname(state)

        return dbloc

us_states = {
    'Alabama': 'AL',
    'Alaska': 'AK',
    'Arizona': 'AZ',
    'Arkansas': 'AR',
    'California': 'CA',
    'Colorado': 'CO',
    'Connecticut': 'CT',
    'Delaware': 'DE',
    'District of Columbia': 'DC',
    'Florida': 'FL',
    'Georgia': 'GA',
    'Hawaii': 'HI',
    'Idaho': 'ID',
    'Illinois': 'IL',
    'Indiana': 'IN',
    'Iowa': 'IA',
    'Kansas': 'KS',
    'Kentucky': 'KY',
    'Louisiana': 'LA',
    'Maine': 'ME',
    'Maryland': 'MD',
    'Massachusetts': 'MA',
    'Michigan': 'MI',
    'Minnesota': 'MN',
    'Mississippi': 'MS',
    'Missouri': 'MO',
    'Montana': 'MT',
    'Nebraska': 'NE',
    'Nevada': 'NV',
    'New Hampshire': 'NH',
    'New Jersey': 'NJ',
    'New Mexico': 'NM',
    'New York': 'NY',
    'North Carolina': 'NC',
    'North Dakota': 'ND',
    'Ohio': 'OH',
    'Oklahoma': 'OK',
    'Oregon': 'OR',
    'Pennsylvania': 'PA',
    'Rhode Island': 'RI',
    'South Carolina': 'SC',
    'South Dakota': 'SD',
    'Tennessee': 'TN',
    'Texas': 'TX',
    'Utah': 'UT',
    'Vermont': 'VT',
    'Virginia': 'VA',
    'Washington': 'WA',
    'West Virginia': 'WV',
    'Wisconsin': 'WI',
    'Wyoming': 'WY'
}
<commit_msg>Add logging when database country lookup fails<commit_after># file ddisearch/geo/utils.py
#
# Copyright 2014 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import re
from django.conf import settings
from ddisearch.geo.models import Location, GeonamesCountry, GeonamesContinent
from ddisearch.geo.geonames import GeonamesClient, GeonamesException
from ddisearch.geo.altnames import alternate_names

logger = logging.getLogger(__name__)

class CodebookGeocoder(object):
    '''Utility class to geocode the geographical coverage terms in
    a :class:`~dddisearch.ddi.models.CodeBook` in order to
    enable browsing records by geography and plotting locations on a map.
    Geographical information is stored in the database using
    :class:`~ddisearch.geo.models.Location`, and the corresponding geonames
    id is added to the xml.
    '''

    name_paren_re = re.compile('^(?P<name>[A-Z][a-zA-Z ]+) \((?P<restriction>[A-Za-z ]+)\)$')

    def __init__(self):
        # initialize the geocoder for reuse
        self.geonames = GeonamesClient(username=settings.GEONAMES_USERNAME)

        # save the continents for easy lookup
        continents = GeonamesContinent.objects.all()
        self.continents = dict([(c.name, c.geonames_id) for c in continents])

    def code_locations(self, cb):
        '''Code the geographic coverage locations in a codebook'''

        # in theory this could give us some help, but is often not set
        if cb.geo_unit:
            logger.debug('geogUnit = %s', '; '.join(cb.geo_unit))

        # check if geographic coverage includes "global" (indicates not US-only)
        is_global = any([geog.val.lower() == 'global' for geog in cb.geo_coverage])
        includes_us = any([geog.val == 'United States' for geog in cb.geo_coverage])

        # check if there are at least three US states in this record,
        # to help determine if we should assume US
        current_us_states = [geog.val for geog in cb.geo_coverage if geog.val in us_states]

        # If not global and includes U.S. and at least one state
        # OR if not global and includes more than three states,
        # restrict geocoded results to U.S. locations

        # set a flag for whether assume US is true for *entire* document
        if (not is_global and includes_us and current_us_states) \
          or (not is_global and len(current_us_states) >= 3):
            doc_assume_US = True
        else:
            doc_assume_US = False

        # loop through geographical coverage terms and look them up,
        # setting geonames id on the geogCover element
        for geog in cb.geo_coverage:

            # reset assume US flag to whatever the document assumption about US is,
            # since it could potentially change for individual locations with
            # U.S. states mentioned
            assume_US = doc_assume_US

            # skip coverage of global - nothing to code
            if geog.val == 'Global':
                continue

            logger.info('geogCover %s', geog.val)

            # special case: first check if we have a continent
            # (geonames not looking these up so well; for some reason
            # 'Europe' is geocoded as 'Minsk' and 'Africa' as 'Camayenne')
            if geog.val in self.continents:
                geog.id = 'geonames:%d' % self.continents[geog.val]

                # make sure continent is present in Locations db table
                if Location.objects.filter(name=geog.val, feature_code='CONT').count() == 0:
                    loc = self.geonames.geocode(name_equals=geog.val, feature_code='CONT')
                    dbloc = self.location_from_geoname(loc)

                continue

            geo_options = {}

            # clean names that are formatted this way:
            #   Portland (Maine)
            #   Hiroshima (prefecture)
            #   New York (state)
            # if (###) is a state, use it to filter; otherwise ignore
            match = self.name_paren_re.match(geog.val)
            if match:
                matchinfo = match.groupdict()
                geogname = matchinfo['name']
                restriction = matchinfo['restriction']
                # if the parenthetical text matches a U.S. state, use that in the lookup
                if restriction in us_states:
                    assume_US = True

                # sometimes used to clarify ambiguous names, e.g. New York (state)
                elif restriction == 'state' and assume_US:
                    geo_options['feature_code'] = 'ADM1'
                    # NOTE: this is the feature code for states in U.S.
                    # may need to generalize more to handle other countries
            else:
                geogname = geog.val
                restriction = None

            # if this place name is in our list of items with known alternate names,
            # lookup as if it were the other name
            if geogname in alternate_names:
                geogname = alternate_names[geogname]
            elif geog.val in alternate_names:
                geogname = alternate_names[geog.val]

            # first check if the name is in the country list
            # (except for assume US, to avoid coding Georgia, US as Republic of Georgia)
            dbloc = None
            if not assume_US:
                dbloc = self.lookup_country(geogname)

            # next check in the db, in case we've already looked this place up
            # NOTE: skipping this even though it will require more geocoding,
            # because adding duplicate logic here to restrict by location,
            # admin code, etc. seems problematic

            # if we still don't have a location, use the geocoder
            if dbloc is None:
                # use US-biased geocoder if we have hit a threshold where
                # we want to assume US (with certain exceptions)
                # NOTE: there may well be other locations like Puerto Rico!

                if assume_US and geogname != 'Puerto Rico':
                    geo_options['country'] = 'US'

                if assume_US and geogname in us_states and not restriction == 'state':
                    # FIXME: what is this doing?
                    geo_options['admin_code1'] = us_states[geogname]
                elif restriction is not None and restriction in us_states:
                    # if we have a restriction that matches a U.S. state
                    # (e.g. "Portland (Maine"), pass that along to the geocoder
                    geo_options['admin_code1'] = us_states[restriction]

                # special case: if record includes U.S. and only one state,
                # and current place is neither of those, look within the state first
                elif includes_us and len(current_us_states) == 1 and \
                  geogname != 'United States' and geogname not in current_us_states:
                    geo_options['admin_code1'] = us_states[current_us_states[0]]

                loc = self.lookup_name(geogname, geo_options)
                # if no location was found, warn and skip
                if not loc:
                    logger.warn('No geonames result found for %s', geogname)
                    continue

                logger.debug('geonames result: %s', unicode(loc))
                logger.debug(loc.raw)

                # check if geonames id is already in the db
                db_locations = Location.objects.filter(geonames_id=loc.raw['geonameId'])
                if db_locations.count():
                    dbloc = db_locations[0]
                else:
                    # if not, create new db location from geonames lookup
                    dbloc = self.location_from_geoname(loc)

            # set geonames id in the xml
            geog.id = 'geonames:%d' % dbloc.geonames_id
            logger.info('setting geonames id to %s (%s, %s, %s)',
                        geog.id, dbloc.name, dbloc.country_code,
                         dbloc.continent_code)

    def lookup_country(self, geogname):
        # lookup a name to see if matches one of our known countries
        country = GeonamesCountry.objects.filter(name=geogname).first()
        if country:
            try:
                loc = self.geonames.get_by_id(country.geonames_id)
                logger.debug('Found a country match for %s, using geonames %s',
                             geogname, country.geonames_id)
                dbloc = self.location_from_geoname(loc)
                return dbloc
            except GeonamesException as err:
                # it's possible to get an error if geonames id no
                # longer exists
                logger.warn('Failed to load Geonames %s id %s: %s',
                            geogname, country.geonames_id, err)

    def lookup_location(self, geogname, assume_US=False):
        # lookup a name to see if it matches a location in the database
        country_filter = {}
        # if conditions are met where we want to assume US, restrict to
        # country code US except for Puerto Rico
        if assume_US and geogname != 'Puerto Rico':
            country_filter['country_code'] = 'US'
        db_locations = Location.objects.filter(name=geogname, **country_filter)
        # if we *don't* have reason to assume US and we get Georgia,
        # look for non-US match
        # NOTE: possibly redundant with new lookup_country logic?
        if not assume_US and geogname == 'Georgia':
            # NOTE: we could use similar logic to give a country bias to geocoder
            db_locations = db_locations.exclude(country_code='US')

        # NOTE: historic German and Italian states are handled
        # via alternate names list, and coded at country level

        # If there is one and only one match, use it; otherwise defer to geocoder
        # to determine which place to use
        if db_locations.count() == 1:
            # store db location so we can put geonames id into the xml
            dbloc = db_locations[0]

            logger.debug('Found a location match for %s, using %s',
                geogname, dbloc)

            return dbloc

    def lookup_name(self, geogname, geo_options):
        # attempt to geocode the geographic name

        # first try for exact match, with feature class of A (country, state, region)
        loc = self.geonames.geocode(name_equals=geogname, feature_class='A',
                                    **geo_options)

        # if that doesn't work, try with exact name and broader feature class (P = city, village)
        if not loc:
            loc = self.geonames.geocode(name_equals=geogname, feature_class=['A', 'P'],
                                        **geo_options)

        # if that doesn't work, try with looser name match but smaller feature class
        if not loc:
            loc = self.geonames.geocode(name=geogname, feature_class='A',
                                        **geo_options)

        # if that still fails, try a looser name match
        if not loc:
            loc = self.geonames.geocode(name=geogname, feature_class=['A', 'P'],
                                        **geo_options)

        # could return none if no match
        return loc

    def location_from_geoname(self, loc):
        '''Create and return a :class:`ddisearch.geo.models.Location`
        based on a location as returned by
        :class:`geopy.geocoders.GeoNames`'''

        # check if requested location is already in the db by geoname id
        # if already present, do nothing
        dbloc = Location.objects.filter(geonames_id=loc.raw['geonameId'])
        if dbloc:
            return dbloc[0]

        # determine continent code based on country
        country_code = loc.raw.get('countryCode', None)

        # special case: for historic places without country code in geonames
        # where we have added local country codes, pull from country db
        # OR where we have overridden country code
        # (i.e. to differentiate Serbia and Montenegro from Serbia)
        geonames_id = int(loc.raw['geonameId'])
        # USSR, Yugoslavia, Serbia and Montenegro
        if geonames_id in [8354411, 8505035, 8505033]:
            country = GeonamesCountry.objects.get(geonames_id=geonames_id)
            country_code = country.code

        continent_code = None
        if country_code is not None:
            try:
                c = GeonamesCountry.objects.get(code=country_code)
                continent_code = c.continent
            except:
                pass
        # special case for continents
        elif loc.raw['fcode'] == 'CONT':
            try:
                c = GeonamesContinent.objects.get(geonames_id=loc.raw['geonameId'])
                continent_code = c.code
            except:
                pass

        # get state code if there is one
        state_code = None
        admin_code = loc.raw.get('adminCode1', None)
        if admin_code and admin_code != '00':
            state_code = admin_code

        dbloc = Location(name=loc.raw['name'],
                         geonames_id=loc.raw['geonameId'],
                         latitude=loc.latitude,
                         longitude=loc.longitude,
                         country_code=country_code,
                         feature_code=loc.raw['fcode'],
                         continent_code=continent_code,
                         state_code=state_code)
        dbloc.save()

        # for cities with state code, make sure the state location is in db
        # so that browse page will not 404
        # TODO: other codes that belong here?
        # FIXME: do we really want ISL ? or probably not?
        # - perhaps restrict via allowed feature codes on initial geocode request
        if dbloc.feature_code in ['PPLA', 'PPLA2', 'ISL', 'ADM2'] and dbloc.state_code:
            if not Location.objects.filter(country_code=dbloc.country_code,
                                       state_code=dbloc.state_code,
                                       feature_code='ADM1').count():
                state = self.geonames.geocode(country=dbloc.country_code,
                                              feature_code='ADM1',
                                              admin_code1=dbloc.state_code)

                if state:
                    self.location_from_geoname(state)

        return dbloc

us_states = {
    'Alabama': 'AL',
    'Alaska': 'AK',
    'Arizona': 'AZ',
    'Arkansas': 'AR',
    'California': 'CA',
    'Colorado': 'CO',
    'Connecticut': 'CT',
    'Delaware': 'DE',
    'District of Columbia': 'DC',
    'Florida': 'FL',
    'Georgia': 'GA',
    'Hawaii': 'HI',
    'Idaho': 'ID',
    'Illinois': 'IL',
    'Indiana': 'IN',
    'Iowa': 'IA',
    'Kansas': 'KS',
    'Kentucky': 'KY',
    'Louisiana': 'LA',
    'Maine': 'ME',
    'Maryland': 'MD',
    'Massachusetts': 'MA',
    'Michigan': 'MI',
    'Minnesota': 'MN',
    'Mississippi': 'MS',
    'Missouri': 'MO',
    'Montana': 'MT',
    'Nebraska': 'NE',
    'Nevada': 'NV',
    'New Hampshire': 'NH',
    'New Jersey': 'NJ',
    'New Mexico': 'NM',
    'New York': 'NY',
    'North Carolina': 'NC',
    'North Dakota': 'ND',
    'Ohio': 'OH',
    'Oklahoma': 'OK',
    'Oregon': 'OR',
    'Pennsylvania': 'PA',
    'Rhode Island': 'RI',
    'South Carolina': 'SC',
    'South Dakota': 'SD',
    'Tennessee': 'TN',
    'Texas': 'TX',
    'Utah': 'UT',
    'Vermont': 'VT',
    'Virginia': 'VA',
    'Washington': 'WA',
    'West Virginia': 'WV',
    'Wisconsin': 'WI',
    'Wyoming': 'WY'
}
<|endoftext|>


<commit_before>#!/usr/bin/python
# web service, so return only JSON (no HTML)
from flask import Flask, jsonify, request, url_for, make_response, render_template, send_file
import sqlite3
import interface_db as d
import os.path
import urlparse
import argparse
import textwrap
import functools    # need to wrap own decorators to comply with flask views
import zipfile
from io import BytesIO
try:
    from flask.ext.cors import CORS  # The typical way to import flask-cors
except ImportError:
    # Path hack allows examples to be run without installation.
    import os
    parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    os.sys.path.insert(0, parentdir)

    from flask_cors import CORS

app = Flask(__name__)
cors = CORS(app)
port = 5000

def parse_args(ns=None):
    """parse arguments from command line and return as namespace object"""
    parser = argparse.ArgumentParser(
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description=textwrap.dedent("""\
            serve a central database with benchmark information

        Generated database:
            Database should be created by populate_db.py, with each task
            organized as a table. A task is a collection of related benchmarks
            that are commonly run together."""),
            usage="%(prog)s [OPTIONS]")

    parser.add_argument("-d", "--database",
            default="results.db",
            help="name of database to store results in; default: %(default)s")
    parser.add_argument("-p", "--port",
            default=5000,
            type=int,
            help="port number to listen on; default: %(default)s")
    params = parser.parse_args(namespace=ns)
    global database, port
    database = os.path.expanduser(params.database)
    port = params.port
    return params

def catch_operation_errors(func):
    @functools.wraps(func)
    def task_checker(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except IOError as e:
            return jsonify({'status': 'File does not exist! ({})'.format(e)})
        except IndexError:
            return jsonify({'status': 'Index out of bounds! (likely table index)'})
        except sqlite3.OperationalError as e:
            return jsonify({'status': 'Malformed request! ({})'.format(e)})
    return task_checker

# each URI should be a noun since it is a resource (vs a verb for most library functions)

@app.route('/')
@app.route('/tasks', methods=['GET'])
@catch_operation_errors
def get_tasks():
    database = parse_db()
    return jsonify({'tasks':d.list_tasks(database), 'database':database})

@app.route('/db', methods=['GET'])
def get_database():
    return jsonify({'database':database})

@app.route('/param', methods=['GET'])
@catch_operation_errors
def get_param_desc():
    database = parse_db()
    tasks = parse_tasks()
    param = request.args.get('p')
    mode = request.args.get('m', 'range')   # by default give ranges, overriden if param is text
    try:
        (param_type, param_val) = d.describe_param(param, mode, tasks, database)
    except ValueError as e:
        return jsonify({'status': 'Parameter value error: {}'.format(e)})
    return jsonify({'status': 'OK',
    'database':database,
    'tasks':tasks,
    'param': param,
    'type': param_type,
    'val': param_val})

@app.route('/params/', methods=['GET'])
@catch_operation_errors
def get_shared_params():
    database = parse_db()
    tasks = parse_tasks()
    params = d.describe_tasks(tasks, database)
    return jsonify({'status': 'OK', 'database':database, 'tasks':tasks, 'params': params})

@app.route('/data', methods=['GET'])
@catch_operation_errors
def get_filtered_data():
    (exception, payload) = parse_data()
    if exception:
        return payload
    else:
        (databaes, tasks, params, data) = payload
        return jsonify({'status': 'OK',
            'database':database,
            'tasks':tasks,
            'params':params,
            'data':data})

@app.route('/csv', methods=['GET'])
@catch_operation_errors
def get_csv_data():
    """Return a zipped archive of csv files for selected tasks"""
    (exception, payload) = parse_data()
    if exception:
        return payload
    else:
        (database, tasks, params, data) = payload
        memory_file = BytesIO()
        with zipfile.ZipFile(memory_file, 'a', zipfile.ZIP_DEFLATED) as zf:
            t = 0
            for csvf in d.export_data_csv(params, data):
                zf.writestr("benchmark_results/" + tasks[t].replace('/','.'), csvf.getvalue())
                t += 1

        # prepare to send over network
        memory_file.seek(0)
        return send_file(memory_file, attachment_filename="benchmark_results.zip", as_attachment=True)

@app.route('/view')
def get_view():
    database = parse_db()
    tasks = {task_name: task_context for (task_name, task_context) in [t.split('|',1) for t in d.list_tasks(database)]}
    queried_tasks = parse_tasks()
    x = y = filters = ""
    # only pass other argument values if valid tasks selected
    if queried_tasks:
        x = request.args.get('x')
        y = request.args.get('y')
        (temp, filters) = parse_filters(verbose=True)

    return render_template('viewer.html',
                            database=database,
                            tasks=tasks,
                            queried_tasks=queried_tasks,
                            queried_x = x,
                            queried_y = y,
                            filters = filters)

@app.errorhandler(404)
def not_found(error):
    resp = jsonify({'error': 'Not found'})
    resp.status_code = 404
    return resp

# should always be run before all other querying since it determines where to look from
def parse_db():
    db = request.args.get('db')
    if db:
        return os.path.expanduser(db)
    # default to the global database
    return database

def parse_tasks():
    tasks = request.args.getlist('t')
    if tasks and tasks[0].isdigit():
        all_tasks = d.list_tasks(database)
        tasks = [all_tasks[int(t)] for t in tasks]
    return tasks

def parse_filters(verbose=False):
    """
    Parse filter from current request query string and return the filtered parameters and filters in a list

    verbose mode returns filters without splitting out the type
    """
    filter_param = None
    filter_method = None
    filters = []
    filter_args = []
    filter_params = []
    for arg in urlparse.parse_qsl(request.query_string):
        if arg[0][0] != 'f':
            continue
        # new filter parameter
        if arg[0] == 'fp':
            # previous filter ready to be built
            if filter_param and filter_method and filter_args:
                filters.append(d.Task_filter(filter_param, filter_method, filter_args))
                filter_args = []    # clear arguments; important!
                print("{}: {}".format(filters[-1], filters[-1].args))
                filter_params.append(filter_param)
            # split out the optional type following parameter name
            if verbose:
                filter_param = arg[1]
            else:
                filter_param = arg[1].split()[0]

        if arg[0] == 'fm':
            filter_method = arg[1]
        if arg[0] == 'fa':
            filter_args.append(arg[1])
    # last filter to be added
    if (not filters or filter_param != filters[-1].param) and filter_args:
        filters.append(d.Task_filter(filter_param, filter_method, filter_args))
        print("{}: {}".format(filters[-1], filters[-1].args))
        filter_params.append(filter_param)

    return filter_params,filters

def parse_data():
    """
    Parses request for data and returns a 2-tuple payload.

    First item is True for an exception occurance, with the 2nd item being the json response for error.
    Else false for no exception occruance, with second item being a 3-tuple.
    """
    database = parse_db()
    # split to get name only in case type is also given
    x_param = request.args.get('x')
    y_param = request.args.get('y')

    if not x_param:
        return (True, jsonify({'status': 'Missing x parameter!'}))
    if not y_param:
        return (True, jsonify({'status': 'Missing y parameter!'}))
    x_param = x_param.split()[0]
    y_param = y_param.split()[0]

    try:
        (filtered_params, filters) = parse_filters()
    except IndexError:
        return (True, jsonify({'status': 'Incomplete filter arguments!'}))
    except ValueError:
        return (True, jsonify({'status': 'Unsupported filter method!'}))

    tasks = parse_tasks()

    (params, data) = d.retrieve_data(x_param, y_param, filters, tasks, database)
    return (False, (database, tasks, params, data))

if __name__ == '__main__':
    parse_args()
    app.run(host='0.0.0.0', port=port)
<commit_msg>Server and viewer now only talk in about database names, database directory path must be set as a --root_directory argument when starting the server for security reasons<commit_after>#!/usr/bin/python
# web service, so return only JSON (no HTML)
from flask import Flask, jsonify, request, url_for, make_response, render_template, send_file
import sqlite3
import interface_db as d
import os.path
import urlparse
import argparse
import textwrap
import functools    # need to wrap own decorators to comply with flask views
import zipfile
from io import BytesIO
try:
    from flask.ext.cors import CORS  # The typical way to import flask-cors
except ImportError:
    # Path hack allows examples to be run without installation.
    import os
    parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    os.sys.path.insert(0, parentdir)

    from flask_cors import CORS

app = Flask(__name__)
cors = CORS(app)
port = 5000
database = None
root_directory = None

def parse_args(ns=None):
    """parse arguments from command line and return as namespace object"""
    parser = argparse.ArgumentParser(
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description=textwrap.dedent("""\
            serve a central database with benchmark information

        Generated database:
            Database should be created by populate_db.py, with each task
            organized as a table. A task is a collection of related benchmarks
            that are commonly run together."""),
            usage="%(prog)s [OPTIONS]")

    parser.add_argument("-d", "--database",
            default="results.db",
            help="name of database to store results in; default: %(default)s")
    parser.add_argument("-r", "--root_directory",
            default="~/benchtracker_data/",
            help="name of the directory to store databases in; default: %(default)s")
    parser.add_argument("-p", "--port",
            default=5000,
            type=int,
            help="port number to listen on; default: %(default)s")
    params = parser.parse_args(namespace=ns)
    global database, port, root_directory
    root_directory = os.path.expanduser(params.root_directory)
    database = os.path.expanduser(params.database)
    port = params.port
    return params

def catch_operation_errors(func):
    @functools.wraps(func)
    def task_checker(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except IOError as e:
            return jsonify({'status': 'File does not exist! ({})'.format(e)})
        except IndexError:
            return jsonify({'status': 'Index out of bounds! (likely table index)'})
        except sqlite3.OperationalError as e:
            return jsonify({'status': 'Malformed request! ({})'.format(e)})
    return task_checker

# each URI should be a noun since it is a resource (vs a verb for most library functions)

@app.route('/')
@app.route('/tasks', methods=['GET'])
@catch_operation_errors
def get_tasks():
    database = parse_db()
    return jsonify({'tasks':d.list_tasks(database), 'database':database})

@app.route('/db', methods=['GET'])
def get_database():
    return jsonify({'database':database})

@app.route('/param', methods=['GET'])
@catch_operation_errors
def get_param_desc():
    database = parse_db()
    tasks = parse_tasks()
    param = request.args.get('p')
    mode = request.args.get('m', 'range')   # by default give ranges, overriden if param is text
    try:
        (param_type, param_val) = d.describe_param(param, mode, tasks, real_db(database))
    except ValueError as e:
        return jsonify({'status': 'Parameter value error: {}'.format(e)})
    return jsonify({'status': 'OK',
    'database':database,
    'tasks':tasks,
    'param': param,
    'type': param_type,
    'val': param_val})

@app.route('/params/', methods=['GET'])
@catch_operation_errors
def get_shared_params():
    database = parse_db()
    tasks = parse_tasks()
    params = d.describe_tasks(tasks, real_db(database))
    return jsonify({'status': 'OK', 'database':database, 'tasks':tasks, 'params': params})

@app.route('/data', methods=['GET'])
@catch_operation_errors
def get_filtered_data():
    (exception, payload) = parse_data()
    if exception:
        return payload
    else:
        (databaes, tasks, params, data) = payload
        return jsonify({'status': 'OK',
            'database':database,
            'tasks':tasks,
            'params':params,
            'data':data})

@app.route('/csv', methods=['GET'])
@catch_operation_errors
def get_csv_data():
    """Return a zipped archive of csv files for selected tasks"""
    (exception, payload) = parse_data()
    if exception:
        return payload
    else:
        (database, tasks, params, data) = payload
        memory_file = BytesIO()
        with zipfile.ZipFile(memory_file, 'a', zipfile.ZIP_DEFLATED) as zf:
            t = 0
            for csvf in d.export_data_csv(params, data):
                zf.writestr("benchmark_results/" + tasks[t].replace('/','.'), csvf.getvalue())
                t += 1

        # prepare to send over network
        memory_file.seek(0)
        return send_file(memory_file, attachment_filename="benchmark_results.zip", as_attachment=True)

@app.route('/view')
@catch_operation_errors
def get_view():
    database = parse_db()
    print(real_db(database))
    tasks = {task_name: task_context for (task_name, task_context) in [t.split('|',1) for t in d.list_tasks(real_db(database))]}
    queried_tasks = parse_tasks()
    x = y = filters = ""
    # only pass other argument values if valid tasks selected
    if queried_tasks:
        x = request.args.get('x')
        y = request.args.get('y')
        (temp, filters) = parse_filters(verbose=True)

    return render_template('viewer.html',
                            database=database,
                            tasks=tasks,
                            queried_tasks=queried_tasks,
                            queried_x = x,
                            queried_y = y,
                            filters = filters)

@app.errorhandler(404)
def not_found(error):
    resp = jsonify({'error': 'Not found'})
    resp.status_code = 404
    return resp

# library call knows path to database, web viewer doesn't for security reasons
def real_db(relative_db):
    return os.path.join(root_directory, relative_db)

# should always be run before all other querying since it determines where to look from
def parse_db():
    db = request.args.get('db')
    if db:
        return db
    # default to the global database
    return database

def parse_tasks():
    tasks = request.args.getlist('t')
    if tasks and tasks[0].isdigit():
        all_tasks = d.list_tasks(database)
        tasks = [all_tasks[int(t)] for t in tasks]
    return tasks

def parse_filters(verbose=False):
    """
    Parse filter from current request query string and return the filtered parameters and filters in a list

    verbose mode returns filters without splitting out the type
    """
    filter_param = None
    filter_method = None
    filters = []
    filter_args = []
    filter_params = []
    for arg in urlparse.parse_qsl(request.query_string):
        if arg[0][0] != 'f':
            continue
        # new filter parameter
        if arg[0] == 'fp':
            # previous filter ready to be built
            if filter_param and filter_method and filter_args:
                filters.append(d.Task_filter(filter_param, filter_method, filter_args))
                filter_args = []    # clear arguments; important!
                print("{}: {}".format(filters[-1], filters[-1].args))
                filter_params.append(filter_param)
            # split out the optional type following parameter name
            if verbose:
                filter_param = arg[1]
            else:
                filter_param = arg[1].split()[0]

        if arg[0] == 'fm':
            filter_method = arg[1]
        if arg[0] == 'fa':
            filter_args.append(arg[1])
    # last filter to be added
    if (not filters or filter_param != filters[-1].param) and filter_args:
        filters.append(d.Task_filter(filter_param, filter_method, filter_args))
        print("{}: {}".format(filters[-1], filters[-1].args))
        filter_params.append(filter_param)

    return filter_params,filters

def parse_data():
    """
    Parses request for data and returns a 2-tuple payload.

    First item is True for an exception occurance, with the 2nd item being the json response for error.
    Else false for no exception occruance, with second item being a 3-tuple.
    """
    database = parse_db()
    # split to get name only in case type is also given
    x_param = request.args.get('x')
    y_param = request.args.get('y')

    if not x_param:
        return (True, jsonify({'status': 'Missing x parameter!'}))
    if not y_param:
        return (True, jsonify({'status': 'Missing y parameter!'}))
    x_param = x_param.split()[0]
    y_param = y_param.split()[0]

    try:
        (filtered_params, filters) = parse_filters()
    except IndexError:
        return (True, jsonify({'status': 'Incomplete filter arguments!'}))
    except ValueError:
        return (True, jsonify({'status': 'Unsupported filter method!'}))

    tasks = parse_tasks()

    (params, data) = d.retrieve_data(x_param, y_param, filters, tasks, real_db(database))
    return (False, (database, tasks, params, data))

if __name__ == '__main__':
    parse_args()
    app.run(host='0.0.0.0', port=port)
<|endoftext|>