<commit_before>import requests
import re
import os.path
import time
import hashlib
import json

from imboclient.header import authenticate
from imboclient.url import image
from imboclient.url import images
from imboclient.url import user as userUrl
from imboclient.url import status
from imboclient.url import metadata
from imboclient.url import accesstoken

py_version = 3

try:
    # py3
    from urllib.parse import urlparse
except ImportError:
    # py2
    from urlparse import urlparse
    py_version = 2

class Client:
    def __init__(self, server_urls, public_key, private_key, version=None, user=None, access_token_generator=accesstoken.AccessToken):
        self.server_urls = self._parse_urls(server_urls)
        self.access_token_generator = access_token_generator

        self._public_key = public_key
        self._private_key = private_key
        self._version = version
        self._user = user

    def metadata_url(self, image_identifier):
        return metadata.UrlMetadata(self._pick_url(image_identifier),
                                    self._public_key,
                                    self._private_key,
                                    image_identifier,
                                    user=self._user,
                                    access_token_generator=self.access_token_generator)

    def status_url(self):
        return status.UrlStatus(self._pick_url(),
                                self._public_key,
                                self._private_key,
                                access_token_generator=self.access_token_generator)

    def user_url(self):
        return userUrl.UrlUser(self._pick_url(),
                               self._public_key,
                               self._private_key,
                               access_token_generator=self.access_token_generator)

    def images_url(self):
        return images.UrlImages(self._pick_url(),
                                self._public_key,
                                self._private_key,
                                user=self._user,
                                access_token_generator=self.access_token_generator)

    def image_url(self, image_identifier):
        return image.UrlImage(self._pick_url(image_identifier),
                              self._public_key,
                              self._private_key,
                              image_identifier,
                              user=self._user,
                              access_token_generator=self.access_token_generator)

    def add_image(self, path):
        image_file_data = self._image_file_data(path)

        url = self.images_url().url()
        headers = self._auth_headers('POST', url)

        def http_add_image(self):
            return requests.post(url, data=image_file_data,  headers=headers)

        return self._wrap_result_json(http_add_image, [200, 201], 'Could not add image.')

    def add_image_from_string(self, image):
        image_url = self.images_url().url()
        headers = self._auth_headers('POST', image_url)

        def http_add_image_from_string(self):
            return requests.post(image_url, data=image, headers=headers)

        return self._wrap_result_json(http_add_image_from_string, [200, 201], 'Could not add image.')

    def add_image_from_url(self, image_url):
        def http_add_image_from_url(self):
            return requests.get(image_url)

        image_string = self._wrap_result(http_add_image_from_url, [200], 'Could not get fetch image data to add').content

        return self.add_image_from_string(image_string)

    def image_exists(self, imbo_id):
        return self.image_identifier_exists(imbo_id)

    def image_identifier_exists(self, image_identifier):
        result = self.head_image(image_identifier)
        return result.status_code == requests.codes.ok

    def head_image(self, image_identifier):
        def http_head_image(self):
            return requests.head(self.image_url(image_identifier).url())

        return self._wrap_result(http_head_image, [200, 404], 'Could not get image headers.')

    def delete_image(self, image_identifier):
        delete_url = self.image_url(image_identifier).url()
        headers = self._auth_headers('DELETE', delete_url)

        def http_delete_image(self):
            return requests.delete(delete_url, headers = headers)

        return self._wrap_result_json(http_delete_image, [200, 404], 'Could not delete image.')

    def edit_metadata(self, image_identifier, metadata):
        edit_metadata_url = self.metadata_url(image_identifier).url()
        metadata = json.dumps(metadata).encode('utf-8')
        headers = self._auth_headers('POST', edit_metadata_url)

        headers['Accept'] = 'application/json'
        headers['Content-Type'] = 'application/json'
        headers['Content-MD5'] = hashlib.md5(metadata).hexdigest()

        def http_edit_metadata(self):
            return requests.post(edit_metadata_url, data=metadata, headers=headers)

        return self._wrap_result_json(http_edit_metadata, [200], 'Could not edit metadata.')

    def replace_metadata(self, image_identifier, metadata):
        replace_metadata_url = self.metadata_url(image_identifier).url()
        metadata = json.dumps(metadata).encode('utf-8')
        headers = self._auth_headers('PUT', replace_metadata_url)

        headers['Accept'] = 'application/json'
        headers['Content-Type'] = 'application/json'
        headers['Content-MD5'] = hashlib.md5(metadata).hexdigest()

        def http_replace_metadata(self):
            return requests.put(replace_metadata_url, data=metadata, headers=headers)

        return self._wrap_result_json(http_replace_metadata, [200], 'Could not replace metadata.')

    def delete_metadata(self, image_identifier):
        delete_metadata_url = self.metadata_url(image_identifier).url()
        headers = self._auth_headers('DELETE', delete_metadata_url)

        def http_delete_metadata(self):
            return requests.delete(delete_metadata_url, headers=headers)

        return self._wrap_result_json(http_delete_metadata, [200], 'Could not delete metadata.')

    def num_images(self):
        user_url = self.user_url().url()

        def http_num_images(self):
            return requests.get(user_url, headers={'Accept': 'application/json'})

        user_data_decoded = self._wrap_result_json(http_num_images, [200], 'Could not get number of images.')

        try:
            return user_data_decoded['numImages']
        except KeyError as error:
            raise self.ImboInternalError(str(error) + ' Could not extract number of images from Imbo response.')

    def images(self, query = None):
        images_url = images.UrlImages(self._pick_url(), self._public_key, self._private_key)

        if query:
            images_url.add_query(query)

        def http_images(self):
            return requests.get(images_url.url(), headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_images, [200], 'Could not get images.')

    def image_data(self, image_identifier):
        image_url = self.image_url(image_identifier).url()
        return self.image_data_from_url(image_url)

    def image_data_from_url(self, url):
        def http_image_data(self):
            return requests.get(url)

        return self._wrap_result(http_image_data, [200], 'Could not get image data.')

    def image_properties(self, image_identifier):
        headers = self.head_image(image_identifier).headers

        try:
            return {
                    "x-imbo-originalwidth": headers['x-imbo-originalwidth'],
                    "x-imbo-originalheight": headers["x-imbo-originalheight"],
                    "x-imbo-originalfilesize": headers["x-imbo-originalfilesize"],
                    "x-imbo-originalmimetype": headers["x-imbo-originalmimetype"],
                    "x-imbo-originalextension": headers["x-imbo-originalextension"]
                    }
        except KeyError as key_error:
            raise self.ImboInternalError(str(key_error) + ' Imbo failed returning image properties.')

    def image_identifier(self, path):
        if self._validate_local_file(path):
            return self._generate_image_identifier(open(path, 'rb').read())

        raise ValueError("Either the path is invalid or empty file")

    def image_identifier_from_string(self, image):
        return self._generate_image_identifier(image)

    def server_status(self):
        url = self.status_url().url()

        def http_server_status(self):
            return requests.get(url, headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_server_status, [200], 'Failed getting server status.')

    def user_info(self):
        url = self.user_url().url()

        def http_user_info(self):
            return requests.get(url, headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_user_info, [200], 'Failed getting user info.')

    def _pick_url(self, key=None):
        if key:
            return self.server_urls[ord(key[0]) % len(self.server_urls)]
        else:
            return self.server_urls[0]

    def _parse_urls(self, urls):
        def should_remove_port(self, url_parts):
            return url_parts.port and (url_parts.scheme == 'http' and url_parts.port == 80 or (url_parts.scheme == 'https' and url_parts.port == 443))

        parsed_urls = []
        pattern = re.compile("https?://")

        if isinstance(urls, str) or (py_version == 2 and isinstance(urls, basestring)):
            urls = [urls]

        for url in urls:
            if not pattern.match(url):
                url = 'http://' + url

            parts = urlparse(url)

            if should_remove_port(self, parts):
                url = parts.scheme + '://' + parts.hostname + '/' + parts.path

            parsed_urls.append(url.rstrip('/'))

        return parsed_urls

    def _auth_headers(self, method, url):
        return authenticate.Authenticate(self._public_key, self._private_key, method, url, self._current_timestamp()).headers()

    def _wrap_result_json(self, function, success_status_codes, error):
        response = self._wrap_result(function, success_status_codes, error)

        try:
            response_json = response.json()
        except ValueError as value_error:
            raise self.ImboInternalError(error + ' The response from Imbo could not be parsed as JSON: \'' + response.text + '\'')

        return response_json

    def _wrap_result(self, function, success_status_codes, error):
        try:
            response = function(self)
        except requests.exceptions.RequestException as request_error:
            raise self.ImboTransportError(error + ', HTTP library returned error: ' + str(request_error))

        if response.status_code in success_status_codes:
            return response
        else:
            raise self.ImboInternalError(error + ' Imbo returned HTTP ' + str(response.status_code) + ' and body \'' + response.text + '\'')

    @classmethod
    def _current_timestamp(cls):
        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

    @classmethod
    def _generate_image_identifier(cls, content):
        return hashlib.md5(content).hexdigest()

    @classmethod
    def _image_file_data(cls, path):
        return open(path, 'rb').read()

    @classmethod
    def _validate_local_file(cls, path):
        return os.path.isfile(path) and os.path.getsize(path) > 0

    class ImboTransportError(Exception):
       pass

    class ImboInternalError(Exception):
       pass

<commit_msg>Remove unused version parameter, PR #38 from matslindh/remove-unused-version-parameter<commit_after>import requests
import re
import os.path
import time
import hashlib
import json

from imboclient.header import authenticate
from imboclient.url import image
from imboclient.url import images
from imboclient.url import user as userUrl
from imboclient.url import status
from imboclient.url import metadata
from imboclient.url import accesstoken

py_version = 3

try:
    # py3
    from urllib.parse import urlparse
except ImportError:
    # py2
    from urlparse import urlparse
    py_version = 2

class Client:
    def __init__(self, server_urls, public_key, private_key, user=None, access_token_generator=accesstoken.AccessToken):
        self.server_urls = self._parse_urls(server_urls)
        self.access_token_generator = access_token_generator

        self._public_key = public_key
        self._private_key = private_key
        self._user = user

    def metadata_url(self, image_identifier):
        return metadata.UrlMetadata(self._pick_url(image_identifier),
                                    self._public_key,
                                    self._private_key,
                                    image_identifier,
                                    user=self._user,
                                    access_token_generator=self.access_token_generator)

    def status_url(self):
        return status.UrlStatus(self._pick_url(),
                                self._public_key,
                                self._private_key,
                                access_token_generator=self.access_token_generator)

    def user_url(self):
        return userUrl.UrlUser(self._pick_url(),
                               self._public_key,
                               self._private_key,
                               access_token_generator=self.access_token_generator)

    def images_url(self):
        return images.UrlImages(self._pick_url(),
                                self._public_key,
                                self._private_key,
                                user=self._user,
                                access_token_generator=self.access_token_generator)

    def image_url(self, image_identifier):
        return image.UrlImage(self._pick_url(image_identifier),
                              self._public_key,
                              self._private_key,
                              image_identifier,
                              user=self._user,
                              access_token_generator=self.access_token_generator)

    def add_image(self, path):
        image_file_data = self._image_file_data(path)

        url = self.images_url().url()
        headers = self._auth_headers('POST', url)

        def http_add_image(self):
            return requests.post(url, data=image_file_data,  headers=headers)

        return self._wrap_result_json(http_add_image, [200, 201], 'Could not add image.')

    def add_image_from_string(self, image):
        image_url = self.images_url().url()
        headers = self._auth_headers('POST', image_url)

        def http_add_image_from_string(self):
            return requests.post(image_url, data=image, headers=headers)

        return self._wrap_result_json(http_add_image_from_string, [200, 201], 'Could not add image.')

    def add_image_from_url(self, image_url):
        def http_add_image_from_url(self):
            return requests.get(image_url)

        image_string = self._wrap_result(http_add_image_from_url, [200], 'Could not get fetch image data to add').content

        return self.add_image_from_string(image_string)

    def image_exists(self, imbo_id):
        return self.image_identifier_exists(imbo_id)

    def image_identifier_exists(self, image_identifier):
        result = self.head_image(image_identifier)
        return result.status_code == requests.codes.ok

    def head_image(self, image_identifier):
        def http_head_image(self):
            return requests.head(self.image_url(image_identifier).url())

        return self._wrap_result(http_head_image, [200, 404], 'Could not get image headers.')

    def delete_image(self, image_identifier):
        delete_url = self.image_url(image_identifier).url()
        headers = self._auth_headers('DELETE', delete_url)

        def http_delete_image(self):
            return requests.delete(delete_url, headers = headers)

        return self._wrap_result_json(http_delete_image, [200, 404], 'Could not delete image.')

    def edit_metadata(self, image_identifier, metadata):
        edit_metadata_url = self.metadata_url(image_identifier).url()
        metadata = json.dumps(metadata).encode('utf-8')
        headers = self._auth_headers('POST', edit_metadata_url)

        headers['Accept'] = 'application/json'
        headers['Content-Type'] = 'application/json'
        headers['Content-MD5'] = hashlib.md5(metadata).hexdigest()

        def http_edit_metadata(self):
            return requests.post(edit_metadata_url, data=metadata, headers=headers)

        return self._wrap_result_json(http_edit_metadata, [200], 'Could not edit metadata.')

    def replace_metadata(self, image_identifier, metadata):
        replace_metadata_url = self.metadata_url(image_identifier).url()
        metadata = json.dumps(metadata).encode('utf-8')
        headers = self._auth_headers('PUT', replace_metadata_url)

        headers['Accept'] = 'application/json'
        headers['Content-Type'] = 'application/json'
        headers['Content-MD5'] = hashlib.md5(metadata).hexdigest()

        def http_replace_metadata(self):
            return requests.put(replace_metadata_url, data=metadata, headers=headers)

        return self._wrap_result_json(http_replace_metadata, [200], 'Could not replace metadata.')

    def delete_metadata(self, image_identifier):
        delete_metadata_url = self.metadata_url(image_identifier).url()
        headers = self._auth_headers('DELETE', delete_metadata_url)

        def http_delete_metadata(self):
            return requests.delete(delete_metadata_url, headers=headers)

        return self._wrap_result_json(http_delete_metadata, [200], 'Could not delete metadata.')

    def num_images(self):
        user_url = self.user_url().url()

        def http_num_images(self):
            return requests.get(user_url, headers={'Accept': 'application/json'})

        user_data_decoded = self._wrap_result_json(http_num_images, [200], 'Could not get number of images.')

        try:
            return user_data_decoded['numImages']
        except KeyError as error:
            raise self.ImboInternalError(str(error) + ' Could not extract number of images from Imbo response.')

    def images(self, query = None):
        images_url = images.UrlImages(self._pick_url(), self._public_key, self._private_key)

        if query:
            images_url.add_query(query)

        def http_images(self):
            return requests.get(images_url.url(), headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_images, [200], 'Could not get images.')

    def image_data(self, image_identifier):
        image_url = self.image_url(image_identifier).url()
        return self.image_data_from_url(image_url)

    def image_data_from_url(self, url):
        def http_image_data(self):
            return requests.get(url)

        return self._wrap_result(http_image_data, [200], 'Could not get image data.')

    def image_properties(self, image_identifier):
        headers = self.head_image(image_identifier).headers

        try:
            return {
                    "x-imbo-originalwidth": headers['x-imbo-originalwidth'],
                    "x-imbo-originalheight": headers["x-imbo-originalheight"],
                    "x-imbo-originalfilesize": headers["x-imbo-originalfilesize"],
                    "x-imbo-originalmimetype": headers["x-imbo-originalmimetype"],
                    "x-imbo-originalextension": headers["x-imbo-originalextension"]
                    }
        except KeyError as key_error:
            raise self.ImboInternalError(str(key_error) + ' Imbo failed returning image properties.')

    def image_identifier(self, path):
        if self._validate_local_file(path):
            return self._generate_image_identifier(open(path, 'rb').read())

        raise ValueError("Either the path is invalid or empty file")

    def image_identifier_from_string(self, image):
        return self._generate_image_identifier(image)

    def server_status(self):
        url = self.status_url().url()

        def http_server_status(self):
            return requests.get(url, headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_server_status, [200], 'Failed getting server status.')

    def user_info(self):
        url = self.user_url().url()

        def http_user_info(self):
            return requests.get(url, headers={'Accept': 'application/json'})

        return self._wrap_result_json(http_user_info, [200], 'Failed getting user info.')

    def _pick_url(self, key=None):
        if key:
            return self.server_urls[ord(key[0]) % len(self.server_urls)]
        else:
            return self.server_urls[0]

    def _parse_urls(self, urls):
        def should_remove_port(self, url_parts):
            return url_parts.port and (url_parts.scheme == 'http' and url_parts.port == 80 or (url_parts.scheme == 'https' and url_parts.port == 443))

        parsed_urls = []
        pattern = re.compile("https?://")

        if isinstance(urls, str) or (py_version == 2 and isinstance(urls, basestring)):
            urls = [urls]

        for url in urls:
            if not pattern.match(url):
                url = 'http://' + url

            parts = urlparse(url)

            if should_remove_port(self, parts):
                url = parts.scheme + '://' + parts.hostname + '/' + parts.path

            parsed_urls.append(url.rstrip('/'))

        return parsed_urls

    def _auth_headers(self, method, url):
        return authenticate.Authenticate(self._public_key, self._private_key, method, url, self._current_timestamp()).headers()

    def _wrap_result_json(self, function, success_status_codes, error):
        response = self._wrap_result(function, success_status_codes, error)

        try:
            response_json = response.json()
        except ValueError as value_error:
            raise self.ImboInternalError(error + ' The response from Imbo could not be parsed as JSON: \'' + response.text + '\'')

        return response_json

    def _wrap_result(self, function, success_status_codes, error):
        try:
            response = function(self)
        except requests.exceptions.RequestException as request_error:
            raise self.ImboTransportError(error + ', HTTP library returned error: ' + str(request_error))

        if response.status_code in success_status_codes:
            return response
        else:
            raise self.ImboInternalError(error + ' Imbo returned HTTP ' + str(response.status_code) + ' and body \'' + response.text + '\'')

    @classmethod
    def _current_timestamp(cls):
        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

    @classmethod
    def _generate_image_identifier(cls, content):
        return hashlib.md5(content).hexdigest()

    @classmethod
    def _image_file_data(cls, path):
        return open(path, 'rb').read()

    @classmethod
    def _validate_local_file(cls, path):
        return os.path.isfile(path) and os.path.getsize(path) > 0

    class ImboTransportError(Exception):
       pass

    class ImboInternalError(Exception):
       pass

<|endoftext|>


<commit_before>from wikidataStuff.WikidataStuff import WikidataStuff as WDS
import pywikibot
import importer_utils as utils
from os import path

MAPPING_DIR = "mappings"
PROPS = utils.load_json(path.join(MAPPING_DIR, "props_general.json"))

class Uploader(object):

    TEST_ITEM = "Q4115189"

    def make_labels(self):
        labels = self.data["labels"]
        new_labels = {}
        for item in labels:
            new_labels[item] = {'language': item, 'value': labels[item]}
        return new_labels

    def make_aliases(self):
        aliases = self.data["aliases"]
        new_aliases = []
        for item in aliases:
            language = item
            for value in aliases[language]:
                new_alias = {"language": language, "value": value}
                new_aliases.append(new_alias)
        return new_aliases

    def add_labels(self, target_item, labels, log):
        for label in labels:
            target_item.get()
            name = labels[label]['value']
            lang = labels[label]['language']
            self.wdstuff.addLabelOrAlias(
                lang, name, target_item, summary=self.summary)
            if log:
                t_id = target_item.getID()
                message = t_id + " ADDED LABEL " + lang + " " + name
                log.logit(message)

    def add_descriptions(self, target_item, descriptions, log):
        for description in descriptions:
            target_item.get()
            name = descriptions[description]['value']
            lang = descriptions[description]['language']
            if (not target_item.descriptions or
                    lang not in target_item.descriptions):
                descs = {lang: name}
                target_item.editDescriptions(descs)
                pywikibot.output("Added description: " + name)
                if log:
                    t_id = target_item.getID()
                    log.logit(t_id + " ADDED DESCRIPTION " + lang + " " + name)

    def add_aliases(self, target_item, aliases, log):
        for alias in aliases:
            target_item.get()
            name = alias["value"]
            language = alias["language"]
            t_aliases = target_item.aliases
            if language in t_aliases and name in t_aliases[language]:
                return
            else:
                target_item.editAliases({language: [name]})
            if log:
                t_id = target_item.getID()
                log.logit(t_id + " ADDED ALIAS " + language + " " + name)

    def make_descriptions(self):
        descriptions = self.data["descriptions"]
        new_descriptions = {}
        for item in descriptions:
            new_descriptions[item] = {
                'language': item, 'value': descriptions[item]}
        return new_descriptions

    def make_image_item(self, filename):
        commonssite = pywikibot.Site("commons", "commons")
        imagelink = pywikibot.Link(
            filename, source=commonssite, defaultNamespace=6)
        return pywikibot.FilePage(imagelink)

    def make_coords_item(self, coordstuple):
        """
        Default precision, such as used by
        http://pywikibot.readthedocs.io/en/latest/_modules/scripts/claimit/
        """
        DEFAULT_PREC = 0.0001
        return pywikibot.Coordinate(
            coordstuple[0], coordstuple[1], precision=DEFAULT_PREC)

    def make_quantity_item(self, quantity, repo):
        value = quantity['quantity_value']
        if 'unit' in quantity:
            unit = "http://www.wikidata.org/entity/" + quantity['unit']
        else:
            unit = None
        return pywikibot.WbQuantity(value, unit, site=repo)

    def make_time_item(self, quantity, repo):
        """
        This only works for full years.
        TODO
        Make it work for year range!
        """
        value = quantity['time_value']
        return pywikibot.WbTime(**value)

    def make_q_item(self, qnumber):
        return self.wdstuff.QtoItemPage(qnumber)

    def item_has_prop(self, property_name, wd_item):
        """
        This is different from WikidataStuff has_claim()
        because it checks whether the property exists,
        not if the statement matches.
        If the target item already uses the property,
        a new claim will not be added even if it's different.
        """
        if PROPS[property_name] in wd_item.claims:
            return True
        else:
            return False

    def make_pywikibot_item(self, value, prop=None):
        val_item = None
        if type(value) is list and len(value) == 1:
            value = value[0]
        if utils.string_is_q_item(value):
            val_item = self.make_q_item(value)
        elif prop == PROPS["image"]:
            if self.item_has_prop("image", self.wd_item) is False:
                if utils.file_is_on_commons(value):
                    val_item = self.make_image_item(value)
        elif utils.tuple_is_coords(value) and prop == PROPS["coordinates"]:
            val_item = self.make_coords_item(value)
        elif isinstance(value, dict) and 'quantity_value' in value:
            val_item = self.make_quantity_item(value, self.repo)
        elif isinstance(value, dict) and 'time_value' in value:
            val_item = self.make_time_item(value, self.repo)
        elif prop == PROPS["commonscat"] and utils.commonscat_exists(value):
            val_item = value
        else:
            val_item = value
        return val_item

    def make_statement(self, value):
        return self.wdstuff.Statement(value)

    def make_url_reference(self, uri):
        prop = PROPS["reference_url"]
        ref = self.wdstuff.Reference(
            source_test=self.wdstuff.make_simple_claim(prop, uri))
        return ref

    def make_stated_in_reference(self, ref_dict):
        prop = ref_dict["source"]["prop"]
        prop_date = ref_dict["published"]["prop"]
        date = ref_dict["published"]["value"]
        date_item = pywikibot.WbTime(**date)
        source_item = self.wdstuff.QtoItemPage(ref_dict["source"]["value"])
        source_claim = self.wdstuff.make_simple_claim(prop, source_item)
        if "reference_url" in ref_dict:
            ref_url = ref_dict["reference_url"]["value"]
            ref_url_prop = ref_dict["reference_url"]["prop"]
            ref_url_claim = self.wdstuff.make_simple_claim(
                ref_url_prop, ref_url)
            ref = self.wdstuff.Reference(
                source_test=[source_claim, ref_url_claim],
                source_notest=self.wdstuff.make_simple_claim(prop_date, date_item))
        else:
            ref = self.wdstuff.Reference(
                source_test=[source_claim],
                source_notest=self.wdstuff.make_simple_claim(prop_date, date_item))
        return ref

    def add_claims(self, wd_item, claims, log):
        if wd_item:
            for claim in claims:
                prop = claim
                for x in claims[claim]:
                    value = x['value']
                    if value != "":
                        ref = None
                        quals = x['quals']
                        refs = x['refs']
                        wd_claim = self.make_pywikibot_item(value, prop)
                        if wd_claim is None:
                            continue
                        wd_value = self.make_statement(wd_claim)
                        if any(quals):
                            for qual in quals:
                                value = self.make_pywikibot_item(
                                    quals[qual], qual)
                                qualifier = self.wdstuff.Qualifier(qual, value)
                                wd_value.addQualifier(qualifier)
                        if len(refs) > 0:
                            for ref in refs:
                                """
                                This only works if it's a url.
                                If we have references of different sort,
                                this will have to be appended.
                                """
                                if utils.is_valid_url(ref):
                                    ref = self.make_url_reference(ref)
                                else:
                                    ref = self.make_stated_in_reference(ref)
                        if wd_value:
                            self.wdstuff.addNewClaim(
                                prop, wd_value, wd_item, ref, summary=self.summary)
                            if log:
                                t_id = wd_item.getID()
                                message = t_id + " ADDED CLAIM " + prop
                                log.logit(message)

    def create_new_item(self, log):
        item = self.wdstuff.make_new_item({}, summary=self.summary)
        if log:
            t_id = item.getID()
            message = t_id + " CREATE"
            log.logit(message)
        return item

    def get_username(self):
        """
        Get Wikidata login that will be used to upload.
        """
        return pywikibot.config.usernames["wikidata"]["wikidata"]

    def upload(self):
        if self.data["upload"] is False:
            print("SKIPPING ITEM")
            return
        labels = self.make_labels()
        descriptions = self.make_descriptions()
        aliases = self.make_aliases()
        claims = self.data["statements"]
        self.add_labels(self.wd_item, labels, self.log)
        self.add_aliases(self.wd_item, aliases, self.log)
        self.add_descriptions(self.wd_item, descriptions, self.log)
        self.add_claims(self.wd_item, claims, self.log)

    def set_wd_item(self):
        """
        Determine WD item to manipulate.

        In live mode, if data object has associated WD item,
        edit it. Otherwise, create a new WD item.
        In sandbox mode, all edits are done on the WD Sandbox item.
        """
        if self.live:
            if self.data["wd-item"] is None:
                self.wd_item = self.create_new_item(self.log)
                self.wd_item_q = self.wd_item.getID()
            else:
                item_q = self.data["wd-item"]
                self.wd_item = self.wdstuff.QtoItemPage(item_q)
                self.wd_item_q = item_q
        else:
            self.wd_item = self.wdstuff.QtoItemPage(self.TEST_ITEM)
            self.wd_item_q = self.TEST_ITEM

    def __init__(self, monument_object, log=None, tablename=None, live=False):
        self.log = False
        self.summary = "#COH #WLM #" + tablename
        self.live = live
        print("User: " + self.get_username())
        print("Edit summary: " + self.summary)
        if self.live:
            print("LIVE MODE")
        else:
            print("SANDBOX MODE: " + self.TEST_ITEM)
        print("---------------")
        if log is not None:
            self.log = log
        self.data = monument_object.wd_item
        site = pywikibot.Site("wikidata", "wikidata")
        self.repo = site.data_repository()
        self.wdstuff = WDS(self.repo)
        self.set_wd_item()
<commit_msg>use add_description() from wdstuff (#33)<commit_after>from wikidataStuff.WikidataStuff import WikidataStuff as WDS
import pywikibot
import importer_utils as utils
from os import path

MAPPING_DIR = "mappings"
PROPS = utils.load_json(path.join(MAPPING_DIR, "props_general.json"))

class Uploader(object):

    TEST_ITEM = "Q4115189"

    def make_labels(self):
        labels = self.data["labels"]
        new_labels = []
        for item in labels:
            new_labels.append({'language': item, 'value': labels[item]})
        return new_labels

    def make_aliases(self):
        aliases = self.data["aliases"]
        new_aliases = []
        for item in aliases:
            language = item
            for value in aliases[language]:
                new_alias = {"language": language, "value": value}
                new_aliases.append(new_alias)
        return new_aliases

    def add_labels(self, target_item, labels, aliases, log):
        """
        Add labels and aliases.

        Normally, if the WD item already has a label in $lang
        and the data object has another one, the new one
        will be automatically added as an alias. Otherwise
        (no existing label), it will be added as a label.
        However, the data model supports an optional [aliases]
        field as well.
        Its contents will be added directly as an alias, without checking
        if it could be a label first.
        """
        print(labels)
        for label in labels:
            target_item.get()
            label_content = label['value']
            language = label['language']
            self.wdstuff.addLabelOrAlias(
                language, label_content, target_item)
            if log:
                t_id = target_item.getID()
                message = "{} ADDED LABEL {} {}".format(
                    t_id, language, label_content)
                log.logit(message)
        for alias in aliases:
            target_item.get()
            name = alias["value"]
            language = alias["language"]
            t_aliases = target_item.aliases
            if language in t_aliases and name in t_aliases[language]:
                return
            else:
                summary = "Added [{}] alias to [[{}]], {}".format(
                    language, target_item.title(), self.summary)
                target_item.editAliases(
                    {language: [name]}, summary=summary)
            if log:
                t_id = target_item.getID()
                message = "{} ADDED ALIAS {} {}".format(
                    t_id, language, name)
                log.logit(message)

    def add_descriptions(self, target_item, descriptions, log):
        for description in descriptions:
            target_item.get()
            desc_content = descriptions[description]['value']
            lang = descriptions[description]['language']
            self.wdstuff.add_description(
                lang, desc_content, target_item)
            if log:
                t_id = target_item.getID()
                message = "{} ADDED DESCRIPTION {} {}".format(
                    t_id, lang, desc_content)
                log.logit(message)

    def make_descriptions(self):
        descriptions = self.data["descriptions"]
        new_descriptions = {}
        for item in descriptions:
            new_descriptions[item] = {
                'language': item, 'value': descriptions[item]}
        return new_descriptions

    def make_image_item(self, filename):
        commonssite = pywikibot.Site("commons", "commons")
        imagelink = pywikibot.Link(
            filename, source=commonssite, defaultNamespace=6)
        return pywikibot.FilePage(imagelink)

    def make_coords_item(self, coordstuple):
        """
        Default precision, such as used by
        http://pywikibot.readthedocs.io/en/latest/_modules/scripts/claimit/
        """
        DEFAULT_PREC = 0.0001
        return pywikibot.Coordinate(
            coordstuple[0], coordstuple[1], precision=DEFAULT_PREC)

    def make_quantity_item(self, quantity, repo):
        value = quantity['quantity_value']
        if 'unit' in quantity:
            unit = "http://www.wikidata.org/entity/" + quantity['unit']
        else:
            unit = None
        return pywikibot.WbQuantity(value, unit, site=repo)

    def make_time_item(self, quantity, repo):
        """
        This only works for full years.
        TODO
        Make it work for year range!
        """
        value = quantity['time_value']
        return pywikibot.WbTime(**value)

    def make_q_item(self, qnumber):
        return self.wdstuff.QtoItemPage(qnumber)

    def item_has_prop(self, property_name, wd_item):
        """
        This is different from WikidataStuff has_claim()
        because it checks whether the property exists,
        not if the statement matches.
        If the target item already uses the property,
        a new claim will not be added even if it's different.
        """
        if PROPS[property_name] in wd_item.claims:
            return True
        else:
            return False

    def make_pywikibot_item(self, value, prop=None):
        val_item = None
        if type(value) is list and len(value) == 1:
            value = value[0]
        if utils.string_is_q_item(value):
            val_item = self.make_q_item(value)
        elif prop == PROPS["image"]:
            if self.item_has_prop("image", self.wd_item) is False:
                if utils.file_is_on_commons(value):
                    val_item = self.make_image_item(value)
        elif utils.tuple_is_coords(value) and prop == PROPS["coordinates"]:
            val_item = self.make_coords_item(value)
        elif isinstance(value, dict) and 'quantity_value' in value:
            val_item = self.make_quantity_item(value, self.repo)
        elif isinstance(value, dict) and 'time_value' in value:
            val_item = self.make_time_item(value, self.repo)
        elif prop == PROPS["commonscat"] and utils.commonscat_exists(value):
            val_item = value
        else:
            val_item = value
        return val_item

    def make_statement(self, value):
        return self.wdstuff.Statement(value)

    def make_url_reference(self, uri):
        prop = PROPS["reference_url"]
        ref = self.wdstuff.Reference(
            source_test=self.wdstuff.make_simple_claim(prop, uri))
        return ref

    def make_stated_in_reference(self, ref_dict):
        prop = ref_dict["source"]["prop"]
        prop_date = ref_dict["published"]["prop"]
        date = ref_dict["published"]["value"]
        date_item = pywikibot.WbTime(**date)
        source_item = self.wdstuff.QtoItemPage(ref_dict["source"]["value"])
        source_claim = self.wdstuff.make_simple_claim(prop, source_item)
        if "reference_url" in ref_dict:
            ref_url = ref_dict["reference_url"]["value"]
            ref_url_prop = ref_dict["reference_url"]["prop"]
            ref_url_claim = self.wdstuff.make_simple_claim(
                ref_url_prop, ref_url)
            ref = self.wdstuff.Reference(
                source_test=[source_claim, ref_url_claim],
                source_notest=self.wdstuff.make_simple_claim(prop_date, date_item))
        else:
            ref = self.wdstuff.Reference(
                source_test=[source_claim],
                source_notest=self.wdstuff.make_simple_claim(prop_date, date_item))
        return ref

    def add_claims(self, wd_item, claims, log):
        if wd_item:
            for claim in claims:
                prop = claim
                for x in claims[claim]:
                    value = x['value']
                    if value != "":
                        ref = None
                        quals = x['quals']
                        refs = x['refs']
                        wd_claim = self.make_pywikibot_item(value, prop)
                        if wd_claim is None:
                            continue
                        wd_value = self.make_statement(wd_claim)
                        if any(quals):
                            for qual in quals:
                                value = self.make_pywikibot_item(
                                    quals[qual], qual)
                                qualifier = self.wdstuff.Qualifier(qual, value)
                                wd_value.addQualifier(qualifier)
                        for ref in refs:
                            """
                            This only works if it's a url.
                            If we have references of different sort,
                            this will have to be appended.
                            """
                            if utils.is_valid_url(ref):
                                ref = self.make_url_reference(ref)
                            else:
                                ref = self.make_stated_in_reference(ref)
                        if wd_value:
                            self.wdstuff.addNewClaim(
                                prop, wd_value, wd_item, ref)
                            if log:
                                t_id = wd_item.getID()
                                message = "{} ADDED CLAIM {}".format(
                                    t_id, prop)
                                log.logit(message)

    def create_new_item(self, log):
        item = self.wdstuff.make_new_item({})
        if log:
            t_id = item.getID()
            message = "{} CREATE".format(t_id)
            log.logit(message)
        return item

    def get_username(self):
        """
        Get Wikidata login that will be used to upload.
        """
        return pywikibot.config.usernames["wikidata"]["wikidata"]

    def upload(self):
        if self.data["upload"] is False:
            print("SKIPPING ITEM")
            return
        labels = self.make_labels()
        descriptions = self.make_descriptions()
        aliases = self.make_aliases()
        claims = self.data["statements"]
        self.add_labels(self.wd_item, labels, aliases, self.log)
        self.add_descriptions(self.wd_item, descriptions, self.log)
        self.add_claims(self.wd_item, claims, self.log)

    def set_wd_item(self):
        """
        Determine WD item to manipulate.

        In live mode, if data object has associated WD item,
        edit it. Otherwise, create a new WD item.
        In sandbox mode, all edits are done on the WD Sandbox item.
        """
        if self.live:
            if self.data["wd-item"] is None:
                self.wd_item = self.create_new_item(self.log)
                self.wd_item_q = self.wd_item.getID()
            else:
                item_q = self.data["wd-item"]
                self.wd_item = self.wdstuff.QtoItemPage(item_q)
                self.wd_item_q = item_q
        else:
            self.wd_item = self.wdstuff.QtoItemPage(self.TEST_ITEM)
            self.wd_item_q = self.TEST_ITEM

    def __init__(self, monument_object, log=None, tablename=None, live=False):
        self.log = False
        self.summary = "#COH #WLM #{}".format(tablename)
        self.live = live
        print("User: {}".format(self.get_username()))
        print("Edit summary: {}".format(self.summary))
        if self.live:
            print("LIVE MODE")
        else:
            print("SANDBOX MODE: {}".format(self.TEST_ITEM))
        print("---------------")
        if log is not None:
            self.log = log
        self.data = monument_object.wd_item
        site = pywikibot.Site("wikidata", "wikidata")
        self.repo = site.data_repository()
        self.wdstuff = WDS(self.repo, edit_summary=self.summary)
        self.set_wd_item()
<|endoftext|>


<commit_before>from flask import (request, g, Blueprint, make_response, Response)
from flask import jsonify as flask_jsonify
from flask.ext.restful import reqparse
from sqlalchemy import asc, func
from sqlalchemy.orm.exc import NoResultFound

from inbox.models import (Message, Block, Part, Thread, Namespace,
                          Contact, Calendar, Event, Transaction,
                          DataProcessingCache, Category, MessageCategory)
from inbox.models.event import RecurringEvent, RecurringEventOverride
from inbox.models.category import EPOCH
from inbox.models.backends.generic import GenericAccount
from inbox.api.sending import (send_draft, send_raw_mime, send_draft_copy,
                               update_draft_on_send)
from inbox.api.update import update_message, update_thread
from inbox.api.kellogs import APIEncoder
from inbox.api import filtering
from inbox.api.validation import (valid_account, get_attachments, get_calendar,
                                  get_recipients, get_draft, valid_public_id,
                                  valid_event, valid_event_update, timestamp,
                                  bounded_str, view, strict_parse_args,
                                  limit, offset, ValidatableArgument,
                                  strict_bool, validate_draft_recipients,
                                  valid_delta_object_types, valid_display_name,
                                  noop_event_update, valid_category_type,
                                  comma_separated_email_list,
                                  get_sending_draft)
from inbox.config import config
from inbox.contacts.algorithms import (calculate_contact_scores,
                                       calculate_group_scores,
                                       calculate_group_counts, is_stale)
import inbox.contacts.crud
from inbox.contacts.search import ContactSearchClient
from inbox.sendmail.base import (create_message_from_json, update_draft,
                                 delete_draft, create_draft_from_mime,
                                 SendMailException)
from inbox.ignition import engine_manager
from inbox.models.action_log import schedule_action
from inbox.models.session import new_session, session_scope
from inbox.search.base import get_search_client, SearchBackendException
from inbox.transactions import delta_sync
from inbox.api.err import (err, APIException, NotFoundError, InputError,
                           AccountDoesNotExistError, log_exception)
from inbox.events.ical import generate_rsvp, send_rsvp
from inbox.events.util import removed_participants
from inbox.util.blockstore import get_from_blockstore
from inbox.util.misc import imap_folder_path
from inbox.actions.backends.generic import remote_delete_sent
from inbox.crispin import writable_connection_pool
from inbox.auth.gmail import GmailAuthHandler

app = Blueprint(
    'jobscore_custom_api',
    __name__,
    url_prefix='/c')

app.log_exception = log_exception

@app.before_request
def start():
    request.environ['log_context'] = {
        'endpoint': request.endpoint,
    }
    # g.encoder = APIEncoder(g.namespace.public_id, is_n1=is_n1)
    g.parser = reqparse.RequestParser(argument_class=ValidatableArgument)

@app.errorhandler(APIException)
def handle_input_error(error):
    # these "errors" are normal, so we don't need to save a traceback
    request.environ['log_context']['error'] = error.__class__.__name__
    request.environ['log_context']['error_message'] = error.message
    response = flask_jsonify(message=error.message,
                             type='invalid_request_error')
    response.status_code = error.status_code
    return response

@app.errorhandler(Exception)
def handle_generic_error(error):
    log_exception(sys.exc_info())
    response = flask_jsonify(message=error.message,
                             type='api_error')
    response.status_code = 500
    return response

@app.route('/auth_callback')
def auth_callback():
    g.parser.add_argument('authorization_code', type=bounded_str,
        location='args', required=True)
    g.parser.add_argument('email', type=bounded_str, location='args')
    g.parser.add_argument('target', type=int, location='args')
    args = strict_parse_args(g.parser, request.args)

    shard = args.get('target', 0) >> 48

    with session_scope(shard) as db_session:
        account = db_session.query(Account).filter_by(email_address=args['email']).first()
        if account is not None:
            return 'Account is already registered', 400

        auth_handler = handler_from_provider('gmail')

        args = {
            'client_id': GmailAuthHandler.OAUTH_CLIENT_ID,
            'client_secret': GmailAuthHandler.OAUTH_CLIENT_SECRET,
            'redirect_uri': GmailAuthHandler.OAUTH_REDIRECT_URI,
            'code': args['authorization_code'],
            'grant_type': 'authorization_code'
        }

        headers = {'Content-type': 'application/x-www-form-urlencoded',
                   'Accept': 'text/plain'}

        data = urllib.urlencode(args)
        resp_dict = requests.post(auth_handler.OAUTH_ACCESS_TOKEN_URL, data=data, headers=headers).json()

        if u'error' in resp_dict:
            return 'Internal error: ' + str(resp_dict['error']), 500

        access_token = resp_dict['access_token']
        validation_dict = auth_handler.validate_token(access_token)
        userinfo_dict = auth_handler._get_user_info(access_token)

        resp_dict.update(validation_dict)
        resp_dict.update(userinfo_dict)
        resp_dict['contacts'] = True
        resp_dict['events'] = True

        auth_info = { 'provider': 'gmail' }
        auth_info.update(resp_dict)

        account = auth_handler.create_account(args['email'], auth_info)
        try:
            if auth_handler.verify_account(account):
                db_session.add(account)
                db_session.commit()
        except NotSupportedError as e:
            return 'Internal error: ' + str(resp_dict['error']), 500

        resp = simplejson.dumps({ 'account_id': account.public_id, 'namespace_id': account.namespace.public_id })
        return make_response( (resp, 201, { 'Content-Type': 'application/json' }))
<commit_msg>Rename variable, fix bug; fix imports<commit_after>import sys
import urllib
import requests
import simplejson
from flask import (request, g, Blueprint, make_response, Response)
from flask import jsonify as flask_jsonify
from flask.ext.restful import reqparse
from sqlalchemy import asc, func
from sqlalchemy.orm.exc import NoResultFound

from inbox.models import (Message, Block, Part, Thread, Namespace,
                          Contact, Calendar, Event, Transaction,
                          DataProcessingCache, Category, MessageCategory)
from inbox.models.event import RecurringEvent, RecurringEventOverride
from inbox.models.category import EPOCH
from inbox.models.backends.generic import GenericAccount
from inbox.api.sending import (send_draft, send_raw_mime, send_draft_copy,
                               update_draft_on_send)
from inbox.api.update import update_message, update_thread
from inbox.api.kellogs import APIEncoder
from inbox.api import filtering
from inbox.api.validation import (valid_account, get_attachments, get_calendar,
                                  get_recipients, get_draft, valid_public_id,
                                  valid_event, valid_event_update, timestamp,
                                  bounded_str, view, strict_parse_args,
                                  limit, offset, ValidatableArgument,
                                  strict_bool, validate_draft_recipients,
                                  valid_delta_object_types, valid_display_name,
                                  noop_event_update, valid_category_type,
                                  comma_separated_email_list,
                                  get_sending_draft)
from inbox.config import config
from inbox.contacts.algorithms import (calculate_contact_scores,
                                       calculate_group_scores,
                                       calculate_group_counts, is_stale)
import inbox.contacts.crud
from inbox.contacts.search import ContactSearchClient
from inbox.sendmail.base import (create_message_from_json, update_draft,
                                 delete_draft, create_draft_from_mime,
                                 SendMailException)
from inbox.ignition import engine_manager
from inbox.models.action_log import schedule_action
from inbox.models.session import new_session, session_scope
from inbox.search.base import get_search_client, SearchBackendException
from inbox.transactions import delta_sync
from inbox.api.err import (err, APIException, NotFoundError, InputError,
                           AccountDoesNotExistError, log_exception)
from inbox.events.ical import generate_rsvp, send_rsvp
from inbox.events.util import removed_participants
from inbox.util.blockstore import get_from_blockstore
from inbox.util.misc import imap_folder_path
from inbox.actions.backends.generic import remote_delete_sent
from inbox.crispin import writable_connection_pool
from inbox.auth.gmail import GmailAuthHandler
from inbox.models import Account
from inbox.auth.base import handler_from_provider

app = Blueprint(
    'jobscore_custom_api',
    __name__,
    url_prefix='/c')

app.log_exception = log_exception

@app.before_request
def start():
    request.environ['log_context'] = {
        'endpoint': request.endpoint,
    }
    # g.encoder = APIEncoder(g.namespace.public_id, is_n1=is_n1)
    g.parser = reqparse.RequestParser(argument_class=ValidatableArgument)

@app.errorhandler(APIException)
def handle_input_error(error):
    # these "errors" are normal, so we don't need to save a traceback
    request.environ['log_context']['error'] = error.__class__.__name__
    request.environ['log_context']['error_message'] = error.message
    response = flask_jsonify(message=error.message,
                             type='invalid_request_error')
    response.status_code = error.status_code
    return response

@app.errorhandler(Exception)
def handle_generic_error(error):
    log_exception(sys.exc_info())
    response = flask_jsonify(message=error.message,
                             type='api_error')
    response.status_code = 500
    return response

@app.route('/auth_callback')
def auth_callback():
    g.parser.add_argument('authorization_code', type=bounded_str,
        location='args', required=True)
    g.parser.add_argument('email', required=True, type=bounded_str, location='args')
    g.parser.add_argument('target', type=int, location='args')
    args = strict_parse_args(g.parser, request.args)

    shard = args.get('target', 0) >> 48

    with session_scope(shard) as db_session:
        account = db_session.query(Account).filter_by(email_address=args['email']).first()
        if account is not None:
            return 'Account is already registered', 400

        auth_handler = handler_from_provider('gmail')

        request_args = {
            'client_id': GmailAuthHandler.OAUTH_CLIENT_ID,
            'client_secret': GmailAuthHandler.OAUTH_CLIENT_SECRET,
            'redirect_uri': GmailAuthHandler.OAUTH_REDIRECT_URI,
            'code': args['authorization_code'],
            'grant_type': 'authorization_code'
        }

        headers = {'Content-type': 'application/x-www-form-urlencoded',
                   'Accept': 'text/plain'}

        data = urllib.urlencode(request_args)
        resp_dict = requests.post(auth_handler.OAUTH_ACCESS_TOKEN_URL, data=data, headers=headers).json()

        # import pdb; pdb.set_trace();

        if u'error' in resp_dict:
            return 'Internal error: ' + str(resp_dict['error']), 500

        access_token = resp_dict['access_token']
        validation_dict = auth_handler.validate_token(access_token)
        userinfo_dict = auth_handler._get_user_info(access_token)

        resp_dict.update(validation_dict)
        resp_dict.update(userinfo_dict)
        resp_dict['contacts'] = True
        resp_dict['events'] = True

        auth_info = { 'provider': 'gmail' }
        auth_info.update(resp_dict)

        account = auth_handler.create_account(args['email'], auth_info)
        try:
            if auth_handler.verify_account(account):
                db_session.add(account)
                db_session.commit()
        except NotSupportedError as e:
            return 'Internal error: ' + str(resp_dict['error']), 500

        resp = simplejson.dumps({
            'account_id': account.public_id,
            'namespace_id': account.namespace.public_id
        })

        return make_response( (resp, 201, { 'Content-Type': 'application/json' }))
<|endoftext|>


<commit_before>#!/usr/bin/python
import json
import datetime
from collections import defaultdict

from flask import Flask, g, request

from sqlalchemy import func
from sqlalchemy.orm.exc import NoResultFound

from inbox.ignition import main_engine
from inbox.models.session import InboxSession
from inbox.models import Account, backend_module_registry
from inbox.api.err import err
from inbox.util.itert import partition

engine = main_engine(pool_size=5)

app = Flask(__name__, static_url_path='')
app.debug = False

@app.before_request
def start():
    g.db_session = InboxSession(engine)

@app.after_request
def finish(response):
    g.db_session.close()
    return response

@app.route('/')
def root():
        return app.send_static_file('index.html')

@app.route('/accounts', methods=['GET'])
def accounts():
    accounts = g.db_session.query(Account).all()
    accounts_info = []

    imap, eas = partition(lambda a: a.provider == 'eas', accounts)

    if imap:
        accounts_info.extend(calculate_imap_status(g.db_session, imap))

    if eas and 'eas' in backend_module_registry:
        accounts_info.extend(calculate_eas_status(g.db_session, eas))

    return json.dumps(accounts_info, cls=DateTimeJSONEncoder)

def calculate_imap_status(db_session, accts):
        from inbox.models.backends.imap import ImapFolderSyncStatus, ImapUid

        ids = [a.id for a in accts]

        metrics = db_session.query(
            ImapFolderSyncStatus.account_id,
            ImapFolderSyncStatus._metrics).filter(
            ImapFolderSyncStatus.account_id.in_(ids)).all()

        numuids = db_session.query(
            ImapUid.account_id, func.count(ImapUid.id)).group_by(
            ImapUid.account_id).all()

        remote = defaultdict(int)
        for m in metrics:
            remote[m[0]] += m[1].get('remote_uid_count', 0)

        local = defaultdict(int)
        for m in numuids:
            local[m[0]] = m[1]

        accounts_info = []
        for a in accts:
            status = a.sync_status
            status.update(remote_count=remote[a.id], local_count=local[a.id])

            accounts_info.append(status)

        return accounts_info

def calculate_eas_status(db_session, accts):
        from inbox.models.backends.eas import EASFolderSyncStatus, EASUid

        ids = [a.id for a in accts]

        metrics = db_session.query(
            EASFolderSyncStatus.account_id,
            EASFolderSyncStatus._metrics).filter(
            EASFolderSyncStatus.account_id.in_(ids)).all()

        numuids = db_session.query(
            EASUid.account_id, func.count(EASUid.id)).group_by(
            EASUid.account_id).all()

        remote = defaultdict(int)
        for m in metrics:
            remote[m[0]] += m[1].get('total_remote_count', 0)

        local = defaultdict(int)
        for m in numuids:
            local[m[0]] = m[1]

        accounts_info = []
        for a in accts:
            status = a.sync_status
            status.update(remote_count=remote[a.id], local_count=local[a.id])

            accounts_info.append(status)

        return accounts_info

@app.route('/account/<account_id>', methods=['GET'])
def account(account_id):
    try:
        account = g.db_session.query(Account).get(account_id)
    except NoResultFound:
        return err(404, 'No account with id `{0}`'.format(account_id))

    if 'action' in request.args:
        action = request.args.get('action', None)
        if action == 'stop':
            if account.sync_enabled:
                account.stop_sync()
                g.db_session.add(account)
                g.db_session.commit()
        elif action == 'start':
            account.start_sync()
            g.db_session.add(account)
            g.db_session.commit()

    if account:
        folders_info = [foldersyncstatus.metrics for foldersyncstatus in
                        account.foldersyncstatuses]
        sync_status = account.sync_status
    else:
        folders_info = []
        sync_status = {}

    return json.dumps({"account": sync_status,
                       "folders": folders_info}, cls=DateTimeJSONEncoder)

class DateTimeJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return str(obj)
        else:
            return super(DateTimeJSONEncoder, self).default(obj)

if __name__ == '__main__':
    from setproctitle import setproctitle
    import os
    setproctitle('monocle')
    os.environ['DEBUG'] = 'true' if app.debug else 'false'
    #app.run(host='127.0.0.1')
    app.run(host='0.0.0.0', debug=True)
<commit_msg>Fix typo.<commit_after>#!/usr/bin/python
import json
import datetime
from collections import defaultdict

from flask import Flask, g, request

from sqlalchemy import func
from sqlalchemy.orm.exc import NoResultFound

from inbox.ignition import main_engine
from inbox.models.session import InboxSession
from inbox.models import Account, backend_module_registry
from inbox.api.err import err
from inbox.util.itert import partition

engine = main_engine(pool_size=5)

app = Flask(__name__, static_url_path='')
app.debug = False

@app.before_request
def start():
    g.db_session = InboxSession(engine)

@app.after_request
def finish(response):
    g.db_session.close()
    return response

@app.route('/')
def root():
        return app.send_static_file('index.html')

@app.route('/accounts', methods=['GET'])
def accounts():
    accounts = g.db_session.query(Account).all()
    accounts_info = []

    imap, eas = partition(lambda a: a.provider == 'eas', accounts)

    if imap:
        accounts_info.extend(calculate_imap_status(g.db_session, imap))

    if eas and 'eas' in backend_module_registry:
        accounts_info.extend(calculate_eas_status(g.db_session, eas))

    return json.dumps(accounts_info, cls=DateTimeJSONEncoder)

def calculate_imap_status(db_session, accts):
        from inbox.models.backends.imap import ImapFolderSyncStatus, ImapUid

        ids = [a.id for a in accts]

        metrics = db_session.query(
            ImapFolderSyncStatus.account_id,
            ImapFolderSyncStatus._metrics).filter(
            ImapFolderSyncStatus.account_id.in_(ids)).all()

        numuids = db_session.query(
            ImapUid.account_id, func.count(ImapUid.id)).group_by(
            ImapUid.account_id).all()

        remote = defaultdict(int)
        for m in metrics:
            remote[m[0]] += m[1].get('remote_uid_count', 0)

        local = defaultdict(int)
        for m in numuids:
            local[m[0]] = m[1]

        accounts_info = []
        for a in accts:
            status = a.sync_status
            status.update(remote_count=remote[a.id], local_count=local[a.id])

            accounts_info.append(status)

        return accounts_info

def calculate_eas_status(db_session, accts):
        from inbox.models.backends.eas import EASFolderSyncStatus, EASUid

        ids = [a.id for a in accts]

        metrics = db_session.query(
            EASFolderSyncStatus.account_id,
            EASFolderSyncStatus._metrics).filter(
            EASFolderSyncStatus.account_id.in_(ids)).all()

        numuids = db_session.query(
            EASUid.easaccount_id, func.count(EASUid.id)).group_by(
            EASUid.easaccount_id).all()

        remote = defaultdict(int)
        for m in metrics:
            remote[m[0]] += m[1].get('total_remote_count', 0)

        local = defaultdict(int)
        for m in numuids:
            local[m[0]] = m[1]

        accounts_info = []
        for a in accts:
            status = a.sync_status
            status.update(remote_count=remote[a.id], local_count=local[a.id])

            accounts_info.append(status)

        return accounts_info

@app.route('/account/<account_id>', methods=['GET'])
def account(account_id):
    try:
        account = g.db_session.query(Account).get(account_id)
    except NoResultFound:
        return err(404, 'No account with id `{0}`'.format(account_id))

    if 'action' in request.args:
        action = request.args.get('action', None)
        if action == 'stop':
            if account.sync_enabled:
                account.stop_sync()
                g.db_session.add(account)
                g.db_session.commit()
        elif action == 'start':
            account.start_sync()
            g.db_session.add(account)
            g.db_session.commit()

    if account:
        folders_info = [foldersyncstatus.metrics for foldersyncstatus in
                        account.foldersyncstatuses]
        sync_status = account.sync_status
    else:
        folders_info = []
        sync_status = {}

    return json.dumps({"account": sync_status,
                       "folders": folders_info}, cls=DateTimeJSONEncoder)

class DateTimeJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return str(obj)
        else:
            return super(DateTimeJSONEncoder, self).default(obj)

if __name__ == '__main__':
    from setproctitle import setproctitle
    import os
    setproctitle('monocle')
    os.environ['DEBUG'] = 'true' if app.debug else 'false'
    app.run(host='127.0.0.1')
<|endoftext|>


<commit_before>"""

"""
import re
import sys
from types import ModuleType
from typing import Any, Dict, List, Optional, Tuple, Type, Union

import pkg_resources
from packaging.version import Version

from .errors import ConfigurationError
from .limits import GRANULARITIES, RateLimitItem

SEPARATORS = re.compile(r"[,;|]{1}")
SINGLE_EXPR = re.compile(
    r"""
    \s*([0-9]+)
    \s*(/|\s*per\s*)
    \s*([0-9]+)
    *\s*(hour|minute|second|day|month|year)s?\s*""",
    re.IGNORECASE | re.VERBOSE,
)
EXPR = re.compile(
    r"^{SINGLE}(:?{SEPARATORS}{SINGLE})*$".format(
        SINGLE=SINGLE_EXPR.pattern, SEPARATORS=SEPARATORS.pattern
    ),
    re.IGNORECASE | re.VERBOSE,
)

class LazyDependency:
    """
    Simple utility that provides an :attr:`dependency`
    to the child class to fetch any dependencies
    without having to import them explicitly.
    """

    DEPENDENCIES: Union[Dict[str, Optional[Version]], List[str]] = []
    FAIL_ON_MISSING_DEPENDENCY: bool = True
    """
    The python modules this class has a dependency on.
    Used to lazily populate the :attr:`dependencies`
    """

    def __init__(self) -> None:
        self._dependencies: Dict[str, Optional[ModuleType]] = {}

    @property
    def dependencies(self) -> Dict[str, Any]:
        """
        Cached mapping of the modules this storage depends on.
        This is done so that the module is only imported lazily
        when the storage is instantiated.
        """
        if not getattr(self, "_dependencies", {}):
            dependencies = {}
            mapping: Dict[str, Optional[Version]]

            if isinstance(self.DEPENDENCIES, list):
                mapping = {dependency: None for dependency in self.DEPENDENCIES}
            else:
                mapping = self.DEPENDENCIES

            for name, minimum_version in mapping.items():
                dependency, version = get_dependency(name)

                if self.FAIL_ON_MISSING_DEPENDENCY:
                    if not dependency:
                        raise ConfigurationError(
                            f"{name} prerequisite not available"
                        )  # pragma: no cover
                    if minimum_version and version and version < minimum_version:
                        raise ConfigurationError(
                            f"The minimum version of {minimum_version} of {name} could not be found"
                        )
                if minimum_version and not (version and version >= minimum_version):
                    dependency = None
                dependencies[name] = dependency
            self._dependencies = dependencies
        return self._dependencies

def get_dependency(module_path: str) -> Tuple[Optional[ModuleType], Optional[Version]]:
    """
    safe function to import a module at runtime
    """
    try:
        if module_path not in sys.modules:
            __import__(module_path)
        root = module_path.split(".")[0]
        version = getattr(sys.modules[root], "__version__", "0.0.0")
        return sys.modules[module_path], Version(version)
    except ImportError:  # pragma: no cover
        return None, None

def get_package_data(path: str) -> bytes:
    return pkg_resources.resource_string(__name__, path)

def parse_many(limit_string: str) -> List[RateLimitItem]:
    """
    parses rate limits in string notation containing multiple rate limits
    (e.g. ``1/second; 5/minute``)

    :param limit_string: rate limit string using :ref:`ratelimit-string`
    :raise ValueError: if the string notation is invalid.

    """

    if not (isinstance(limit_string, str) and EXPR.match(limit_string)):
        raise ValueError("couldn't parse rate limit string '%s'" % limit_string)
    limits = []

    for limit in SEPARATORS.split(limit_string):
        match = SINGLE_EXPR.match(limit)

        if match:
            amount, _, multiples, granularity_string = match.groups()
            granularity = granularity_from_string(granularity_string)
            limits.append(
                granularity(int(amount), multiples and int(multiples) or None)
            )

    return limits

def parse(limit_string: str) -> RateLimitItem:
    """
    parses a single rate limit in string notation
    (e.g. ``1/second`` or ``1 per second``)

    :param limit_string: rate limit string using :ref:`ratelimit-string`
    :raise ValueError: if the string notation is invalid.

    """

    return list(parse_many(limit_string))[0]

def granularity_from_string(granularity_string: str) -> Type[RateLimitItem]:
    """

    :param granularity_string:
    :raise ValueError:
    """

    for granularity in GRANULARITIES.values():
        if granularity.check_granularity_string(granularity_string):
            return granularity
    raise ValueError("no granularity matched for %s" % granularity_string)
<commit_msg>Tighten utils type annotations<commit_after>"""

"""
import re
import sys
from types import ModuleType
from typing import Dict, List, Optional, Tuple, Type, Union

import pkg_resources
from packaging.version import Version

from .errors import ConfigurationError
from .limits import GRANULARITIES, RateLimitItem

SEPARATORS = re.compile(r"[,;|]{1}")
SINGLE_EXPR = re.compile(
    r"""
    \s*([0-9]+)
    \s*(/|\s*per\s*)
    \s*([0-9]+)
    *\s*(hour|minute|second|day|month|year)s?\s*""",
    re.IGNORECASE | re.VERBOSE,
)
EXPR = re.compile(
    r"^{SINGLE}(:?{SEPARATORS}{SINGLE})*$".format(
        SINGLE=SINGLE_EXPR.pattern, SEPARATORS=SEPARATORS.pattern
    ),
    re.IGNORECASE | re.VERBOSE,
)

class LazyDependency:
    """
    Simple utility that provides an :attr:`dependency`
    to the child class to fetch any dependencies
    without having to import them explicitly.
    """

    DEPENDENCIES: Union[Dict[str, Optional[Version]], List[str]] = []
    FAIL_ON_MISSING_DEPENDENCY: bool = True
    """
    The python modules this class has a dependency on.
    Used to lazily populate the :attr:`dependencies`
    """

    def __init__(self) -> None:
        self._dependencies: Dict[str, Optional[ModuleType]] = {}

    @property
    def dependencies(self) -> Dict[str, Optional[ModuleType]]:
        """
        Cached mapping of the modules this storage depends on.
        This is done so that the module is only imported lazily
        when the storage is instantiated.
        """
        if not getattr(self, "_dependencies", {}):
            dependencies = {}
            mapping: Dict[str, Optional[Version]]

            if isinstance(self.DEPENDENCIES, list):
                mapping = {dependency: None for dependency in self.DEPENDENCIES}
            else:
                mapping = self.DEPENDENCIES

            for name, minimum_version in mapping.items():
                dependency, version = get_dependency(name)

                if self.FAIL_ON_MISSING_DEPENDENCY:
                    if not dependency:
                        raise ConfigurationError(
                            f"{name} prerequisite not available"
                        )  # pragma: no cover
                    if minimum_version and version and version < minimum_version:
                        raise ConfigurationError(
                            f"The minimum version of {minimum_version} of {name} could not be found"
                        )
                if minimum_version and not (version and version >= minimum_version):
                    dependency = None
                dependencies[name] = dependency
            self._dependencies = dependencies
        return self._dependencies

def get_dependency(module_path: str) -> Tuple[Optional[ModuleType], Optional[Version]]:
    """
    safe function to import a module at runtime
    """
    try:
        if module_path not in sys.modules:
            __import__(module_path)
        root = module_path.split(".")[0]
        version = getattr(sys.modules[root], "__version__", "0.0.0")
        return sys.modules[module_path], Version(version)
    except ImportError:  # pragma: no cover
        return None, None

def get_package_data(path: str) -> bytes:
    return pkg_resources.resource_string(__name__, path)

def parse_many(limit_string: str) -> List[RateLimitItem]:
    """
    parses rate limits in string notation containing multiple rate limits
    (e.g. ``1/second; 5/minute``)

    :param limit_string: rate limit string using :ref:`ratelimit-string`
    :raise ValueError: if the string notation is invalid.

    """

    if not (isinstance(limit_string, str) and EXPR.match(limit_string)):
        raise ValueError("couldn't parse rate limit string '%s'" % limit_string)
    limits = []

    for limit in SEPARATORS.split(limit_string):
        match = SINGLE_EXPR.match(limit)

        if match:
            amount, _, multiples, granularity_string = match.groups()
            granularity = granularity_from_string(granularity_string)
            limits.append(
                granularity(int(amount), multiples and int(multiples) or None)
            )

    return limits

def parse(limit_string: str) -> RateLimitItem:
    """
    parses a single rate limit in string notation
    (e.g. ``1/second`` or ``1 per second``)

    :param limit_string: rate limit string using :ref:`ratelimit-string`
    :raise ValueError: if the string notation is invalid.

    """

    return list(parse_many(limit_string))[0]

def granularity_from_string(granularity_string: str) -> Type[RateLimitItem]:
    """

    :param granularity_string:
    :raise ValueError:
    """

    for granularity in GRANULARITIES.values():
        if granularity.check_granularity_string(granularity_string):
            return granularity
    raise ValueError("no granularity matched for %s" % granularity_string)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from utilUdpCmd import recvCommand, procCommand, getSetting
from utilSetting import CSetting
from utilComRelay import comrelay, comReopen
from utilLogger import CUtilLogger
import socket

#--- selection of import based on the package ---
''' 1. with python3-serial'''
import serial
''' 2. without python3-serial'''
#from dummySerial import CDummySerial
#serial = CDummySerial()
#-----------------

'''
v0.8 2015/12/08
  - avoid udp send error when the network is not available
v0.7 2015/12/02
  - change prefix string from "1:" to "rcvd1," for csv handling
  - save Log feature
v0.6 2015/11/28
  - add 1st line for auto start at /etc/rc.local
v0.5 2015/11/21
  - tweak Seiral timeout to fix slowness of the communication
v0.4 2015/11/21
  - fix bug on monport > TypeError: an integer is required
v0.3 2015/11/16
  - can change combaud (but not using comReopen())
  - add get/set settings (combaud)
v0.2 2015/11/13
  - add dummySerial.py
v0.1 2015/11/13
  - can get/set settings (monitor ip, monitor port, com delay[msec])
'''

# HACKME: something does not sit right concerning "setting" passing (g_setting)

g_setting = CSetting() # for having g_setting as global
logger = CUtilLogger()

def main():
	# command udp setting
	cmdip = "" # INADDR_ANY
	g_setting = getSetting() # HACKME:
	cmdport = g_setting.getCmdPort()
	cmdsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	cmdsock.bind((cmdip, cmdport))
	cmdsock.setblocking(0)
	# monitor udp setting (no binding)
	monip = g_setting.getMonip()
	monport = g_setting.getMonport()
	monsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	# COM setting
	con1=serial.Serial('/dev/ttyUSB1', 9600, timeout=0.001)
	con2=serial.Serial('/dev/ttyUSB0', 9600, timeout=0.001)

	print "cmdport=", cmdport

	rcvd1 = ""
	rcvd2 = ""
	rcvcmd = ""

# TODO: 0z > add COM delay feature

	while 1:
		#HACKME: ---- calling getSetting(), then, g_setting.getXXX() is not a good style
		g_setting = getSetting()
		monip = g_setting.getMonip()
		monport = g_setting.getMonport()

		if g_setting.getBaudChange() == True:
#			comReopen( con1, con2, g_setting.getCombaud() )
			con1.close()
			con2.close()
			combaud = g_setting.getCombaud()
			con1=serial.Serial('/dev/ttyUSB1', combaud, timeout=0.001)
			con2=serial.Serial('/dev/ttyUSB0', combaud, timeout=0.001)
			g_setting.setBaudChange(False)

		rcvd1,isNL = comrelay(rcvd1, con1, con2)
		if isNL == True: # new line
			text = "rcvd1," + rcvd1
			try:
				monsock.sendto(text, (monip, int(monport)))
			except socket.error, msg:
				print str(msg[0])
			logger.add(text)
			rcvd1 = ""

		rcvd2,isNL = comrelay(rcvd2, con2, con1)
		if isNL == True:
			text = "rcvd2," + rcvd2
			try:
				monsock.sendto(text, (monip, int(monport)))
			except socket.error, msg:
				print str(msg[0])
			logger.add(text)
			rcvd2 = ""

		rcvcmd,rcvd = recvCommand(cmdsock, rcvcmd)
		if rcvd == True and "\n" in rcvcmd:
			procCommand(rcvcmd)
			rcvcmd = ""

if __name__ == '__main__':
	main()

# TODO: 0m> fix for UDP fail when RPi2 does not get IP address > to work only COM relay
<commit_msg>show concrete error message<commit_after>#!/usr/bin/env python
from utilUdpCmd import recvCommand, procCommand, getSetting
from utilSetting import CSetting
from utilComRelay import comrelay, comReopen
from utilLogger import CUtilLogger
import socket

#--- selection of import based on the package ---
''' 1. with python3-serial'''
import serial
''' 2. without python3-serial'''
#from dummySerial import CDummySerial
#serial = CDummySerial()
#-----------------

'''
v0.8 2015/12/08
  - avoid udp send error when the network is not available
v0.7 2015/12/02
  - change prefix string from "1:" to "rcvd1," for csv handling
  - save Log feature
v0.6 2015/11/28
  - add 1st line for auto start at /etc/rc.local
v0.5 2015/11/21
  - tweak Seiral timeout to fix slowness of the communication
v0.4 2015/11/21
  - fix bug on monport > TypeError: an integer is required
v0.3 2015/11/16
  - can change combaud (but not using comReopen())
  - add get/set settings (combaud)
v0.2 2015/11/13
  - add dummySerial.py
v0.1 2015/11/13
  - can get/set settings (monitor ip, monitor port, com delay[msec])
'''

# HACKME: something does not sit right concerning "setting" passing (g_setting)

g_setting = CSetting() # for having g_setting as global
logger = CUtilLogger()

def main():
	# command udp setting
	cmdip = "" # INADDR_ANY
	g_setting = getSetting() # HACKME:
	cmdport = g_setting.getCmdPort()
	cmdsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	cmdsock.bind((cmdip, cmdport))
	cmdsock.setblocking(0)
	# monitor udp setting (no binding)
	monip = g_setting.getMonip()
	monport = g_setting.getMonport()
	monsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	# COM setting
	con1=serial.Serial('/dev/ttyUSB1', 9600, timeout=0.001)
	con2=serial.Serial('/dev/ttyUSB0', 9600, timeout=0.001)

	print "cmdport=", cmdport

	rcvd1 = ""
	rcvd2 = ""
	rcvcmd = ""

# TODO: 0z > add COM delay feature

	while 1:
		#HACKME: ---- calling getSetting(), then, g_setting.getXXX() is not a good style
		g_setting = getSetting()
		monip = g_setting.getMonip()
		monport = g_setting.getMonport()

		if g_setting.getBaudChange() == True:
#			comReopen( con1, con2, g_setting.getCombaud() )
			con1.close()
			con2.close()
			combaud = g_setting.getCombaud()
			con1=serial.Serial('/dev/ttyUSB1', combaud, timeout=0.001)
			con2=serial.Serial('/dev/ttyUSB0', combaud, timeout=0.001)
			g_setting.setBaudChange(False)

		rcvd1,isNL = comrelay(rcvd1, con1, con2)
		if isNL == True: # new line
			text = "rcvd1," + rcvd1
			try:
				monsock.sendto(text, (monip, int(monport)))
			except socket.error, msg:
				print "udp network is not available." + "err:" + str(msg[0])
			logger.add(text)
			rcvd1 = ""

		rcvd2,isNL = comrelay(rcvd2, con2, con1)
		if isNL == True:
			text = "rcvd2," + rcvd2
			try:
				monsock.sendto(text, (monip, int(monport)))
			except socket.error, msg:
				print "udp network is not available." + "err:" + str(msg[0])
			logger.add(text)
			rcvd2 = ""

		rcvcmd,rcvd = recvCommand(cmdsock, rcvcmd)
		if rcvd == True and "\n" in rcvcmd:
			procCommand(rcvcmd)
			rcvcmd = ""

if __name__ == '__main__':
	main()

# TODO: 0m> fix for UDP fail when RPi2 does not get IP address > to work only COM relay
<|endoftext|>


<commit_before>import numpy as np
import math
import scipy.interpolate as si
import iso_obj as io

class iso_grid_tefflogg:
	'''A class which allows isochrones to be queried for many objects at the same time
		- set up to work in {Teff, logg, feh} space
		- Asumes a regular grid in '''

	def __init__(self, filename, metal_col=0, Mi_col=1, logage_col=2, teff_col=3, logg_col=4, Jac_col=5, verbose=False):
		self.metal_col=metal_col
		self.teff_col=teff_col
		self.logg_col=logg_col

		self.verbose=verbose

		iso_array=np.loadtxt(filename)
		self.metal_dict={}

		self.metal_interp=None

		self.teff_step=0.
		self.teff_min=0.
		self.teff_gridlen=0.

		self.logg_step=0.
		self.logg_min=0.
		self.logg_gridlen=0.

		self.register(iso_array)

		abs_mags={}
		abs_mags["r"]=iso_array[:,6]
		abs_mags["i"]=iso_array[:,7]
		abs_mags["Ha"]=iso_array[:,8]

		self.iso_array2=io.iso_objs(iso_array[:,Mi_col], iso_array[:,logage_col], iso_array[:,metal_col], iso_array[:,teff_col], iso_array[:,logg_col],
						 abs_mags, iso_array[:,Jac_col])

	def register(self, array):

		if self.verbose:
			print "Registering"

		for i in range(array.shape[0]):

			if array[i,self.metal_col] not in self.metal_dict:
				self.metal_dict[ array[i,self.metal_col] ]=i
#				if self.metal_step==None and i!=0:
#					self.metal_step=i
		if self.verbose:
			print "metal dict:", self.metal_dict

		self.metal_interp=si.interp1d(sorted(self.metal_dict.keys()), sorted(self.metal_dict.values()), kind='nearest', bounds_error=False)

		self.teff_min=array[0,self.teff_col]
		for i in range(sorted(self.metal_dict.values())[1]):
			if array[i,self.teff_col]!=self.teff_min:
				self.teff_step=array[i,self.teff_col]-self.teff_min
				self.teff_gridlen=i
				break
		if self.verbose:
			print "Teff grid:", self.teff_min, self.teff_step, self.teff_gridlen

		self.logg_min=array[0,self.logg_col]
		for i in range(self.teff_gridlen):
			if array[i,self.logg_col]!=self.logg_min:
				self.logg_step=array[i,self.logg_col]-self.logg_min
				self.logg_gridlen=i
				break
		if self.verbose:
			print "logg grid:", self.logg_min, self.logg_step, self.logg_gridlen

		self.teff_max=np.max(array[:,self.teff_col])
		self.logg_max=np.max(array[:,self.logg_col])

	def query(self, feh, teff, logg):

		if teff>self.teff_max or teff<self.teff_min or logg>self.logg_max or logg<self.logg_min:
			raise IndexError()

		rows=(self.metal_interp(feh)
			+ ( np.rint((teff-self.teff_min)/self.teff_step)*self.teff_gridlen )
			+ ( np.rint((logg-self.logg_min)/self.logg_step)*self.logg_gridlen )).astype(int)

		return self.iso_array2.subset(rows)

<commit_msg>Working out which columns contain required photom from file header<commit_after>import numpy as np
import math
import scipy.interpolate as si
import iso_obj as io

class iso_grid_tefflogg:
	'''A class which allows isochrones to be queried for many objects at the same time
		- set up to work in {Teff, logg, feh} space
		- Asumes a regular grid in '''

	def __init__(self, filename, metal_col=0, Mi_col=1, logage_col=2, teff_col=3, logg_col=4, Jac_col=5, bands=["r", "i", "Ha"], verbose=False):
		self.metal_col=metal_col
		self.teff_col=teff_col
		self.logg_col=logg_col

		self.verbose=verbose

		iso_array=np.loadtxt(filename)
		self.metal_dict={}

		self.metal_interp=None

		self.teff_step=0.
		self.teff_min=0.
		self.teff_gridlen=0.

		self.logg_step=0.
		self.logg_min=0.
		self.logg_gridlen=0.

		self.register(iso_array)

		abs_mags={}
		with open(filename, 'r') as f:
			first_line = f.readline().split()

		for band in bands:
			abs_mags[band]=iso_array[:,first_line.index(band)-1]

		self.iso_array2=io.iso_objs(iso_array[:,Mi_col], iso_array[:,logage_col], iso_array[:,metal_col], iso_array[:,teff_col], iso_array[:,logg_col],
						 abs_mags, iso_array[:,Jac_col])

	def register(self, array):

		if self.verbose:
			print "Registering"

		for i in range(array.shape[0]):

			if array[i,self.metal_col] not in self.metal_dict:
				self.metal_dict[ array[i,self.metal_col] ]=i
#				if self.metal_step==None and i!=0:
#					self.metal_step=i
		if self.verbose:
			print "metal dict:", self.metal_dict

		self.metal_interp=si.interp1d(sorted(self.metal_dict.keys()), sorted(self.metal_dict.values()), kind='nearest', bounds_error=False)

		self.teff_min=array[0,self.teff_col]
		for i in range(sorted(self.metal_dict.values())[1]):
			if array[i,self.teff_col]!=self.teff_min:
				self.teff_step=array[i,self.teff_col]-self.teff_min
				self.teff_gridlen=i
				break
		if self.verbose:
			print "Teff grid:", self.teff_min, self.teff_step, self.teff_gridlen

		self.logg_min=array[0,self.logg_col]
		for i in range(self.teff_gridlen):
			if array[i,self.logg_col]!=self.logg_min:
				self.logg_step=array[i,self.logg_col]-self.logg_min
				self.logg_gridlen=i
				break
		if self.verbose:
			print "logg grid:", self.logg_min, self.logg_step, self.logg_gridlen

		self.teff_max=np.max(array[:,self.teff_col])
		self.logg_max=np.max(array[:,self.logg_col])

	def query(self, feh, teff, logg):

		if teff>self.teff_max or teff<self.teff_min or logg>self.logg_max or logg<self.logg_min:
			raise IndexError()

		rows=(self.metal_interp(feh)
			+ ( np.rint((teff-self.teff_min)/self.teff_step)*self.teff_gridlen )
			+ ( np.rint((logg-self.logg_min)/self.logg_step)*self.logg_gridlen )).astype(int)

		return self.iso_array2.subset(rows)

<|endoftext|>


<commit_before>import urllib2
from urlparse import urljoin
from locksmith.common import apicall, UNPUBLISHED, PUBLISHED, NEEDS_UPDATE

from celery.task import task

from django.conf import settings

ReplicatedApiNames = getattr(settings, 'LOCKSMITH_REPLICATED_APIS', [])

@task(max_retries=5)
def push_key(key, replicate_too=True):
    if replicate_too:
        for kps in key.pub_statuses.filter(api__push_enabled=True):
            if kps.api.name in ReplicatedApiNames:
                replicate_key.delay(key, kps.api)

    endpoints = {UNPUBLISHED: 'create_key/', NEEDS_UPDATE: 'update_key/'}
    dirty = key.pub_statuses.exclude(status=PUBLISHED).filter(
              api__push_enabled=True).select_related()
    if not dirty:
        print u"Skipping push_key for {k} because all KeyPublicationStatus objects are PUBLISHED.".format(k=key.key)

    # Retrying immediately on failure would allow a broken or unresponsive
    # api to prevent other, properly functioning apis from receiving the key.
    # Thus we use retry_flag to delay the task retry until after attempting
    # to push to all apis.
    retry_flag = False
    for kps in dirty:
        if kps.api.name in ReplicatedApiNames:
            # Skip this API because we've queued a replicate_key task above
            print u"push_key for {k} ignoring {a} because it uses replicate_key.".format(k=key.key, a=kps.api.name)
            continue

        endpoint = urljoin(kps.api.url, endpoints[kps.status])
        try:
            apicall(endpoint, kps.api.signing_key, api=kps.api.name,
                    key=kps.key.key, email=kps.key.email, status=kps.key.status)
            print 'sent key {k} to {a}'.format(k=key.key, a=kps.api.name)
            kps.status = PUBLISHED
            kps.save()

        except Exception as e:
            ctx = {
                'a': str(kps.api.name),
                'k': str(key.key),
                'e': str(e.read()) if isinstance(e, urllib2.HTTPError) else str(e)
            }
            print 'Caught exception while pushing key {k} to {a}: {e}'.format(**ctx)
            print 'Will retry'
            retry_flag = True

    if retry_flag:
        push_key.retry()

@task(max_retries=5)
def replicate_key(key, api):
    kps = key.pub_statuses.get(api=api)
    endpoint = urljoin(kps.api.url, 'replicate_key/{k}/'.format(k=key.key))
    try:
        apicall(endpoint, kps.api.signing_key,
                api=kps.api.name,
                key=kps.key.key,
                email=kps.key.email,
                status=kps.key.status)
        print 'replicated key {k} to {a} with status {s}'.format(k=key.key, s=key.status, a=kps.api.name)
        kps.status = PUBLISHED
        kps.save()

    except Exception as e:
        ctx = {
            'a': str(kps.api.name),
            'k': str(key.key),
            'e': str(e.read()) if isinstance(e, urllib2.HTTPError) else str(e)
        }
        print 'Caught exception while pushing key {k} to {a}: {e}'.format(**ctx)
        print 'Will retry'
        replicate_key.retry_flag()

<commit_msg>s/retry_flag/retry/<commit_after>import urllib2
from urlparse import urljoin
from locksmith.common import apicall, UNPUBLISHED, PUBLISHED, NEEDS_UPDATE

from celery.task import task

from django.conf import settings

ReplicatedApiNames = getattr(settings, 'LOCKSMITH_REPLICATED_APIS', [])

@task(max_retries=5)
def push_key(key, replicate_too=True):
    if replicate_too:
        for kps in key.pub_statuses.filter(api__push_enabled=True):
            if kps.api.name in ReplicatedApiNames:
                replicate_key.delay(key, kps.api)

    endpoints = {UNPUBLISHED: 'create_key/', NEEDS_UPDATE: 'update_key/'}
    dirty = key.pub_statuses.exclude(status=PUBLISHED).filter(
              api__push_enabled=True).select_related()
    if not dirty:
        print u"Skipping push_key for {k} because all KeyPublicationStatus objects are PUBLISHED.".format(k=key.key)

    # Retrying immediately on failure would allow a broken or unresponsive
    # api to prevent other, properly functioning apis from receiving the key.
    # Thus we use retry_flag to delay the task retry until after attempting
    # to push to all apis.
    retry_flag = False
    for kps in dirty:
        if kps.api.name in ReplicatedApiNames:
            # Skip this API because we've queued a replicate_key task above
            print u"push_key for {k} ignoring {a} because it uses replicate_key.".format(k=key.key, a=kps.api.name)
            continue

        endpoint = urljoin(kps.api.url, endpoints[kps.status])
        try:
            apicall(endpoint, kps.api.signing_key, api=kps.api.name,
                    key=kps.key.key, email=kps.key.email, status=kps.key.status)
            print 'sent key {k} to {a}'.format(k=key.key, a=kps.api.name)
            kps.status = PUBLISHED
            kps.save()

        except Exception as e:
            ctx = {
                'a': str(kps.api.name),
                'k': str(key.key),
                'e': str(e.read()) if isinstance(e, urllib2.HTTPError) else str(e)
            }
            print 'Caught exception while pushing key {k} to {a}: {e}'.format(**ctx)
            print 'Will retry'
            retry_flag = True

    if retry_flag:
        push_key.retry()

@task(max_retries=5)
def replicate_key(key, api):
    kps = key.pub_statuses.get(api=api)
    endpoint = urljoin(kps.api.url, 'replicate_key/{k}/'.format(k=key.key))
    try:
        apicall(endpoint, kps.api.signing_key,
                api=kps.api.name,
                key=kps.key.key,
                email=kps.key.email,
                status=kps.key.status)
        print 'replicated key {k} to {a} with status {s}'.format(k=key.key, s=key.status, a=kps.api.name)
        kps.status = PUBLISHED
        kps.save()

    except Exception as e:
        ctx = {
            'a': str(kps.api.name),
            'k': str(key.key),
            'e': str(e.read()) if isinstance(e, urllib2.HTTPError) else str(e)
        }
        print 'Caught exception while pushing key {k} to {a}: {e}'.format(**ctx)
        print 'Will retry'
        replicate_key.retry()

<|endoftext|>


<commit_before>from locksmith.common import apicall, UNPUBLISHED, PUBLISHED, NEEDS_UPDATE

from celery.task import task

@task(max_retries=5)
def push_key(key):
    endpoints = {UNPUBLISHED: 'create_key/', NEEDS_UPDATE: 'update_key/'}
    dirty = key.pub_statuses.exclude(status=PUBLISHED).filter(
              api__push_enabled=True).select_related()
    for kps in dirty:
        endpoint = urljoin(kps.api.url, endpoints[kps.status])
        try:
            apicall(endpoint, kps.api.signing_key, api=kps.api.name,
                    key=kps.key.key, email=kps.key.email, status=kps.key.status)
        except:
            push_key.retry()
        actions[kps.status] += 1
        kps.status = PUBLISHED
        kps.save()
<commit_msg>fix task<commit_after>from urlparse import urljoin
from locksmith.common import apicall, UNPUBLISHED, PUBLISHED, NEEDS_UPDATE

from celery.task import task

@task(max_retries=5)
def push_key(key):
    endpoints = {UNPUBLISHED: 'create_key/', NEEDS_UPDATE: 'update_key/'}
    dirty = key.pub_statuses.exclude(status=PUBLISHED).filter(
              api__push_enabled=True).select_related()
    for kps in dirty:
        endpoint = urljoin(kps.api.url, endpoints[kps.status])
        try:
            apicall(endpoint, kps.api.signing_key, api=kps.api.name,
                    key=kps.key.key, email=kps.key.email, status=kps.key.status)
        except:
            push_key.retry()
        actions[kps.status] += 1
        kps.status = PUBLISHED
        kps.save()
<|endoftext|>