<commit_before>"""
South Africa-specific Form helpers
"""

from django.core.validators import EMPTY_VALUES
from django.forms import ValidationError
from django.forms.fields import Field, RegexField
from django.utils.checksums import luhn
from django.utils.translation import gettext as _
import re
from datetime import date

id_re = re.compile(r'^(?P<yy>\d\d)(?P<mm>\d\d)(?P<dd>\d\d)(?P<mid>\d{4})(?P<end>\d{3})')

class ZAIDField(Field):
    """A form field for South African ID numbers -- the checksum is validated
    using the Luhn checksum, and uses a simlistic (read: not entirely accurate)
    check for the birthdate
    """
    default_error_messages = {
        'invalid': _(u'Enter a valid South African ID number'),
    }

    def clean(self, value):
        super(ZAIDField, self).clean(value)

        if value in EMPTY_VALUES:
            return u''

        # strip spaces and dashes
        value = value.strip().replace(' ', '').replace('-', '')

        match = re.match(id_re, value)

        if not match:
            raise ValidationError(self.error_messages['invalid'])

        g = match.groupdict()

        try:
            # The year 2000 is conveniently a leapyear.
            # This algorithm will break in xx00 years which aren't leap years
            # There is no way to guess the century of a ZA ID number
            d = date(int(g['yy']) + 2000, int(g['mm']), int(g['dd']))
        except ValueError:
            raise ValidationError(self.error_messages['invalid'])

        if not luhn(value):
            raise ValidationError(self.error_messages['invalid'])

        return value

class ZAPostCodeField(RegexField):
    default_error_messages = {
        'invalid': _(u'Enter a valid South African postal code'),
    }

    def __init__(self, *args, **kwargs):
        super(ZAPostCodeField, self).__init__(r'^\d{4}$',
            max_length=None, min_length=None, *args, **kwargs)
<commit_msg>Fixed #12595 -- Fixed bad arguments handling in localflavor form fields.<commit_after>"""
South Africa-specific Form helpers
"""

from django.core.validators import EMPTY_VALUES
from django.forms import ValidationError
from django.forms.fields import CharField, RegexField
from django.utils.checksums import luhn
from django.utils.translation import gettext as _
import re
from datetime import date

id_re = re.compile(r'^(?P<yy>\d\d)(?P<mm>\d\d)(?P<dd>\d\d)(?P<mid>\d{4})(?P<end>\d{3})')

class ZAIDField(CharField):
    """A form field for South African ID numbers -- the checksum is validated
    using the Luhn checksum, and uses a simlistic (read: not entirely accurate)
    check for the birthdate
    """
    default_error_messages = {
        'invalid': _(u'Enter a valid South African ID number'),
    }

    def clean(self, value):
        super(ZAIDField, self).clean(value)

        if value in EMPTY_VALUES:
            return u''

        # strip spaces and dashes
        value = value.strip().replace(' ', '').replace('-', '')

        match = re.match(id_re, value)

        if not match:
            raise ValidationError(self.error_messages['invalid'])

        g = match.groupdict()

        try:
            # The year 2000 is conveniently a leapyear.
            # This algorithm will break in xx00 years which aren't leap years
            # There is no way to guess the century of a ZA ID number
            d = date(int(g['yy']) + 2000, int(g['mm']), int(g['dd']))
        except ValueError:
            raise ValidationError(self.error_messages['invalid'])

        if not luhn(value):
            raise ValidationError(self.error_messages['invalid'])

        return value

class ZAPostCodeField(RegexField):
    default_error_messages = {
        'invalid': _(u'Enter a valid South African postal code'),
    }

    def __init__(self, max_length=None, min_length=None, *args, **kwargs):
        super(ZAPostCodeField, self).__init__(r'^\d{4}$',
            max_length, min_length, *args, **kwargs)
<|endoftext|>


<commit_before>from werkzeug import generate_password_hash, check_password_hash
from couchdb import ResourceNotFound, ResourceConflict
import shortuuid, datetime, hashlib, threading, json, time, copy

from totalimpact.providers.provider import ProviderFactory
from totalimpact.providers.provider import ProviderTimeout
from totalimpact import default_settings
from totalimpact.utils import Retry

# Master lock to ensure that only a single thread can write
# to the DB at one time to avoid document conflicts

import logging
logger = logging.getLogger('ti.models')

class NotAuthenticatedError(Exception):
    pass

def closest(target, collection) :
    return min((abs(target - i), i) for i in collection)[1]

class ItemFactory():

    all_static_meta = ProviderFactory.get_all_static_meta()

    @classmethod
    def get_item(cls, tiid, myrefsets, dao):
        res = dao.view("queues/by_tiid_with_snaps")
        rows = res[[tiid,0]:[tiid,"zzzzzzzzzzzzzz"]].rows

        if not rows:
            return None
        else:
            item = rows[0]["value"]
            most_recent_snaps = {}
            for row in rows[1:]:
                snap = row["value"]
                # they are in reverse order of creation, so overwrite
                # and most recent one will be on top.
                most_recent_snaps[snap["metric_name"]] = snap

            snaps = most_recent_snaps.values()

            try:
                item = cls.build_item_for_client(item, snaps, myrefsets)
            except Exception, e:
                item = None
                logger.error("Exception %s: Unable to build item %s, %s, %s" % (e.__repr__(), tiid, str(item), str(snaps)))
                raise
        return item

    @classmethod
    def build_item_for_client(cls, item, snaps, myrefsets):
        item["biblio"]['genre'] = cls.decide_genre(item['aliases'])
        item["metrics"] = {} #not using what is in stored item for this

        # need year to calculate normalization below
        try:
            year = item["biblio"]["year"]
            if year < 2002:
                year = 2002
        except KeyError:
            year = 99 # hack so that it won't match anything.  what else to do?

        for snap in snaps:
            metric_name = snap["metric_name"]
            if metric_name in cls.all_static_meta.keys():
                # add snap values. this line can go away once all snap values migrated into items in db.
                item = cls.add_snap_data(item, snap)

                #delete the raw history from what we return to the client for now
                del item["metrics"][metric_name]["values"]["raw_history"]

                # add static data
                item["metrics"][metric_name]["static_meta"] = cls.all_static_meta[metric_name]

                # add normalization values
                normalized_values = cls.get_normalized_values(item["biblio"]['genre'], year, metric_name, snap["value"], myrefsets)
                item["metrics"][metric_name]["values"].update(normalized_values)

        return item

    @classmethod
    def add_snap_data(cls, item, snap):
        metrics = item.setdefault("metrics", {})

        this_metric = metrics.setdefault(snap["metric_name"], {})
        this_metric["provenance_url"] = snap["drilldown_url"]

        this_metric_values = this_metric.setdefault("values", {})
        this_metric_values["raw"] = snap["value"]

        this_metric_values_raw_history = this_metric_values.setdefault("raw_history", {})
        this_metric_values_raw_history[snap["created"]] = snap["value"]
        return item

    @classmethod
    def build_snap(cls, tiid, metric_value_drilldown, metric_name):
        # if the alphabet below changes, need to update couch queue lookups
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')

        snap = {}
        snap["_id"] = shortuuid.uuid()[0:24]
        snap["type"] = "metric_snap"
        snap["metric_name"] = metric_name
        snap["tiid"] = tiid
        snap["created"] = datetime.datetime.now().isoformat()
        (value, drilldown_url) = metric_value_drilldown
        snap["value"] = value
        snap["drilldown_url"] = drilldown_url
        return snap

    @classmethod
    def make(cls):

        now = datetime.datetime.now().isoformat()
        # if the alphabet below changes, need to update couch queue lookups
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')

        item = {}
        item["_id"] = shortuuid.uuid()[0:24]
        item["aliases"] = {}
        item["biblio"] = {}
        item["last_modified"] = now
        item["created"] = now
        item["type"] = "item"
        return item

    @classmethod
    def decide_genre(self, alias_dict):
        '''Uses available aliases to decide the item's genre'''
        if "doi" in alias_dict:
            if "10.5061/dryad." in "".join(alias_dict["doi"]):
                return "dataset"
            else:
                return "article"
        elif "pmid" in alias_dict:
            return "article"
        elif "github" in alias_dict:
            return "software"
        elif "url" in alias_dict:
            joined_urls = "".join(alias_dict["url"])
            if "slideshare.net" in joined_urls:
                return "slides"
            elif "github.com" in joined_urls:
                return "software"
            else:
                return "webpage"
        else:
            return "unknown"

    @classmethod
    def alias_tuples_from_dict(self, aliases_dict):
        """
        Convert from aliases dict we use in items, to a list of alias tuples.

        The providers need the tuples list, which look like this:
        [(doi, 10.123), (doi, 10.345), (pmid, 1234567)]
        """
        alias_tuples = []
        for ns, ids in aliases_dict.iteritems():
            if isinstance(ids, basestring): # it's a date, not a list of ids
                alias_tuples.append((ns, ids))
            else:
                for id in ids:
                    alias_tuples.append((ns, id))
        return alias_tuples

    @classmethod
    def alias_dict_from_tuples(self, aliases_tuples):
        alias_dict = {}
        for (ns, ids) in aliases_tuples:
            if ns in alias_dict:
                alias_dict[ns] += [ids]
            else:
                alias_dict[ns] = [ids]
        return alias_dict

    @classmethod
    def merge_alias_dicts(self, aliases1, aliases2):
        #logger.debug("in MERGE ALIAS DICTS with %s and %s" %(aliases1, aliases2))
        merged_aliases = copy.deepcopy(aliases1)
        for ns, nid_list in aliases2.iteritems():
            for nid in nid_list:
                try:
                    if not nid in merged_aliases[ns]:
                        merged_aliases[ns].append(nid)
                except KeyError: # no ids for that namespace yet. make it.
                    merged_aliases[ns] = [nid]
        return merged_aliases

    @classmethod
    def get_metric_names(self, providers_config):
        full_metric_names = []
        providers = ProviderFactory.get_providers(providers_config)
        for provider in providers:
            metric_names = provider.metric_names()
            for metric_name in metric_names:
                full_metric_names.append(provider.provider_name + ':' + metric_name)
        return full_metric_names

    @classmethod
    def get_normalized_values(cls, genre, year, metric_name, value, myrefsets):
        # Will be passed None as myrefsets type when loading items in reference collections :)
        if not myrefsets:
            return {}

        # for now, only normalize articles
        if genre != "article":
            return {}

        # treat the f1000 "Yes" as a 1 for normalization
        if value=="Yes":
            value = 1

        response = {}
        for refsetname in myrefsets:
            try:
                fencepost_values = myrefsets[refsetname][str(year)][metric_name].keys()
                myclosest = closest(value, fencepost_values)
                response[refsetname] = myrefsets[refsetname][str(year)][metric_name][myclosest]
            except KeyError:
                #logger.info("No good lookup in %s %s for %s" %(refsetname, str(year), metric_name))
                pass

        return response

    @classmethod
    def retrieve_items(cls, tiids, myrefsets, myredis, mydao):
        something_currently_updating = False
        items = []
        for tiid in tiids:
            try:
                item = cls.get_item(tiid, myrefsets, mydao)
            except (LookupError, AttributeError), e:
                logger.warning("Got an error looking up tiid '{tiid}'; error: {error}".format(
                        tiid=tiid, error=e.__repr__()))
                raise

            if not item:
                logger.warning("Looks like there's no item with tiid '{tiid}': ".format(
                        tiid=tiid))
                raise LookupError

            item["currently_updating"] = cls.is_currently_updating(tiid, myredis)
            something_currently_updating = something_currently_updating or item["currently_updating"]

            items.append(item)
        return (items, something_currently_updating)

    @classmethod
    def is_currently_updating(cls, tiid, myredis):
        num_providers_left = myredis.get_num_providers_left(tiid)
        if num_providers_left:
            currently_updating = myredis.get_num_providers_left(tiid) > 0
        else: # not in redis, maybe because it expired.  Assume it is not currently updating.
            currently_updating = False
        return currently_updating

class MemberItems():

    def __init__(self, provider, redis):
        self.provider = provider
        self.redis = redis

    def start_update(self, str):
        pages = self.provider.paginate(str)
        hash = hashlib.md5(str.encode('utf-8')).hexdigest()
        t = threading.Thread(target=self._update,
                            args=(pages, hash),
                            name=hash[0:4]+"_memberitems_thread")
        t.daemon = True
        t.start()
        return hash

    def get_sync(self, query):
        ret = {}
        start = time.time()
        ret = {
            "memberitems": self.provider.member_items(query),
            "pages": 1,
            "complete": 1
        }
        logger.debug("got {num_memberitems} synchronous memberitems for query '{query}' in {elapsed} seconds.".format(
            num_memberitems=len(ret["memberitems"]),
            query=query,
            elapsed=round(time.time() - start, 2)
        ))
        return ret

    def get_async(self, query_hash):
        query_status_str = self.redis.get(query_hash)
        start = time.time()

        try:
            ret = json.loads(query_status_str)
        except TypeError:
            # if redis returns None, the update hasn't started yet (likely still
            # parsing the input string; give the client some information, though:
            ret = {"memberitems": [], "pages": 1, "complete": 0 }

        logger.debug("have finished {num_memberitems} asynchronous memberitems for query hash '{query_hash}' in {elapsed} seconds.".format(
                num_memberitems=len(ret["memberitems"]),
                query_hash=query_hash,
                elapsed=round(time.time() - start, 2)
            ))

        return ret

    @Retry(3, ProviderTimeout, 0.1)
    def _update(self, pages, key):

        status = {
            "memberitems": [],
            "pages": len(pages),
            "complete": 0
        }
        for page in pages:
            status["memberitems"].append(self.provider.member_items(page))
            status["complete"] += 1
            self.redis.set(key, json.dumps(status))

        return True

class UserFactory():

    @classmethod
    def get(cls, id, dao, key):
        try:
            print id
            doc = dao.db[id]
            print dict(doc)
        except ResourceNotFound:
            raise KeyError("User doesn't exist.")

        else:
            if doc["key"] == key:
                return doc
            else:
                raise NotAuthenticatedError

    @classmethod
    def put(cls, userdict,  password, dao):

        if "_id" not in userdict.keys() or "colls" not in userdict.keys():
            raise AttributeError

        try:
            doc = cls.get(userdict["_id"], dao, password)
            userdict["_rev"] = doc["_rev"]
        except KeyError:
            pass # no worries, we'll just make a new user.

        userdict["type"] = "user"
        dao.db.save(userdict)
        return userdict

<commit_msg>use item[metrics] if it is there<commit_after>from werkzeug import generate_password_hash, check_password_hash
from couchdb import ResourceNotFound, ResourceConflict
import shortuuid, datetime, hashlib, threading, json, time, copy

from totalimpact.providers.provider import ProviderFactory
from totalimpact.providers.provider import ProviderTimeout
from totalimpact import default_settings
from totalimpact.utils import Retry

# Master lock to ensure that only a single thread can write
# to the DB at one time to avoid document conflicts

import logging
logger = logging.getLogger('ti.models')

class NotAuthenticatedError(Exception):
    pass

def closest(target, collection) :
    return min((abs(target - i), i) for i in collection)[1]

class ItemFactory():

    all_static_meta = ProviderFactory.get_all_static_meta()

    @classmethod
    def get_item(cls, tiid, myrefsets, dao):
        res = dao.view("queues/by_tiid_with_snaps")
        rows = res[[tiid,0]:[tiid,"zzzzzzzzzzzzzz"]].rows

        if not rows:
            return None
        else:
            item = rows[0]["value"]
            most_recent_snaps = {}
            for row in rows[1:]:
                snap = row["value"]
                # they are in reverse order of creation, so overwrite
                # and most recent one will be on top.
                most_recent_snaps[snap["metric_name"]] = snap

            snaps = most_recent_snaps.values()

            try:
                item = cls.build_item_for_client(item, snaps, myrefsets)
            except Exception, e:
                item = None
                logger.error("Exception %s: Unable to build item %s, %s, %s" % (e.__repr__(), tiid, str(item), str(snaps)))
                raise
        return item

    @classmethod
    def build_item_for_client(cls, item, snaps, myrefsets):
        item["biblio"]['genre'] = cls.decide_genre(item['aliases'])
        #item["metrics"] = {} #not using what is in stored item for this

        # need year to calculate normalization below
        try:
            year = item["biblio"]["year"]
            if year < 2002:
                year = 2002
        except KeyError:
            year = 99 # hack so that it won't match anything.  what else to do?

        for snap in snaps:
            metric_name = snap["metric_name"]
            if metric_name in cls.all_static_meta.keys():
                # add snap values. this line can go away once all snap values migrated into items in db.
                item = cls.add_snap_data(item, snap)

                #delete the raw history from what we return to the client for now
                del item["metrics"][metric_name]["values"]["raw_history"]

                # add static data
                item["metrics"][metric_name]["static_meta"] = cls.all_static_meta[metric_name]

                # add normalization values
                normalized_values = cls.get_normalized_values(item["biblio"]['genre'], year, metric_name, snap["value"], myrefsets)
                item["metrics"][metric_name]["values"].update(normalized_values)

        return item

    @classmethod
    def add_snap_data(cls, item, snap):
        metrics = item.setdefault("metrics", {})

        this_metric = metrics.setdefault(snap["metric_name"], {})
        this_metric["provenance_url"] = snap["drilldown_url"]

        this_metric_values = this_metric.setdefault("values", {})
        this_metric_values["raw"] = snap["value"]

        this_metric_values_raw_history = this_metric_values.setdefault("raw_history", {})
        this_metric_values_raw_history[snap["created"]] = snap["value"]
        return item

    @classmethod
    def build_snap(cls, tiid, metric_value_drilldown, metric_name):
        # if the alphabet below changes, need to update couch queue lookups
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')

        snap = {}
        snap["_id"] = shortuuid.uuid()[0:24]
        snap["type"] = "metric_snap"
        snap["metric_name"] = metric_name
        snap["tiid"] = tiid
        snap["created"] = datetime.datetime.now().isoformat()
        (value, drilldown_url) = metric_value_drilldown
        snap["value"] = value
        snap["drilldown_url"] = drilldown_url
        return snap

    @classmethod
    def make(cls):

        now = datetime.datetime.now().isoformat()
        # if the alphabet below changes, need to update couch queue lookups
        shortuuid.set_alphabet('abcdefghijklmnopqrstuvwxyz1234567890')

        item = {}
        item["_id"] = shortuuid.uuid()[0:24]
        item["aliases"] = {}
        item["biblio"] = {}
        item["last_modified"] = now
        item["created"] = now
        item["type"] = "item"
        return item

    @classmethod
    def decide_genre(self, alias_dict):
        '''Uses available aliases to decide the item's genre'''
        if "doi" in alias_dict:
            if "10.5061/dryad." in "".join(alias_dict["doi"]):
                return "dataset"
            else:
                return "article"
        elif "pmid" in alias_dict:
            return "article"
        elif "github" in alias_dict:
            return "software"
        elif "url" in alias_dict:
            joined_urls = "".join(alias_dict["url"])
            if "slideshare.net" in joined_urls:
                return "slides"
            elif "github.com" in joined_urls:
                return "software"
            else:
                return "webpage"
        else:
            return "unknown"

    @classmethod
    def alias_tuples_from_dict(self, aliases_dict):
        """
        Convert from aliases dict we use in items, to a list of alias tuples.

        The providers need the tuples list, which look like this:
        [(doi, 10.123), (doi, 10.345), (pmid, 1234567)]
        """
        alias_tuples = []
        for ns, ids in aliases_dict.iteritems():
            if isinstance(ids, basestring): # it's a date, not a list of ids
                alias_tuples.append((ns, ids))
            else:
                for id in ids:
                    alias_tuples.append((ns, id))
        return alias_tuples

    @classmethod
    def alias_dict_from_tuples(self, aliases_tuples):
        alias_dict = {}
        for (ns, ids) in aliases_tuples:
            if ns in alias_dict:
                alias_dict[ns] += [ids]
            else:
                alias_dict[ns] = [ids]
        return alias_dict

    @classmethod
    def merge_alias_dicts(self, aliases1, aliases2):
        #logger.debug("in MERGE ALIAS DICTS with %s and %s" %(aliases1, aliases2))
        merged_aliases = copy.deepcopy(aliases1)
        for ns, nid_list in aliases2.iteritems():
            for nid in nid_list:
                try:
                    if not nid in merged_aliases[ns]:
                        merged_aliases[ns].append(nid)
                except KeyError: # no ids for that namespace yet. make it.
                    merged_aliases[ns] = [nid]
        return merged_aliases

    @classmethod
    def get_metric_names(self, providers_config):
        full_metric_names = []
        providers = ProviderFactory.get_providers(providers_config)
        for provider in providers:
            metric_names = provider.metric_names()
            for metric_name in metric_names:
                full_metric_names.append(provider.provider_name + ':' + metric_name)
        return full_metric_names

    @classmethod
    def get_normalized_values(cls, genre, year, metric_name, value, myrefsets):
        # Will be passed None as myrefsets type when loading items in reference collections :)
        if not myrefsets:
            return {}

        # for now, only normalize articles
        if genre != "article":
            return {}

        # treat the f1000 "Yes" as a 1 for normalization
        if value=="Yes":
            value = 1

        response = {}
        for refsetname in myrefsets:
            try:
                fencepost_values = myrefsets[refsetname][str(year)][metric_name].keys()
                myclosest = closest(value, fencepost_values)
                response[refsetname] = myrefsets[refsetname][str(year)][metric_name][myclosest]
            except KeyError:
                #logger.info("No good lookup in %s %s for %s" %(refsetname, str(year), metric_name))
                pass

        return response

    @classmethod
    def retrieve_items(cls, tiids, myrefsets, myredis, mydao):
        something_currently_updating = False
        items = []
        for tiid in tiids:
            try:
                item = cls.get_item(tiid, myrefsets, mydao)
            except (LookupError, AttributeError), e:
                logger.warning("Got an error looking up tiid '{tiid}'; error: {error}".format(
                        tiid=tiid, error=e.__repr__()))
                raise

            if not item:
                logger.warning("Looks like there's no item with tiid '{tiid}': ".format(
                        tiid=tiid))
                raise LookupError

            item["currently_updating"] = cls.is_currently_updating(tiid, myredis)
            something_currently_updating = something_currently_updating or item["currently_updating"]

            items.append(item)
        return (items, something_currently_updating)

    @classmethod
    def is_currently_updating(cls, tiid, myredis):
        num_providers_left = myredis.get_num_providers_left(tiid)
        if num_providers_left:
            currently_updating = myredis.get_num_providers_left(tiid) > 0
        else: # not in redis, maybe because it expired.  Assume it is not currently updating.
            currently_updating = False
        return currently_updating

class MemberItems():

    def __init__(self, provider, redis):
        self.provider = provider
        self.redis = redis

    def start_update(self, str):
        pages = self.provider.paginate(str)
        hash = hashlib.md5(str.encode('utf-8')).hexdigest()
        t = threading.Thread(target=self._update,
                            args=(pages, hash),
                            name=hash[0:4]+"_memberitems_thread")
        t.daemon = True
        t.start()
        return hash

    def get_sync(self, query):
        ret = {}
        start = time.time()
        ret = {
            "memberitems": self.provider.member_items(query),
            "pages": 1,
            "complete": 1
        }
        logger.debug("got {num_memberitems} synchronous memberitems for query '{query}' in {elapsed} seconds.".format(
            num_memberitems=len(ret["memberitems"]),
            query=query,
            elapsed=round(time.time() - start, 2)
        ))
        return ret

    def get_async(self, query_hash):
        query_status_str = self.redis.get(query_hash)
        start = time.time()

        try:
            ret = json.loads(query_status_str)
        except TypeError:
            # if redis returns None, the update hasn't started yet (likely still
            # parsing the input string; give the client some information, though:
            ret = {"memberitems": [], "pages": 1, "complete": 0 }

        logger.debug("have finished {num_memberitems} asynchronous memberitems for query hash '{query_hash}' in {elapsed} seconds.".format(
                num_memberitems=len(ret["memberitems"]),
                query_hash=query_hash,
                elapsed=round(time.time() - start, 2)
            ))

        return ret

    @Retry(3, ProviderTimeout, 0.1)
    def _update(self, pages, key):

        status = {
            "memberitems": [],
            "pages": len(pages),
            "complete": 0
        }
        for page in pages:
            status["memberitems"].append(self.provider.member_items(page))
            status["complete"] += 1
            self.redis.set(key, json.dumps(status))

        return True

class UserFactory():

    @classmethod
    def get(cls, id, dao, key):
        try:
            print id
            doc = dao.db[id]
            print dict(doc)
        except ResourceNotFound:
            raise KeyError("User doesn't exist.")

        else:
            if doc["key"] == key:
                return doc
            else:
                raise NotAuthenticatedError

    @classmethod
    def put(cls, userdict,  password, dao):

        if "_id" not in userdict.keys() or "colls" not in userdict.keys():
            raise AttributeError

        try:
            doc = cls.get(userdict["_id"], dao, password)
            userdict["_rev"] = doc["_rev"]
        except KeyError:
            pass # no worries, we'll just make a new user.

        userdict["type"] = "user"
        dao.db.save(userdict)
        return userdict

<|endoftext|>


<commit_before>from __future__ import print_function
import sys
import os
import glob
import tempfile
import shutil
import time
import netrc
import json
from six.moves.urllib import request
from six.moves.urllib.parse import urlparse, urljoin
from six.moves.urllib.error import HTTPError, URLError
from subprocess import Popen, PIPE, check_call

from w3lib.form import encode_multipart

from scrapy.command import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.http import basic_auth_header
from scrapy.utils.python import retry_on_eintr
from scrapy.utils.conf import get_config, closest_scrapy_cfg

_SETUP_PY_TEMPLATE = \
"""# Automatically created by: scrapy deploy

from setuptools import setup, find_packages

setup(
    name         = 'project',
    version      = '1.0',
    packages     = find_packages(),
    entry_points = {'scrapy': ['settings = %(settings)s']},
)
"""

class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] [ [target] | -l | -L <target> ]"

    def short_desc(self):
        return "Deploy project in Scrapyd target"

    def long_desc(self):
        return "Deploy the current project into the given Scrapyd server " \
            "(known as target)"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-p", "--project",
            help="the project name in the target")
        parser.add_option("-v", "--version",
            help="the version to deploy. Defaults to current timestamp")
        parser.add_option("-l", "--list-targets", action="store_true", \
            help="list available targets")
        parser.add_option("-d", "--debug", action="store_true",
            help="debug mode (do not remove build dir)")
        parser.add_option("-L", "--list-projects", metavar="TARGET", \
            help="list available projects on TARGET")
        parser.add_option("--egg", metavar="FILE",
            help="use the given egg, instead of building it")
        parser.add_option("--build-egg", metavar="FILE",
            help="only build the egg, don't deploy it")

    def run(self, args, opts):
        try:
            import setuptools
        except ImportError:
            raise UsageError("setuptools not installed")

        request.install_opener(request.build_opener(HTTPRedirectHandler))

        if opts.list_targets:
            for name, target in _get_targets().items():
                print("%-20s %s" % (name, target['url']))
            return

        if opts.list_projects:
            target = _get_target(opts.list_projects)
            req = request.Request(_url(target, 'listprojects.json'))
            _add_auth_header(req, target)
            f = request.urlopen(req)
            projects = json.loads(f.read())['projects']
            print(os.linesep.join(projects))
            return

        tmpdir = None

        if opts.build_egg: # build egg only
            egg, tmpdir = _build_egg()
            _log("Writing egg to %s" % opts.build_egg)
            shutil.copyfile(egg, opts.build_egg)
        else: # buld egg and deploy
            target_name = _get_target_name(args)
            target = _get_target(target_name)
            project = _get_project(target, opts)
            version = _get_version(target, opts)
            if opts.egg:
                _log("Using egg: %s" % opts.egg)
                egg = opts.egg
            else:
                _log("Packing version %s" % version)
                egg, tmpdir = _build_egg()
            if not _upload_egg(target, egg, project, version):
                self.exitcode = 1

        if tmpdir:
            if opts.debug:
                _log("Output dir not removed: %s" % tmpdir)
            else:
                shutil.rmtree(tmpdir)

def _log(message):
    sys.stderr.write(message + os.linesep)

def _get_target_name(args):
    if len(args) > 1:
        raise UsageError("Too many arguments: %s" % ' '.join(args))
    elif args:
        return args[0]
    elif len(args) < 1:
        return 'default'

def _get_project(target, opts):
    project = opts.project or target.get('project')
    if not project:
        raise UsageError("Missing project")
    return project

def _get_option(section, option, default=None):
    cfg = get_config()
    return cfg.get(section, option) if cfg.has_option(section, option) \
        else default

def _get_targets():
    cfg = get_config()
    baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
    targets = {}
    if 'url' in baset:
        targets['default'] = baset
    for x in cfg.sections():
        if x.startswith('deploy:'):
            t = baset.copy()
            t.update(cfg.items(x))
            targets[x[7:]] = t
    return targets

def _get_target(name):
    try:
        return _get_targets()[name]
    except KeyError:
        raise UsageError("Unknown target: %s" % name)

def _url(target, action):
    return urljoin(target['url'], action)

def _get_version(target, opts):
    version = opts.version or target.get('version')
    if version == 'HG':
        p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE)
        d = 'r%s' % p.communicate()[0]
        p = Popen(['hg', 'branch'], stdout=PIPE)
        b = p.communicate()[0].strip('\n')
        return '%s-%s' % (d, b)
    elif version == 'GIT':
        p = Popen(['git', 'describe', '--always'], stdout=PIPE)
        d = p.communicate()[0].strip('\n')
        p = Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=PIPE)
        b = p.communicate()[0].strip('\n')
        return '%s-%s' % (d, b)
    elif version:
        return version
    else:
        return str(int(time.time()))

def _upload_egg(target, eggpath, project, version):
    with open(eggpath, 'rb') as f:
        eggdata = f.read()
    data = {
        'project': project,
        'version': version,
        'egg': ('project.egg', eggdata),
    }
    body, boundary = encode_multipart(data)
    url = _url(target, 'addversion.json')
    headers = {
        'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
        'Content-Length': str(len(body)),
    }
    req = request.Request(url, body, headers)
    _add_auth_header(req, target)
    _log('Deploying to project "%s" in %s' % (project, url))
    return _http_post(req)

def _add_auth_header(request, target):
    if 'username' in target:
        u, p = target.get('username'), target.get('password', '')
        request.add_header('Authorization', basic_auth_header(u, p))
    else: # try netrc
        try:
            host = urlparse(target['url']).hostname
            a = netrc.netrc().authenticators(host)
            request.add_header('Authorization', basic_auth_header(a[0], a[2]))
        except (netrc.NetrcParseError, IOError, TypeError):
            pass

def _http_post(request):
    try:
        f = request.urlopen(request)
        _log("Server response (%s):" % f.code)
        print(f.read())
        return True
    except HTTPError as e:
        _log("Deploy failed (%s):" % e.code)
        print(e.read())
    except URLError as e:
        _log("Deploy failed: %s" % e)

def _build_egg():
    closest = closest_scrapy_cfg()
    os.chdir(os.path.dirname(closest))
    if not os.path.exists('setup.py'):
        settings = get_config().get('settings', 'default')
        _create_default_setup_py(settings=settings)
    d = tempfile.mkdtemp(prefix="scrapydeploy-")
    o = open(os.path.join(d, "stdout"), "wb")
    e = open(os.path.join(d, "stderr"), "wb")
    retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
    o.close()
    e.close()
    egg = glob.glob(os.path.join(d, '*.egg'))[0]
    return egg, d

def _create_default_setup_py(**kwargs):
    with open('setup.py', 'w') as f:
        f.write(_SETUP_PY_TEMPLATE % kwargs)

class HTTPRedirectHandler(request.HTTPRedirectHandler):

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        newurl = newurl.replace(' ', '%20')
        if code in (301, 307):
            return request.Request(newurl,
                                   data=req.get_data(),
                                   headers=req.headers,
                                   origin_req_host=req.get_origin_req_host(),
                                   unverifiable=True)
        elif code in (302, 303):
            newheaders = dict((k, v) for k, v in req.headers.items()
                              if k.lower() not in ("content-length", "content-type"))
            return request.Request(newurl,
                                   headers=newheaders,
                                   origin_req_host=req.get_origin_req_host(),
                                   unverifiable=True)
        else:
            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
<commit_msg>Avoid name clashing in deploy.py, fixes #1143<commit_after>from __future__ import print_function
import sys
import os
import glob
import tempfile
import shutil
import time
import netrc
import json
from six.moves.urllib import request
from six.moves.urllib.parse import urlparse, urljoin
from six.moves.urllib.error import HTTPError, URLError
from subprocess import Popen, PIPE, check_call

from w3lib.form import encode_multipart

from scrapy.command import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.http import basic_auth_header
from scrapy.utils.python import retry_on_eintr
from scrapy.utils.conf import get_config, closest_scrapy_cfg

_SETUP_PY_TEMPLATE = \
"""# Automatically created by: scrapy deploy

from setuptools import setup, find_packages

setup(
    name         = 'project',
    version      = '1.0',
    packages     = find_packages(),
    entry_points = {'scrapy': ['settings = %(settings)s']},
)
"""

class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return "[options] [ [target] | -l | -L <target> ]"

    def short_desc(self):
        return "Deploy project in Scrapyd target"

    def long_desc(self):
        return "Deploy the current project into the given Scrapyd server " \
            "(known as target)"

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option("-p", "--project",
            help="the project name in the target")
        parser.add_option("-v", "--version",
            help="the version to deploy. Defaults to current timestamp")
        parser.add_option("-l", "--list-targets", action="store_true", \
            help="list available targets")
        parser.add_option("-d", "--debug", action="store_true",
            help="debug mode (do not remove build dir)")
        parser.add_option("-L", "--list-projects", metavar="TARGET", \
            help="list available projects on TARGET")
        parser.add_option("--egg", metavar="FILE",
            help="use the given egg, instead of building it")
        parser.add_option("--build-egg", metavar="FILE",
            help="only build the egg, don't deploy it")

    def run(self, args, opts):
        try:
            import setuptools
        except ImportError:
            raise UsageError("setuptools not installed")

        request.install_opener(request.build_opener(HTTPRedirectHandler))

        if opts.list_targets:
            for name, target in _get_targets().items():
                print("%-20s %s" % (name, target['url']))
            return

        if opts.list_projects:
            target = _get_target(opts.list_projects)
            req = request.Request(_url(target, 'listprojects.json'))
            _add_auth_header(req, target)
            f = request.urlopen(req)
            projects = json.loads(f.read())['projects']
            print(os.linesep.join(projects))
            return

        tmpdir = None

        if opts.build_egg: # build egg only
            egg, tmpdir = _build_egg()
            _log("Writing egg to %s" % opts.build_egg)
            shutil.copyfile(egg, opts.build_egg)
        else: # buld egg and deploy
            target_name = _get_target_name(args)
            target = _get_target(target_name)
            project = _get_project(target, opts)
            version = _get_version(target, opts)
            if opts.egg:
                _log("Using egg: %s" % opts.egg)
                egg = opts.egg
            else:
                _log("Packing version %s" % version)
                egg, tmpdir = _build_egg()
            if not _upload_egg(target, egg, project, version):
                self.exitcode = 1

        if tmpdir:
            if opts.debug:
                _log("Output dir not removed: %s" % tmpdir)
            else:
                shutil.rmtree(tmpdir)

def _log(message):
    sys.stderr.write(message + os.linesep)

def _get_target_name(args):
    if len(args) > 1:
        raise UsageError("Too many arguments: %s" % ' '.join(args))
    elif args:
        return args[0]
    elif len(args) < 1:
        return 'default'

def _get_project(target, opts):
    project = opts.project or target.get('project')
    if not project:
        raise UsageError("Missing project")
    return project

def _get_option(section, option, default=None):
    cfg = get_config()
    return cfg.get(section, option) if cfg.has_option(section, option) \
        else default

def _get_targets():
    cfg = get_config()
    baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {}
    targets = {}
    if 'url' in baset:
        targets['default'] = baset
    for x in cfg.sections():
        if x.startswith('deploy:'):
            t = baset.copy()
            t.update(cfg.items(x))
            targets[x[7:]] = t
    return targets

def _get_target(name):
    try:
        return _get_targets()[name]
    except KeyError:
        raise UsageError("Unknown target: %s" % name)

def _url(target, action):
    return urljoin(target['url'], action)

def _get_version(target, opts):
    version = opts.version or target.get('version')
    if version == 'HG':
        p = Popen(['hg', 'tip', '--template', '{rev}'], stdout=PIPE)
        d = 'r%s' % p.communicate()[0]
        p = Popen(['hg', 'branch'], stdout=PIPE)
        b = p.communicate()[0].strip('\n')
        return '%s-%s' % (d, b)
    elif version == 'GIT':
        p = Popen(['git', 'describe', '--always'], stdout=PIPE)
        d = p.communicate()[0].strip('\n')
        p = Popen(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stdout=PIPE)
        b = p.communicate()[0].strip('\n')
        return '%s-%s' % (d, b)
    elif version:
        return version
    else:
        return str(int(time.time()))

def _upload_egg(target, eggpath, project, version):
    with open(eggpath, 'rb') as f:
        eggdata = f.read()
    data = {
        'project': project,
        'version': version,
        'egg': ('project.egg', eggdata),
    }
    body, boundary = encode_multipart(data)
    url = _url(target, 'addversion.json')
    headers = {
        'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
        'Content-Length': str(len(body)),
    }
    req = request.Request(url, body, headers)
    _add_auth_header(req, target)
    _log('Deploying to project "%s" in %s' % (project, url))
    return _http_post(req)

def _add_auth_header(req, target):
    if 'username' in target:
        u, p = target.get('username'), target.get('password', '')
        req.add_header('Authorization', basic_auth_header(u, p))
    else: # try netrc
        try:
            host = urlparse(target['url']).hostname
            a = netrc.netrc().authenticators(host)
            req.add_header('Authorization', basic_auth_header(a[0], a[2]))
        except (netrc.NetrcParseError, IOError, TypeError):
            pass

def _http_post(req):
    try:
        f = request.urlopen(req)
        _log("Server response (%s):" % f.code)
        print(f.read())
        return True
    except HTTPError as e:
        _log("Deploy failed (%s):" % e.code)
        print(e.read())
    except URLError as e:
        _log("Deploy failed: %s" % e)

def _build_egg():
    closest = closest_scrapy_cfg()
    os.chdir(os.path.dirname(closest))
    if not os.path.exists('setup.py'):
        settings = get_config().get('settings', 'default')
        _create_default_setup_py(settings=settings)
    d = tempfile.mkdtemp(prefix="scrapydeploy-")
    o = open(os.path.join(d, "stdout"), "wb")
    e = open(os.path.join(d, "stderr"), "wb")
    retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
    o.close()
    e.close()
    egg = glob.glob(os.path.join(d, '*.egg'))[0]
    return egg, d

def _create_default_setup_py(**kwargs):
    with open('setup.py', 'w') as f:
        f.write(_SETUP_PY_TEMPLATE % kwargs)

class HTTPRedirectHandler(request.HTTPRedirectHandler):

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        newurl = newurl.replace(' ', '%20')
        if code in (301, 307):
            return request.Request(newurl,
                                   data=req.get_data(),
                                   headers=req.headers,
                                   origin_req_host=req.get_origin_req_host(),
                                   unverifiable=True)
        elif code in (302, 303):
            newheaders = dict((k, v) for k, v in req.headers.items()
                              if k.lower() not in ("content-length", "content-type"))
            return request.Request(newurl,
                                   headers=newheaders,
                                   origin_req_host=req.get_origin_req_host(),
                                   unverifiable=True)
        else:
            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import time

import iotbx.phil
from scitbx.array_family import flex

help_message = """

Examples::

  i19.stability_fft image_*.cbf

"""

phil_scope = iotbx.phil.parse(
    """\
plot = False
  .type = bool
  .help = "Make some plots"
remove_spots = False
  .type = bool
  .help = "Remove signal pixels from images before analysis"
scan_range = None
  .help = "Scan range to use for analysis"
  .type = ints(size=2)
exposure_time = None
  .type = float
  .help = "Override exposure time if not correctly written in headers"
output_file = "i19_stability.dat"
  .type = path
  .help = "Name for the output file"

""",
    process_includes=True,
)

def stability_fft(imageset, params):
    from i19.util.time_analysis import fft

    scan = imageset.get_scan()
    detector = imageset.get_detector()[0]
    if params.exposure_time:
        exposure = params.exposure_time
    else:
        exposure = scan.get_exposure_times()[0]

    trusted = detector.get_trusted_range()

    indices = imageset.indices()

    if params.scan_range:
        start, end = params.scan_range
        indices = indices[start:end]
    else:
        start = 0
        end = indices[-1] + 1

    counts = flex.double(len(indices))

    t0 = time.time()

    if params.remove_spots:
        from dials.algorithms.spot_finding.factory import SpotFinderFactory
        from dials.algorithms.spot_finding.factory import phil_scope
        from dxtbx import datablock

        spot_params = phil_scope.fetch(source=iotbx.phil.parse("")).extract()
        threshold_function = SpotFinderFactory.configure_threshold(
            spot_params, datablock.DataBlock([imageset])
        )
    else:
        threshold_function = None

    for i in indices:
        pixels = imageset.get_raw_data(i)[0].as_double()

        negative = pixels < 0
        hot = pixels > int(round(trusted[1]))
        bad = negative | hot

        if threshold_function:
            peak_pixels = threshold_function.compute_threshold(pixels, ~bad)
            good = pixels.select((~bad & ~peak_pixels).iselection())
        else:
            good = pixels.select((~bad).iselection())

        counts[i - start] = flex.sum(good)

    t1 = time.time()

    print("Read data for %d images in %.1fs" % (len(indices), t1 - t0))

    # scale data to give sensible FFT values

    mean = flex.sum(counts) / counts.size()
    counts /= mean

    power = fft(counts)

    f_hz = 1.0 / exposure
    f_scale = f_hz / counts.size()

    print("Sample frequency: %.2f Hz" % f_hz)
    print("Writing output to: %s" % params.output_file)

    fout = open(params.output_file, "w")
    for j in range(power.size()):
        fout.write("%f %f\n" % (j * f_scale, power[j]))
    fout.close()

def main():
    from dials.util.options import OptionParser
    from dials.util.options import flatten_datablocks

    usage = "%prog [options] image_*.cbf"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_datablocks=True,
        read_datablocks_from_images=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    datablocks = flatten_datablocks(params.input.datablock)

    if len(datablocks) == 0:
        parser.print_help()
        exit()

    datablock = datablocks[0]
    imageset = datablock.extract_imagesets()[0]
    stability_fft(imageset, params)

if __name__ == "__main__":
    main()
<commit_msg>Remove unused parameter<commit_after>from __future__ import absolute_import, division, print_function

import time

import iotbx.phil
from scitbx.array_family import flex

help_message = """

Examples::

  i19.stability_fft image_*.cbf

"""

phil_scope = iotbx.phil.parse(
    """\
plot = False
  .type = bool
  .help = "Make some plots"
remove_spots = False
  .type = bool
  .help = "Remove signal pixels from images before analysis"
scan_range = None
  .help = "Scan range to use for analysis"
  .type = ints(size=2)
exposure_time = None
  .type = float
  .help = "Override exposure time if not correctly written in headers"
output_file = "i19_stability.dat"
  .type = path
  .help = "Name for the output file"

""",
    process_includes=True,
)

def stability_fft(imageset, params):
    from i19.util.time_analysis import fft

    scan = imageset.get_scan()
    detector = imageset.get_detector()[0]
    if params.exposure_time:
        exposure = params.exposure_time
    else:
        exposure = scan.get_exposure_times()[0]

    trusted = detector.get_trusted_range()

    indices = imageset.indices()

    if params.scan_range:
        start, end = params.scan_range
        indices = indices[start:end]
    else:
        start = 0
        end = indices[-1] + 1

    counts = flex.double(len(indices))

    t0 = time.time()

    if params.remove_spots:
        from dials.algorithms.spot_finding.factory import SpotFinderFactory
        from dials.algorithms.spot_finding.factory import phil_scope

        spot_params = phil_scope.fetch(source=iotbx.phil.parse("")).extract()
        threshold_function = SpotFinderFactory.configure_threshold(spot_params)
    else:
        threshold_function = None

    for i in indices:
        pixels = imageset.get_raw_data(i)[0].as_double()

        negative = pixels < 0
        hot = pixels > int(round(trusted[1]))
        bad = negative | hot

        if threshold_function:
            peak_pixels = threshold_function.compute_threshold(pixels, ~bad)
            good = pixels.select((~bad & ~peak_pixels).iselection())
        else:
            good = pixels.select((~bad).iselection())

        counts[i - start] = flex.sum(good)

    t1 = time.time()

    print("Read data for %d images in %.1fs" % (len(indices), t1 - t0))

    # scale data to give sensible FFT values

    mean = flex.sum(counts) / counts.size()
    counts /= mean

    power = fft(counts)

    f_hz = 1.0 / exposure
    f_scale = f_hz / counts.size()

    print("Sample frequency: %.2f Hz" % f_hz)
    print("Writing output to: %s" % params.output_file)

    fout = open(params.output_file, "w")
    for j in range(power.size()):
        fout.write("%f %f\n" % (j * f_scale, power[j]))
    fout.close()

def main():
    from dials.util.options import OptionParser
    from dials.util.options import flatten_datablocks

    usage = "%prog [options] image_*.cbf"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_datablocks=True,
        read_datablocks_from_images=True,
        epilog=help_message,
    )

    params, options = parser.parse_args(show_diff_phil=True)
    datablocks = flatten_datablocks(params.input.datablock)

    if len(datablocks) == 0:
        parser.print_help()
        exit()

    datablock = datablocks[0]
    imageset = datablock.extract_imagesets()[0]
    stability_fft(imageset, params)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>version_info = (0, 9, 0)
__version__ = '.'.join(map(str, version_info[:3])) + ''.join(version_info[3:])
<commit_msg>bump to 0.9.1.dev0<commit_after>version_info = (0, 9, 1, 'dev0')
__version__ = '.'.join(map(str, version_info[:3])) + ''.join(version_info[3:])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
TraceView API library

:copyright: (c) 2014 by Daniel Riti.
:license: MIT, see LICENSE for more details.

"""

__title__ = 'traceview'
__version__ = '0.1.0'
__author__ = 'Daniel Riti'
__license__ = 'MIT'

from .organization import Organization
from .discovery import Action, App, Browser, Controller, Domain, Host
from .discovery import Layer, Metric, Region
from .errors import Error
from .latency import Latency

class TraceView(object):
    """ The :class:`TraceView <TraceView>` object.

    Provides access to TraceView API resources.

    :param api_key: The TraceView API access key.

    Usage::

      >>> import traceview
      >>> tv = traceview.TraceView("API KEY HERE")

    """

    def __init__(self, api_key):
        self.api_key = api_key

        #: Get all :py:class:`Actions <traceview.discovery.Action>` that have
        #: been traced.
        self.actions = Action(self.api_key)

        #: Get all available :py:class:`Apps <traceview.discovery.App>`.
        self.apps = App(self.api_key)

        #: Get all :py:class:`Browsers <traceview.discovery.Browser>` used by
        #: end users.
        self.browsers = Browser(self.api_key)

        #: Get all :py:class:`Controllers <traceview.discovery.Controller>` that
        #: have been traced.
        self.controllers = Controller(self.api_key)

        #: Get all :py:class:`Domains <traceview.discovery.Domain>` that have
        #: been traced.
        self.domains = Domain(self.api_key)

        #: See :py:class:`Errors <traceview.errors.Rate>`.
        self.errors = Error(self.api_key)

        #: Get all :py:class:`Hosts <traceview.discovery.Host>` that have been
        #: traced.
        self.hosts = Host(self.api_key)

        #: Get server or client latency information. For more details, check out
        #: :py:class:`Series <traceview.latency.Series>`,
        #: :py:class:`Summary <traceview.latency.Summary>`, and
        #: :py:class:`ByLayer <traceview.latency.ByLayer>`.
        self.latency = Latency(self.api_key)

        #: Get all recent :py:class:`Layers <traceview.discovery.Layer>` for an
        #: application.
        self.layers = Layer(self.api_key)

        #: Get all available :py:class:`Host Metrics <traceview.discovery.Metric>`
        #: that have been collected.
        self.metrics = Metric(self.api_key)

        #: Get :py:class:`Organization <traceview.organization.Organization>`
        #: and :py:class:`User <traceview.organization.User>` information.
        self.organization = Organization(self.api_key)

        #: Get all geographical :py:class:`Regions <traceview.discovery.Region>`
        #: of end users.
        self.regions = Region(self.api_key)
<commit_msg>Bump version number #3<commit_after># -*- coding: utf-8 -*-

"""
TraceView API library

:copyright: (c) 2014 by Daniel Riti.
:license: MIT, see LICENSE for more details.

"""

__title__ = 'traceview'
__version__ = '0.2.0'
__author__ = 'Daniel Riti'
__license__ = 'MIT'

from .organization import Organization
from .discovery import Action, App, Browser, Controller, Domain, Host
from .discovery import Layer, Metric, Region
from .errors import Error
from .latency import Latency

class TraceView(object):
    """ The :class:`TraceView <TraceView>` object.

    Provides access to TraceView API resources.

    :param api_key: The TraceView API access key.

    Usage::

      >>> import traceview
      >>> tv = traceview.TraceView("API KEY HERE")

    """

    def __init__(self, api_key):
        self.api_key = api_key

        #: Get all :py:class:`Actions <traceview.discovery.Action>` that have
        #: been traced.
        self.actions = Action(self.api_key)

        #: Get all available :py:class:`Apps <traceview.discovery.App>`.
        self.apps = App(self.api_key)

        #: Get all :py:class:`Browsers <traceview.discovery.Browser>` used by
        #: end users.
        self.browsers = Browser(self.api_key)

        #: Get all :py:class:`Controllers <traceview.discovery.Controller>` that
        #: have been traced.
        self.controllers = Controller(self.api_key)

        #: Get all :py:class:`Domains <traceview.discovery.Domain>` that have
        #: been traced.
        self.domains = Domain(self.api_key)

        #: See :py:class:`Errors <traceview.errors.Rate>`.
        self.errors = Error(self.api_key)

        #: Get all :py:class:`Hosts <traceview.discovery.Host>` that have been
        #: traced.
        self.hosts = Host(self.api_key)

        #: Get server or client latency information. For more details, check out
        #: :py:class:`Series <traceview.latency.Series>`,
        #: :py:class:`Summary <traceview.latency.Summary>`, and
        #: :py:class:`ByLayer <traceview.latency.ByLayer>`.
        self.latency = Latency(self.api_key)

        #: Get all recent :py:class:`Layers <traceview.discovery.Layer>` for an
        #: application.
        self.layers = Layer(self.api_key)

        #: Get all available :py:class:`Host Metrics <traceview.discovery.Metric>`
        #: that have been collected.
        self.metrics = Metric(self.api_key)

        #: Get :py:class:`Organization <traceview.organization.Organization>`
        #: and :py:class:`User <traceview.organization.User>` information.
        self.organization = Organization(self.api_key)

        #: Get all geographical :py:class:`Regions <traceview.discovery.Region>`
        #: of end users.
        self.regions = Region(self.api_key)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

"""Youtubedlg module to handle settings. """

from __future__ import unicode_literals

import json
import os.path

from .utils import (
    os_path_expanduser,
    os_path_exists,
    encode_tuple,
    decode_tuple,
    check_path
)

from .formats import (
    OUTPUT_FORMATS,
    FORMATS
)

class OptionsManager(object):

    """Handles youtubedlg options.

    This class is responsible for storing and retrieving the options.

    Attributes:
        SETTINGS_FILENAME (string): Filename of the settings file.
        SENSITIVE_KEYS (tuple): Contains the keys that we don't want
            to store on the settings file. (SECURITY ISSUES).

    Args:
        config_path (string): Absolute path where OptionsManager
            should store the settings file.

    Note:
        See load_default() method for available options.

    Example:
        Access the options using the 'options' variable.

        opt_manager = OptionsManager('.')
        opt_manager.options['save_path'] = '~/Downloads'

    """

    SETTINGS_FILENAME = 'settings.json'
    SENSITIVE_KEYS = ('sudo_password', 'password', 'video_password')

    def __init__(self, config_path):
        self.config_path = config_path
        self.settings_file = os.path.join(config_path, self.SETTINGS_FILENAME)
        self.options = dict()
        self.load_default()
        self.load_from_file()

    def load_default(self):
        """Load the default options.

        Note:
            This method is automatically called by the constructor.

        Options Description:

            save_path (string): Path where youtube-dl should store the
                downloaded file. Default is $HOME.

            video_format (string): Video format to download.
                When this options is set to '0' youtube-dl will choose
                the best video format available for the given URL.

            second_video_format (string): Video format to mix with the first
                one (-f 18+17).

            to_audio (boolean): If True youtube-dl will post process the
                video file.

            keep_video (boolen): If True youtube-dl will keep the video file
                after post processing it.

            audio_format (string): Audio format of the post processed file.
                Available values are "mp3", "wav", "aac", "m4a", "vorbis", "opus".

            audio_quality (string): Audio quality of the post processed file.
                Available values are "9", "5", "0". The lowest the value the
                better the quality.

            restrict_filenames (boolean): If True youtube-dl will restrict
                the downloaded file filename to ASCII characters only.

            output_format (int): This option sets the downloaded file
                output template. See formats.OUTPUT_FORMATS for more info.

            output_template (string): Can be any output template supported
                by youtube-dl.

            playlist_start (int): Playlist index to start downloading.

            playlist_end (int): Playlist index to stop downloading.

            max_downloads (int): Maximum number of video files to download
                from the given playlist.

            min_filesize (float): Minimum file size of the video file.
                If the video file is smaller than the given size then
                youtube-dl will abort the download process.

            max_filesize (float): Maximum file size of the video file.
                If the video file is larger than the given size then
                youtube-dl will abort the download process.

            min_filesize_unit (string): Minimum file size unit.
                Available values: '', 'k', 'm', 'g', 'y', 'p', 'e', 'z', 'y'.

            max_filesize_unit (string): Maximum file size unit.
                See 'min_filesize_unit' option for available values.

            write_subs (boolean): If True youtube-dl will try to download
                the subtitles file for the given URL.

            write_all_subs (boolean): If True youtube-dl will try to download
                all the available subtitles files for the given URL.

            write_auto_subs (boolean): If True youtube-dl will try to download
                the automatic subtitles file for the given URL.

            embed_subs (boolean): If True youtube-dl will merge the subtitles
                file with the video. (ONLY mp4 files).

            subs_lang (string): Language of the subtitles file to download.
                Needs 'write_subs' option.

            ignore_errors (boolean): If True youtube-dl will ignore the errors
                and continue the download process.

            open_dl_dir (boolean): If True youtube-dlg will open the
                destination folder after download process has been completed.

            write_description (boolean): If True youtube-dl will write video
                description to a .description file.

            write_info (boolean): If True youtube-dl will write video
                metadata to a .info.json file.

            write_thumbnail (boolean): If True youtube-dl will write
                thumbnail image to disk.

            retries (int): Number of youtube-dl retries.

            user_agent (string): Specify a custom user agent for youtube-dl.

            referer (string): Specify a custom referer to use if the video
                access is restricted to one domain.

            proxy (string): Use the specified HTTP/HTTPS proxy.

            shutdown (boolean): If True youtube-dlg will turn the computer
                off after the download process has been completed.

            sudo_password (string): SUDO password for the shutdown process if
                the user does not have elevated privileges.

            username (string): Username to login with.

            password (string): Password to login with.

            video_password (string): Video password for the given URL.

            youtubedl_path (string): Absolute path to the youtube-dl binary.
                Default is the self.config_path. You can change this option
                to point on /usr/local/bin etc.. if you want to use the
                youtube-dl binary on your system. This is also the directory
                where youtube-dlg will auto download the youtube-dl if not
                exists so you should make sure you have write access if you
                want to update the youtube-dl binary from within youtube-dlg.

            cmd_args (string): String that contains extra youtube-dl options
                seperated by spaces.

            enable_log (boolean): If True youtube-dlg will enable
                the LogManager. See main() function under __init__().

            log_time (boolean): See logmanager.LogManager add_time attribute.

            workers_number (int): Number of download workers that download manager
                will spawn. Must be greater than zero.

            locale_name (string): Locale name (e.g. ru_RU).

            main_win_size (tuple): Main window size (width, height).
                If window becomes to small the program will reset its size.
                See _settings_are_valid method MIN_FRAME_SIZE.

            opts_win_size (tuple): Options window size (width, height).
                If window becomes to small the program will reset its size.
                See _settings_are_valid method MIN_FRAME_SIZE.

            save_path_dirs (list): List that contains temporary save paths.

            selected_video_formats (list): List that contains the selected
                video formats to display on the main window.

            selected_audio_formats (list): List that contains the selected
                audio formats to display on the main window.

            selected_format (string): Current format selected on the main window.

        """
        #TODO Remove old options & check options validation
        self.options = {
            'save_path': os_path_expanduser('~'),
            'save_path_dirs': [],
            'video_format': '0',
            'second_video_format': '0',
            'to_audio': False,
            'keep_video': False,
            'audio_format': '',
            'audio_quality': '5',
            'restrict_filenames': False,
            'output_format': 1,
            'output_template': os.path.join('%(uploader)s', '%(title)s.%(ext)s'),
            'playlist_start': 1,
            'playlist_end': 0,
            'max_downloads': 0,
            'min_filesize': 0,
            'max_filesize': 0,
            'min_filesize_unit': '',
            'max_filesize_unit': '',
            'write_subs': False,
            'write_all_subs': False,
            'write_auto_subs': False,
            'embed_subs': False,
            'subs_lang': 'en',
            'ignore_errors': True,
            'open_dl_dir': True,
            'write_description': False,
            'write_info': False,
            'write_thumbnail': False,
            'retries': 10,
            'user_agent': '',
            'referer': '',
            'proxy': '',
            'shutdown': False,
            'sudo_password': '',
            'username': '',
            'password': '',
            'video_password': '',
            'youtubedl_path': self.config_path,
            'cmd_args': '',
            'enable_log': True,
            'log_time': False,
            'workers_number': 3,
            'locale_name': 'en_US',
            'main_win_size': (710, 490),
            'opts_win_size': (640, 470),
            'selected_video_formats': [],
            'selected_audio_formats': [],
            'selected_format': '0'
        }

    def load_from_file(self):
        """Load options from settings file. """
        if not os_path_exists(self.settings_file):
            return

        with open(self.settings_file, 'rb') as settings_file:
            try:
                options = json.load(settings_file)

                if self._settings_are_valid(options):
                    self.options = options
            except:
                self.load_default()

    def save_to_file(self):
        """Save options to settings file. """
        check_path(self.config_path)

        with open(self.settings_file, 'wb') as settings_file:
            options = self._get_options()
            json.dump(options,
                      settings_file,
                      indent=4,
                      separators=(',', ': '))

    def _settings_are_valid(self, settings_dictionary):
        """Check settings.json dictionary.

        Args:
            settings_dictionary (dict): Options dictionary loaded
                from the settings file. See load_from_file() method.

        Returns:
            True if settings.json dictionary is valid, else False.

        """
        VALID_VIDEO_FORMAT = ('0', '17', '36', '5', '34', '35', '43', '44', '45',
            '46', '18', '22', '37', '38', '160', '133', '134', '135', '136','137',
            '264', '138', '242', '243', '244', '247', '248', '271', '272', '82',
            '83', '84', '85', '100', '101', '102', '139', '140', '141', '171', '172')

        VALID_AUDIO_FORMAT = ('mp3', 'wav', 'aac', 'm4a', 'vorbis', 'opus', '')

        VALID_AUDIO_QUALITY = ('0', '5', '9')

        VALID_FILESIZE_UNIT = ('', 'k', 'm', 'g', 't', 'p', 'e', 'z', 'y')

        VALID_SUB_LANGUAGE = ('en', 'gr', 'pt', 'fr', 'it', 'ru', 'es', 'de')

        MIN_FRAME_SIZE = 100

        # Decode string formatted tuples back to normal tuples
        settings_dictionary['main_win_size'] = decode_tuple(settings_dictionary['main_win_size'])
        settings_dictionary['opts_win_size'] = decode_tuple(settings_dictionary['opts_win_size'])

        for key in self.options:
            if key not in settings_dictionary:
                return False

            if type(self.options[key]) != type(settings_dictionary[key]):
                return False

        # Check if each key has a valid value
        rules_dict = {
            'video_format': FORMATS.keys(),
            'second_video_format': VALID_VIDEO_FORMAT,
            'audio_format': VALID_AUDIO_FORMAT,
            'audio_quality': VALID_AUDIO_QUALITY,
            'output_format': OUTPUT_FORMATS.keys(),
            'min_filesize_unit': VALID_FILESIZE_UNIT,
            'max_filesize_unit': VALID_FILESIZE_UNIT,
            'subs_lang': VALID_SUB_LANGUAGE
        }

        for key, valid_list in rules_dict.items():
            if settings_dictionary[key] not in valid_list:
                return False

        # Check workers number value
        if settings_dictionary['workers_number'] < 1:
            return False

        # Check main-options frame size
        for size in settings_dictionary['main_win_size']:
            if size < MIN_FRAME_SIZE:
                return False

        for size in settings_dictionary['opts_win_size']:
            if size < MIN_FRAME_SIZE:
                return False

        return True

    def _get_options(self):
        """Return options dictionary without SENSITIVE_KEYS. """
        temp_options = self.options.copy()

        for key in self.SENSITIVE_KEYS:
            temp_options[key] = ''

        # Encode normal tuples to string formatted tuples
        temp_options['main_win_size'] = encode_tuple(temp_options['main_win_size'])
        temp_options['opts_win_size'] = encode_tuple(temp_options['opts_win_size'])

        return temp_options

<commit_msg>Change default options<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

"""Youtubedlg module to handle settings. """

from __future__ import unicode_literals

import json
import os.path

from .utils import (
    os_path_expanduser,
    os_path_exists,
    encode_tuple,
    decode_tuple,
    check_path
)

from .formats import (
    OUTPUT_FORMATS,
    FORMATS
)

class OptionsManager(object):

    """Handles youtubedlg options.

    This class is responsible for storing and retrieving the options.

    Attributes:
        SETTINGS_FILENAME (string): Filename of the settings file.
        SENSITIVE_KEYS (tuple): Contains the keys that we don't want
            to store on the settings file. (SECURITY ISSUES).

    Args:
        config_path (string): Absolute path where OptionsManager
            should store the settings file.

    Note:
        See load_default() method for available options.

    Example:
        Access the options using the 'options' variable.

        opt_manager = OptionsManager('.')
        opt_manager.options['save_path'] = '~/Downloads'

    """

    SETTINGS_FILENAME = 'settings.json'
    SENSITIVE_KEYS = ('sudo_password', 'password', 'video_password')

    def __init__(self, config_path):
        self.config_path = config_path
        self.settings_file = os.path.join(config_path, self.SETTINGS_FILENAME)
        self.options = dict()
        self.load_default()
        self.load_from_file()

    def load_default(self):
        """Load the default options.

        Note:
            This method is automatically called by the constructor.

        Options Description:

            save_path (string): Path where youtube-dl should store the
                downloaded file. Default is $HOME.

            video_format (string): Video format to download.
                When this options is set to '0' youtube-dl will choose
                the best video format available for the given URL.

            second_video_format (string): Video format to mix with the first
                one (-f 18+17).

            to_audio (boolean): If True youtube-dl will post process the
                video file.

            keep_video (boolen): If True youtube-dl will keep the video file
                after post processing it.

            audio_format (string): Audio format of the post processed file.
                Available values are "mp3", "wav", "aac", "m4a", "vorbis", "opus".

            audio_quality (string): Audio quality of the post processed file.
                Available values are "9", "5", "0". The lowest the value the
                better the quality.

            restrict_filenames (boolean): If True youtube-dl will restrict
                the downloaded file filename to ASCII characters only.

            output_format (int): This option sets the downloaded file
                output template. See formats.OUTPUT_FORMATS for more info.

            output_template (string): Can be any output template supported
                by youtube-dl.

            playlist_start (int): Playlist index to start downloading.

            playlist_end (int): Playlist index to stop downloading.

            max_downloads (int): Maximum number of video files to download
                from the given playlist.

            min_filesize (float): Minimum file size of the video file.
                If the video file is smaller than the given size then
                youtube-dl will abort the download process.

            max_filesize (float): Maximum file size of the video file.
                If the video file is larger than the given size then
                youtube-dl will abort the download process.

            min_filesize_unit (string): Minimum file size unit.
                Available values: '', 'k', 'm', 'g', 'y', 'p', 'e', 'z', 'y'.

            max_filesize_unit (string): Maximum file size unit.
                See 'min_filesize_unit' option for available values.

            write_subs (boolean): If True youtube-dl will try to download
                the subtitles file for the given URL.

            write_all_subs (boolean): If True youtube-dl will try to download
                all the available subtitles files for the given URL.

            write_auto_subs (boolean): If True youtube-dl will try to download
                the automatic subtitles file for the given URL.

            embed_subs (boolean): If True youtube-dl will merge the subtitles
                file with the video. (ONLY mp4 files).

            subs_lang (string): Language of the subtitles file to download.
                Needs 'write_subs' option.

            ignore_errors (boolean): If True youtube-dl will ignore the errors
                and continue the download process.

            open_dl_dir (boolean): If True youtube-dlg will open the
                destination folder after download process has been completed.

            write_description (boolean): If True youtube-dl will write video
                description to a .description file.

            write_info (boolean): If True youtube-dl will write video
                metadata to a .info.json file.

            write_thumbnail (boolean): If True youtube-dl will write
                thumbnail image to disk.

            retries (int): Number of youtube-dl retries.

            user_agent (string): Specify a custom user agent for youtube-dl.

            referer (string): Specify a custom referer to use if the video
                access is restricted to one domain.

            proxy (string): Use the specified HTTP/HTTPS proxy.

            shutdown (boolean): If True youtube-dlg will turn the computer
                off after the download process has been completed.

            sudo_password (string): SUDO password for the shutdown process if
                the user does not have elevated privileges.

            username (string): Username to login with.

            password (string): Password to login with.

            video_password (string): Video password for the given URL.

            youtubedl_path (string): Absolute path to the youtube-dl binary.
                Default is the self.config_path. You can change this option
                to point on /usr/local/bin etc.. if you want to use the
                youtube-dl binary on your system. This is also the directory
                where youtube-dlg will auto download the youtube-dl if not
                exists so you should make sure you have write access if you
                want to update the youtube-dl binary from within youtube-dlg.

            cmd_args (string): String that contains extra youtube-dl options
                seperated by spaces.

            enable_log (boolean): If True youtube-dlg will enable
                the LogManager. See main() function under __init__().

            log_time (boolean): See logmanager.LogManager add_time attribute.

            workers_number (int): Number of download workers that download manager
                will spawn. Must be greater than zero.

            locale_name (string): Locale name (e.g. ru_RU).

            main_win_size (tuple): Main window size (width, height).
                If window becomes to small the program will reset its size.
                See _settings_are_valid method MIN_FRAME_SIZE.

            opts_win_size (tuple): Options window size (width, height).
                If window becomes to small the program will reset its size.
                See _settings_are_valid method MIN_FRAME_SIZE.

            save_path_dirs (list): List that contains temporary save paths.

            selected_video_formats (list): List that contains the selected
                video formats to display on the main window.

            selected_audio_formats (list): List that contains the selected
                audio formats to display on the main window.

            selected_format (string): Current format selected on the main window.

        """
        #TODO Remove old options & check options validation
        self.options = {
            'save_path': os_path_expanduser('~'),
            'save_path_dirs': [],
            'video_format': '0',
            'second_video_format': '0',
            'to_audio': False,
            'keep_video': False,
            'audio_format': '',
            'audio_quality': '5',
            'restrict_filenames': False,
            'output_format': 1,
            'output_template': os.path.join('%(uploader)s', '%(title)s.%(ext)s'),
            'playlist_start': 1,
            'playlist_end': 0,
            'max_downloads': 0,
            'min_filesize': 0,
            'max_filesize': 0,
            'min_filesize_unit': '',
            'max_filesize_unit': '',
            'write_subs': False,
            'write_all_subs': False,
            'write_auto_subs': False,
            'embed_subs': False,
            'subs_lang': 'en',
            'ignore_errors': True,
            'open_dl_dir': False,
            'write_description': False,
            'write_info': False,
            'write_thumbnail': False,
            'retries': 10,
            'user_agent': '',
            'referer': '',
            'proxy': '',
            'shutdown': False,
            'sudo_password': '',
            'username': '',
            'password': '',
            'video_password': '',
            'youtubedl_path': self.config_path,
            'cmd_args': '',
            'enable_log': True,
            'log_time': True,
            'workers_number': 3,
            'locale_name': 'en_US',
            'main_win_size': (710, 490),
            'opts_win_size': (640, 470),
            'selected_video_formats': [],
            'selected_audio_formats': [],
            'selected_format': '0'
        }

    def load_from_file(self):
        """Load options from settings file. """
        if not os_path_exists(self.settings_file):
            return

        with open(self.settings_file, 'rb') as settings_file:
            try:
                options = json.load(settings_file)

                if self._settings_are_valid(options):
                    self.options = options
            except:
                self.load_default()

    def save_to_file(self):
        """Save options to settings file. """
        check_path(self.config_path)

        with open(self.settings_file, 'wb') as settings_file:
            options = self._get_options()
            json.dump(options,
                      settings_file,
                      indent=4,
                      separators=(',', ': '))

    def _settings_are_valid(self, settings_dictionary):
        """Check settings.json dictionary.

        Args:
            settings_dictionary (dict): Options dictionary loaded
                from the settings file. See load_from_file() method.

        Returns:
            True if settings.json dictionary is valid, else False.

        """
        VALID_VIDEO_FORMAT = ('0', '17', '36', '5', '34', '35', '43', '44', '45',
            '46', '18', '22', '37', '38', '160', '133', '134', '135', '136','137',
            '264', '138', '242', '243', '244', '247', '248', '271', '272', '82',
            '83', '84', '85', '100', '101', '102', '139', '140', '141', '171', '172')

        VALID_AUDIO_FORMAT = ('mp3', 'wav', 'aac', 'm4a', 'vorbis', 'opus', '')

        VALID_AUDIO_QUALITY = ('0', '5', '9')

        VALID_FILESIZE_UNIT = ('', 'k', 'm', 'g', 't', 'p', 'e', 'z', 'y')

        VALID_SUB_LANGUAGE = ('en', 'gr', 'pt', 'fr', 'it', 'ru', 'es', 'de')

        MIN_FRAME_SIZE = 100

        # Decode string formatted tuples back to normal tuples
        settings_dictionary['main_win_size'] = decode_tuple(settings_dictionary['main_win_size'])
        settings_dictionary['opts_win_size'] = decode_tuple(settings_dictionary['opts_win_size'])

        for key in self.options:
            if key not in settings_dictionary:
                return False

            if type(self.options[key]) != type(settings_dictionary[key]):
                return False

        # Check if each key has a valid value
        rules_dict = {
            'video_format': FORMATS.keys(),
            'second_video_format': VALID_VIDEO_FORMAT,
            'audio_format': VALID_AUDIO_FORMAT,
            'audio_quality': VALID_AUDIO_QUALITY,
            'output_format': OUTPUT_FORMATS.keys(),
            'min_filesize_unit': VALID_FILESIZE_UNIT,
            'max_filesize_unit': VALID_FILESIZE_UNIT,
            'subs_lang': VALID_SUB_LANGUAGE
        }

        for key, valid_list in rules_dict.items():
            if settings_dictionary[key] not in valid_list:
                return False

        # Check workers number value
        if settings_dictionary['workers_number'] < 1:
            return False

        # Check main-options frame size
        for size in settings_dictionary['main_win_size']:
            if size < MIN_FRAME_SIZE:
                return False

        for size in settings_dictionary['opts_win_size']:
            if size < MIN_FRAME_SIZE:
                return False

        return True

    def _get_options(self):
        """Return options dictionary without SENSITIVE_KEYS. """
        temp_options = self.options.copy()

        for key in self.SENSITIVE_KEYS:
            temp_options[key] = ''

        # Encode normal tuples to string formatted tuples
        temp_options['main_win_size'] = encode_tuple(temp_options['main_win_size'])
        temp_options['opts_win_size'] = encode_tuple(temp_options['opts_win_size'])

        return temp_options

<|endoftext|>


<commit_before>#
# __init__.py
#
# Copyright (C) AB Strakt 2001, All rights reserved
#
# $Id: __init__.py,v 1.4 2004/07/22 12:01:25 martin Exp $
#
"""
pyOpenSSL - A simple wrapper around the OpenSSL library
"""
import rand, crypto, SSL, tsafe
from version import __version__
<commit_msg>Hack up module import like mad<commit_after>#
# __init__.py
#
# Copyright (C) AB Strakt 2001, All rights reserved
#
# $Id: __init__.py,v 1.4 2004/07/22 12:01:25 martin Exp $
#
"""
pyOpenSSL - A simple wrapper around the OpenSSL library
"""

import sys
orig = sys.getdlopenflags()
sys.setdlopenflags(2|256)
from OpenSSL import crypto
sys.setdlopenflags(orig)
del sys, orig

from OpenSSL import rand, SSL, tsafe
from OpenSSL.version import __version__

__all__ = [
    'rand', 'crypto', 'SSL', 'tsafe', '__version__']
<|endoftext|>


<commit_before># Webhooks for teamcity integration

import logging
from typing import Any, Dict, List, Optional

import ujson
from django.db.models import Q
from django.http import HttpRequest, HttpResponse

from zerver.decorator import api_key_only_webhook_view
from zerver.lib.actions import check_send_private_message, \
    check_send_stream_message
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_error, json_success
from zerver.models import Realm, UserProfile

def guess_zulip_user_from_teamcity(teamcity_username: str, realm: Realm) -> Optional[UserProfile]:
    try:
        # Try to find a matching user in Zulip
        # We search a user's full name, short name,
        # and beginning of email address
        user = UserProfile.objects.filter(
            Q(full_name__iexact=teamcity_username) |
            Q(short_name__iexact=teamcity_username) |
            Q(email__istartswith=teamcity_username),
            is_active=True,
            realm=realm).order_by("id")[0]
        return user
    except IndexError:
        return None

def get_teamcity_property_value(property_list: List[Dict[str, str]], name: str) -> Optional[str]:
    for property in property_list:
        if property['name'] == name:
            return property['value']
    return None

@api_key_only_webhook_view('Teamcity')
@has_request_variables
def api_teamcity_webhook(request, user_profile, payload=REQ(argument_type='body'),
                         stream=REQ(default='teamcity')):
    # type: (HttpRequest, UserProfile, Dict[str, Any], str) -> HttpResponse
    message = payload['build']

    build_name = message['buildFullName']
    build_url = message['buildStatusUrl']
    changes_url = build_url + '&tab=buildChangesDiv'
    build_number = message['buildNumber']
    build_result = message['buildResult']
    build_result_delta = message['buildResultDelta']
    build_status = message['buildStatus']

    if build_result == 'success':
        if build_result_delta == 'fixed':
            status = 'has been fixed! :thumbsup:'
        else:
            status = 'was successful! :thumbsup:'
    elif build_result == 'failure':
        if build_result_delta == 'broken':
            status = 'is broken with status %s! :thumbsdown:' % (build_status,)
        else:
            status = 'is still broken with status %s! :thumbsdown:' % (build_status,)
    elif build_result == 'running':
        status = 'has started.'
    else:
        status = '(has no message specified for status %s)' % (build_status,)

    template = (
        u'%s build %s %s\n'
        u'Details: [changes](%s), [build log](%s)')

    body = template % (build_name, build_number, status, changes_url, build_url)
    topic = build_name

    # Check if this is a personal build, and if so try to private message the user who triggered it.
    if get_teamcity_property_value(message['teamcityProperties'], 'env.BUILD_IS_PERSONAL') == 'true':
        # The triggeredBy field gives us the teamcity user full name, and the
        # "teamcity.build.triggeredBy.username" property gives us the teamcity username.
        # Let's try finding the user email from both.
        teamcity_fullname = message['triggeredBy'].split(';')[0]
        teamcity_user = guess_zulip_user_from_teamcity(teamcity_fullname, user_profile.realm)

        if teamcity_user is None:
            teamcity_shortname = get_teamcity_property_value(message['teamcityProperties'],
                                                             'teamcity.build.triggeredBy.username')
            if teamcity_shortname is not None:
                teamcity_user = guess_zulip_user_from_teamcity(teamcity_shortname, user_profile.realm)

        if teamcity_user is None:
            # We can't figure out who started this build - there's nothing we can do here.
            logging.info("Teamcity webhook couldn't find a matching Zulip user for "
                         "Teamcity user '%s' or '%s'" % (teamcity_fullname, teamcity_shortname))
            return json_success()

        body = "Your personal build of " + body
        check_send_private_message(user_profile, request.client, teamcity_user, body)

        return json_success()

    check_send_stream_message(user_profile, request.client, stream, topic, body)
    return json_success()
<commit_msg>mypy: Use Python 3 type syntax in zerver/webhooks/teamcity/view.py.<commit_after># Webhooks for teamcity integration

import logging
from typing import Any, Dict, List, Optional

import ujson
from django.db.models import Q
from django.http import HttpRequest, HttpResponse

from zerver.decorator import api_key_only_webhook_view
from zerver.lib.actions import check_send_private_message, \
    check_send_stream_message
from zerver.lib.request import REQ, has_request_variables
from zerver.lib.response import json_error, json_success
from zerver.models import Realm, UserProfile

def guess_zulip_user_from_teamcity(teamcity_username: str, realm: Realm) -> Optional[UserProfile]:
    try:
        # Try to find a matching user in Zulip
        # We search a user's full name, short name,
        # and beginning of email address
        user = UserProfile.objects.filter(
            Q(full_name__iexact=teamcity_username) |
            Q(short_name__iexact=teamcity_username) |
            Q(email__istartswith=teamcity_username),
            is_active=True,
            realm=realm).order_by("id")[0]
        return user
    except IndexError:
        return None

def get_teamcity_property_value(property_list: List[Dict[str, str]], name: str) -> Optional[str]:
    for property in property_list:
        if property['name'] == name:
            return property['value']
    return None

@api_key_only_webhook_view('Teamcity')
@has_request_variables
def api_teamcity_webhook(request: HttpRequest, user_profile: UserProfile,
                         payload: Dict[str, Any]=REQ(argument_type='body'),
                         stream: str=REQ(default='teamcity')) -> HttpResponse:
    message = payload['build']

    build_name = message['buildFullName']
    build_url = message['buildStatusUrl']
    changes_url = build_url + '&tab=buildChangesDiv'
    build_number = message['buildNumber']
    build_result = message['buildResult']
    build_result_delta = message['buildResultDelta']
    build_status = message['buildStatus']

    if build_result == 'success':
        if build_result_delta == 'fixed':
            status = 'has been fixed! :thumbsup:'
        else:
            status = 'was successful! :thumbsup:'
    elif build_result == 'failure':
        if build_result_delta == 'broken':
            status = 'is broken with status %s! :thumbsdown:' % (build_status,)
        else:
            status = 'is still broken with status %s! :thumbsdown:' % (build_status,)
    elif build_result == 'running':
        status = 'has started.'
    else:
        status = '(has no message specified for status %s)' % (build_status,)

    template = (
        u'%s build %s %s\n'
        u'Details: [changes](%s), [build log](%s)')

    body = template % (build_name, build_number, status, changes_url, build_url)
    topic = build_name

    # Check if this is a personal build, and if so try to private message the user who triggered it.
    if get_teamcity_property_value(message['teamcityProperties'], 'env.BUILD_IS_PERSONAL') == 'true':
        # The triggeredBy field gives us the teamcity user full name, and the
        # "teamcity.build.triggeredBy.username" property gives us the teamcity username.
        # Let's try finding the user email from both.
        teamcity_fullname = message['triggeredBy'].split(';')[0]
        teamcity_user = guess_zulip_user_from_teamcity(teamcity_fullname, user_profile.realm)

        if teamcity_user is None:
            teamcity_shortname = get_teamcity_property_value(message['teamcityProperties'],
                                                             'teamcity.build.triggeredBy.username')
            if teamcity_shortname is not None:
                teamcity_user = guess_zulip_user_from_teamcity(teamcity_shortname, user_profile.realm)

        if teamcity_user is None:
            # We can't figure out who started this build - there's nothing we can do here.
            logging.info("Teamcity webhook couldn't find a matching Zulip user for "
                         "Teamcity user '%s' or '%s'" % (teamcity_fullname, teamcity_shortname))
            return json_success()

        body = "Your personal build of " + body
        check_send_private_message(user_profile, request.client, teamcity_user, body)

        return json_success()

    check_send_stream_message(user_profile, request.client, stream, topic, body)
    return json_success()
<|endoftext|>


<commit_before>__version__ = {'major': 0, 'minor': 5, 'patch': 6}

class ModelProxy(object):
    pass

def make_passwordmanager():
    from cryptacular.bcrypt import BCRYPTPasswordManager
    from cryptacular.core import DelegatingPasswordManager
    from ziggurat_foundations.utils import PlaceholderPasswordChecker
    return DelegatingPasswordManager(
        preferred=BCRYPTPasswordManager(),
        fallbacks=(PlaceholderPasswordChecker(),)
    )

# this maps models together during runtime so we can access them in methods:
#
# def foo(self):
#     query(self, self.OtherModel)
#

def ziggurat_model_init(*k, **kw):
    """
    This function handles attaching model to service if model has one specified as `_ziggurat_service`,
    Also attached a proxy object holding all model definitions that services might use

    :param k:
    :param kw:
    :return:
    """
    models = ModelProxy()
    for cls2 in k:
        setattr(models, cls2.__name__, cls2)

    for cls in k:
        if cls.__name__ == 'User':
            if kw.get('passwordmanager'):
                cls.passwordmanager = kw['passwordmanager']
            else:
                cls.passwordmanager = make_passwordmanager()

        for cls2 in k:
            setattr(models, cls2.__name__, cls2)
        setattr(cls, "_ziggurat_models", models)
        # if model has a manager attached attached the class also to manager
        if hasattr(cls, '_ziggurat_service'):
            setattr(cls._ziggurat_service, 'model', cls)
            setattr(cls._ziggurat_service, 'models_proxy', models)<commit_msg>version: 0.6.0<commit_after>__version__ = {'major': 0, 'minor': 6, 'patch': 0}

class ModelProxy(object):
    pass

def make_passwordmanager():
    from cryptacular.bcrypt import BCRYPTPasswordManager
    from cryptacular.core import DelegatingPasswordManager
    from ziggurat_foundations.utils import PlaceholderPasswordChecker
    return DelegatingPasswordManager(
        preferred=BCRYPTPasswordManager(),
        fallbacks=(PlaceholderPasswordChecker(),)
    )

# this maps models together during runtime so we can access them in methods:
#
# def foo(self):
#     query(self, self.OtherModel)
#

def ziggurat_model_init(*k, **kw):
    """
    This function handles attaching model to service if model has one specified as `_ziggurat_service`,
    Also attached a proxy object holding all model definitions that services might use

    :param k:
    :param kw:
    :return:
    """
    models = ModelProxy()
    for cls2 in k:
        setattr(models, cls2.__name__, cls2)

    for cls in k:
        if cls.__name__ == 'User':
            if kw.get('passwordmanager'):
                cls.passwordmanager = kw['passwordmanager']
            else:
                cls.passwordmanager = make_passwordmanager()

        for cls2 in k:
            setattr(models, cls2.__name__, cls2)
        setattr(cls, "_ziggurat_models", models)
        # if model has a manager attached attached the class also to manager
        if hasattr(cls, '_ziggurat_service'):
            setattr(cls._ziggurat_service, 'model', cls)
            setattr(cls._ziggurat_service, 'models_proxy', models)
<|endoftext|>