<commit_before>"""
github3.models
==============

This module provides the basic models used in github3.py

"""

from datetime import datetime
from json import dumps

class GitHubCore(object):
    """The :class:`GitHubCore <GitHubCore>` object. This class provides some
    basic attributes to other classes that are very useful to have.
    """
    def __init__(self, session=None):
        self._session = session
        if self._session:
            setattr(self._session, '_remain', 5000)
        self._github_url = 'https://api.github.com'
        self._time_format = '%Y-%m-%dT%H:%M:%SZ'
        self._remaining = 5000

    def __repr__(self):
        return '<github3-core at 0x%x>' % id(self)

    def _json(self, request, status_code):
        if request.status_code == status_code:
            return request.json if request.content else True
        if request.status_code >= 400:
            raise Error(request)
        return None

    def _getr(self, url, status_code=200, **kwargs):
        # In the rare instance we care about the entire response.
        req = None
        if self._remaining > 0:
            req = self._session.get(url, **kwargs)
            if req.status_code != status_code or req.status_code >= 400:
                raise Error(req)
        return req

    def _boolean(self, request, status_code):
        if request.status_code == status_code:
            return True
        if request.status_code != 404 and request.status_code >= 400:
            raise Error(request)
        return False

    def _delete(self, url, status_code=204, **kwargs):
        req = False
        if self._remaining > 0:
            req = self._session.delete(url, **kwargs)
            req = self._boolean(req, status_code)
        return req

    def _get(self, url, status_code=200, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.get(url, **kwargs)
            if status_code == 204:
                # We're not expecting any json back
                # If we left it as a simple _json() call there would be a
                #  TypeError since requests doesn't handle that cleanly
                req = self._boolean(req, status_code)
            else:
                req = self._json(req, status_code)
        return req

    def _patch(self, url, data=None, status_code=200, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.patch(url, data, **kwargs)
            req = self._json(req, status_code)
        return req

    def _post(self, url, data=None, status_code=201, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.post(url, data, **kwargs)
            req = self._json(req, status_code)
        return req

    def _put(self, url, data=None, status_code=204, **kwargs):
        req = False
        if self._remaining > 0:
            kwargs.update(headers={'Content-Length': '0'})
            req = self._session.put(url, data, **kwargs)
            req = self._boolean(req, status_code)
        return req

    def _strptime(self, time_str):
        """Converts an ISO 8601 formatted string into a datetime object."""
        if time_str:
            return datetime.strptime(time_str, self._time_format)
        else:
            return None

    @property
    def ratelimit_remaining(self):
        """Number of requests before GitHub imposes a ratelimit."""
        json = self._get(self._github_url + '/rate_limit')
        return json.get('rate', {}).get('remaining', 5000)

class BaseComment(GitHubCore):
    """The :class:`BaseComment <BaseComment>` object. A basic class for Gist,
    Issue and Pull Request Comments."""
    def __init__(self, comment, session):
        super(BaseComment, self).__init__(session)
        self._update_(comment)

    def __repr__(self):
        return '<github3-comment at 0x%x>' % id(self)

    def _update_(self, comment):
        self._id = comment.get('id')
        self._body = comment.get('body')
        self._bodyt = comment.get('body_text')
        self._bodyh = comment.get('body_html')
        self._created = self._strptime(comment.get('created_at'))
        self._updated = self._strptime(comment.get('updated_at'))

        self._api = comment.get('url')
        if comment.get('_links'):
            self._url = comment['_links'].get('html')
            self._pull = comment['_links'].get('pull_request')

        self._path = comment.get('path')
        self._pos = comment.get('position')
        self._cid = comment.get('commit_id')

    @property
    def body(self):
        """Body of the comment. (As written by the commenter)"""
        return self._body

    @property
    def body_html(self):
        """Body of the comment formatted as html."""
        return self._bodyh

    @property
    def body_text(self):
        """Body of the comment formatted as plain-text. (Stripped of markdown,
        etc.)"""
        return self._bodyt

    @property
    def created_at(self):
        """datetime object representing when the comment was created."""
        return self._created

    def delete(self):
        """Delete this comment.

        :returns: bool
        """
        return self._delete(self._api)

    def edit(self, body):
        """Edit this comment.

        :param body: (required), new body of the comment, Markdown formatted
        :type body: str
        :returns: bool
        """
        if body:
            json = self._patch(self._api, dumps({'body': body}))
            if json:
                self._update_(json)
                return True
        return False

    @property
    def id(self):
        """Unique ID of the comment."""
        return self._id

    @property
    def user(self):
        """:class:`User <user.User>` who created the comment."""
        return self._user

class BaseCommit(GitHubCore):
    """The :class:`BaseCommit <BaseCommit>` object. This serves as the base for
    the various types of commit objects returned by the API.
    """
    def __init__(self, commit, session):
        super(BaseCommit, self).__init__(session)
        self._api = commit.get('url')
        self._sha = commit.get('sha')
        self._msg = commit.get('message')
        self._parents = commit.get('parents', [])

    @property
    def message(self):
        """Commit message"""
        return self._msg

    @property
    def parents(self):
        """List of parents to this commit."""
        return self._parents

    @property
    def sha(self):
        """SHA of this commit."""
        return self._sha

class BaseAccount(GitHubCore):
    """The :class:`BaseAccount <BaseAccount>` object. This is used to do the
    heavy lifting for :class:`Organization <org.Organization>` and :class:`User
    <user.User>` objects.
    """
    def __init__(self, acct, session):
        super(BaseAccount, self).__init__(session)
        self._update_(acct)

    def __repr__(self):
        return '<BaseAccount [%s:%s]>' % (self._login, self._name)

    def _update_(self, acct):
        # Public information
        ## e.g. https://api.github.com/users/self._login
        self._type = None
        if acct.get('type'):
            self._type = acct.get('type')
        self._api = acct.get('url', '')

        self._avatar = acct.get('avatar_url', '')
        self._blog = acct.get('blog', '')
        self._company = acct.get('company', '')

        self._created = None
        if acct.get('created_at'):
            self._created = self._strptime(acct.get('created_at'))
        self._email = acct.get('email')

        ## The number of people following this acct
        self._followers = acct.get('followers', 0)

        ## The number of people this acct follows
        self._following = acct.get('following', 0)

        self._id = acct.get('id', 0)
        self._location = acct.get('location', '')
        self._login = acct.get('login', '')

        ## e.g. first_name last_name
        self._name = acct.get('name', '')

        ## The number of public_repos
        self._public_repos = acct.get('public_repos', 0)

        ## e.g. https://github.com/self._login
        self._url = acct.get('html_url', '')

        ## The number of private repos
        if self._type == 'Organization':
            self._private_repos = acct.get('private_repos', 0)

        self._bio = acct.get('bio')
        if self._type == 'User':

            ## The number of people this acct folows
            self._grav_id = acct.get('gravatar_id', '')
            self._hire = acct.get('hireable', False)

            ## The number of public_gists
            self._public_gists = acct.get('public_gists', 0)

            # Private information
            self._disk = acct.get('disk_usage', 0)

            self._owned_private_repos = acct.get('owned_private_repos', 0)
            self._private_gists = acct.get('total_private_gists', 0)
            self._private_repos = acct.get('total_private_repos', 0)

    @property
    def avatar_url(self):
        """URL of the avatar at gravatar"""
        return self._avatar

    @property
    def bio(self):
        """Markdown formatted biography"""
        return self._bio

    @property
    def blog(self):
        """URL of the blog"""
        return self._blog

    @property
    def company(self):
        """Name of the company"""
        return self._company

    @property
    def created_at(self):
        """datetime object representing the date the account was created"""
        return self._created

    @property
    def email(self):
        """E-mail address of the user/org"""
        return self._email

    @property
    def followers(self):
        """Number of followers"""
        return self._followers

    @property
    def following(self):
        """Number of people the user is following"""
        return self._following

    @property
    def html_url(self):
        """URL of the user/org's profile"""
        return self._url

    @property
    def id(self):
        """Unique ID of the user/org"""
        return self._id

    @property
    def location(self):
        """Location of the user/org"""
        return self._location

    @property
    def login(self):
        """login name of the user/org"""
        return self._login

    @property
    def name(self):
        """Real name of the user/org"""
        return self._name

    @property
    def public_repos(self):
        """Number of public repos owned by the user/org"""
        return self._public_repos

class Error(BaseException):
    def __init__(self, resp):
        super(Error, self).__init__()
        self._code = resp.status_code
        error = resp.json
        self._message = error.get('message')
        self._errors = []
        if error.get('errors'):
            self._errors = error.get('errors')

    def __repr__(self):
        return '<Error [%s]>' % (self._message or self._code)

    def __str__(self):
        if not self._errors:
            return '{0} {1}'.format(self._code, self._message)
        else:
            return '{0} {1}: {2}'.format(self._code, self._message,
                ', '.join(self._errors))

    @property
    def code(self):
        return self._code

    @property
    def errors(self):
        return self._errors

    @property
    def message(self):
        return self._message
<commit_msg>subclass from Exception rather than BaseException<commit_after>"""
github3.models
==============

This module provides the basic models used in github3.py

"""

from datetime import datetime
from json import dumps

class GitHubCore(object):
    """The :class:`GitHubCore <GitHubCore>` object. This class provides some
    basic attributes to other classes that are very useful to have.
    """
    def __init__(self, session=None):
        self._session = session
        if self._session:
            setattr(self._session, '_remain', 5000)
        self._github_url = 'https://api.github.com'
        self._time_format = '%Y-%m-%dT%H:%M:%SZ'
        self._remaining = 5000

    def __repr__(self):
        return '<github3-core at 0x%x>' % id(self)

    def _json(self, request, status_code):
        if request.status_code == status_code:
            return request.json if request.content else True
        if request.status_code >= 400:
            raise Error(request)
        return None

    def _getr(self, url, status_code=200, **kwargs):
        # In the rare instance we care about the entire response.
        req = None
        if self._remaining > 0:
            req = self._session.get(url, **kwargs)
            if req.status_code != status_code or req.status_code >= 400:
                raise Error(req)
        return req

    def _boolean(self, request, status_code):
        if request.status_code == status_code:
            return True
        if request.status_code != 404 and request.status_code >= 400:
            raise Error(request)
        return False

    def _delete(self, url, status_code=204, **kwargs):
        req = False
        if self._remaining > 0:
            req = self._session.delete(url, **kwargs)
            req = self._boolean(req, status_code)
        return req

    def _get(self, url, status_code=200, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.get(url, **kwargs)
            if status_code == 204:
                # We're not expecting any json back
                # If we left it as a simple _json() call there would be a
                #  TypeError since requests doesn't handle that cleanly
                req = self._boolean(req, status_code)
            else:
                req = self._json(req, status_code)
        return req

    def _patch(self, url, data=None, status_code=200, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.patch(url, data, **kwargs)
            req = self._json(req, status_code)
        return req

    def _post(self, url, data=None, status_code=201, **kwargs):
        req = None
        if self._remaining > 0:
            req = self._session.post(url, data, **kwargs)
            req = self._json(req, status_code)
        return req

    def _put(self, url, data=None, status_code=204, **kwargs):
        req = False
        if self._remaining > 0:
            kwargs.update(headers={'Content-Length': '0'})
            req = self._session.put(url, data, **kwargs)
            req = self._boolean(req, status_code)
        return req

    def _strptime(self, time_str):
        """Converts an ISO 8601 formatted string into a datetime object."""
        if time_str:
            return datetime.strptime(time_str, self._time_format)
        else:
            return None

    @property
    def ratelimit_remaining(self):
        """Number of requests before GitHub imposes a ratelimit."""
        json = self._get(self._github_url + '/rate_limit')
        return json.get('rate', {}).get('remaining', 5000)

class BaseComment(GitHubCore):
    """The :class:`BaseComment <BaseComment>` object. A basic class for Gist,
    Issue and Pull Request Comments."""
    def __init__(self, comment, session):
        super(BaseComment, self).__init__(session)
        self._update_(comment)

    def __repr__(self):
        return '<github3-comment at 0x%x>' % id(self)

    def _update_(self, comment):
        self._id = comment.get('id')
        self._body = comment.get('body')
        self._bodyt = comment.get('body_text')
        self._bodyh = comment.get('body_html')
        self._created = self._strptime(comment.get('created_at'))
        self._updated = self._strptime(comment.get('updated_at'))

        self._api = comment.get('url')
        if comment.get('_links'):
            self._url = comment['_links'].get('html')
            self._pull = comment['_links'].get('pull_request')

        self._path = comment.get('path')
        self._pos = comment.get('position')
        self._cid = comment.get('commit_id')

    @property
    def body(self):
        """Body of the comment. (As written by the commenter)"""
        return self._body

    @property
    def body_html(self):
        """Body of the comment formatted as html."""
        return self._bodyh

    @property
    def body_text(self):
        """Body of the comment formatted as plain-text. (Stripped of markdown,
        etc.)"""
        return self._bodyt

    @property
    def created_at(self):
        """datetime object representing when the comment was created."""
        return self._created

    def delete(self):
        """Delete this comment.

        :returns: bool
        """
        return self._delete(self._api)

    def edit(self, body):
        """Edit this comment.

        :param body: (required), new body of the comment, Markdown formatted
        :type body: str
        :returns: bool
        """
        if body:
            json = self._patch(self._api, dumps({'body': body}))
            if json:
                self._update_(json)
                return True
        return False

    @property
    def id(self):
        """Unique ID of the comment."""
        return self._id

    @property
    def user(self):
        """:class:`User <user.User>` who created the comment."""
        return self._user

class BaseCommit(GitHubCore):
    """The :class:`BaseCommit <BaseCommit>` object. This serves as the base for
    the various types of commit objects returned by the API.
    """
    def __init__(self, commit, session):
        super(BaseCommit, self).__init__(session)
        self._api = commit.get('url')
        self._sha = commit.get('sha')
        self._msg = commit.get('message')
        self._parents = commit.get('parents', [])

    @property
    def message(self):
        """Commit message"""
        return self._msg

    @property
    def parents(self):
        """List of parents to this commit."""
        return self._parents

    @property
    def sha(self):
        """SHA of this commit."""
        return self._sha

class BaseAccount(GitHubCore):
    """The :class:`BaseAccount <BaseAccount>` object. This is used to do the
    heavy lifting for :class:`Organization <org.Organization>` and :class:`User
    <user.User>` objects.
    """
    def __init__(self, acct, session):
        super(BaseAccount, self).__init__(session)
        self._update_(acct)

    def __repr__(self):
        return '<BaseAccount [%s:%s]>' % (self._login, self._name)

    def _update_(self, acct):
        # Public information
        ## e.g. https://api.github.com/users/self._login
        self._type = None
        if acct.get('type'):
            self._type = acct.get('type')
        self._api = acct.get('url', '')

        self._avatar = acct.get('avatar_url', '')
        self._blog = acct.get('blog', '')
        self._company = acct.get('company', '')

        self._created = None
        if acct.get('created_at'):
            self._created = self._strptime(acct.get('created_at'))
        self._email = acct.get('email')

        ## The number of people following this acct
        self._followers = acct.get('followers', 0)

        ## The number of people this acct follows
        self._following = acct.get('following', 0)

        self._id = acct.get('id', 0)
        self._location = acct.get('location', '')
        self._login = acct.get('login', '')

        ## e.g. first_name last_name
        self._name = acct.get('name', '')

        ## The number of public_repos
        self._public_repos = acct.get('public_repos', 0)

        ## e.g. https://github.com/self._login
        self._url = acct.get('html_url', '')

        ## The number of private repos
        if self._type == 'Organization':
            self._private_repos = acct.get('private_repos', 0)

        self._bio = acct.get('bio')
        if self._type == 'User':

            ## The number of people this acct folows
            self._grav_id = acct.get('gravatar_id', '')
            self._hire = acct.get('hireable', False)

            ## The number of public_gists
            self._public_gists = acct.get('public_gists', 0)

            # Private information
            self._disk = acct.get('disk_usage', 0)

            self._owned_private_repos = acct.get('owned_private_repos', 0)
            self._private_gists = acct.get('total_private_gists', 0)
            self._private_repos = acct.get('total_private_repos', 0)

    @property
    def avatar_url(self):
        """URL of the avatar at gravatar"""
        return self._avatar

    @property
    def bio(self):
        """Markdown formatted biography"""
        return self._bio

    @property
    def blog(self):
        """URL of the blog"""
        return self._blog

    @property
    def company(self):
        """Name of the company"""
        return self._company

    @property
    def created_at(self):
        """datetime object representing the date the account was created"""
        return self._created

    @property
    def email(self):
        """E-mail address of the user/org"""
        return self._email

    @property
    def followers(self):
        """Number of followers"""
        return self._followers

    @property
    def following(self):
        """Number of people the user is following"""
        return self._following

    @property
    def html_url(self):
        """URL of the user/org's profile"""
        return self._url

    @property
    def id(self):
        """Unique ID of the user/org"""
        return self._id

    @property
    def location(self):
        """Location of the user/org"""
        return self._location

    @property
    def login(self):
        """login name of the user/org"""
        return self._login

    @property
    def name(self):
        """Real name of the user/org"""
        return self._name

    @property
    def public_repos(self):
        """Number of public repos owned by the user/org"""
        return self._public_repos

class Error(Exception):
    def __init__(self, resp):
        super(Error, self).__init__()
        self._code = resp.status_code
        error = resp.json
        self._message = error.get('message')
        self._errors = []
        if error.get('errors'):
            self._errors = error.get('errors')

    def __repr__(self):
        return '<Error [%s]>' % (self._message or self._code)

    def __str__(self):
        if not self._errors:
            return '{0} {1}'.format(self._code, self._message)
        else:
            return '{0} {1}: {2}'.format(self._code, self._message,
                ', '.join(self._errors))

    @property
    def code(self):
        return self._code

    @property
    def errors(self):
        return self._errors

    @property
    def message(self):
        return self._message
<|endoftext|>


<commit_before>from typing import List, Dict
import numbers, re
from datetime import date, datetime
from etk.field_types import FieldType
from etk.etk_exceptions import ISODateError

class KGSchema(object):
    """
    This class define the schema for a knowledge graph object.
    Create a knowledge graph schema according to the master config the user defined in myDIG UI
    """

    def __init__(self, config: Dict) -> None:
        """
        Record a mapping about each fields and its type from config file

        Args:
            config: Dict
        """

        self.fields_dict = dict()
        try:
            for field in config["fields"]:
                if config["fields"][field]["type"] == "kg_id":
                    self.fields_dict[field] = FieldType.KG_ID
                elif config["fields"][field]["type"] == "number":
                    self.fields_dict[field] = FieldType.NUMBER
                elif config["fields"][field]["type"] == "date":
                    self.fields_dict[field] = FieldType.DATE
                elif config["fields"][field]["type"] == "location":
                    self.fields_dict[field] = FieldType.LOCATION
                else:
                    self.fields_dict[field] = FieldType.STRING

        except KeyError as key:
            print(str(key) + " not in config")

    @property
    def fields(self) -> List[str]:
        """
        Return a list of all fields that are defined in master config

        Args:

        Returns: List of fields
        """
        return list(self.fields_dict.keys())

    def has_field(self, field_name: str) -> bool:
        """
        Return true if the schema has the field, otherwise false

        Args:
            field_name: str

        Returns: bool
        """
        return field_name in self.fields_dict

    def field_type(self, field_name: str) -> FieldType:
        """
        Return the type of a field defined in schema, if field not defined, return None

        Args:
            field_name: str

        Returns: FieldType
        """
        return self.fields_dict.get(field_name, None)

        # Amandeep: The code below does not do what the description of the function says, replaced with line above
        # if self.has_field(field_name):
        #     return self.fields_dict[field_name]
        # else:
        #     print(field_name + " field not defined")

    @staticmethod
    def iso_date(d) -> str:
        """
        Return iso format of a date

        Args:
            d:
        Returns: str

        """
        if isinstance(d, datetime):
            return d.isoformat()
        elif isinstance(d, date):
            return datetime.combine(d, datetime.min.time()).isoformat()
        else:
            try:
                datetime.strptime(d, '%Y-%m-%dT%H:%M:%S')
                return d
            except ValueError:
                try:
                    datetime.strptime(d, '%Y-%m-%d')
                    return d + "T00:00:00"
                except ValueError:
                    pass
        raise ISODateError("Can not convert value to ISO format for kg")

    def is_valid(self, field_name, value) -> (bool, object):
        """
        Return true if the value type matches or can be coerced to the defined type in schema, otherwise false.
        If field not defined, return none

        Args:
            field_name: str
            value:

        Returns: bool, value, where the value may have been coerced to the required type.
        """

        if self.has_field(field_name):
            if self.fields_dict[field_name] == FieldType.KG_ID:
                return True, value

            if self.fields_dict[field_name] == FieldType.NUMBER:
                if isinstance(value, numbers.Number):
                    return True, value
                else:
                    # todo: Should try to parse the string as a number
                    return False, value
            if self.fields_dict[field_name] == FieldType.STRING:
                if isinstance(value, str):
                    return True, value.strip()
                else:
                    return True, str(value).strip()

            if self.fields_dict[field_name] == FieldType.DATE:
                valid, d = self.is_date(value)
                if valid:
                    return True, d.isoformat()
                else:
                    return False, value

            if self.fields_dict[field_name] == FieldType.LOCATION:
                valid, l = self.is_location(value)
                if valid:
                    return True, l
                else:
                    return False, value
        else:
            print('{} not found in KG Schema'.format(field_name))
            return False, value
            # Amandeep: returning False instead of printing a message
            # print(field_name + " field not defined")

    @staticmethod
    def is_date(v) -> (bool, date):
        """
        Boolean function for checking if v is a date

        Args:
            v:
        Returns: bool

        """
        if isinstance(v, date):
            return True, v
        try:
            reg = r'^([0-9]{4})(?:-(0[1-9]|1[0-2])(?:-(0[1-9]|[1-2][0-9]|3[0-1])(?:T' \
                  r'([0-5][0-9])(?::([0-5][0-9])(?::([0-5][0-9]))?)?)?)?)?$'
            match = re.match(reg, v)
            if match:
                groups = match.groups()
                patterns = ['%Y', '%m', '%d', '%H', '%M', '%S']
                d = datetime.strptime('-'.join([x for x in groups if x]),
                                      '-'.join([patterns[i] for i in range(len(patterns)) if groups[i]]))
                return True, d
        except:
            pass
        return False, v

    @staticmethod
    def is_location(v) -> (bool, str):
        """
        Boolean function for checking if v is a location format

        Args:
            v:
        Returns: bool

        """

        def convert2float(value):
            try:
                float_num = float(value)
                return float_num
            except ValueError:
                return False

        if not isinstance(v, str):
            return False, v
        split_lst = v.split(":")
        if len(split_lst) != 5:
            return False, v
        if convert2float(split_lst[3]):
            longitude = abs(convert2float(split_lst[3]))
            if longitude > 90:
                return False, v
        if convert2float(split_lst[4]):
            latitude = abs(convert2float(split_lst[3]))
            if latitude > 180:
                return False, v
        return True, v
<commit_msg>if Schema type is number, and the value type is string, try to convert<commit_after>from typing import List, Dict
import numbers, re
from datetime import date, datetime
from etk.field_types import FieldType
from etk.etk_exceptions import ISODateError

class KGSchema(object):
    """
    This class define the schema for a knowledge graph object.
    Create a knowledge graph schema according to the master config the user defined in myDIG UI
    """

    def __init__(self, config: Dict) -> None:
        """
        Record a mapping about each fields and its type from config file

        Args:
            config: Dict
        """

        self.fields_dict = dict()
        try:
            for field in config["fields"]:
                if config["fields"][field]["type"] == "kg_id":
                    self.fields_dict[field] = FieldType.KG_ID
                elif config["fields"][field]["type"] == "number":
                    self.fields_dict[field] = FieldType.NUMBER
                elif config["fields"][field]["type"] == "date":
                    self.fields_dict[field] = FieldType.DATE
                elif config["fields"][field]["type"] == "location":
                    self.fields_dict[field] = FieldType.LOCATION
                else:
                    self.fields_dict[field] = FieldType.STRING

        except KeyError as key:
            print(str(key) + " not in config")

    @property
    def fields(self) -> List[str]:
        """
        Return a list of all fields that are defined in master config

        Args:

        Returns: List of fields
        """
        return list(self.fields_dict.keys())

    def has_field(self, field_name: str) -> bool:
        """
        Return true if the schema has the field, otherwise false

        Args:
            field_name: str

        Returns: bool
        """
        return field_name in self.fields_dict

    def field_type(self, field_name: str) -> FieldType:
        """
        Return the type of a field defined in schema, if field not defined, return None

        Args:
            field_name: str

        Returns: FieldType
        """
        return self.fields_dict.get(field_name, None)

        # Amandeep: The code below does not do what the description of the function says, replaced with line above
        # if self.has_field(field_name):
        #     return self.fields_dict[field_name]
        # else:
        #     print(field_name + " field not defined")

    @staticmethod
    def iso_date(d) -> str:
        """
        Return iso format of a date

        Args:
            d:
        Returns: str

        """
        if isinstance(d, datetime):
            return d.isoformat()
        elif isinstance(d, date):
            return datetime.combine(d, datetime.min.time()).isoformat()
        else:
            try:
                datetime.strptime(d, '%Y-%m-%dT%H:%M:%S')
                return d
            except ValueError:
                try:
                    datetime.strptime(d, '%Y-%m-%d')
                    return d + "T00:00:00"
                except ValueError:
                    pass
        raise ISODateError("Can not convert value to ISO format for kg")

    @staticmethod
    def parse_number(txt_num):
        if isinstance(txt_num, str) or isinstance(txt_num, numbers.Number):

            if isinstance(txt_num, numbers.Number):
                return str(txt_num)

            try:
                # this will fail spectacularly if the data originated in europe,
                # where they use ',' inplace of '.' and vice versa for numbers

                text = txt_num.strip().replace('\n', '').replace('\t', '').replace(',', '')
                num = str(float(text)) if '.' in text else str(int(text))
                return num
            except:
                pass
        return None

    def is_valid(self, field_name, value) -> (bool, object):
        """
        Return true if the value type matches or can be coerced to the defined type in schema, otherwise false.
        If field not defined, return none

        Args:
            field_name: str
            value:

        Returns: bool, value, where the value may have been coerced to the required type.
        """

        if self.has_field(field_name):
            if self.fields_dict[field_name] == FieldType.KG_ID:
                return True, value

            if self.fields_dict[field_name] == FieldType.NUMBER:
                if isinstance(value, numbers.Number):
                    return True, value
                else:
                    converted_number = self.parse_number(value)
                    return (False, value) if not converted_number else (True, value)
            if self.fields_dict[field_name] == FieldType.STRING:
                if isinstance(value, str):
                    return True, value.strip()
                else:
                    return True, str(value).strip()

            if self.fields_dict[field_name] == FieldType.DATE:
                valid, d = self.is_date(value)
                if valid:
                    return True, d.isoformat()
                else:
                    return False, value

            if self.fields_dict[field_name] == FieldType.LOCATION:
                valid, l = self.is_location(value)
                if valid:
                    return True, l
                else:
                    return False, value
        else:
            print('{} not found in KG Schema'.format(field_name))
            return False, value
            # Amandeep: returning False instead of printing a message
            # print(field_name + " field not defined")

    @staticmethod
    def is_date(v) -> (bool, date):
        """
        Boolean function for checking if v is a date

        Args:
            v:
        Returns: bool

        """
        if isinstance(v, date):
            return True, v
        try:
            reg = r'^([0-9]{4})(?:-(0[1-9]|1[0-2])(?:-(0[1-9]|[1-2][0-9]|3[0-1])(?:T' \
                  r'([0-5][0-9])(?::([0-5][0-9])(?::([0-5][0-9]))?)?)?)?)?$'
            match = re.match(reg, v)
            if match:
                groups = match.groups()
                patterns = ['%Y', '%m', '%d', '%H', '%M', '%S']
                d = datetime.strptime('-'.join([x for x in groups if x]),
                                      '-'.join([patterns[i] for i in range(len(patterns)) if groups[i]]))
                return True, d
        except:
            pass
        return False, v

    @staticmethod
    def is_location(v) -> (bool, str):
        """
        Boolean function for checking if v is a location format

        Args:
            v:
        Returns: bool

        """

        def convert2float(value):
            try:
                float_num = float(value)
                return float_num
            except ValueError:
                return False

        if not isinstance(v, str):
            return False, v
        split_lst = v.split(":")
        if len(split_lst) != 5:
            return False, v
        if convert2float(split_lst[3]):
            longitude = abs(convert2float(split_lst[3]))
            if longitude > 90:
                return False, v
        if convert2float(split_lst[4]):
            latitude = abs(convert2float(split_lst[3]))
            if latitude > 180:
                return False, v
        return True, v
<|endoftext|>


<commit_before>"""A simple root of supervisor tree and a generic container."""

___all__ = [
    'GRACEFUL_EXIT',
    'SERVER_MAKER',
    'prepare',
]

import argparse
import logging
import os
import signal
from contextlib import ExitStack

import curio

from garage import asyncs
from garage import components
from garage.okay import OKAY, NOT_OKAY
from garage.startups.logging import LoggingComponent
from startup import Startup, startup

GRACEFUL_EXIT = __name__ + ':graceful_exit'
GRACEFUL_PERIOD = 5  # Unit: seconds

SERVER_MAKER = __name__ + ':server_maker'
SERVER_MAKERS = __name__ + ':server_makers'

LOG = logging.getLogger(__name__)

def prepare(*, prog=None, description, comps, verbose=1):
    parser = argparse.ArgumentParser(prog=prog, description=description)

    startup.set(components.MAIN, main)
    startup.set(components.PARSER, parser)
    startup(copy_first_stage_vars)

    next_startup = Startup()
    parser.set_defaults(next_startup=next_startup)

    next_startup.set(GRACEFUL_EXIT, curio.Event())

    # Overcome the limitation that startup requires >0 writes.
    next_startup.set(SERVER_MAKER, None)
    next_startup(collect_server_maker)

    # First-stage startup
    components.bind(LoggingComponent(verbose=verbose))

    # Second-stage startup
    for comp in components.find_closure(*comps, ignore=[GRACEFUL_EXIT]):
        components.bind(comp, next_startup=next_startup)

def copy_first_stage_vars(args: components.ARGS, parser: components.PARSER):
    next_startup = args.next_startup
    next_startup.set(components.ARGS, args)
    next_startup.set(components.PARSER, parser)

def collect_server_maker(server_makers: [SERVER_MAKER]) -> SERVER_MAKERS:
    return tuple(filter(None, server_makers))

# The root node of the supervisor tree
async def init(graceful_exit, server_makers):
    okay = NOT_OKAY
    async with asyncs.TaskStack() as servers:
        LOG.info('start servers: pid=%d', os.getpid())
        for server_maker in server_makers:
            await servers.spawn(server_maker())
        # Also spawn default signal handler
        await servers.spawn(signal_handler(graceful_exit, GRACEFUL_PERIOD))
        # Now let's wait for the servers...
        async with curio.wait(servers) as wait_servers:
            # When one server exits, normally or not, we bring down all
            # other servers
            server_task = await wait_servers.next_done()
            try:
                await server_task.join()
                LOG.info('server exit: %r', server_task)
                okay = OKAY
            except curio.TaskError:
                LOG.exception('server crash: %r', server_task)
            LOG.info('stop servers')
            # curio.wait() will cancel all the remaining tasks
    LOG.info('exit')
    return okay

async def signal_handler(graceful_exit, graceful_period):
    # Exploit the fact that when one of the server task exits, the init
    # task will bring down all other server tasks
    async with curio.SignalSet(signal.SIGINT, signal.SIGTERM) as sigset:
        sig = await sigset.wait()
        LOG.info('receive signal: %s', sig)
        if sig is signal.SIGINT:
            LOG.info('notify graceful exit')
            await graceful_exit.set()
        elif sig is signal.SIGTERM:
            return
        else:
            raise AssertionError('unknown signal: %s' % sig)
        async with curio.ignore_after(graceful_period):
            sig = await sigset.wait()
            LOG.info('receive signal again: %s', sig)
            return
        LOG.info('exceed graceful period %f', graceful_period)

def main(args):
    with ExitStack() as exit_stack:
        next_startup = args.next_startup
        next_startup.set(components.EXIT_STACK, exit_stack)
        varz = next_startup.call()
        graceful_exit = varz[GRACEFUL_EXIT]
        server_makers = varz[SERVER_MAKERS]
        del varz
        okay = curio.run(init(graceful_exit, server_makers))
        return 0 if okay else 1
<commit_msg>Let TaskStack cancel tasks<commit_after>"""A simple root of supervisor tree and a generic container."""

___all__ = [
    'GRACEFUL_EXIT',
    'SERVER_MAKER',
    'prepare',
]

import argparse
import logging
import os
import signal
from contextlib import ExitStack

import curio

from garage import asyncs
from garage import components
from garage.okay import OKAY, NOT_OKAY
from garage.startups.logging import LoggingComponent
from startup import Startup, startup

GRACEFUL_EXIT = __name__ + ':graceful_exit'
GRACEFUL_PERIOD = 5  # Unit: seconds

SERVER_MAKER = __name__ + ':server_maker'
SERVER_MAKERS = __name__ + ':server_makers'

LOG = logging.getLogger(__name__)

def prepare(*, prog=None, description, comps, verbose=1):
    parser = argparse.ArgumentParser(prog=prog, description=description)

    startup.set(components.MAIN, main)
    startup.set(components.PARSER, parser)
    startup(copy_first_stage_vars)

    next_startup = Startup()
    parser.set_defaults(next_startup=next_startup)

    next_startup.set(GRACEFUL_EXIT, curio.Event())

    # Overcome the limitation that startup requires >0 writes.
    next_startup.set(SERVER_MAKER, None)
    next_startup(collect_server_maker)

    # First-stage startup
    components.bind(LoggingComponent(verbose=verbose))

    # Second-stage startup
    for comp in components.find_closure(*comps, ignore=[GRACEFUL_EXIT]):
        components.bind(comp, next_startup=next_startup)

def copy_first_stage_vars(args: components.ARGS, parser: components.PARSER):
    next_startup = args.next_startup
    next_startup.set(components.ARGS, args)
    next_startup.set(components.PARSER, parser)

def collect_server_maker(server_makers: [SERVER_MAKER]) -> SERVER_MAKERS:
    return tuple(filter(None, server_makers))

# The root node of the supervisor tree
async def init(graceful_exit, server_makers):
    okay = NOT_OKAY
    async with asyncs.TaskStack() as servers:
        LOG.info('start servers: pid=%d', os.getpid())
        for server_maker in server_makers:
            await servers.spawn(server_maker())
        # Also spawn default signal handler
        await servers.spawn(signal_handler(graceful_exit, GRACEFUL_PERIOD))
        # Now let's wait for the servers...
        server_task = await curio.wait(servers).next_done()
        # When one server exits, normally or not, we bring down all
        # other servers
        try:
            await server_task.join()
            LOG.info('server exit: %r', server_task)
            okay = OKAY
        except curio.TaskError:
            LOG.exception('server crash: %r', server_task)
        LOG.info('stop servers')
        # TaskStack will cancel all the remaining tasks
    LOG.info('exit')
    return okay

async def signal_handler(graceful_exit, graceful_period):
    # Exploit the fact that when one of the server task exits, the init
    # task will bring down all other server tasks
    async with curio.SignalSet(signal.SIGINT, signal.SIGTERM) as sigset:
        sig = await sigset.wait()
        LOG.info('receive signal: %s', sig)
        if sig is signal.SIGINT:
            LOG.info('notify graceful exit')
            await graceful_exit.set()
        elif sig is signal.SIGTERM:
            return
        else:
            raise AssertionError('unknown signal: %s' % sig)
        async with curio.ignore_after(graceful_period):
            sig = await sigset.wait()
            LOG.info('receive signal again: %s', sig)
            return
        LOG.info('exceed graceful period %f', graceful_period)

def main(args):
    with ExitStack() as exit_stack:
        next_startup = args.next_startup
        next_startup.set(components.EXIT_STACK, exit_stack)
        varz = next_startup.call()
        graceful_exit = varz[GRACEFUL_EXIT]
        server_makers = varz[SERVER_MAKERS]
        del varz
        okay = curio.run(init(graceful_exit, server_makers))
        return 0 if okay else 1
<|endoftext|>


<commit_before>from socket import error as SocketError
try:
    from select import poll, POLLIN
except ImportError:  # `poll` doesn't exist on OSX and other platforms
    poll = False
    try:
        from select import select
    except ImportError:  # `select` doesn't exist on AppEngine.
        select = False

def is_connection_dropped(conn):  # Platform-specific
    """
    Returns True if the connection is dropped and should be closed.

    :param conn:
        :class:`httplib.HTTPConnection` object.

    Note: For platforms like AppEngine, this will always return ``False`` to
    let the platform handle connection recycling transparently for us.
    """
    sock = getattr(conn, 'sock', False)
    if not sock: # Platform-specific: AppEngine
        return False

    if not poll:
        if not select: # Platform-specific: AppEngine
            return False

        try:
            return select([sock], [], [], 0.0)[0]
        except SocketError:
            return True

    # This version is better on platforms that support it.
    p = poll()
    p.register(sock, POLLIN)
    for (fno, ev) in p.poll(0.0):
        if fno == sock.fileno():
            # Either data is buffered (bad), or the connection is dropped.
            return True

<commit_msg>make it more clear why we are closing connections<commit_after>from socket import error as SocketError
try:
    from select import poll, POLLIN
except ImportError:  # `poll` doesn't exist on OSX and other platforms
    poll = False
    try:
        from select import select
    except ImportError:  # `select` doesn't exist on AppEngine.
        select = False

def is_connection_dropped(conn):  # Platform-specific
    """
    Returns True if the connection is dropped and should be closed.

    :param conn:
        :class:`httplib.HTTPConnection` object.

    Note: For platforms like AppEngine, this will always return ``False`` to
    let the platform handle connection recycling transparently for us.
    """
    sock = getattr(conn, 'sock', False)
    if sock is False: # Platform-specific: AppEngine
        return False

    if sock is None: # Connection already closed by e.g. httplib.
        return False

    if not poll:
        if not select: # Platform-specific: AppEngine
            return False

        try:
            return select([sock], [], [], 0.0)[0]
        except SocketError:
            return True

    # This version is better on platforms that support it.
    p = poll()
    p.register(sock, POLLIN)
    for (fno, ev) in p.poll(0.0):
        if fno == sock.fileno():
            # Either data is buffered (bad), or the connection is dropped.
            return True

<|endoftext|>


<commit_before>import pygame, sys, math
from Player import *
from Enemy import *
from Dirt import *
from Timer import *
from Score import *
from Level import *
pygame.init()

clock = pygame.time.Clock()

width = 768
height = 640
size = width, height
screen = pygame.display.set_mode(size)

bgColor = 0,0,0

level = Level("Digger level1.lvl")

enemies = [Enemy("Enemy-Pew.png")]

player = Player()
walls = level.walls

while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT: sys.exit()
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_UP:
                player.go("up")
            if event.key == pygame.K_DOWN:
                player.go("down")
            if event.key == pygame.K_RIGHT:
                player.go("right")
            if event.key == pygame.K_LEFT:
                player.go("left")
        if event.type == pygame.KEYUP:
            if event.key == pygame.K_UP:
                player.go("stop up")
            if event.key == pygame.K_DOWN:
                player.go("stop down")
            if event.key == pygame.K_RIGHT:
                player.go("stop right")
            if event.key == pygame.K_LEFT:
                player.go("stop left")
    else:
        if event.type == pygame.MOUSEMOTION:
            pygame.mouse.set_visible(False)
            player.goMouse(event.pos)

    player.move()
    player.screenCollide(size)
    for wall in walls:
        player.bounceWall(wall)

    bgColor = r,g,b
    screen.fill(bgColor)
    for ball in balls:
        screen.blit(ball.image, ball.rect)
    screen.blit(player.image, player.rect)
    for wall in walls:
        screen.blit(wall.image, wall.rect)
    pygame.display.flip()
    clock.tick(60)
<commit_msg>PEW PEW pew<commit_after>import pygame, sys, math
from Player import *
from Enemy import *
from Dirt import *
from Timer import *
from Score import *
from Level import *
pygame.init()

clock = pygame.time.Clock()

width = 768
height = 640
size = width, height
screen = pygame.display.set_mode(size)

bgColor = 0,0,0

level = Level("Digger level1.lvl")

enemies = [Enemy("Enemy-Pew.png")]

player = Player()
walls = level.walls

while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT: sys.exit()
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_UP:
                player.move("up")
            if event.key == pygame.K_DOWN:
                player.move("down")
            if event.key == pygame.K_RIGHT:
                player.move("right")
            if event.key == pygame.K_LEFT:
                player.move("left")
        if event.type == pygame.KEYUP:
            if event.key == pygame.K_UP:
                player.move("stop up")
            if event.key == pygame.K_DOWN:
                player.move("stop down")
            if event.key == pygame.K_RIGHT:
                player.move("stop right")
            if event.key == pygame.K_LEFT:
                player.move("stop left")

    player.move()
    player.screenCollide(width)
    for wall in walls:
        player.bounceWall(wall)

    bgColor = r,g,b = 0,0,0
    screen.fill(bgColor)
    for enemy in enemies:
        screen.blit(enemy.image, enemy.rect)
    screen.blit(player.image, player.rect)
    for wall in walls:
        screen.blit(wall.image, wall.rect)
    pygame.display.flip()
    clock.tick(60)
<|endoftext|>


<commit_before>#!/usr/bin/python2

from __future__ import print_function

from sklearn.feature_extraction.text import TfidfVectorizer
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from sklearn import svm
from sklearn.multiclass import OneVsRestClassifier
import logging
import webbrowser
import argparse
import sys

from settings import LOGFILE_URI, DATABASE_URI, LOG_LEVEL, CLIENT_ID, CLIENT_SECRET, CLIENT_ACCESSCODE, SUBREDDIT, REDIRECT_URI, LOG_FORMAT
import model
import praw

# This allows for Python 3 compatibility by replacing input() on Python 2
if sys.version_info[:2] <= (2, 7):
    input = raw_input

responses = {
    'bad': '''
Hi! Your post doesn't seem very suitable for /r/learnprogramming. This may be because you didn't include a code sample, provided very little detail or linked content that doesn't seem relevant. You can improve your post by:

- [Asking Questions The Smart Way](http://catb.org/~esr/faqs/smart-questions.html)
- Avoiding posting links without any explanation, discussion or question
- Using code pastebins (images don't count!)
- Reviewing the post guidelines on the sidebar

Don't worry about this message if you think your post **is** relevant - it may just be an error in my classifier - but please check the resources above anyway to make sure that your post is good.

*I am a bot for /r/learnprogramming using supervised learning to detect common questions. If this message is irrelevant or unhelpful, please report an issue so I can improve my classifier!*
    ''',
    'faq': '''
Hello! Your post seems similar to one of the common questions answered in the [/r/learnprogramming FAQ](https://www.reddit.com/r/learnprogramming/wiki/faq), and you might find an answer to your question there.

Links that may be useful to you:

- [Programming Resources](https://www.reddit.com/r/learnprogramming/wiki/online)
- [/r/cscareerquestions](https://www.reddit.com/r/cscareerquestions)
- [Getting Started with Programming](https://www.reddit.com/r/learnprogramming/wiki/gettingstarted)

*I am a bot for /r/learnprogramming using supervised learning to detect common questions. Your post was recognised as an faq question. If this seems irrelevant or unhelpful, please report an issue on GitHub!*
    '''
}

class Classifier(object):
    """
    Wrapper for the vectorizer and classifier that handles training of both.
    """
    def __init__(self, training_values, training_targets):
        self.vectorizer = TfidfVectorizer()
        self.classifier = OneVsRestClassifier(svm.LinearSVC(class_weight='balanced'))
        training_values = self.vectorizer.fit_transform(training_values).toarray()
        self.classifier.fit(training_values, training_targets)

    def classify(self, text):
        transformed_text = self.vectorizer.transform([text]).toarray()
        return self.classifier.predict(transformed_text)

    def get_probability(self, text):
        transformed_text = self.vectorizer.transform([text]).toarray()
        return self.classifier.decision_function(transformed_text)

def connect_to_database(uri):
    engine = create_engine(uri)
    return sessionmaker(bind=engine)

def get_reddit_client():
    reddit = praw.Reddit(user_agent='all platforms:Learn Programming Bot:v0.2.0-pre (by /u/Aurora0001, contact at github.com/Aurora0001/LearnProgrammingBot/issues)')
    reddit.set_oauth_app_info(client_id=CLIENT_ID, client_secret=CLIENT_SECRET, redirect_uri=REDIRECT_URI)
    return reddit

def run_bot(args):
    logging.basicConfig(filename=LOGFILE_URI, level=LOG_LEVEL, format=LOG_FORMAT)

    logging.info('Connecting to database {}'.format(DATABASE_URI))
    Session = connect_to_database(DATABASE_URI)
    logging.info('Database connection OK')

    session = Session()
    data = session.query(model.Corpus).all()

    data_values = [col.title + ' ' + col.text for col in data]
    data_targets = [col.category for col in data]

    logging.info('Training classifier with {} values'.format(len(data_values)))
    classifier = Classifier(data_values, data_targets)
    logging.info('Classifier trained')

    logging.info('Connecting to reddit...')
    reddit = get_reddit_client()

    logging.info('Authorizing...')
    access_information = reddit.get_access_information(CLIENT_ACCESSCODE)
    reddit.set_access_credentials(**access_information)
    logging.info('Logged in successfully.')

    for message in praw.helpers.submission_stream(reddit, SUBREDDIT, limit=5, verbosity=0):
        message_text = message.title + ' ' + message.selftext
        pred = classifier.classify(message_text)[0]
        if pred != 'good':
            try:
                message.add_comment(responses[pred])
            except praw.errors.RateLimitExceeded:
                # TODO:
                # Ideally, errors should actually be handled properly. Perhaps a dequeue could be used
                # to store all the posts which failed, which could be retried every minute (or so)
                logging.error('Rate limit exceeded, cannot post to thread {}'.format(message.title))

def train_id(args):
    train_bot(args, True)

def train_batch(args):
    train_bot(args, False)

def train_bot(args, by_id):
    reddit = get_reddit_client()
    if by_id:
        messages = [reddit.get_submission(submission_id=args.id)]
    else:
        messages = reddit.get_subreddit(SUBREDDIT).get_new(limit=args.limit)
    for message in messages:
        print(message.title)
        print('----------')
        print(message.selftext)
        print('')
        message_type = input('Enter category: ')
        Session = connect_to_database(DATABASE_URI)
        session = Session()
        session.add(model.Corpus(title=message.title, text=message.selftext, category=message_type))
        session.commit()

def create_token(args):
    reddit = get_reddit_client()
    url = reddit.get_authorize_url('uniqueKey', 'identity,submit,read', True)
    webbrowser.open(url)
    print('                   !!!                    ')
    print('Please copy the access code that you are redirected to ')
    print('like this: http://praw.readthedocs.org/en/latest/_images/CodeUrl.png')
    print('You need to put it in settings.py as CLIENT_ACCESSCODE')
    print('                   !!!                    ')

def classify_item(args):
    reddit = get_reddit_client()
    post = reddit.get_submission(submission_id=args.id)

    Session = connect_to_database(DATABASE_URI)
    session = Session()

    data = session.query(model.Corpus).all()
    data_values = [col.title + ' ' + col.text for col in data]
    data_targets = [col.category for col in data]

    classifier = Classifier(data_values, data_targets)
    post_text = post.title + ' ' + post.selftext
    classification = classifier.classify(post_text)[0]
    probability = classifier.get_probability(post_text)[0]
    print('p({}) = {}'.format(classification, max(probability)))

def initialise_database(args):
    engine = create_engine(DATABASE_URI)
    model.Corpus.metadata.create_all(engine)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_run = subparsers.add_parser('run', help='runs the bot')
    parser_run.set_defaults(func=run_bot)
    parser_train = subparsers.add_parser('train', help='adds training data to the bot (using a specific id)')
    parser_train.add_argument('--id', type=str, required=True, help='the submission id of the post to review')
    parser_train.set_defaults(func=train_id)
    parser_batch = subparsers.add_parser('train-batch', help='adds training data to the bot in batches')
    parser_batch.add_argument('--limit', type=int, required=True, help='the maximum number of posts to fetch')
    parser_batch.set_defaults(func=train_batch)
    parser_token = subparsers.add_parser('create-token', help='gets an access token with your client id/secret')
    parser_token.set_defaults(func=create_token)
    parser_init = subparsers.add_parser('init', help='initialises the database, ready to insert training data')
    parser_init.set_defaults(func=initialise_database)
    parser_classify = subparsers.add_parser('classify', help='classifies a specific post using the trained data')
    parser_classify.add_argument('--id', type=str, required=True, help='the submission id of the post to classify')
    parser_classify.set_defaults(func=classify_item)
    args = parser.parse_args(sys.argv[1:])
    args.func(args)
<commit_msg>Refine probability display for classify subcommand<commit_after>#!/usr/bin/python2

from __future__ import print_function

from sklearn.feature_extraction.text import TfidfVectorizer
from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from sklearn import svm
from sklearn.multiclass import OneVsRestClassifier
import logging
import webbrowser
import argparse
import sys

from settings import LOGFILE_URI, DATABASE_URI, LOG_LEVEL, CLIENT_ID, CLIENT_SECRET, CLIENT_ACCESSCODE, SUBREDDIT, REDIRECT_URI, LOG_FORMAT
import model
import praw

# This allows for Python 3 compatibility by replacing input() on Python 2
if sys.version_info[:2] <= (2, 7):
    input = raw_input

responses = {
    'bad': '''
Hi! Your post doesn't seem very suitable for /r/learnprogramming. This may be because you didn't include a code sample, provided very little detail or linked content that doesn't seem relevant. You can improve your post by:

- [Asking Questions The Smart Way](http://catb.org/~esr/faqs/smart-questions.html)
- Avoiding posting links without any explanation, discussion or question
- Using code pastebins (images don't count!)
- Reviewing the post guidelines on the sidebar

Don't worry about this message if you think your post **is** relevant - it may just be an error in my classifier - but please check the resources above anyway to make sure that your post is good.

*I am a bot for /r/learnprogramming using supervised learning to detect common questions. If this message is irrelevant or unhelpful, please report an issue so I can improve my classifier!*
    ''',
    'faq': '''
Hello! Your post seems similar to one of the common questions answered in the [/r/learnprogramming FAQ](https://www.reddit.com/r/learnprogramming/wiki/faq), and you might find an answer to your question there.

Links that may be useful to you:

- [Programming Resources](https://www.reddit.com/r/learnprogramming/wiki/online)
- [/r/cscareerquestions](https://www.reddit.com/r/cscareerquestions)
- [Getting Started with Programming](https://www.reddit.com/r/learnprogramming/wiki/gettingstarted)

*I am a bot for /r/learnprogramming using supervised learning to detect common questions. Your post was recognised as an faq question. If this seems irrelevant or unhelpful, please report an issue on GitHub!*
    '''
}

class Classifier(object):
    """
    Wrapper for the vectorizer and classifier that handles training of both.
    """
    def __init__(self, training_values, training_targets):
        self.vectorizer = TfidfVectorizer()
        self.classifier = OneVsRestClassifier(svm.LinearSVC(class_weight='balanced'))
        training_values = self.vectorizer.fit_transform(training_values).toarray()
        self.classifier.fit(training_values, training_targets)

    def classify(self, text):
        transformed_text = self.vectorizer.transform([text]).toarray()
        return self.classifier.predict(transformed_text)

    def get_probability(self, text):
        transformed_text = self.vectorizer.transform([text]).toarray()
        return self.classifier.decision_function(transformed_text)

def connect_to_database(uri):
    engine = create_engine(uri)
    return sessionmaker(bind=engine)

def get_reddit_client():
    reddit = praw.Reddit(user_agent='all platforms:Learn Programming Bot:v0.2.0-pre (by /u/Aurora0001, contact at github.com/Aurora0001/LearnProgrammingBot/issues)')
    reddit.set_oauth_app_info(client_id=CLIENT_ID, client_secret=CLIENT_SECRET, redirect_uri=REDIRECT_URI)
    return reddit

def run_bot(args):
    logging.basicConfig(filename=LOGFILE_URI, level=LOG_LEVEL, format=LOG_FORMAT)

    logging.info('Connecting to database {}'.format(DATABASE_URI))
    Session = connect_to_database(DATABASE_URI)
    logging.info('Database connection OK')

    session = Session()
    data = session.query(model.Corpus).all()

    data_values = [col.title + ' ' + col.text for col in data]
    data_targets = [col.category for col in data]

    logging.info('Training classifier with {} values'.format(len(data_values)))
    classifier = Classifier(data_values, data_targets)
    logging.info('Classifier trained')

    logging.info('Connecting to reddit...')
    reddit = get_reddit_client()

    logging.info('Authorizing...')
    access_information = reddit.get_access_information(CLIENT_ACCESSCODE)
    reddit.set_access_credentials(**access_information)
    logging.info('Logged in successfully.')

    for message in praw.helpers.submission_stream(reddit, SUBREDDIT, limit=5, verbosity=0):
        message_text = message.title + ' ' + message.selftext
        pred = classifier.classify(message_text)[0]
        if pred != 'good':
            try:
                message.add_comment(responses[pred])
            except praw.errors.RateLimitExceeded:
                # TODO:
                # Ideally, errors should actually be handled properly. Perhaps a dequeue could be used
                # to store all the posts which failed, which could be retried every minute (or so)
                logging.error('Rate limit exceeded, cannot post to thread {}'.format(message.title))

def train_id(args):
    train_bot(args, True)

def train_batch(args):
    train_bot(args, False)

def train_bot(args, by_id):
    reddit = get_reddit_client()
    if by_id:
        messages = [reddit.get_submission(submission_id=args.id)]
    else:
        messages = reddit.get_subreddit(SUBREDDIT).get_new(limit=args.limit)
    for message in messages:
        print(message.title)
        print('----------')
        print(message.selftext)
        print('')
        message_type = input('Enter category: ')
        Session = connect_to_database(DATABASE_URI)
        session = Session()
        session.add(model.Corpus(title=message.title, text=message.selftext, category=message_type))
        session.commit()

def create_token(args):
    reddit = get_reddit_client()
    url = reddit.get_authorize_url('uniqueKey', 'identity,submit,read', True)
    webbrowser.open(url)
    print('                   !!!                    ')
    print('Please copy the access code that you are redirected to ')
    print('like this: http://praw.readthedocs.org/en/latest/_images/CodeUrl.png')
    print('You need to put it in settings.py as CLIENT_ACCESSCODE')
    print('                   !!!                    ')

def classify_item(args):
    reddit = get_reddit_client()
    post = reddit.get_submission(submission_id=args.id)

    Session = connect_to_database(DATABASE_URI)
    session = Session()

    data = session.query(model.Corpus).all()
    data_values = [col.title + ' ' + col.text for col in data]
    data_targets = [col.category for col in data]

    classifier = Classifier(data_values, data_targets)
    post_text = post.title + ' ' + post.selftext
    classification = classifier.classify(post_text)[0]
    probability = classifier.get_probability(post_text)[0]
    print('p({}) = {}'.format(classification, max(probability)))
    print('All probabilities: {}'.format(probability))
    print('p(event) = -1 means that the classifier is certain that the post is not in this category.')
    print('p(event) = 1 means that the classifier is certain that the post is in this category.')

def initialise_database(args):
    engine = create_engine(DATABASE_URI)
    model.Corpus.metadata.create_all(engine)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_run = subparsers.add_parser('run', help='runs the bot')
    parser_run.set_defaults(func=run_bot)
    parser_train = subparsers.add_parser('train', help='adds training data to the bot (using a specific id)')
    parser_train.add_argument('--id', type=str, required=True, help='the submission id of the post to review')
    parser_train.set_defaults(func=train_id)
    parser_batch = subparsers.add_parser('train-batch', help='adds training data to the bot in batches')
    parser_batch.add_argument('--limit', type=int, required=True, help='the maximum number of posts to fetch')
    parser_batch.set_defaults(func=train_batch)
    parser_token = subparsers.add_parser('create-token', help='gets an access token with your client id/secret')
    parser_token.set_defaults(func=create_token)
    parser_init = subparsers.add_parser('init', help='initialises the database, ready to insert training data')
    parser_init.set_defaults(func=initialise_database)
    parser_classify = subparsers.add_parser('classify', help='classifies a specific post using the trained data')
    parser_classify.add_argument('--id', type=str, required=True, help='the submission id of the post to classify')
    parser_classify.set_defaults(func=classify_item)
    args = parser.parse_args(sys.argv[1:])
    args.func(args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Generated by Django 1.9.6 on 2016-08-04 11:40
from __future__ import unicode_literals

from django.db import migrations, models
import wagtail.wagtailcore.blocks
import wagtail.wagtailcore.fields

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0041_GA_tracking_code'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='sitesettings',
            name='content_rotation',
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='content_rotation_end_date',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='content_rotation_start_date',
            field=models.DateTimeField(blank=True, null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='f',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='m',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='sa',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='su',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='th',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='time',
            field=wagtail.wagtailcore.fields.StreamField([(b'time', wagtail.wagtailcore.blocks.TimeBlock(required=False))], blank=True, null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='tu',
            field=models.BooleanField(default=False),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='w',
            field=models.BooleanField(default=False),
        ),
        migrations.AlterField(
            model_name='sitelanguage',
            name='locale',
            field=models.CharField(choices=[(b'af', b'Afrikaans'), (b'ar', b'Arabic'), (b'ast', b'Asturian'), (b'az', b'Azerbaijani'), (b'bg', b'Bulgarian'), (b'be', b'Belarusian'), (b'bn', b'Bengali'), (b'br', b'Breton'), (b'bs', b'Bosnian'), (b'ca', b'Catalan'), (b'cs', b'Czech'), (b'cy', b'Welsh'), (b'da', b'Danish'), (b'de', b'German'), (b'el', b'Greek'), (b'en', b'English'), (b'en-au', b'Australian English'), (b'en-gb', b'British English'), (b'eo', b'Esperanto'), (b'es', b'Spanish'), (b'es-ar', b'Argentinian Spanish'), (b'es-co', b'Colombian Spanish'), (b'es-mx', b'Mexican Spanish'), (b'es-ni', b'Nicaraguan Spanish'), (b'es-ve', b'Venezuelan Spanish'), (b'et', b'Estonian'), (b'eu', b'Basque'), (b'fa', b'Persian'), (b'fi', b'Finnish'), (b'fr', b'French'), (b'fy', b'Frisian'), (b'ga', b'Irish'), (b'gd', b'Scottish Gaelic'), (b'gl', b'Galician'), (b'he', b'Hebrew'), (b'hi', b'Hindi'), (b'hr', b'Croatian'), (b'hu', b'Hungarian'), (b'ia', b'Interlingua'), (b'id', b'Indonesian'), (b'io', b'Ido'), (b'is', b'Icelandic'), (b'it', b'Italian'), (b'ja', b'Japanese'), (b'ka', b'Georgian'), (b'kk', b'Kazakh'), (b'km', b'Khmer'), (b'kn', b'Kannada'), (b'ko', b'Korean'), (b'lb', b'Luxembourgish'), (b'lt', b'Lithuanian'), (b'lv', b'Latvian'), (b'mk', b'Macedonian'), (b'ml', b'Malayalam'), (b'mn', b'Mongolian'), (b'mr', b'Marathi'), (b'my', b'Burmese'), (b'nb', b'Norwegian Bokmal'), (b'ne', b'Nepali'), (b'nl', b'Dutch'), (b'nn', b'Norwegian Nynorsk'), (b'os', b'Ossetic'), (b'pa', b'Punjabi'), (b'pl', b'Polish'), (b'pt', b'Portuguese'), (b'pt-br', b'Brazilian Portuguese'), (b'ro', b'Romanian'), (b'ru', b'Russian'), (b'sk', b'Slovak'), (b'sl', b'Slovenian'), (b'sq', b'Albanian'), (b'sr', b'Serbian'), (b'sr-latn', b'Serbian Latin'), (b'sv', b'Swedish'), (b'sw', b'Swahili'), (b'ta', b'Tamil'), (b'te', b'Telugu'), (b'th', b'Thai'), (b'tr', b'Turkish'), (b'tt', b'Tatar'), (b'udm', b'Udmurt'), (b'uk', b'Ukrainian'), (b'ur', b'Urdu'), (b'vi', b'Vietnamese'), (b'zh-hans', b'Simplified Chinese'), (b'zh-hant', b'Traditional Chinese'), (b'zu', 'Zulu'), (b'xh', 'Xhosa'), (b'st', 'Sotho'), (b've', 'Venda'), (b'tn', 'Tswana'), (b'ts', 'Tsonga'), (b'ss', 'Swati'), (b'nr', 'Ndebele')], help_text='Site language', max_length=255, verbose_name='language name'),
        ),
    ]
<commit_msg>add migrations<commit_after># -*- coding: utf-8 -*-
# Generated by Django 1.9.6 on 2016-08-05 15:21
from __future__ import unicode_literals

from django.db import migrations, models
import wagtail.wagtailcore.blocks
import wagtail.wagtailcore.fields

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0041_GA_tracking_code'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='sectionpage',
            name='featured_in_homepage_rotation',
        ),
        migrations.RemoveField(
            model_name='sitesettings',
            name='content_rotation',
        ),
        migrations.RemoveField(
            model_name='sitesettings',
            name='content_rotation_time',
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='content_rotation_end_date',
            field=models.DateTimeField(blank=True, help_text=b'The date rotation will end', null=True),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='content_rotation_start_date',
            field=models.DateTimeField(blank=True, help_text=b'The date rotation will begin', null=True),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='friday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Friday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='monday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Monday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='saturday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Saturday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='sunday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Sunday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='thursday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Thursday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='time',
            field=wagtail.wagtailcore.fields.StreamField([(b'time', wagtail.wagtailcore.blocks.TimeBlock(required=False))], blank=True, help_text=b'The time/s content will be rotated', null=True),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='tuesday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Tuesday'),
        ),
        migrations.AddField(
            model_name='sectionpage',
            name='wednesday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Wednesday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='content_rotation_end_date',
            field=models.DateTimeField(blank=True, help_text=b'The date rotation will end', null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='content_rotation_start_date',
            field=models.DateTimeField(blank=True, help_text=b'The date rotation will begin', null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='friday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Friday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='monday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Monday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='saturday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Saturday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='sunday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Sunday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='thursday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Thursday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='time',
            field=wagtail.wagtailcore.fields.StreamField([(b'time', wagtail.wagtailcore.blocks.TimeBlock(required=False))], blank=True, help_text=b'The time/s content will be rotated', null=True),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='tuesday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Tuesday'),
        ),
        migrations.AddField(
            model_name='sitesettings',
            name='wednesday_rotation',
            field=models.BooleanField(default=False, verbose_name=b'Wednesday'),
        ),
        migrations.AlterField(
            model_name='sitelanguage',
            name='locale',
            field=models.CharField(choices=[(b'af', b'Afrikaans'), (b'ar', b'Arabic'), (b'ast', b'Asturian'), (b'az', b'Azerbaijani'), (b'bg', b'Bulgarian'), (b'be', b'Belarusian'), (b'bn', b'Bengali'), (b'br', b'Breton'), (b'bs', b'Bosnian'), (b'ca', b'Catalan'), (b'cs', b'Czech'), (b'cy', b'Welsh'), (b'da', b'Danish'), (b'de', b'German'), (b'el', b'Greek'), (b'en', b'English'), (b'en-au', b'Australian English'), (b'en-gb', b'British English'), (b'eo', b'Esperanto'), (b'es', b'Spanish'), (b'es-ar', b'Argentinian Spanish'), (b'es-co', b'Colombian Spanish'), (b'es-mx', b'Mexican Spanish'), (b'es-ni', b'Nicaraguan Spanish'), (b'es-ve', b'Venezuelan Spanish'), (b'et', b'Estonian'), (b'eu', b'Basque'), (b'fa', b'Persian'), (b'fi', b'Finnish'), (b'fr', b'French'), (b'fy', b'Frisian'), (b'ga', b'Irish'), (b'gd', b'Scottish Gaelic'), (b'gl', b'Galician'), (b'he', b'Hebrew'), (b'hi', b'Hindi'), (b'hr', b'Croatian'), (b'hu', b'Hungarian'), (b'ia', b'Interlingua'), (b'id', b'Indonesian'), (b'io', b'Ido'), (b'is', b'Icelandic'), (b'it', b'Italian'), (b'ja', b'Japanese'), (b'ka', b'Georgian'), (b'kk', b'Kazakh'), (b'km', b'Khmer'), (b'kn', b'Kannada'), (b'ko', b'Korean'), (b'lb', b'Luxembourgish'), (b'lt', b'Lithuanian'), (b'lv', b'Latvian'), (b'mk', b'Macedonian'), (b'ml', b'Malayalam'), (b'mn', b'Mongolian'), (b'mr', b'Marathi'), (b'my', b'Burmese'), (b'nb', b'Norwegian Bokmal'), (b'ne', b'Nepali'), (b'nl', b'Dutch'), (b'nn', b'Norwegian Nynorsk'), (b'os', b'Ossetic'), (b'pa', b'Punjabi'), (b'pl', b'Polish'), (b'pt', b'Portuguese'), (b'pt-br', b'Brazilian Portuguese'), (b'ro', b'Romanian'), (b'ru', b'Russian'), (b'sk', b'Slovak'), (b'sl', b'Slovenian'), (b'sq', b'Albanian'), (b'sr', b'Serbian'), (b'sr-latn', b'Serbian Latin'), (b'sv', b'Swedish'), (b'sw', b'Swahili'), (b'ta', b'Tamil'), (b'te', b'Telugu'), (b'th', b'Thai'), (b'tr', b'Turkish'), (b'tt', b'Tatar'), (b'udm', b'Udmurt'), (b'uk', b'Ukrainian'), (b'ur', b'Urdu'), (b'vi', b'Vietnamese'), (b'zh-hans', b'Simplified Chinese'), (b'zh-hant', b'Traditional Chinese'), (b'zu', 'Zulu'), (b'xh', 'Xhosa'), (b'st', 'Sotho'), (b've', 'Venda'), (b'tn', 'Tswana'), (b'ts', 'Tsonga'), (b'ss', 'Swati'), (b'nr', 'Ndebele')], help_text='Site language', max_length=255, verbose_name='language name'),
        ),
    ]
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd

class Trajectories(pd.DataFrame):
    """
    This class is a subclass of the class :class:`pandas.DataFrame`

    It is mainly here to provide utility attributes and syntactic shugar.

    Attributes
    ----------
    t_stamps : ndarray
        unique values of the `t_stamps` index of the `self.trajs` dataframe

    labels : ndarray
        unique values of the `labels` index of the `self.trajs` dataframe

    iter_segments : iterator
        yields a `(label, segment)` pair where `label` is iterated over `self.labels`
        and `segment` is a chunk of `self.trajs`

    """
    def __init__(self, trajs):

        super().__init__(trajs)
        if not isinstance(self, pd.DataFrame):
            raise TypeError('''The constructor argument `trajs`
                            must be a pandas.DataFrame instance''')
    @property
    def t_stamps(self):
        return self.index.get_level_values('t_stamp').unique()

    @property
    def labels(self):
        return self.index.get_level_values('label').unique()

    @property
    def _segment_idxs(self):
        return self.groupby(level='label').groups

    @property
    def iter_segments(self):
        for lbl, idxs in self._segment_idxs:
            yield lbl, self.loc[idxs]

    def get_segments(self):
        """
        Returns a dictionnary with labels as keys and segments as values

        A segment contains all the data from `self.trajs` with

        """
        return {key: segment for key, segment
                in self.iter_segments}

    def check_trajs_df_structure(self, index=None, columns=None):
        """Check wether trajcetories contains a specified structure.

        Parameters
        ----------
        index : list
            Index names (order is important)
        columns : list
            Column names (order does not matter here)

        Raises
        ------
        ValueError in both case
        """

        error_mess = "Trajectories does not contain correct indexes : {}"
        if index and self.index.names != index:
            raise ValueError(error_mess.format(index))

        error_mess = "Trajectories does not contain correct columns : {}"
        if columns:
            columns = set(columns)
            if not columns.issubset(set(self.columns)):
                raise ValueError(error_mess.format(columns))

    def relabel_fromzero(self, level, inplace=False):

        old_lbls = self.index.get_level_values(level)
        nu_lbls = old_lbls.values.astype(np.uint16).copy()
        for n, uv in enumerate(old_lbls.unique()):
            nu_lbls[old_lbls == uv] = n

        if not inplace:
            trajs = Trajectories(self.copy())
        else:
            trajs = self
        trajs['new_label'] = nu_lbls
        trajs.set_index('new_label', append=True, inplace=True)
        trajs.reset_index(level, drop=True, inplace=True)
        names = list(self.index.names)
        names[names.index('new_label')] = level
        trajs.index.set_names(names, inplace=True)
        return trajs
<commit_msg>Add show method to Trajectories<commit_after>import numpy as np
import pandas as pd

class Trajectories(pd.DataFrame):
    """
    This class is a subclass of the class :class:`pandas.DataFrame`

    It is mainly here to provide utility attributes and syntactic shugar.

    Attributes
    ----------
    t_stamps : ndarray
        unique values of the `t_stamps` index of the `self.trajs` dataframe

    labels : ndarray
        unique values of the `labels` index of the `self.trajs` dataframe

    iter_segments : iterator
        yields a `(label, segment)` pair where `label` is iterated over `self.labels`
        and `segment` is a chunk of `self.trajs`

    """
    def __init__(self, trajs):

        super().__init__(trajs)
        if not isinstance(self, pd.DataFrame):
            raise TypeError('''The constructor argument `trajs`
                            must be a pandas.DataFrame instance''')

    @property
    def t_stamps(self):
        return self.index.get_level_values('t_stamp').unique()

    @property
    def labels(self):
        return self.index.get_level_values('label').unique()

    @property
    def _segment_idxs(self):
        return self.groupby(level='label').groups

    @property
    def iter_segments(self):
        for lbl, idxs in self._segment_idxs:
            yield lbl, self.loc[idxs]

    def get_segments(self):
        """
        Returns a dictionnary with labels as keys and segments as values

        A segment contains all the data from `self.trajs` with

        """
        return {key: segment for key, segment
                in self.iter_segments}

    def check_trajs_df_structure(self, index=None, columns=None):
        """Check wether trajcetories contains a specified structure.

        Parameters
        ----------
        index : list
            Index names (order is important)
        columns : list
            Column names (order does not matter here)

        Raises
        ------
        ValueError in both case
        """

        error_mess = "Trajectories does not contain correct indexes : {}"
        if index and self.index.names != index:
            raise ValueError(error_mess.format(index))

        error_mess = "Trajectories does not contain correct columns : {}"
        if columns:
            columns = set(columns)
            if not columns.issubset(set(self.columns)):
                raise ValueError(error_mess.format(columns))

    def relabel_fromzero(self, level, inplace=False):

        old_lbls = self.index.get_level_values(level)
        nu_lbls = old_lbls.values.astype(np.uint16).copy()
        for n, uv in enumerate(old_lbls.unique()):
            nu_lbls[old_lbls == uv] = n

        if not inplace:
            trajs = Trajectories(self.copy())
        else:
            trajs = self
        trajs['new_label'] = nu_lbls
        trajs.set_index('new_label', append=True, inplace=True)
        trajs.reset_index(level, drop=True, inplace=True)
        names = list(self.index.names)
        names[names.index('new_label')] = level
        trajs.index.set_names(names, inplace=True)
        return trajs

    def show(self, xaxis='t',
             yaxis='x',
             groupby_args={'level': "label"},
             ax=None):
        """Show trajectories

        Parameters
        ----------
        xaxis : str
        yaxis : str
        groupby : dict
            How to group trajectories
        ax : matplotlib Axes
            None will create a new one.

        Returns
        -------
        matplotlib axis instance

        Examples
        --------
        >>> from sktracker import data
        >>> from sktracker.tracker.solver import ByFrameSolver
        >>> true_trajs = data.brownian_trajectories_generator(p_disapear=0.1)
        >>> solver = ByFrameSolver.for_brownian_motion(true_trajs, max_speed=2)
        >>> trajs = solver.track(progress_bar=False)
        >>> fig, (ax1, ax2) = plt.subplots(nrows=2)
        >>> ax1 = trajs.show(xaxis='t', yaxis='x', groupby_args={'level': "label"}, ax=ax1)
        >>> ax2 = trajs.show(xaxis='t', yaxis='x', groupby_args={'by': "true_label"}, ax=ax2)

        """

        import matplotlib.pyplot as plt

        if ax is None:
            ax = plt.gca()

        gp = self.groupby(**groupby_args).groups
        for k, v in gp.items():
            traj = self.loc[v]
            ax.plot(traj[xaxis], traj[yaxis], '-o')

        ax.set_xlabel(xaxis)
        ax.set_ylabel(yaxis)
        ax.set_title(str(groupby_args))

        return ax
<|endoftext|>