<commit_before>import argparse
import flickrapi
# import code
# import json
import os
import pyexiv2
import time
import urllib
import webbrowser

class FlickrSavr(object):
    """This is a digital preservation experiment.  It crawls your Flickr
    Photos and saves them to disk with all the Flickr metadata stored
    in the EXIF.  The idea here is, most preservation methods keep the
    photos and then keep a separate database (in whatever format)
    elsewhere.  I thought I could have all the date stored in the
    photo so that it is not only coupled with the primary data but
    could be reconstructed from arbitrary collections of photos across
    people.  It's just an experiment...it's not complete and I make no
    promises.

    3600 queries per hour is all thats allowed (thats 1 query per
    second). Each photo takes 3 queries to get it's metadata...so
    that's a cap of 1200 photos per hour.  It takes (by rough estimate)
    3 seconds to query 3 times, download, and write to disk.  So by
    most estimates you won't overrun the limit...that said, we'll sleep
    200 ms between photos just to be nice.

    """

    def __init__(self,
                 key,
                 secret,
                 nsid,
                 basepath,
                 verbose,
                 force):
        """do

        Arguments:
        - `key`:
        - `secret`:
        - `ndsid`:
        - `basepath`:
        - `verbose`:
        - `force`:
        """
        # auth
        self.api_key = key
        self.api_secret = secret
        self.nsid = nsid
        self.basepath = os.path.join(basepath, "nsid", self.nsid)
        self.verbose = not verbose
        self.force = force

        self.photo_count = 0
        self.photo_page = 0
        self.photo_total = 0
        self.per_page = 500

        self.flickr = flickrapi.FlickrAPI(self.api_key,
                                          self.api_secret,
                                          format='parsed-json')

        # Only do this if we don't have a valid token already
        if not self.flickr.token_valid(perms=unicode('write')):

            # Get a request token
            self.flickr.get_request_token(oauth_callback='oob')

            # Open a browser at the authentication URL. Do this however
            # you want, as long as the user visits that URL.
            authorize_url = self.flickr.auth_url(perms=unicode('write'))
            webbrowser.open_new_tab(authorize_url)

            # Get the verifier code from the user. Do this however you
            # want, as long as the user gives the application the code.
            verifier = unicode(raw_input('Verifier code: '))

            # Trade the request token for an access token
            self.flickr.get_access_token(verifier)

        # dir
        if not os.path.exists(self.basepath):
            os.makedirs(self.basepath)

        # search: this has a 'pages' field with how many photos left
        # accounting per page..max is 500
        extras = "%s,%s,%s,%s,%s,%s,%s,%s," % ("url_o",
                                               "geo",
                                               "tags",
                                               "machine_tags",
                                               "views",
                                               "description",
                                               "date_upload",
                                               "date_taken")

        photos = self.flickr.photos_search(user_id=self.nsid,
                                           per_page=self.per_page,
                                           extras=extras)
        photos = photos['photos']

        self.photo_total = photos['perpage'] * photos['pages']
        self.photo_page = 0

        for i in range(photos['perpage']):
            self.photo_count = i
            photo = photos['photo'][i]
            self.get_photo(photo)

        # this page counter is for the next page actually
        for page in range(photos['pages'])[1:]:
            self.photo_page = page
            photos = self.flickr.photos_search(user_id=self.nsid,
                                               page=str(self.photo_page + 1),
                                               per_page=self.per_page,
                                               extras=extras)
            photos = photos['photos']
            for i in range(photos['perpage']):
                self.photo_count = i
                photo = photos['photo'][i]
                self.get_photo(photo)
        return

    def get_photo(self, photo):
        """Get the data around a photo.

        Arguments:
        - `self`:
        - `photo`:
        """

        # Check if we have it already
        url = photo['url_o']
        fname = os.path.join(self.get_date_path(photo),
                             photo['id'] + url[-4:])
        exists = not self.force and os.path.isfile(fname)
        if exists:
            self.print_status_count(True)
            return

        # get image
        resp = urllib.urlopen(url)
        image_data = resp.read()

        # Open output file in binary mode, write, and close.
        with open(fname, 'wb') as f:
            f.write(image_data)
        self.print_status_count()

        # get more metadata
        favorites = []
        favs = self.flickr.photos_getFavorites(photo_id=photo['id'])
        for person in favs['photo']['person']:
            s = "Flickr.favorites:%s:%s:%s" % (person['username'],
                                               person['nsid'],
                                               person['favedate'])
            favorites.append(s)
        comments = []
        comms = self.flickr.photos_comments_getList(photo_id=photo['id'])
        try:
            for comment in comms['comments']['comment']:
                c = "Flickr.comments:%s:%s:%s:%s" % (comment['author'],
                                                     comment['authorname'],
                                                     comment['datecreate'],
                                                     comment['_content'])
                comments.append(c)
        except:
            pass
        pools = []
        # TODO: this call failed...maybe try to wrap all the bad ids somewhere
        # for later handling
        pool = self.flickr.photos_getAllContexts(photo_id=photo['id'])
        try:
            for sets in pool['set']:
                c = "Flickr.sets:%s:$s" % (sets['title'],
                                           sets['id'])
                pools.append(c)
        except:
            pass
        # write exif
        metadata = pyexiv2.ImageMetadata(fname)
        metadata.read()
        key = 'Iptc.Application2.Keywords'
        vals = ["Flickr.id:%s" % photo['id'],
                "Flickr.owner:%s" % self.nsid,
                "Flickr.title:%s" % photo['title'],
                "Flickr.description:%s" % photo['description']['_content'],
                "Flickr.isfamily:%d" % photo['isfamily'],
                "Flickr.isfriend:%d" % photo['isfriend'],
                "Flickr.views:%s" % photo['views'],
                "Flickr.date_upload:%s" % photo['dateupload'],
                "Flickr.date_taken:%s" % photo['datetaken']]
        if ('latitude' in photo and 'longitude' in photo):
            coords = ["Flickr.latitude:%s" % photo['latitude'],
                      "Flickr.longitude:%s" % photo['longitude']]
            vals += coords
        if ('accuracy' in photo):
            accuracy = ["Flickr.accuracy:%s" % photo['accuracy']]
            vals += accuracy
        if ('tags' in photo):
            tags = ["Flickr.tags:%s" % photo['tags']]
            vals += tags
        # join all into vals
        vals.extend(favorites)
        metadata[key] = vals
        metadata.iptc_charset = 'utf-8'
        metadata.write()
        time.sleep(0.1)

    def get_date_path(self, photo):
        datetaken = photo['datetaken']
        date = datetaken.split(' ')[0]
        parsed = date.split('-')
        path = os.path.join(self.basepath, parsed[0], parsed[1], date)
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def print_status(self, s):
        if self.verbose:
            print s
        return

    def print_status_count(self, exists=False):
        condition = ""
        if exists:
            condition = "(file exists)"
        if self.verbose:
            self.print_status("%s / %s %s" %
                              (1 + self.photo_count + (self.per_page *
                                                       self.photo_page),
                               self.photo_total,
                               condition))
        return

def main():
    desc = 'Download a Flickr Account.'
    parser = argparse.ArgumentParser(prog='FlickrSavr',
                                     usage='%(prog)s key secret nsid',
                                     description=desc,
                                     epilog='thats how its done.')
    parser.add_argument('key', help='Flickr API Key')
    parser.add_argument('secret', help='Flickr API Secret')
    parser.add_argument('nsid', help='Flickr Account NSID')
    parser.add_argument('-b', '--basepath',
                        nargs=1,
                        default='',
                        help='Basedirectory to use for storing files.')
    parser.add_argument("-f",
                        "--force",
                        help="Force download if file already exists. ",
                        action="store_true")
    parser.add_argument("-q",
                        "--quiet",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()
    FlickrSavr(args.key, args.secret, args.nsid, args.basepath[0],
               args.quiet, args.force)

if __name__ == "__main__":
    main()
<commit_msg>Fixed iteration bug.<commit_after>import argparse
import flickrapi
# import code
# import json
import os
import pyexiv2
import time
import urllib
import webbrowser

class FlickrSavr(object):
    """This is a digital preservation experiment.  It crawls your Flickr
    Photos and saves them to disk with all the Flickr metadata stored
    in the EXIF.  The idea here is, most preservation methods keep the
    photos and then keep a separate database (in whatever format)
    elsewhere.  I thought I could have all the date stored in the
    photo so that it is not only coupled with the primary data but
    could be reconstructed from arbitrary collections of photos across
    people.  It's just an experiment...it's not complete and I make no
    promises.

    3600 queries per hour is all thats allowed (thats 1 query per
    second). Each photo takes 3 queries to get it's metadata...so
    that's a cap of 1200 photos per hour.  It takes (by rough estimate)
    3 seconds to query 3 times, download, and write to disk.  So by
    most estimates you won't overrun the limit...that said, we'll sleep
    200 ms between photos just to be nice.

    """

    def __init__(self,
                 key,
                 secret,
                 nsid,
                 basepath,
                 verbose,
                 force):
        """do

        Arguments:
        - `key`:
        - `secret`:
        - `ndsid`:
        - `basepath`:
        - `verbose`:
        - `force`:
        """
        # auth
        self.api_key = key
        self.api_secret = secret
        self.nsid = nsid
        self.basepath = os.path.join(basepath, "nsid", self.nsid)
        self.verbose = not verbose
        self.force = force

        self.photo_count = 0
        self.photo_page = 0
        self.photo_total = 0
        self.per_page = 500

        self.flickr = flickrapi.FlickrAPI(self.api_key,
                                          self.api_secret,
                                          format='parsed-json')

        # Only do this if we don't have a valid token already
        if not self.flickr.token_valid(perms=unicode('write')):

            # Get a request token
            self.flickr.get_request_token(oauth_callback='oob')

            # Open a browser at the authentication URL. Do this however
            # you want, as long as the user visits that URL.
            authorize_url = self.flickr.auth_url(perms=unicode('write'))
            webbrowser.open_new_tab(authorize_url)

            # Get the verifier code from the user. Do this however you
            # want, as long as the user gives the application the code.
            verifier = unicode(raw_input('Verifier code: '))

            # Trade the request token for an access token
            self.flickr.get_access_token(verifier)

        # dir
        if not os.path.exists(self.basepath):
            os.makedirs(self.basepath)

        # search: this has a 'pages' field with how many photos left
        # accounting per page..max is 500
        extras = "%s,%s,%s,%s,%s,%s,%s,%s," % ("url_o",
                                               "geo",
                                               "tags",
                                               "machine_tags",
                                               "views",
                                               "description",
                                               "date_upload",
                                               "date_taken")

        photos = self.flickr.photos_search(user_id=self.nsid,
                                           per_page=self.per_page,
                                           extras=extras)
        photos = photos['photos']

        self.photo_total = photos['perpage'] * photos['pages']
        self.photo_page = 0

        for i in range(photos['perpage']):
            self.photo_count = i
            photo = photos['photo'][i]
            self.get_photo(photo)

        # this page counter is for the next page actually
        for page in range(photos['pages'])[1:]:
            self.photo_page = page
            photos = self.flickr.photos_search(user_id=self.nsid,
                                               page=str(self.photo_page + 1),
                                               per_page=self.per_page,
                                               extras=extras)
            photos = photos['photos']
            print photos['perpage']
            for i in range(len(photos['photo'])):
                self.photo_count = i
                photo = photos['photo'][i]
                self.get_photo(photo)
        return

    def get_photo(self, photo):
        """Get the data around a photo.

        Arguments:
        - `self`:
        - `photo`:
        """

        # Check if we have it already
        url = photo['url_o']
        fname = os.path.join(self.get_date_path(photo),
                             photo['id'] + url[-4:])
        exists = not self.force and os.path.isfile(fname)
        if exists:
            self.print_status_count(True)
            return

        # get image
        resp = urllib.urlopen(url)
        image_data = resp.read()

        # Open output file in binary mode, write, and close.
        with open(fname, 'wb') as f:
            f.write(image_data)
        self.print_status_count()

        # get more metadata
        favorites = []
        favs = self.flickr.photos_getFavorites(photo_id=photo['id'])
        for person in favs['photo']['person']:
            s = "Flickr.favorites:%s:%s:%s" % (person['username'],
                                               person['nsid'],
                                               person['favedate'])
            favorites.append(s)
        comments = []
        comms = self.flickr.photos_comments_getList(photo_id=photo['id'])
        try:
            for comment in comms['comments']['comment']:
                c = "Flickr.comments:%s:%s:%s:%s" % (comment['author'],
                                                     comment['authorname'],
                                                     comment['datecreate'],
                                                     comment['_content'])
                comments.append(c)
        except:
            pass
        pools = []
        # TODO: this call failed...maybe try to wrap all the bad ids somewhere
        # for later handling
        pool = self.flickr.photos_getAllContexts(photo_id=photo['id'])
        try:
            for sets in pool['set']:
                c = "Flickr.sets:%s:$s" % (sets['title'],
                                           sets['id'])
                pools.append(c)
        except:
            pass
        # write exif
        metadata = pyexiv2.ImageMetadata(fname)
        metadata.read()
        key = 'Iptc.Application2.Keywords'
        vals = ["Flickr.id:%s" % photo['id'],
                "Flickr.owner:%s" % self.nsid,
                "Flickr.title:%s" % photo['title'],
                "Flickr.description:%s" % photo['description']['_content'],
                "Flickr.isfamily:%d" % photo['isfamily'],
                "Flickr.isfriend:%d" % photo['isfriend'],
                "Flickr.views:%s" % photo['views'],
                "Flickr.date_upload:%s" % photo['dateupload'],
                "Flickr.date_taken:%s" % photo['datetaken']]
        if ('latitude' in photo and 'longitude' in photo):
            coords = ["Flickr.latitude:%s" % photo['latitude'],
                      "Flickr.longitude:%s" % photo['longitude']]
            vals += coords
        if ('accuracy' in photo):
            accuracy = ["Flickr.accuracy:%s" % photo['accuracy']]
            vals += accuracy
        if ('tags' in photo):
            tags = ["Flickr.tags:%s" % photo['tags']]
            vals += tags
        # join all into vals
        vals.extend(favorites)
        metadata[key] = vals
        metadata.iptc_charset = 'utf-8'
        metadata.write()
        time.sleep(0.1)

    def get_date_path(self, photo):
        datetaken = photo['datetaken']
        date = datetaken.split(' ')[0]
        parsed = date.split('-')
        path = os.path.join(self.basepath, parsed[0], parsed[1], date)
        if not os.path.exists(path):
            os.makedirs(path)
        return path

    def print_status(self, s):
        if self.verbose:
            print s
        return

    def print_status_count(self, exists=False):
        condition = ""
        if exists:
            condition = "(file exists)"
        if self.verbose:
            self.print_status("%s / %s %s" %
                              (1 + self.photo_count + (self.per_page *
                                                       self.photo_page),
                               self.photo_total,
                               condition))
        return

def main():
    desc = 'Download a Flickr Account.'
    parser = argparse.ArgumentParser(prog='FlickrSavr',
                                     usage='%(prog)s key secret nsid',
                                     description=desc,
                                     epilog='thats how its done.')
    parser.add_argument('key', help='Flickr API Key')
    parser.add_argument('secret', help='Flickr API Secret')
    parser.add_argument('nsid', help='Flickr Account NSID')
    parser.add_argument('-b', '--basepath',
                        nargs=1,
                        default='',
                        help='Basedirectory to use for storing files.')
    parser.add_argument("-f",
                        "--force",
                        help="Force download if file already exists. ",
                        action="store_true")
    parser.add_argument("-q",
                        "--quiet",
                        help="increase output verbosity",
                        action="store_true")
    args = parser.parse_args()
    FlickrSavr(args.key, args.secret, args.nsid, args.basepath[0],
               args.quiet, args.force)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_approle_role_secret
version_added: "4.0.0"
short_description: Hashicorp Vault approle role secret id manager
description:
    - Create, update and delete approle secrets.
options:
    state:
        description:
            - present or absent
        default: present
    name:
        description:
            - secret name.
    mount_point:
        description:
            - mount point for role
        default: approle
    secret_id:
        description:
            - Custom SecretID to be attached to the role.
    cidr_list:
        description:
            - Comma-separated string or list of CIDR blocks.
    metadata:
        description:
            - Metadata to be tied to the secret.
    wrap_ttl:
        description:
            - Wrap TTL.
    secret:
        description:
            - secret id for delete.
extends_documentation_fragment: hashivault
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_approle_role_secret:
        name: ashley
        state: present
      register: vault_approle_role_secret_create
    - debug: msg="Role secret id is {{vault_approle_role_secret_create.id}}"

- hosts: localhost
  tasks:
    - hashivault_approle_role_secret:
        name: robert
        state: present
        secret_id: '{{ lookup("password", "/dev/null length=32 chars=ascii_letters,digits") }}'
      register: vault_approle_role_custom_secret_create
    - debug: msg="Role custom secret id is {{vault_approle_role_custom_secret_create.id}}"
'''

def main():
    argspec = hashivault_argspec()
    argspec['state'] = dict(required=False, choices=['present', 'absent'], default='present')
    argspec['name'] = dict(required=True, type='str')
    argspec['mount_point'] = dict(required=False, type='str', default='approle')
    argspec['secret_id'] = dict(required=False, type='str')
    argspec['cidr_list'] = dict(required=False, type='str')
    argspec['metadata'] = dict(required=False, type='dict')
    argspec['wrap_ttl'] = dict(required=False, type='str')
    argspec['secret'] = dict(required=False, type='str', default='notspecified')
    module = hashivault_init(argspec, supports_check_mode=True)
    result = hashivault_approle_role_secret_create(module)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_approle_role_secret_create(module):
    params = module.params
    state = params.get('state')
    name = params.get('name')
    mount_point = params.get('mount_point')

    client = hashivault_auth_client(params)
    if state == 'present':
        custom_secret_id = params.get('secret_id')
        cidr_list = params.get('cidr_list')
        metadata = params.get('metadata')
        wrap_ttl = params.get('wrap_ttl')
        if custom_secret_id is not None:
            if module.check_mode:
                try:
                    client.get_role_secret_id(name, custom_secret_id, mount_point=mount_point)
                except Exception:
                    return {'changed': True}
                return {'changed': False}
            result = client.create_role_custom_secret_id(role_name=name,
                                                         mount_point=mount_point,
                                                         secret_id=custom_secret_id,
                                                         meta=metadata)
        else:
            if module.check_mode:
                return {'changed': True}
            result = client.create_role_secret_id(role_name=name,
                                                  mount_point=mount_point,
                                                  meta=metadata,
                                                  cidr_list=cidr_list,
                                                  wrap_ttl=wrap_ttl)

        if wrap_ttl is None:
            response_key = 'data'
        else:
            response_key = 'wrap_info'

        return {'changed': True, response_key: result.get(response_key, {})}
    elif state == 'absent':
        secret = params.get('secret')
        if module.check_mode:
            try:
                client.get_role_secret_id(name, secret, mount_point=mount_point)
            except Exception:
                return {'changed': False}
            return {'changed': True}
        else:
            client.delete_role_secret_id(name, secret, mount_point=mount_point)
        return {'changed': True}
    else:
        return {'failed': True, 'msg': 'Unkown state value: {}'.format(state)}

if __name__ == '__main__':
    main()
<commit_msg>Fix incompatible approle parameter<commit_after>#!/usr/bin/env python

from ansible.module_utils.hashivault import hashivault_argspec
from ansible.module_utils.hashivault import hashivault_auth_client
from ansible.module_utils.hashivault import hashivault_init
from ansible.module_utils.hashivault import hashiwrapper

ANSIBLE_METADATA = {'status': ['stableinterface'], 'supported_by': 'community', 'version': '1.1'}
DOCUMENTATION = '''
---
module: hashivault_approle_role_secret
version_added: "4.0.0"
short_description: Hashicorp Vault approle role secret id manager
description:
    - Create, update and delete approle secrets.
options:
    state:
        description:
            - present or absent
        default: present
    name:
        description:
            - secret name.
    mount_point:
        description:
            - mount point for role
        default: approle
    secret:
        description:
            - Secret id to be added or deleted. (was secret_id)
    cidr_list:
        description:
            - Comma-separated string or list of CIDR blocks.
    metadata:
        description:
            - Metadata to be tied to the secret.
    wrap_ttl:
        description:
            - Wrap TTL.
extends_documentation_fragment: hashivault
'''
EXAMPLES = '''
---
- hosts: localhost
  tasks:
    - hashivault_approle_role_secret:
        name: ashley
        state: present
      register: vault_approle_role_secret_create
    - debug: msg="Role secret id is {{vault_approle_role_secret_create.id}}"

- hosts: localhost
  tasks:
    - hashivault_approle_role_secret:
        name: robert
        state: present
        secret: '{{ lookup("password", "/dev/null length=32 chars=ascii_letters,digits") }}'
      register: vault_approle_role_custom_secret_create
    - debug: msg="Role custom secret id is {{vault_approle_role_custom_secret_create.id}}"
'''

def main():
    argspec = hashivault_argspec()
    argspec['state'] = dict(required=False, choices=['present', 'absent'], default='present')
    argspec['name'] = dict(required=True, type='str')
    argspec['mount_point'] = dict(required=False, type='str', default='approle')
    argspec['cidr_list'] = dict(required=False, type='str')
    argspec['metadata'] = dict(required=False, type='dict')
    argspec['wrap_ttl'] = dict(required=False, type='str')
    argspec['secret'] = dict(required=False, type='str')
    module = hashivault_init(argspec, supports_check_mode=True)
    result = hashivault_approle_role_secret(module)
    if result.get('failed'):
        module.fail_json(**result)
    else:
        module.exit_json(**result)

@hashiwrapper
def hashivault_approle_role_secret(module):
    params = module.params
    state = params.get('state')
    name = params.get('name')
    mount_point = params.get('mount_point')

    client = hashivault_auth_client(params)
    if state == 'present':
        custom_secret_id = params.get('secret')
        if custom_secret_id is None:
            custom_secret_id = params.get('secret_id') # deprecated
        cidr_list = params.get('cidr_list')
        metadata = params.get('metadata')
        wrap_ttl = params.get('wrap_ttl')
        if custom_secret_id is not None:
            if module.check_mode:
                try:
                    client.get_role_secret_id(name, custom_secret_id, mount_point=mount_point)
                except Exception:
                    return {'changed': True}
                return {'changed': False}
            result = client.create_role_custom_secret_id(role_name=name,
                                                         mount_point=mount_point,
                                                         secret_id=custom_secret_id,
                                                         meta=metadata)
        else:
            if module.check_mode:
                return {'changed': True}
            result = client.create_role_secret_id(role_name=name,
                                                  mount_point=mount_point,
                                                  meta=metadata,
                                                  cidr_list=cidr_list,
                                                  wrap_ttl=wrap_ttl)

        if wrap_ttl is None:
            response_key = 'data'
        else:
            response_key = 'wrap_info'

        return {'changed': True, response_key: result.get(response_key, {})}
    elif state == 'absent':
        secret = params.get('secret')
        if module.check_mode:
            try:
                client.get_role_secret_id(name, secret, mount_point=mount_point)
            except Exception:
                return {'changed': False}
            return {'changed': True}
        else:
            client.delete_role_secret_id(name, secret, mount_point=mount_point)
        return {'changed': True}
    else:
        return {'failed': True, 'msg': 'Unkown state value: {}'.format(state)}

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># CloudShell L1 resource autoload XML helper
#
# It should not be necessary to edit this file.
#
# - Generates the autoload XML resource format to return to CloudShell
# - Subresources are also represented with nested instances of this class
# - See example usage in <project>_l1_handler.py

class L1DriverResourceInfo:
    def __init__(self, name, full_address, family, model, map_path=None, serial='-1'):
        """
        :param name: str
        :param full_address: str
        :param family: str
        :param model: str
        :param map_path: str
        :param serial: str
        """
        self.name = name
        self.address = full_address
        self.family = family
        self.model = model
        self.serial = serial
        self.map_path = map_path
        self.subresources = []
        self.attrname2typevaluetuple = {}
        if False:
            self.subresources.append(L1DriverResourceInfo(None, None, None, None))

    def add_subresource(self, subresource):
        """
        :param subresource: L1DriverResourceInfo
        :return: None
        """
        self.subresources.append(subresource)

    def set_attribute(self, name, value, typename='String'):
        """
        :param name: str
        :param value: str
        :param typename: str
        :return: None
        """
        self.attrname2typevaluetuple[name] = (typename, value)

    def get_attribute(self, name):
        """
        :param name: str
        :return: str
        """
        return self.attrname2typevaluetuple[name][1]

    def to_string(self, tabs=''):
        """
        :param tabs: str
        :return: str
        """
        def indent(t, s):
            return t + (('\n' + t).join(s.split('\n'))).strip()

        return indent(tabs,
                      '''<ResourceInfo
        Name="''' + self.name + '''"
        Address="''' + self.address + '''"
        ResourceFamilyName="''' + self.family + '''"
        ResourceModelName="''' + self.model + '''"
        SerialNumber="''' + self.serial + '''">
    <ChildResources>
''' + ('\n'.join(
                          [
                              x.to_string(tabs=tabs + '    ')
                              for x in self.subresources
                              ]
                      )) + '''</ChildResources>
    <ResourceAttributes>''' + ('\n'.join(
                          [
                              '''<Attribute
                                      Name="''' + attrname + '''"
                Type="''' + self.attrname2typevaluetuple[attrname][0] + '''"
                Value="''' + self.attrname2typevaluetuple[attrname][1] + '''" />'''
                              for attrname in self.attrname2typevaluetuple.keys()
                              ]
                      )) + '''
    </ResourceAttributes>
''' +
                      ('''    <ResourceMapping><IncomingMapping>''' + self.map_path + '''</IncomingMapping></ResourceMapping>''' if self.map_path else '') +
                      '''
</ResourceInfo>
                      ''')
<commit_msg>Update l1_driver_resource_info.py<commit_after># CloudShell L1 resource autoload XML helper
#
# It should not be necessary to edit this file.
#
# - Generates the autoload XML resource format to return to CloudShell
# - Subresources are also represented with nested instances of this class
# - See example usage in <project>_l1_handler.py

class L1DriverResourceInfo:
    def __init__(self, name, full_address, family, model, map_path=None, serial='-1'):
        """
        :param name: str
        :param full_address: str
        :param family: str
        :param model: str
        :param map_path: str
        :param serial: str
        """
        self.name = name
        self.address = full_address
        self.family = family
        self.model = model
        self.serial = serial
        self.map_path = map_path
        self.subresources = []
        self.attrname2typevaluetuple = {}
        if False:
            self.subresources.append(L1DriverResourceInfo(None, None, None, None))

    def add_subresource(self, subresource):
        """
        :param subresource: L1DriverResourceInfo
        :return: None
        """
        self.subresources.append(subresource)

    def set_attribute(self, name, value, typename='String'):
        """
        :param name: str
        :param value: str
        :param typename: str
        :return: None
        """
        self.attrname2typevaluetuple[name] = (typename, value)

    def get_attribute(self, name):
        """
        :param name: str
        :return: str
        """
        return self.attrname2typevaluetuple[name][1]

    def to_string(self, tabs=''):
        """
        :param tabs: str
        :return: str
        """
        def indent(t, s):
            return t + (('\n' + t).join(s.split('\n'))).strip()

        return indent(tabs,
                      '''<ResourceInfo
        Name="''' + self.name + '''"
        Address="''' + self.address + '''"
        ResourceFamilyName="''' + self.family + '''"
        ResourceModelName="''' + self.model + '''"
        SerialNumber="''' + self.serial + '''">
    <ChildResources>
''' + ('\n'.join(
                          [
                              x.to_string(tabs=tabs + '    ')
                              for x in self.subresources
                              ]
                      )) + '''    </ChildResources>
    <ResourceAttributes>
''' + (''.join(
                          [
'''        <Attribute
                Name="''' + attrname + '''"
                Type="''' + self.attrname2typevaluetuple[attrname][0] + '''"
                Value="''' + self.attrname2typevaluetuple[attrname][1] + '''" />\n'''
                              for attrname in self.attrname2typevaluetuple.keys()
                              ]
                      )) + '''    </ResourceAttributes>
''' +
                      ('''    <ResourceMapping><IncomingMapping>''' + self.map_path + '''</IncomingMapping></ResourceMapping>''' if self.map_path else '') +
                      '''
</ResourceInfo>
                      ''')
<|endoftext|>


<commit_before>import asyncio
import itertools
import logging
import time

from .telegrambaseclient import TelegramBaseClient
from .. import errors, utils
from ..tl import TLObject, TLRequest, types, functions
from ..errors import MultiError, RPCError

__log__ = logging.getLogger(__name__)
_NOT_A_REQUEST = TypeError('You can only invoke requests, not types!')

class UserMethods(TelegramBaseClient):
    async def __call__(self, request, ordered=False):
        requests = (request if utils.is_list_like(request) else (request,))
        for r in requests:
            if not isinstance(r, TLRequest):
                raise _NOT_A_REQUEST
            await r.resolve(self, utils)

            # Avoid making the request if it's already in a flood wait
            if r.CONSTRUCTOR_ID in self._flood_waited_requests:
                due = self._flood_waited_requests[r.CONSTRUCTOR_ID]
                diff = round(due - time.time())
                if diff <= 3:  # Flood waits below 3 seconds are "ignored"
                    self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
                elif diff <= self.flood_sleep_threshold:
                    __log__.info('Sleeping early for %ds on flood wait', diff)
                    await asyncio.sleep(diff, loop=self._loop)
                    self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
                else:
                    raise errors.FloodWaitError(capture=diff)

        request_index = 0
        self._last_request = time.time()
        for _ in range(self._request_retries):
            try:
                future = self._sender.send(request, ordered=ordered)
                if isinstance(future, list):
                    results = []
                    exceptions = []
                    for f in future:
                        try:
                            result = await f
                        except RPCError as e:
                            exceptions.append(e)
                            results.append(None)
                            continue
                        self.session.process_entities(result)
                        exceptions.append(None)
                        results.append(result)
                        request_index += 1
                    if any(x is not None for x in exceptions):
                        raise MultiError(exceptions, results, requests)
                    else:
                        return results
                else:
                    result = await future
                    self.session.process_entities(result)
                    return result
            except (errors.ServerError, errors.RpcCallFailError) as e:
                __log__.warning('Telegram is having internal issues %s: %s',
                                e.__class__.__name__, e)
            except (errors.FloodWaitError, errors.FloodTestPhoneWaitError) as e:
                if utils.is_list_like(request):
                    request = request[request_index]

                self._flood_waited_requests\
                    [request.CONSTRUCTOR_ID] = time.time() + e.seconds

                if e.seconds <= self.flood_sleep_threshold:
                    __log__.info('Sleeping for %ds on flood wait', e.seconds)
                    await asyncio.sleep(e.seconds, loop=self._loop)
                else:
                    raise
            except (errors.PhoneMigrateError, errors.NetworkMigrateError,
                    errors.UserMigrateError) as e:
                __log__.info('Phone migrated to %d', e.new_dc)
                should_raise = isinstance(e, (
                    errors.PhoneMigrateError,  errors.NetworkMigrateError
                ))
                if should_raise and await self.is_user_authorized():
                    raise
                await self._switch_dc(e.new_dc)

        raise ValueError('Number of retries reached 0')

    # region Public methods

    async def get_me(self, input_peer=False):
        """
        Gets "me" (the self user) which is currently authenticated,
        or None if the request fails (hence, not authenticated).

        Args:
            input_peer (`bool`, optional):
                Whether to return the :tl:`InputPeerUser` version or the normal
                :tl:`User`. This can be useful if you just need to know the ID
                of yourself.

        Returns:
            Your own :tl:`User`.
        """
        if input_peer and self._self_input_peer:
            return self._self_input_peer

        try:
            me = (await self(
                functions.users.GetUsersRequest([types.InputUserSelf()])))[0]

            if not self._self_input_peer:
                self._self_input_peer = utils.get_input_peer(
                    me, allow_self=False
                )

            return self._self_input_peer if input_peer else me
        except errors.UnauthorizedError:
            return None

    async def is_user_authorized(self):
        """
        Returns ``True`` if the user is authorized.
        """
        if self._self_input_peer is not None or self._state.pts != -1:
            return True

        try:
            self._state = await self(functions.updates.GetStateRequest())
            return True
        except errors.RPCError:
            return False

    async def get_entity(self, entity):
        """
        Turns the given entity into a valid Telegram :tl:`User`, :tl:`Chat`
        or :tl:`Channel`. You can also pass a list or iterable of entities,
        and they will be efficiently fetched from the network.

        entity (`str` | `int` | :tl:`Peer` | :tl:`InputPeer`):
            If a username is given, **the username will be resolved** making
            an API call every time. Resolving usernames is an expensive
            operation and will start hitting flood waits around 50 usernames
            in a short period of time.

            If you want to get the entity for a *cached* username, you should
            first `get_input_entity(username) <get_input_entity>` which will
            use the cache), and then use `get_entity` with the result of the
            previous call.

            Similar limits apply to invite links, and you should use their
            ID instead.

            Using phone numbers, exact names, integer IDs or :tl:`Peer`
            rely on a `get_input_entity` first, which in turn needs the
            entity to be in cache, unless a :tl:`InputPeer` was passed.

            Unsupported types will raise ``TypeError``.

            If the entity can't be found, ``ValueError`` will be raised.

        Returns:
            :tl:`User`, :tl:`Chat` or :tl:`Channel` corresponding to the
            input entity. A list will be returned if more than one was given.
        """
        single = not utils.is_list_like(entity)
        if single:
            entity = (entity,)

        # Group input entities by string (resolve username),
        # input users (get users), input chat (get chats) and
        # input channels (get channels) to get the most entities
        # in the less amount of calls possible.
        inputs = []
        for x in entity:
            if isinstance(x, str):
                inputs.append(x)
            else:
                inputs.append(await self.get_input_entity(x))

        users = [x for x in inputs
                 if isinstance(x, (types.InputPeerUser, types.InputPeerSelf))]
        chats = [x.chat_id for x in inputs
                 if isinstance(x, types.InputPeerChat)]
        channels = [x for x in inputs
                    if isinstance(x, types.InputPeerChannel)]
        if users:
            # GetUsersRequest has a limit of 200 per call
            tmp = []
            while users:
                curr, users = users[:200], users[200:]
                tmp.extend(await self(functions.users.GetUsersRequest(curr)))
            users = tmp
        if chats:  # TODO Handle chats slice?
            chats = (await self(
                functions.messages.GetChatsRequest(chats))).chats
        if channels:
            channels = (await self(
                functions.channels.GetChannelsRequest(channels))).chats

        # Merge users, chats and channels into a single dictionary
        id_entity = {
            utils.get_peer_id(x): x
            for x in itertools.chain(users, chats, channels)
        }

        # We could check saved usernames and put them into the users,
        # chats and channels list from before. While this would reduce
        # the amount of ResolveUsername calls, it would fail to catch
        # username changes.
        result = []
        for x in inputs:
            if isinstance(x, str):
                result.append(await self._get_entity_from_string(x))
            elif not isinstance(x, types.InputPeerSelf):
                result.append(id_entity[utils.get_peer_id(x)])
            else:
                result.append(next(
                    u for u in id_entity.values()
                    if isinstance(u, types.User) and u.is_self
                ))

        return result[0] if single else result

    async def get_input_entity(self, peer):
        """
        Turns the given peer into its input entity version. Most requests
        use this kind of :tl:`InputPeer`, so this is the most suitable call
        to make for those cases. **Generally you should let the library do
        its job** and don't worry about getting the input entity first, but
        if you're going to use an entity often, consider making the call:

        >>> import asyncio
        >>> rc = asyncio.get_event_loop().run_until_complete
        >>>
        >>> from telethon import TelegramClient
        >>> client = TelegramClient(...)
        >>> # If you're going to use "username" often in your code
        >>> # (make a lot of calls), consider getting its input entity
        >>> # once, and then using the "user" everywhere instead.
        >>> user = rc(client.get_input_entity('username'))
        >>> # The same applies to IDs, chats or channels.
        >>> chat = rc(client.get_input_entity(-123456789))

        entity (`str` | `int` | :tl:`Peer` | :tl:`InputPeer`):
            If a username or invite link is given, **the library will
            use the cache**. This means that it's possible to be using
            a username that *changed* or an old invite link (this only
            happens if an invite link for a small group chat is used
            after it was upgraded to a mega-group).

            If the username or ID from the invite link is not found in
            the cache, it will be fetched. The same rules apply to phone
            numbers (``'+34 123456789'``).

            If an exact name is given, it must be in the cache too. This
            is not reliable as different people can share the same name
            and which entity is returned is arbitrary, and should be used
            only for quick tests.

            If a positive integer ID is given, the entity will be searched
            in cached users, chats or channels, without making any call.

            If a negative integer ID is given, the entity will be searched
            exactly as either a chat (prefixed with ``-``) or as a channel
            (prefixed with ``-100``).

            If a :tl:`Peer` is given, it will be searched exactly in the
            cache as either a user, chat or channel.

            If the given object can be turned into an input entity directly,
            said operation will be done.

            Unsupported types will raise ``TypeError``.

            If the entity can't be found, ``ValueError`` will be raised.

        Returns:
            :tl:`InputPeerUser`, :tl:`InputPeerChat` or :tl:`InputPeerChannel`
            or :tl:`InputPeerSelf` if the parameter is ``'me'`` or ``'self'``.

            If you need to get the ID of yourself, you should use
            `get_me` with ``input_peer=True``) instead.
        """
        if peer in ('me', 'self'):
            return types.InputPeerSelf()

        try:
            # First try to get the entity from cache, otherwise figure it out
            return self.session.get_input_entity(peer)
        except ValueError:
            pass

        if isinstance(peer, str):
            return utils.get_input_peer(
                await self._get_entity_from_string(peer))

        if not isinstance(peer, int) and (not isinstance(peer, TLObject)
                                          or peer.SUBCLASS_OF_ID != 0x2d45687):
            # Try casting the object into an input peer. Might TypeError.
            # Don't do it if a not-found ID was given (instead ValueError).
            # Also ignore Peer (0x2d45687 == crc32(b'Peer'))'s, lacking hash.
            return utils.get_input_peer(peer)

        raise ValueError(
            'Could not find the input entity for {!r}. Please read https://'
            'telethon.readthedocs.io/en/latest/extra/basic/entities.html to'
            ' find out more details.'
            .format(peer)
        )

    async def get_peer_id(self, peer, add_mark=True):
        """
        Gets the ID for the given peer, which may be anything entity-like.

        This method needs to be ``async`` because `peer` supports usernames,
        invite-links, phone numbers, etc.

        If ``add_mark is False``, then a positive ID will be returned
        instead. By default, bot-API style IDs (signed) are returned.
        """
        if isinstance(peer, int):
            return utils.get_peer_id(peer, add_mark=add_mark)

        try:
            if peer.SUBCLASS_OF_ID in (0x2d45687, 0xc91c90b6):
                # 0x2d45687, 0xc91c90b6 == crc32(b'Peer') and b'InputPeer'
                return utils.get_peer_id(peer)
        except AttributeError:
            pass

        peer = await self.get_input_entity(peer)
        if isinstance(peer, types.InputPeerSelf):
            peer = await self.get_me(input_peer=True)

        return utils.get_peer_id(peer, add_mark=add_mark)

    # endregion

    # region Private methods

    async def _get_entity_from_string(self, string):
        """
        Gets a full entity from the given string, which may be a phone or
        a username, and processes all the found entities on the session.
        The string may also be a user link, or a channel/chat invite link.

        This method has the side effect of adding the found users to the
        session database, so it can be queried later without API calls,
        if this option is enabled on the session.

        Returns the found entity, or raises TypeError if not found.
        """
        phone = utils.parse_phone(string)
        if phone:
            for user in (await self(
                    functions.contacts.GetContactsRequest(0))).users:
                if user.phone == phone:
                    return user
        else:
            username, is_join_chat = utils.parse_username(string)
            if is_join_chat:
                invite = await self(
                    functions.messages.CheckChatInviteRequest(username))

                if isinstance(invite, types.ChatInvite):
                    raise ValueError(
                        'Cannot get entity from a channel (or group) '
                        'that you are not part of. Join the group and retry'
                    )
                elif isinstance(invite, types.ChatInviteAlready):
                    return invite.chat
            elif username:
                if username in ('me', 'self'):
                    return await self.get_me()

                try:
                    result = await self(
                        functions.contacts.ResolveUsernameRequest(username))
                except errors.UsernameNotOccupiedError as e:
                    raise ValueError('No user has "{}" as username'
                                     .format(username)) from e

                try:
                    pid = utils.get_peer_id(result.peer, add_mark=False)
                    if isinstance(result.peer, types.PeerUser):
                        return next(x for x in result.users if x.id == pid)
                    else:
                        return next(x for x in result.chats if x.id == pid)
                except StopIteration:
                    pass
            try:
                # Nobody with this username, maybe it's an exact name/title
                return await self.get_entity(
                    self.session.get_input_entity(string))
            except ValueError:
                pass

        raise ValueError(
            'Cannot find any entity corresponding to "{}"'.format(string)
        )

    async def _get_input_dialog(self, dialog):
        """
        Returns a :tl:`InputDialogPeer`. This is a bit tricky because
        it may or not need access to the client to convert what's given
        into an input entity.
        """
        try:
            if dialog.SUBCLASS_OF_ID == 0xa21c9795:  # crc32(b'InputDialogPeer')
                dialog.peer = await self.get_input_entity(dialog.peer)
                return dialog
            elif dialog.SUBCLASS_OF_ID == 0xc91c90b6:  # crc32(b'InputPeer')
                return types.InputDialogPeer(dialog)
        except AttributeError:
            pass

        return types.InputDialogPeer(await self.get_input_entity(dialog))

    async def _get_input_notify(self, notify):
        """
        Returns a :tl:`InputNotifyPeer`. This is a bit tricky because
        it may or not need access to the client to convert what's given
        into an input entity.
        """
        try:
            if notify.SUBCLASS_OF_ID == 0x58981615:
                if isinstance(notify, types.InputNotifyPeer):
                    notify.peer = await self.get_input_entity(notify.peer)
                return notify
        except AttributeError:
            return types.InputNotifyPeer(await self.get_input_entity(notify))

    # endregion
<commit_msg>Except BotMethodInvalidError on getContacts<commit_after>import asyncio
import itertools
import logging
import time

from .telegrambaseclient import TelegramBaseClient
from .. import errors, utils
from ..tl import TLObject, TLRequest, types, functions
from ..errors import MultiError, RPCError

__log__ = logging.getLogger(__name__)
_NOT_A_REQUEST = TypeError('You can only invoke requests, not types!')

class UserMethods(TelegramBaseClient):
    async def __call__(self, request, ordered=False):
        requests = (request if utils.is_list_like(request) else (request,))
        for r in requests:
            if not isinstance(r, TLRequest):
                raise _NOT_A_REQUEST
            await r.resolve(self, utils)

            # Avoid making the request if it's already in a flood wait
            if r.CONSTRUCTOR_ID in self._flood_waited_requests:
                due = self._flood_waited_requests[r.CONSTRUCTOR_ID]
                diff = round(due - time.time())
                if diff <= 3:  # Flood waits below 3 seconds are "ignored"
                    self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
                elif diff <= self.flood_sleep_threshold:
                    __log__.info('Sleeping early for %ds on flood wait', diff)
                    await asyncio.sleep(diff, loop=self._loop)
                    self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
                else:
                    raise errors.FloodWaitError(capture=diff)

        request_index = 0
        self._last_request = time.time()
        for _ in range(self._request_retries):
            try:
                future = self._sender.send(request, ordered=ordered)
                if isinstance(future, list):
                    results = []
                    exceptions = []
                    for f in future:
                        try:
                            result = await f
                        except RPCError as e:
                            exceptions.append(e)
                            results.append(None)
                            continue
                        self.session.process_entities(result)
                        exceptions.append(None)
                        results.append(result)
                        request_index += 1
                    if any(x is not None for x in exceptions):
                        raise MultiError(exceptions, results, requests)
                    else:
                        return results
                else:
                    result = await future
                    self.session.process_entities(result)
                    return result
            except (errors.ServerError, errors.RpcCallFailError) as e:
                __log__.warning('Telegram is having internal issues %s: %s',
                                e.__class__.__name__, e)
            except (errors.FloodWaitError, errors.FloodTestPhoneWaitError) as e:
                if utils.is_list_like(request):
                    request = request[request_index]

                self._flood_waited_requests\
                    [request.CONSTRUCTOR_ID] = time.time() + e.seconds

                if e.seconds <= self.flood_sleep_threshold:
                    __log__.info('Sleeping for %ds on flood wait', e.seconds)
                    await asyncio.sleep(e.seconds, loop=self._loop)
                else:
                    raise
            except (errors.PhoneMigrateError, errors.NetworkMigrateError,
                    errors.UserMigrateError) as e:
                __log__.info('Phone migrated to %d', e.new_dc)
                should_raise = isinstance(e, (
                    errors.PhoneMigrateError,  errors.NetworkMigrateError
                ))
                if should_raise and await self.is_user_authorized():
                    raise
                await self._switch_dc(e.new_dc)

        raise ValueError('Number of retries reached 0')

    # region Public methods

    async def get_me(self, input_peer=False):
        """
        Gets "me" (the self user) which is currently authenticated,
        or None if the request fails (hence, not authenticated).

        Args:
            input_peer (`bool`, optional):
                Whether to return the :tl:`InputPeerUser` version or the normal
                :tl:`User`. This can be useful if you just need to know the ID
                of yourself.

        Returns:
            Your own :tl:`User`.
        """
        if input_peer and self._self_input_peer:
            return self._self_input_peer

        try:
            me = (await self(
                functions.users.GetUsersRequest([types.InputUserSelf()])))[0]

            if not self._self_input_peer:
                self._self_input_peer = utils.get_input_peer(
                    me, allow_self=False
                )

            return self._self_input_peer if input_peer else me
        except errors.UnauthorizedError:
            return None

    async def is_user_authorized(self):
        """
        Returns ``True`` if the user is authorized.
        """
        if self._self_input_peer is not None or self._state.pts != -1:
            return True

        try:
            self._state = await self(functions.updates.GetStateRequest())
            return True
        except errors.RPCError:
            return False

    async def get_entity(self, entity):
        """
        Turns the given entity into a valid Telegram :tl:`User`, :tl:`Chat`
        or :tl:`Channel`. You can also pass a list or iterable of entities,
        and they will be efficiently fetched from the network.

        entity (`str` | `int` | :tl:`Peer` | :tl:`InputPeer`):
            If a username is given, **the username will be resolved** making
            an API call every time. Resolving usernames is an expensive
            operation and will start hitting flood waits around 50 usernames
            in a short period of time.

            If you want to get the entity for a *cached* username, you should
            first `get_input_entity(username) <get_input_entity>` which will
            use the cache), and then use `get_entity` with the result of the
            previous call.

            Similar limits apply to invite links, and you should use their
            ID instead.

            Using phone numbers, exact names, integer IDs or :tl:`Peer`
            rely on a `get_input_entity` first, which in turn needs the
            entity to be in cache, unless a :tl:`InputPeer` was passed.

            Unsupported types will raise ``TypeError``.

            If the entity can't be found, ``ValueError`` will be raised.

        Returns:
            :tl:`User`, :tl:`Chat` or :tl:`Channel` corresponding to the
            input entity. A list will be returned if more than one was given.
        """
        single = not utils.is_list_like(entity)
        if single:
            entity = (entity,)

        # Group input entities by string (resolve username),
        # input users (get users), input chat (get chats) and
        # input channels (get channels) to get the most entities
        # in the less amount of calls possible.
        inputs = []
        for x in entity:
            if isinstance(x, str):
                inputs.append(x)
            else:
                inputs.append(await self.get_input_entity(x))

        users = [x for x in inputs
                 if isinstance(x, (types.InputPeerUser, types.InputPeerSelf))]
        chats = [x.chat_id for x in inputs
                 if isinstance(x, types.InputPeerChat)]
        channels = [x for x in inputs
                    if isinstance(x, types.InputPeerChannel)]
        if users:
            # GetUsersRequest has a limit of 200 per call
            tmp = []
            while users:
                curr, users = users[:200], users[200:]
                tmp.extend(await self(functions.users.GetUsersRequest(curr)))
            users = tmp
        if chats:  # TODO Handle chats slice?
            chats = (await self(
                functions.messages.GetChatsRequest(chats))).chats
        if channels:
            channels = (await self(
                functions.channels.GetChannelsRequest(channels))).chats

        # Merge users, chats and channels into a single dictionary
        id_entity = {
            utils.get_peer_id(x): x
            for x in itertools.chain(users, chats, channels)
        }

        # We could check saved usernames and put them into the users,
        # chats and channels list from before. While this would reduce
        # the amount of ResolveUsername calls, it would fail to catch
        # username changes.
        result = []
        for x in inputs:
            if isinstance(x, str):
                result.append(await self._get_entity_from_string(x))
            elif not isinstance(x, types.InputPeerSelf):
                result.append(id_entity[utils.get_peer_id(x)])
            else:
                result.append(next(
                    u for u in id_entity.values()
                    if isinstance(u, types.User) and u.is_self
                ))

        return result[0] if single else result

    async def get_input_entity(self, peer):
        """
        Turns the given peer into its input entity version. Most requests
        use this kind of :tl:`InputPeer`, so this is the most suitable call
        to make for those cases. **Generally you should let the library do
        its job** and don't worry about getting the input entity first, but
        if you're going to use an entity often, consider making the call:

        >>> import asyncio
        >>> rc = asyncio.get_event_loop().run_until_complete
        >>>
        >>> from telethon import TelegramClient
        >>> client = TelegramClient(...)
        >>> # If you're going to use "username" often in your code
        >>> # (make a lot of calls), consider getting its input entity
        >>> # once, and then using the "user" everywhere instead.
        >>> user = rc(client.get_input_entity('username'))
        >>> # The same applies to IDs, chats or channels.
        >>> chat = rc(client.get_input_entity(-123456789))

        entity (`str` | `int` | :tl:`Peer` | :tl:`InputPeer`):
            If a username or invite link is given, **the library will
            use the cache**. This means that it's possible to be using
            a username that *changed* or an old invite link (this only
            happens if an invite link for a small group chat is used
            after it was upgraded to a mega-group).

            If the username or ID from the invite link is not found in
            the cache, it will be fetched. The same rules apply to phone
            numbers (``'+34 123456789'``).

            If an exact name is given, it must be in the cache too. This
            is not reliable as different people can share the same name
            and which entity is returned is arbitrary, and should be used
            only for quick tests.

            If a positive integer ID is given, the entity will be searched
            in cached users, chats or channels, without making any call.

            If a negative integer ID is given, the entity will be searched
            exactly as either a chat (prefixed with ``-``) or as a channel
            (prefixed with ``-100``).

            If a :tl:`Peer` is given, it will be searched exactly in the
            cache as either a user, chat or channel.

            If the given object can be turned into an input entity directly,
            said operation will be done.

            Unsupported types will raise ``TypeError``.

            If the entity can't be found, ``ValueError`` will be raised.

        Returns:
            :tl:`InputPeerUser`, :tl:`InputPeerChat` or :tl:`InputPeerChannel`
            or :tl:`InputPeerSelf` if the parameter is ``'me'`` or ``'self'``.

            If you need to get the ID of yourself, you should use
            `get_me` with ``input_peer=True``) instead.
        """
        if peer in ('me', 'self'):
            return types.InputPeerSelf()

        try:
            # First try to get the entity from cache, otherwise figure it out
            return self.session.get_input_entity(peer)
        except ValueError:
            pass

        if isinstance(peer, str):
            return utils.get_input_peer(
                await self._get_entity_from_string(peer))

        if not isinstance(peer, int) and (not isinstance(peer, TLObject)
                                          or peer.SUBCLASS_OF_ID != 0x2d45687):
            # Try casting the object into an input peer. Might TypeError.
            # Don't do it if a not-found ID was given (instead ValueError).
            # Also ignore Peer (0x2d45687 == crc32(b'Peer'))'s, lacking hash.
            return utils.get_input_peer(peer)

        raise ValueError(
            'Could not find the input entity for {!r}. Please read https://'
            'telethon.readthedocs.io/en/latest/extra/basic/entities.html to'
            ' find out more details.'
            .format(peer)
        )

    async def get_peer_id(self, peer, add_mark=True):
        """
        Gets the ID for the given peer, which may be anything entity-like.

        This method needs to be ``async`` because `peer` supports usernames,
        invite-links, phone numbers, etc.

        If ``add_mark is False``, then a positive ID will be returned
        instead. By default, bot-API style IDs (signed) are returned.
        """
        if isinstance(peer, int):
            return utils.get_peer_id(peer, add_mark=add_mark)

        try:
            if peer.SUBCLASS_OF_ID in (0x2d45687, 0xc91c90b6):
                # 0x2d45687, 0xc91c90b6 == crc32(b'Peer') and b'InputPeer'
                return utils.get_peer_id(peer)
        except AttributeError:
            pass

        peer = await self.get_input_entity(peer)
        if isinstance(peer, types.InputPeerSelf):
            peer = await self.get_me(input_peer=True)

        return utils.get_peer_id(peer, add_mark=add_mark)

    # endregion

    # region Private methods

    async def _get_entity_from_string(self, string):
        """
        Gets a full entity from the given string, which may be a phone or
        a username, and processes all the found entities on the session.
        The string may also be a user link, or a channel/chat invite link.

        This method has the side effect of adding the found users to the
        session database, so it can be queried later without API calls,
        if this option is enabled on the session.

        Returns the found entity, or raises TypeError if not found.
        """
        phone = utils.parse_phone(string)
        if phone:
            try:
                for user in (await self(
                        functions.contacts.GetContactsRequest(0))).users:
                    if user.phone == phone:
                        return user
            except errors.BotMethodInvalidError:
                raise ValueError('Cannot get entity by phone number as a '
                                 'bot (try using integer IDs, not strings)')
        else:
            username, is_join_chat = utils.parse_username(string)
            if is_join_chat:
                invite = await self(
                    functions.messages.CheckChatInviteRequest(username))

                if isinstance(invite, types.ChatInvite):
                    raise ValueError(
                        'Cannot get entity from a channel (or group) '
                        'that you are not part of. Join the group and retry'
                    )
                elif isinstance(invite, types.ChatInviteAlready):
                    return invite.chat
            elif username:
                if username in ('me', 'self'):
                    return await self.get_me()

                try:
                    result = await self(
                        functions.contacts.ResolveUsernameRequest(username))
                except errors.UsernameNotOccupiedError as e:
                    raise ValueError('No user has "{}" as username'
                                     .format(username)) from e

                try:
                    pid = utils.get_peer_id(result.peer, add_mark=False)
                    if isinstance(result.peer, types.PeerUser):
                        return next(x for x in result.users if x.id == pid)
                    else:
                        return next(x for x in result.chats if x.id == pid)
                except StopIteration:
                    pass
            try:
                # Nobody with this username, maybe it's an exact name/title
                return await self.get_entity(
                    self.session.get_input_entity(string))
            except ValueError:
                pass

        raise ValueError(
            'Cannot find any entity corresponding to "{}"'.format(string)
        )

    async def _get_input_dialog(self, dialog):
        """
        Returns a :tl:`InputDialogPeer`. This is a bit tricky because
        it may or not need access to the client to convert what's given
        into an input entity.
        """
        try:
            if dialog.SUBCLASS_OF_ID == 0xa21c9795:  # crc32(b'InputDialogPeer')
                dialog.peer = await self.get_input_entity(dialog.peer)
                return dialog
            elif dialog.SUBCLASS_OF_ID == 0xc91c90b6:  # crc32(b'InputPeer')
                return types.InputDialogPeer(dialog)
        except AttributeError:
            pass

        return types.InputDialogPeer(await self.get_input_entity(dialog))

    async def _get_input_notify(self, notify):
        """
        Returns a :tl:`InputNotifyPeer`. This is a bit tricky because
        it may or not need access to the client to convert what's given
        into an input entity.
        """
        try:
            if notify.SUBCLASS_OF_ID == 0x58981615:
                if isinstance(notify, types.InputNotifyPeer):
                    notify.peer = await self.get_input_entity(notify.peer)
                return notify
        except AttributeError:
            return types.InputNotifyPeer(await self.get_input_entity(notify))

    # endregion
<|endoftext|>


<commit_before>import csv
import os

##############################################################################
################################  Constants  #################################
##############################################################################

LOGS_FOLDER_NAME = 'logs'

##############################################################################
#################################  Methods  ##################################
##############################################################################

def retrieveLogsList(folder_name):
    '''
    Saves the logs filenames in a list. Uses the absolute path name to
    avoid storing the folder name as a separate parameter.
    '''
    original_path = os.getcwd()

    # Validate input
    if not isinstance(folder_name,str):
        raise TypeError('Please enter a valid folder name')

    # Change cwd to the logs folder
    os.chdir(os.path.join(os.getcwd(),folder_name))

    files = os.listdir(os.getcwd())
    logs_filenames = [os.path.join(os.getcwd(),f) for f in files
                      if f[-3:] == 'log']

    os.chdir(original_path)
    return logs_filenames

def createLogObject(log_filename):
    '''
    Creates a log object with the desired properties as key value pairs.

    This method is needed since sometimes CURR measurements appear before
    the firmware version is shown and the multipliers cannot be computed.
    '''
    CURR_ROW_NAME = 'CURR'
    ROW_NAME_INDEX = 0
    TIMEUS_INDEX = 1

    # Validate input
    if not isinstance(log_filename,str) or log_filename[-3:] != 'log':
        raise TypeError('Please enter a valid log filename.' +
                        ' Filename entered: "%s"' % (log_filename))

    # Initialize empty log object
    log_object = {'firmware_version' : '',
                        'curr_index' : -1,
                    'timeus_entries' : [],
                   'current_entries' : []}

    # Store desired properties in log_object
    with open(log_filename,'rb') as log:
        log_reader = csv.reader(log)
        for row in log_reader:
            if log_object['firmware_version'] == '':
                log_object['firmware_version'] = findFirmwareVersionInRow(row)
            if log_object['curr_index'] == -1:
                log_object['curr_index'] = findCurrIndexInRow(row)
            if row[ROW_NAME_INDEX] == CURR_ROW_NAME:
                curr_index = log_object['curr_index']
                log_object['timeus_entries'].append(float(row[TIMEUS_INDEX].strip()))
                log_object['current_entries'].append(float(row[curr_index].strip()))

    return log_object

def computeFlightTime(log_object):
    CURRENT_THRESHOLD = 4
    CURRENT_MULTIPLIERS = {'V3.2': 1, 'V3.3': 1.0/100, 'V3.4': 1}
    TIME_MULTIPLIERS = {'V3.2': 1000, 'V3.3': 1, 'V3.4': 1}

    # Validate input
    log_object_valid = 'firmware_version' in log_object and \
                       'curr_index' in log_object and \
                       'timeus_entries' in log_object and \
                       'current_entries' in log_object

    if not isinstance(log_object,dict) or not log_object_valid:
        raise TypeError('Please enter a valid log object.' +
                        ' Object entered: %s' % (log_object))

    current_multiplier = CURRENT_MULTIPLIERS[log_object['firmware_version']]
    time_multiplier = TIME_MULTIPLIERS[log_object['firmware_version']]

    drone_flying = False
    last_takeoff = -1
    flight_time = 0
    for entry_ind in range(len(log_object['timeus_entries'])):
        timeus = time_multiplier*log_object['timeus_entries'][entry_ind]
        current = current_multiplier*log_object['current_entries'][entry_ind]

        if current > CURRENT_THRESHOLD:
            if not drone_flying:
                last_takeoff = timeus
                drone_flying = True
        elif drone_flying:
            flight_time += timeus - last_takeoff
            drone_flying = False

    return flight_time/1e6

def computeTotalFlightTime(logs_list):
    total_flight_time = 0

    if not isinstance(logs_list,list):
        raise TypeError('Please enter a list of log filenames')

    for log_filename in logs_list:
        log_object = createLogObject(log_filename)
        total_flight_time += computeFlightTime(log_object)

    return total_flight_time

##############################################################################
############################  Helper functions  ##############################
##############################################################################

def getEntryIndexFromHeader(header,entry_name):
    entry_index = -1
    try:
        # Index starts at 1 (0 is header name)
        entry_index = header.index(entry_name) + 1
    except ValueError:
        print 'Entry "%s" not found in header: %s' % (entry_name,header)
    return entry_index

def findHeaderInRow(row,header_row_name):
    FMT_ROW_NAME = 'FMT'
    HEADER_FIRST_INDEX = 5

    if (row[0].strip() == FMT_ROW_NAME and row[3].strip() == header_row_name):
        curr_header = [header_element.strip().upper() for header_element
                       in row[HEADER_FIRST_INDEX:]]
        return curr_header
    return []

def findCurrHeaderInRow(row):
    return findHeaderInRow(row,'CURR')

def findCurrIndexInRow(row):
    CURR_ENTRY_NAME = 'CURR'

    curr_index = -1

    curr_header = findCurrHeaderInRow(row)
    if curr_header != []:
        curr_index = getEntryIndexFromHeader(curr_header,CURR_ENTRY_NAME)

    return curr_index

def findFirmwareVersionInRow(row):
    ROW_NAME_INDEX = 0
    MSG_ROW_NAME = 'MSG'
    FIRMWARE_ENTRY_INDEX = -1
    FIRMWARE_LENGTH = 4 # To save only V3.2, V3.3, V3.4
    firmware_string = row[FIRMWARE_ENTRY_INDEX]

    if row[ROW_NAME_INDEX] == MSG_ROW_NAME:
        start_firmware = firmware_string.find('V') # Assuming always starts with V
        return firmware_string[start_firmware:start_firmware+FIRMWARE_LENGTH]
    return ''

if __name__ == '__main__':
    folder_name = LOGS_FOLDER_NAME
    logs_list = retrieveLogsList(folder_name)
    print 'Analyzing %s logs...' % (len(logs_list))
    #print 'Flight time: %s' % (computeFlightTime(logs_list[86]))
    total_flight_time = computeTotalFlightTime(logs_list)
    print 'Total flight time: %s s' % (total_flight_time)
<commit_msg>Allow only .log files (e.g. not .rlog), fix indents<commit_after>import csv
import os

##############################################################################
################################  Constants  #################################
##############################################################################

LOGS_FOLDER_NAME = 'logs'
LOG_FILE_EXTENSION = '.log'

##############################################################################
#################################  Methods  ##################################
##############################################################################

def retrieveLogsList(folder_name):
    '''
    Saves the logs filenames in a list. Uses the absolute path name to
    avoid storing the folder name as a separate parameter.
    '''
    original_path = os.getcwd()

    # Validate input
    if not isinstance(folder_name,str):
        raise TypeError('Please enter a valid folder name')

    # Change cwd to the logs folder
    os.chdir(os.path.join(os.getcwd(),folder_name))

    files = os.listdir(os.getcwd())
    logs_filenames = [os.path.join(os.getcwd(),f) for f in files
                      if f.endswith(LOG_FILE_EXTENSION)]

    os.chdir(original_path)
    return logs_filenames

def createLogObject(log_filename):
    '''
    Creates a log object with the desired properties as key value pairs.

    This method is needed since sometimes CURR measurements appear before
    the firmware version is shown and the multipliers cannot be computed.
    '''
    CURR_ROW_NAME = 'CURR'
    ROW_NAME_INDEX = 0
    TIMEUS_INDEX = 1

    # Validate input
    if not isinstance(log_filename,str) or not(log_filename.endswith(LOG_FILE_EXTENSION)):
        raise TypeError('Please enter a valid log filename.' +
                        ' Filename entered: "%s"' % (log_filename))

    # Initialize empty log object
    log_object = {'firmware_version' : '',
                        'curr_index' : -1,
                    'timeus_entries' : [],
                   'current_entries' : []}

    # Store desired properties in log_object
    with open(log_filename,'rb') as log:
        log_reader = csv.reader(log)
        for row in log_reader:
            if log_object['firmware_version'] == '':
                log_object['firmware_version'] = findFirmwareVersionInRow(row)
            if log_object['curr_index'] == -1:
                log_object['curr_index'] = findCurrIndexInRow(row)
            if row[ROW_NAME_INDEX] == CURR_ROW_NAME:
                curr_index = log_object['curr_index']
                log_object['timeus_entries'].append(float(row[TIMEUS_INDEX].strip()))
                log_object['current_entries'].append(float(row[curr_index].strip()))

    return log_object

def computeFlightTime(log_object):
    CURRENT_THRESHOLD = 4
    CURRENT_MULTIPLIERS = {'V3.2': 1, 'V3.3': 1.0/100, 'V3.4': 1}
    TIME_MULTIPLIERS = {'V3.2': 1000, 'V3.3': 1, 'V3.4': 1}

    # Validate input
    log_object_valid = 'firmware_version' in log_object and \
                       'curr_index' in log_object and \
                       'timeus_entries' in log_object and \
                       'current_entries' in log_object

    if not isinstance(log_object,dict) or not log_object_valid:
        raise TypeError('Please enter a valid log object.' +
                        ' Object entered: %s' % (log_object))

    current_multiplier = CURRENT_MULTIPLIERS[log_object['firmware_version']]
    time_multiplier = TIME_MULTIPLIERS[log_object['firmware_version']]

    drone_flying = False
    last_takeoff = -1
    flight_time = 0
    for entry_ind in range(len(log_object['timeus_entries'])):
        timeus = time_multiplier*log_object['timeus_entries'][entry_ind]
        current = current_multiplier*log_object['current_entries'][entry_ind]

        if current > CURRENT_THRESHOLD:
            if not drone_flying:
                last_takeoff = timeus
                drone_flying = True
        elif drone_flying:
            flight_time += timeus - last_takeoff
            drone_flying = False

    return flight_time/1e6

def computeTotalFlightTime(logs_list):
    total_flight_time = 0

    if not isinstance(logs_list,list):
        raise TypeError('Please enter a list of log filenames')

    for log_filename in logs_list:
        log_object = createLogObject(log_filename)
        total_flight_time += computeFlightTime(log_object)

    return total_flight_time

##############################################################################
############################  Helper functions  ##############################
##############################################################################

def getEntryIndexFromHeader(header,entry_name):
    entry_index = -1
    try:
        # Index starts at 1 (0 is header name)
        entry_index = header.index(entry_name) + 1
    except ValueError:
        print 'Entry "%s" not found in header: %s' % (entry_name,header)
    return entry_index

def findHeaderInRow(row,header_row_name):
    FMT_ROW_NAME = 'FMT'
    HEADER_FIRST_INDEX = 5

    if (row[0].strip() == FMT_ROW_NAME and row[3].strip() == header_row_name):
        curr_header = [header_element.strip().upper() for header_element
                       in row[HEADER_FIRST_INDEX:]]
        return curr_header
    return []

def findCurrHeaderInRow(row):
    return findHeaderInRow(row,'CURR')

def findCurrIndexInRow(row):
    CURR_ENTRY_NAME = 'CURR'

    curr_index = -1

    curr_header = findCurrHeaderInRow(row)
    if curr_header != []:
        curr_index = getEntryIndexFromHeader(curr_header,CURR_ENTRY_NAME)

    return curr_index

def findFirmwareVersionInRow(row):
    ROW_NAME_INDEX = 0
    MSG_ROW_NAME = 'MSG'
    FIRMWARE_ENTRY_INDEX = -1
    FIRMWARE_LENGTH = 4 # To save only V3.2, V3.3, V3.4
    firmware_string = row[FIRMWARE_ENTRY_INDEX]

    if row[ROW_NAME_INDEX] == MSG_ROW_NAME:
        start_firmware = firmware_string.find('V') # Assuming always starts with V
        return firmware_string[start_firmware:start_firmware+FIRMWARE_LENGTH]
    return ''

if __name__ == '__main__':
    folder_name = LOGS_FOLDER_NAME
    logs_list = retrieveLogsList(folder_name)
    print 'Analyzing %s logs...' % (len(logs_list))
    #print 'Flight time: %s' % (computeFlightTime(logs_list[86]))
    total_flight_time = computeTotalFlightTime(logs_list)
    print 'Total flight time: %s s' % (total_flight_time)
<|endoftext|>


<commit_before>"""
Context managers which can distribute workers (functions) across multiple
processes.
"""

from __future__ import absolute_import
from __future__ import unicode_literals

__all__ = ["distribute"]

import logging
import functools

from buckshot import contexts
from buckshot import funcutils

LOG = logging.getLogger(__name__)

def distribute(*func, **opts):
    """Decorator which distributes the wrapped function across multiple
    processes.

    The input function will be wrapped in a new generator which accepts
    an iterable collection of argument tuples. If a function only takes
    one parameter, a flat list of items can be passed in.

    Each value in the iterable will be mapped to subprocess workers which
    executed the original function and return the result to the parent
    process.

    Example:
    >>> @distribute(processes=4)
    ... def foo(x, y):
    ...     return expensive_calulation(x) + another_expensive_calculation(y)
    ...
    >>> values = zip("abc", [1,2,3])  # [(a, 1), (b, 2), (c, 3)]
    >>> for result in foo(values):    # Map each item in `values` to the original function.
    ...     print result

    Keyword Arguments:
        processes (int): Number of worker processes to use. If None,
            the number of CPUs on the host system will be used.
        ordered (bool): If True, results are returned in the order of their
            respective inputs.
        timeout (float): Number of seconds to wait before killing a worker
            process. If None, no timeout is used.
    """
    if func and opts:
        raise ValueError("Cannot provide positional arguments.")

    def wrapper(func):
        funcutils.patch_recursion(func)  # Make

        @functools.wraps(func)
        def inner(*args):
            iterable = args[-1]  # Kind of a hack to work with instance methods.

            with contexts.distributed(func, **opts) as distributed_function:
                for result in distributed_function(iterable):
                    yield result
        return inner

    if func:
        func = func[0]
        return wrapper(func)
    return wrapper
<commit_msg>Added comment to path_recursion() call<commit_after>"""
Context managers which can distribute workers (functions) across multiple
processes.
"""

from __future__ import absolute_import
from __future__ import unicode_literals

__all__ = ["distribute"]

import logging
import functools

from buckshot import contexts
from buckshot import funcutils

LOG = logging.getLogger(__name__)

def distribute(*func, **opts):
    """Decorator which distributes the wrapped function across multiple
    processes.

    The input function will be wrapped in a new generator which accepts
    an iterable collection of argument tuples. If a function only takes
    one parameter, a flat list of items can be passed in.

    Each value in the iterable will be mapped to subprocess workers which
    executed the original function and return the result to the parent
    process.

    Example:
    >>> @distribute(processes=4)
    ... def foo(x, y):
    ...     return expensive_calulation(x) + another_expensive_calculation(y)
    ...
    >>> values = zip("abc", [1,2,3])  # [(a, 1), (b, 2), (c, 3)]
    >>> for result in foo(values):    # Map each item in `values` to the original function.
    ...     print result

    Keyword Arguments:
        processes (int): Number of worker processes to use. If None,
            the number of CPUs on the host system will be used.
        ordered (bool): If True, results are returned in the order of their
            respective inputs.
        timeout (float): Number of seconds to wait before killing a worker
            process. If None, no timeout is used.
    """
    if func and opts:
        raise ValueError("Cannot provide positional arguments.")

    def wrapper(func):
        funcutils.patch_recursion(func)  # Make immediate recursion work.

        @functools.wraps(func)
        def inner(*args):
            iterable = args[-1]  # Kind of a hack to work with instance methods.

            with contexts.distributed(func, **opts) as distributed_function:
                for result in distributed_function(iterable):
                    yield result
        return inner

    if func:
        func = func[0]
        return wrapper(func)
    return wrapper
<|endoftext|>