<commit_before># coding: utf-8

from __future__ import absolute_import
import sys
import socket
import time
import greenlet
from functools import wraps

from tornado.ioloop import IOLoop
from tornado.iostream import IOStream
from tornado.concurrent import Future
from tornado.gen import coroutine, Return
from tornado.netutil import Resolver

IS_PYPY = False
try:
    import __pypy__
    __pypy__
    IS_PYPY = True
except:
    pass

def enable_debug():
    if IS_PYPY:
        sys.stderr.write("settrace api unsupported on pypy")
        sys.stderr.flush()
        return

    import inspect

    def trace_green(event, args):
        src, target = args
        if event == "switch":
            print("from %s switch to %s" % (src, target))
        elif event == "throw":
            print("from %s throw exception to %s" % (src, target))

        if src.gr_frame:
            tracebacks = inspect.getouterframes(src.gr_frame)
            buff = []
            for traceback in tracebacks:
                srcfile, lineno, func_name, codesample = traceback[1:-1]
                trace_line = '''File "%s", line %d, in %s\n%s '''
                buff.append(trace_line %
                            (srcfile, lineno, func_name, "".join(codesample)))

            print("".join(buff))

    greenlet.settrace(trace_green)

class GreenTask(greenlet.greenlet):
    def __init__(self, run, *args, **kwargs):
        super(GreenTask, self).__init__()
        self._run = run
        self._args = args
        self._kwargs = kwargs
        self._future = Future()
        self._result = None
        self._exc_info = ()

    @property
    def args(self):
        return self._args

    @property
    def kwargs(self):
        return self._kwargs

    def run(self):
        try:
            timeout = self.kwargs.pop("timeout", 0)
            if timeout:
                timer = Timeout(timeout)
                timer.start()
            self._result = self._run(*self.args, **self.kwargs)
            self._future.set_result(self._result)
        except:
            self._exc_info = sys.exc_info()
            self._future.set_exc_info(self._exc_info)
        finally:
            if timeout:
                timer.cancel()

    def start(self):
        self.switch()

    def __str__(self):
        func_name = "%s of %s " % (self._run.__name__, self._run.__module__)
        return "<greenlet %s at %s>" % (func_name, hex(id(self)))

    def __repr__(self):
        return self.__str__()

    def wait(self):
        return self._future

    @classmethod
    def spawn(cls_green, *args, **kwargs):
        task = cls_green(*args, **kwargs)
        task.start()
        return task

def synclize(func):
    coro = coroutine(func)

    @wraps(func)
    def _sync_call(*args, **kwargs):
        child_gr = greenlet.getcurrent()
        main = child_gr.parent
        assert main, "only run in child greenlet"

        def callback(future):
            if future.exc_info():
                child_gr.throw(*future.exc_info())
            elif future.exception():
                child_gr.throw(future.exception())
            else:
                child_gr.switch(future.result())

        IOLoop.current().add_future(coro(*args, **kwargs), callback)
        return main.switch()

    return _sync_call

def spawn(callable_obj, *args, **kwargs):
    return GreenTask.spawn(callable_obj, *args, **kwargs).wait()

def green(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        return GreenTask.spawn(func, *args, **kwargs).wait()

    return wrapper

class Waiter(object):
    def __init__(self):
        self._greenlet = greenlet.getcurrent()
        self._main = self._greenlet.parent

    @property
    def greenlet(self):
        return self._greenlet

    def switch(self, value):
        self._greenlet.switch(value)

    def throw(self, *exc_info):
        self._greenlet.throw(*exc_info)

    def get(self):
        return self._main.switch()

    def clear(self):
        pass

def sleep(seconds):
    waiter = Waiter()
    unique = object()
    IOLoop.current().add_timeout(time.time() + seconds, waiter.switch, unique)
    waiter.get()

class TimeoutException(Exception):

    pass

class Timeout(object):
    def __init__(self, deadline, ex=TimeoutException):
        self._greenlet = greenlet.getcurrent()
        self._ex = ex
        self._callback = None
        self._deadline = deadline
        self._delta = time.time() + deadline
        self._ioloop = IOLoop.current()

    def start(self, callback=None):
        errmsg = "%s timeout, deadline is %d seconds" % (
            str(self._greenlet), self._deadline)
        if callback:
            self._callback = self._ioloop.add_timeout(self._delta, callback,
                                                      self._ex(errmsg))
        else:
            self._callback = self._ioloop.add_timeout(
                self._delta, self._greenlet.throw, self._ex(errmsg))

    def cancel(self):
        assert self._callback, "Timeout not started"
        self._ioloop.remove_timeout(self._callback)
        self._greenlet = None

class AsyncSocket(object):
    def __init__(self, sock):
        self._iostream = IOStream(sock)
        self._resolver = Resolver()
        self._readtimeout = 0
        self._connecttimeout = 0

    def set_readtimeout(self, timeout):
        self._readtimeout = timeout

    def set_connecttimeout(self, timeout):
        self._connecttimeout = timeout

    @synclize
    def connect(self, address):
        host, port = address
        timer = None
        try:
            if self._connecttimeout:
                timer = Timeout(self._connecttimeout)
                timer.start()
            resolved_addrs = yield self._resolver.resolve(
                host,
                port,
                family=socket.AF_INET)
            for addr in resolved_addrs:
                family, host_port = addr
                yield self._iostream.connect(host_port)
                break
        except TimeoutException:
            self.close()
            raise
        finally:
            if timer:
                timer.cancel()

    def sendall(self, buff):
        self._iostream.write(buff)

    @synclize
    def read(self, nbytes, partial=False):
        timer = None
        try:
            if self._readtimeout:
                timer = Timeout(self._readtimeout)
                timer.start()
            buff = yield self._iostream.read_bytes(nbytes, partial=partial)
            raise Return(buff)
        except TimeoutException:
            self.close()
            raise
        finally:
            if timer:
                timer.cancel()

    def recv(self, nbytes):
        return self.read(nbytes, partial=True)

    @synclize
    def readline(self, max_bytes=-1):
        timer = None
        if self._readtimeout:
            timer = Timeout(self._readtimeout)
            timer.start()
        try:
            if max_bytes > 0:
                buff = yield self._iostream.read_until('\n',
                                                       max_bytes=max_bytes)
            else:
                buff = yield self._iostream.read_until('\n')
            raise Return(buff)
        except TimeoutException:
            self.close()
            raise
        finally:
            if timer:
                timer.cancel()

    def close(self):
        self._iostream.close()

    def set_nodelay(self, flag):
        self._iostream.set_nodelay(flag)

    def settimeout(self, timeout):
        pass

    def shutdown(self, direction):
        if self._iostream.fileno():
            self._iostream.fileno().shutdown(direction)

    def recv_into(self, buff):
        expected_rbytes = len(buff)
        data = self.read(expected_rbytes, True)
        srcarray = bytearray(data)
        nbytes = len(srcarray)
        buff[0:nbytes] = srcarray
        return nbytes

    def makefile(self, mode, other):
        return self

    def fileno(self):
        return self._iostream.fileno()
<commit_msg>add socket timeout exception<commit_after># coding: utf-8

from __future__ import absolute_import
import sys
import socket
import time
import greenlet
from functools import wraps

from tornado.ioloop import IOLoop
from tornado.iostream import IOStream
from tornado.concurrent import Future
from tornado.gen import coroutine, Return
from tornado.netutil import Resolver

IS_PYPY = False
try:
    import __pypy__
    __pypy__
    IS_PYPY = True
except:
    pass

def enable_debug():
    if IS_PYPY:
        sys.stderr.write("settrace api unsupported on pypy")
        sys.stderr.flush()
        return

    import inspect

    def trace_green(event, args):
        src, target = args
        if event == "switch":
            print("from %s switch to %s" % (src, target))
        elif event == "throw":
            print("from %s throw exception to %s" % (src, target))

        if src.gr_frame:
            tracebacks = inspect.getouterframes(src.gr_frame)
            buff = []
            for traceback in tracebacks:
                srcfile, lineno, func_name, codesample = traceback[1:-1]
                trace_line = '''File "%s", line %d, in %s\n%s '''
                buff.append(trace_line %
                            (srcfile, lineno, func_name, "".join(codesample)))

            print("".join(buff))

    greenlet.settrace(trace_green)

class GreenTask(greenlet.greenlet):
    def __init__(self, run, *args, **kwargs):
        super(GreenTask, self).__init__()
        self._run = run
        self._args = args
        self._kwargs = kwargs
        self._future = Future()
        self._result = None
        self._exc_info = ()

    @property
    def args(self):
        return self._args

    @property
    def kwargs(self):
        return self._kwargs

    def run(self):
        try:
            timeout = self.kwargs.pop("timeout", 0)
            if timeout:
                timer = Timeout(timeout)
                timer.start()
            self._result = self._run(*self.args, **self.kwargs)
            self._future.set_result(self._result)
        except:
            self._exc_info = sys.exc_info()
            self._future.set_exc_info(self._exc_info)
        finally:
            if timeout:
                timer.cancel()

    def start(self):
        self.switch()

    def __str__(self):
        func_name = "%s of %s " % (self._run.__name__, self._run.__module__)
        return "<greenlet %s at %s>" % (func_name, hex(id(self)))

    def __repr__(self):
        return self.__str__()

    def wait(self):
        return self._future

    @classmethod
    def spawn(cls_green, *args, **kwargs):
        task = cls_green(*args, **kwargs)
        task.start()
        return task

def synclize(func):
    coro = coroutine(func)

    @wraps(func)
    def _sync_call(*args, **kwargs):
        child_gr = greenlet.getcurrent()
        main = child_gr.parent
        assert main, "only run in child greenlet"

        def callback(future):
            if future.exc_info():
                child_gr.throw(*future.exc_info())
            elif future.exception():
                child_gr.throw(future.exception())
            else:
                child_gr.switch(future.result())

        IOLoop.current().add_future(coro(*args, **kwargs), callback)
        return main.switch()

    return _sync_call

def spawn(callable_obj, *args, **kwargs):
    return GreenTask.spawn(callable_obj, *args, **kwargs).wait()

def green(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        return GreenTask.spawn(func, *args, **kwargs).wait()

    return wrapper

class Waiter(object):
    def __init__(self):
        self._greenlet = greenlet.getcurrent()
        self._main = self._greenlet.parent

    @property
    def greenlet(self):
        return self._greenlet

    def switch(self, value):
        self._greenlet.switch(value)

    def throw(self, *exc_info):
        self._greenlet.throw(*exc_info)

    def get(self):
        return self._main.switch()

    def clear(self):
        pass

def sleep(seconds):
    waiter = Waiter()
    unique = object()
    IOLoop.current().add_timeout(time.time() + seconds, waiter.switch, unique)
    waiter.get()

class TimeoutException(Exception):

    pass

class Timeout(object):
    def __init__(self, deadline, ex=TimeoutException):
        self._greenlet = greenlet.getcurrent()
        self._ex = ex
        self._callback = None
        self._deadline = deadline
        self._delta = time.time() + deadline
        self._ioloop = IOLoop.current()

    def start(self, callback=None):
        errmsg = "%s timeout, deadline is %d seconds" % (
            str(self._greenlet), self._deadline)
        if callback:
            self._callback = self._ioloop.add_timeout(self._delta, callback,
                                                      self._ex(errmsg))
        else:
            self._callback = self._ioloop.add_timeout(
                self._delta, self._greenlet.throw, self._ex(errmsg))

    def cancel(self):
        assert self._callback, "Timeout not started"
        self._ioloop.remove_timeout(self._callback)
        self._greenlet = None

class AsyncSocket(object):
    def __init__(self, sock):
        self._iostream = IOStream(sock)
        self._resolver = Resolver()
        self._readtimeout = 0
        self._connecttimeout = 0

    def set_readtimeout(self, timeout):
        self._readtimeout = timeout

    def set_connecttimeout(self, timeout):
        self._connecttimeout = timeout

    @synclize
    def connect(self, address):
        host, port = address
        timer = None
        try:
            if self._connecttimeout:
                timer = Timeout(self._connecttimeout)
                timer.start()
            resolved_addrs = yield self._resolver.resolve(
                host,
                port,
                family=socket.AF_INET)
            for addr in resolved_addrs:
                family, host_port = addr
                yield self._iostream.connect(host_port)
                break
        except TimeoutException, e:
            self.close()
            raise socket.timeout(e.message)
        finally:
            if timer:
                timer.cancel()

    def sendall(self, buff):
        self._iostream.write(buff)

    @synclize
    def read(self, nbytes, partial=False):
        timer = None
        try:
            if self._readtimeout:
                timer = Timeout(self._readtimeout)
                timer.start()
            buff = yield self._iostream.read_bytes(nbytes, partial=partial)
            raise Return(buff)
        except TimeoutException, e:
            self.close()
            raise socket.timeout(e.message)
        finally:
            if timer:
                timer.cancel()

    def recv(self, nbytes):
        return self.read(nbytes, partial=True)

    @synclize
    def readline(self, max_bytes=-1):
        timer = None
        if self._readtimeout:
            timer = Timeout(self._readtimeout)
            timer.start()
        try:
            if max_bytes > 0:
                buff = yield self._iostream.read_until('\n',
                                                       max_bytes=max_bytes)
            else:
                buff = yield self._iostream.read_until('\n')
            raise Return(buff)
        except TimeoutException, e:
            self.close()
            raise socket.timeout(e.message)
        finally:
            if timer:
                timer.cancel()

    def close(self):
        self._iostream.close()

    def set_nodelay(self, flag):
        self._iostream.set_nodelay(flag)

    def settimeout(self, timeout):
        pass

    def shutdown(self, direction):
        if self._iostream.fileno():
            self._iostream.fileno().shutdown(direction)

    def recv_into(self, buff):
        expected_rbytes = len(buff)
        data = self.read(expected_rbytes, True)
        srcarray = bytearray(data)
        nbytes = len(srcarray)
        buff[0:nbytes] = srcarray
        return nbytes

    def makefile(self, mode, other):
        return self

    def fileno(self):
        return self._iostream.fileno()
<|endoftext|>


<commit_before>from django.db import models

class HeadTeacher(models.Model):
    first_name = models.CharField(max_length=50, verbose_name=u'First Name')
    last_name = models.CharField(max_length=50, verbose_name=u'Last Name')
    gender = models.CharField(max_length=6, verbose_name=u'Gender')
    msisdn = models.CharField(max_length=20)
    date_of_birth = models.DateField()
    is_zonal_head = models.BooleanField()
    zonal_head_name = models.CharField(max_length=100, verbose_name=u'Zonal Head Name')
    created_at = models.DateTimeField(auto_now_add=True)
    emis = models.ForeignKey('hierarchy.School',
                             null=True,
                             blank=True,
                             verbose_name=u'EMIS Number')

    def __unicode__(self):
        return "%s %s" % (self.first_name, self.last_name)

    class Meta:
        verbose_name = "Head Teacher"

class SchoolData(models.Model):
    emis = models.ForeignKey('hierarchy.School',
                             verbose_name=u'EMIS Number')
    name = models.CharField(max_length=100, verbose_name=u'Name of School')
    classrooms = models.IntegerField()
    teachers = models.IntegerField()
    teachers_g1 = models.IntegerField()
    teachers_g2 = models.IntegerField()
    boys_g2 = models.IntegerField()
    girls_g2 = models.IntegerField()
    created_at = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(HeadTeacher,
                                    verbose_name=u'Teacher')

    def __unicode__(self):
        return "%s" % self.emis

    class Meta:
        verbose_name = "School Data"

class TeacherPerfomanceData(models.Model):
    gender = models.CharField(max_length=6, verbose_name=u'Gender')
    age = models.IntegerField()
    years_experience = models.IntegerField()
    g2_pupils_present = models.IntegerField()
    g2_pupils_registered = models.IntegerField()
    classroom_environment_score = models.IntegerField()
    t_l_materials = models.IntegerField()
    pupils_materials_score = models.IntegerField()
    pupils_books_number = models.IntegerField()
    reading_lesson = models.IntegerField()
    pupil_engagment_score = models.IntegerField()
    attitudes_and_beliefs = models.IntegerField()
    training_subtotal = models.IntegerField()
    local_reading_score = models.IntegerField()
    created_at = models.DateTimeField(auto_now_add=True)
    academic_level = models.IntegerField()  # Used IntField for now
    emis = models.ForeignKey('hierarchy.School',
                             verbose_name=u'EMIS Number')
    created_by = models.ForeignKey(HeadTeacher,
                                   verbose_name=u'Teacher')
<commit_msg>wrote models for LearnerPerfomanceData<commit_after>from django.db import models

class HeadTeacher(models.Model):
    first_name = models.CharField(max_length=50, verbose_name=u'First Name')
    last_name = models.CharField(max_length=50, verbose_name=u'Last Name')
    gender = models.CharField(max_length=6, verbose_name=u'Gender')
    msisdn = models.CharField(max_length=20)
    date_of_birth = models.DateField()
    is_zonal_head = models.BooleanField()
    zonal_head_name = models.CharField(max_length=100, verbose_name=u'Zonal Head Name')
    created_at = models.DateTimeField(auto_now_add=True)
    emis = models.ForeignKey('hierarchy.School',
                             null=True,
                             blank=True,
                             verbose_name=u'EMIS Number')

    def __unicode__(self):
        return "%s %s" % (self.first_name, self.last_name)

    class Meta:
        verbose_name = "Head Teacher"

class SchoolData(models.Model):
    emis = models.ForeignKey('hierarchy.School',
                             verbose_name=u'EMIS Number')
    name = models.CharField(max_length=100, verbose_name=u'Name of School')
    classrooms = models.IntegerField()
    teachers = models.IntegerField()
    teachers_g1 = models.IntegerField()
    teachers_g2 = models.IntegerField()
    boys_g2 = models.IntegerField()
    girls_g2 = models.IntegerField()
    created_at = models.DateTimeField(auto_now_add=True)
    created_by = models.ForeignKey(HeadTeacher,
                                    verbose_name=u'Teacher')

    def __unicode__(self):
        return "%s" % self.emis

    class Meta:
        verbose_name = "School Data"

class TeacherPerfomanceData(models.Model):
    gender = models.CharField(max_length=6, verbose_name=u'Gender')
    age = models.IntegerField()
    years_experience = models.IntegerField()
    g2_pupils_present = models.IntegerField()
    g2_pupils_registered = models.IntegerField()
    classroom_environment_score = models.IntegerField()
    t_l_materials = models.IntegerField()
    pupils_materials_score = models.IntegerField()
    pupils_books_number = models.IntegerField()
    reading_lesson = models.IntegerField()
    pupil_engagment_score = models.IntegerField()
    attitudes_and_beliefs = models.IntegerField()
    training_subtotal = models.IntegerField()
    local_reading_score = models.IntegerField()
    created_at = models.DateTimeField(auto_now_add=True)
    academic_level = models.IntegerField()  # Used IntField for now
    emis = models.ForeignKey('hierarchy.School',
                             verbose_name=u'EMIS Number')
    created_by = models.ForeignKey(HeadTeacher,
                                   verbose_name=u'Teacher')

class LearnerPerfomanceData(models.Model):
    gender = models.CharField(max_length=6, verbose_name=u'Gender')
    total_number_pupils = models.IntegerField()
    phonetic_awareness = models.IntegerField()
    vocabulary = models.IntegerField()
    reading_comprehension = models.IntegerField()
    writing_diction = models.IntegerField()
    below_minimum_results = models.IntegerField()
    minimum_results = models.IntegerField()
    desirable_results = models.IntegerField()
    outstanding_results = models.IntegerField()
    created_at = models.DateTimeField(auto_now_add=True)
    emis = models.ForeignKey('hierarchy.School',
                             verbose_name=u'EMIS Number')
    created_by = models.ForeignKey(HeadTeacher,
                                   verbose_name=u'Teacher')
<|endoftext|>


<commit_before># git describe --tags
__version__ = "0.8.0-10-g15486e6"
<commit_msg>increment version<commit_after># git describe --tags
__version__ = "0.8.0-14-g548f692"
<|endoftext|>


<commit_before>"""
gspread.client
~~~~~~~~~~~~~~

This module contains Client class responsible for communicating with
Google API.

"""

from google.auth.transport.requests import AuthorizedSession

from .exceptions import APIError, SpreadsheetNotFound
from .spreadsheet import Spreadsheet
from .urls import (
    DRIVE_FILES_API_V2_URL,
    DRIVE_FILES_API_V3_COMMENTS_URL,
    DRIVE_FILES_API_V3_URL,
    DRIVE_FILES_UPLOAD_API_V2_URL,
)
from .utils import convert_credentials, extract_id_from_url, finditem

class Client:
    """An instance of this class communicates with Google API.

    :param auth: An OAuth2 credential object. Credential objects
        created by `google-auth <https://github.com/googleapis/google-auth-library-python>`_.

    :param session: (optional) A session object capable of making HTTP requests
        while persisting some parameters across requests.
        Defaults to `google.auth.transport.requests.AuthorizedSession <https://google-auth.readthedocs.io/en/latest/reference/google.auth.transport.requests.html#google.auth.transport.requests.AuthorizedSession>`_.

    >>> c = gspread.Client(auth=OAuthCredentialObject)
    """

    def __init__(self, auth, session=None):
        if auth is not None:
            self.auth = convert_credentials(auth)
            self.session = session or AuthorizedSession(self.auth)
        else:
            self.session = session

    def login(self):
        from google.auth.transport.requests import Request

        self.auth.refresh(Request(self.session))

        self.session.headers.update({"Authorization": "Bearer %s" % self.auth.token})

    def request(
        self,
        method,
        endpoint,
        params=None,
        data=None,
        json=None,
        files=None,
        headers=None,
    ):
        response = getattr(self.session, method)(
            endpoint,
            json=json,
            params=params,
            data=data,
            files=files,
            headers=headers,
        )

        if response.ok:
            return response
        else:
            raise APIError(response)

    def list_spreadsheet_files(self, title=None, folder_id=None):
        files = []
        page_token = ""
        url = DRIVE_FILES_API_V3_URL

        q = 'mimeType="application/vnd.google-apps.spreadsheet"'
        if title:
            q += ' and name = "{}"'.format(title)
        if folder_id:
            q += ' and parents in "{}"'.format(folder_id)

        params = {
            "q": q,
            "pageSize": 1000,
            "supportsAllDrives": True,
            "includeItemsFromAllDrives": True,
            "fields": "kind,nextPageToken,files(id,name,createdTime,modifiedTime)",
        }

        while page_token is not None:
            if page_token:
                params["pageToken"] = page_token

            res = self.request("get", url, params=params).json()
            files.extend(res["files"])
            page_token = res.get("nextPageToken", None)

        return files

    def open(self, title, folder_id=None):
        """Opens a spreadsheet.

        :param str title: A title of a spreadsheet.
        :param str folder_id: (optional) If specified can be used to filter
            spreadsheets by parent folder ID.
        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        If there's more than one spreadsheet with same title the first one
        will be opened.

        :raises gspread.SpreadsheetNotFound: if no spreadsheet with
                                             specified `title` is found.

        >>> gc.open('My fancy spreadsheet')
        """
        try:
            properties = finditem(
                lambda x: x["name"] == title,
                self.list_spreadsheet_files(title, folder_id),
            )

            # Drive uses different terminology
            properties["title"] = properties["name"]

            return Spreadsheet(self, properties)
        except StopIteration:
            raise SpreadsheetNotFound

    def open_by_key(self, key):
        """Opens a spreadsheet specified by `key` (a.k.a Spreadsheet ID).

        :param str key: A key of a spreadsheet as it appears in a URL in a browser.
        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        >>> gc.open_by_key('0BmgG6nO_6dprdS1MN3d3MkdPa142WFRrdnRRUWl1UFE')
        """
        return Spreadsheet(self, {"id": key})

    def open_by_url(self, url):
        """Opens a spreadsheet specified by `url`.

        :param str url: URL of a spreadsheet as it appears in a browser.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        :raises gspread.SpreadsheetNotFound: if no spreadsheet with
                                             specified `url` is found.

        >>> gc.open_by_url('https://docs.google.com/spreadsheet/ccc?key=0Bm...FE&hl')
        """
        return self.open_by_key(extract_id_from_url(url))

    def openall(self, title=None):
        """Opens all available spreadsheets.

        :param str title: (optional) If specified can be used to filter
            spreadsheets by title.

        :returns: a list of :class:`~gspread.models.Spreadsheet` instances.
        """
        spreadsheet_files = self.list_spreadsheet_files(title)

        if title:
            spreadsheet_files = [
                spread for spread in spreadsheet_files if title == spread["name"]
            ]

        return [
            Spreadsheet(self, dict(title=x["name"], **x)) for x in spreadsheet_files
        ]

    def create(self, title, folder_id=None):
        """Creates a new spreadsheet.

        :param str title: A title of a new spreadsheet.

        :param str folder_id: Id of the folder where we want to save
            the spreadsheet.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        """
        payload = {
            "name": title,
            "mimeType": "application/vnd.google-apps.spreadsheet",
        }

        params = {
            "supportsAllDrives": True,
        }

        if folder_id is not None:
            payload["parents"] = [folder_id]

        r = self.request("post", DRIVE_FILES_API_V3_URL, json=payload, params=params)
        spreadsheet_id = r.json()["id"]
        return self.open_by_key(spreadsheet_id)

    def copy(
        self,
        file_id,
        title=None,
        copy_permissions=False,
        folder_id=None,
        copy_comments=True,
    ):
        """Copies a spreadsheet.

        :param str file_id: A key of a spreadsheet to copy.
        :param str title: (optional) A title for the new spreadsheet.

        :param bool copy_permissions: (optional) If True, copy permissions from
            the original spreadsheet to the new spreadsheet.

        :param str folder_id: Id of the folder where we want to save
            the spreadsheet.

        :param bool copy_comments: (optional) If True, copy the comments from
            the original spreadsheet to the new spreadsheet.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        .. versionadded:: 3.1.0

        .. note::

           If you're using custom credentials without the Drive scope, you need to add
           ``https://www.googleapis.com/auth/drive`` to your OAuth scope in order to use
           this method.

           Example::

              scope = [
                  'https://www.googleapis.com/auth/spreadsheets',
                  'https://www.googleapis.com/auth/drive'
              ]

           Otherwise, you will get an ``Insufficient Permission`` error
           when you try to copy a spreadsheet.

        """
        url = "{}/{}/copy".format(DRIVE_FILES_API_V2_URL, file_id)

        payload = {
            "title": title,
            "mimeType": "application/vnd.google-apps.spreadsheet",
        }

        if folder_id is not None:
            payload["parents"] = [{"id": folder_id}]

        params = {"supportsAllDrives": True}
        r = self.request("post", url, json=payload, params=params)
        spreadsheet_id = r.json()["id"]

        new_spreadsheet = self.open_by_key(spreadsheet_id)

        if copy_permissions is True:
            original = self.open_by_key(file_id)

            permissions = original.list_permissions()
            for p in permissions:
                if p.get("deleted"):
                    continue
                try:
                    new_spreadsheet.share(
                        value=p["emailAddress"],
                        perm_type=p["type"],
                        role=p["role"],
                        notify=False,
                    )
                except Exception:
                    pass

        if copy_comments is True:
            source_url = DRIVE_FILES_API_V3_COMMENTS_URL % (file_id)
            page_token = ""
            comments = []
            params = {
                "fields": "comments/content,comments/anchor,nextPageToken",
                "includeDeleted": False,
                "pageSize": 100,  # API limit to maximum 100
            }

            while page_token is not None:
                params["pageToken"] = page_token
                res = self.request("get", source_url, params=params).json()

                comments.extend(res["comments"])
                page_token = res.get("nextPageToken", None)

            destination_url = DRIVE_FILES_API_V3_COMMENTS_URL % (new_spreadsheet.id)
            # requesting some fields in the response is mandatory from the API.
            # choose 'id' randomly out of all the fields, but no need to use it for now.
            params = {"fields": "id"}
            for comment in comments:
                self.request("post", destination_url, json=comment, params=params)

        return new_spreadsheet

    def del_spreadsheet(self, file_id):
        """Deletes a spreadsheet.

        :param str file_id: a spreadsheet ID (a.k.a file ID).
        """
        url = "{}/{}".format(DRIVE_FILES_API_V3_URL, file_id)

        params = {"supportsAllDrives": True}
        self.request("delete", url, params=params)

    def import_csv(self, file_id, data):
        """Imports data into the first page of the spreadsheet.

        :param str data: A CSV string of data.

        Example:

        .. code::

            # Read CSV file contents
            content = open('file_to_import.csv', 'r').read()

            gc.import_csv(spreadsheet.id, content)

        .. note::

           This method removes all other worksheets and then entirely
           replaces the contents of the first worksheet.

        """
        headers = {"Content-Type": "text/csv"}
        url = "{}/{}".format(DRIVE_FILES_UPLOAD_API_V2_URL, file_id)

        self.request(
            "put",
            url,
            data=data,
            params={
                "uploadType": "media",
                "convert": True,
                "supportsAllDrives": True,
            },
            headers=headers,
        )

    def list_permissions(self, file_id):
        """Retrieve a list of permissions for a file.

        :param str file_id: a spreadsheet ID (aka file ID).
        """
        url = "{}/{}/permissions".format(DRIVE_FILES_API_V2_URL, file_id)

        params = {"supportsAllDrives": True}
        r = self.request("get", url, params=params)

        return r.json()["items"]

    def insert_permission(
        self,
        file_id,
        value,
        perm_type,
        role,
        notify=True,
        email_message=None,
        with_link=False,
    ):
        """Creates a new permission for a file.

        :param str file_id: a spreadsheet ID (aka file ID).
        :param value: user or group e-mail address, domain name
            or None for 'default' type.
        :type value: str, None
        :param str perm_type: (optional) The account type.
            Allowed values are: ``user``, ``group``, ``domain``, ``anyone``
        :param str role: (optional) The primary role for this user.
            Allowed values are: ``owner``, ``writer``, ``reader``
        :param str notify: (optional) Whether to send an email to the target
            user/domain.
        :param str email_message: (optional) An email message to be sent
            if ``notify=True``.
        :param bool with_link: (optional) Whether the link is required for this
            permission to be active.

        Examples::

            # Give write permissions to otto@example.com

            gc.insert_permission(
                '0BmgG6nO_6dprnRRUWl1UFE',
                'otto@example.org',
                perm_type='user',
                role='writer'
            )

            # Make the spreadsheet publicly readable

            gc.insert_permission(
                '0BmgG6nO_6dprnRRUWl1UFE',
                None,
                perm_type='anyone',
                role='reader'
            )

        """

        url = "{}/{}/permissions".format(DRIVE_FILES_API_V2_URL, file_id)

        payload = {
            "value": value,
            "type": perm_type,
            "role": role,
            "withLink": with_link,
        }

        params = {
            "sendNotificationEmails": notify,
            "emailMessage": email_message,
            "supportsAllDrives": "true",
        }

        self.request("post", url, json=payload, params=params)

    def remove_permission(self, file_id, permission_id):
        """Deletes a permission from a file.

        :param str file_id: a spreadsheet ID (aka file ID.)
        :param str permission_id: an ID for the permission.
        """
        url = "{}/{}/permissions/{}".format(
            DRIVE_FILES_API_V2_URL, file_id, permission_id
        )

        params = {"supportsAllDrives": True}
        self.request("delete", url, params=params)
<commit_msg>Add timeout to client<commit_after>"""
gspread.client
~~~~~~~~~~~~~~

This module contains Client class responsible for communicating with
Google API.

"""

from google.auth.transport.requests import AuthorizedSession

from .exceptions import APIError, SpreadsheetNotFound
from .spreadsheet import Spreadsheet
from .urls import (
    DRIVE_FILES_API_V2_URL,
    DRIVE_FILES_API_V3_COMMENTS_URL,
    DRIVE_FILES_API_V3_URL,
    DRIVE_FILES_UPLOAD_API_V2_URL,
)
from .utils import convert_credentials, extract_id_from_url, finditem

class Client:
    """An instance of this class communicates with Google API.

    :param auth: An OAuth2 credential object. Credential objects
        created by `google-auth <https://github.com/googleapis/google-auth-library-python>`_.

    :param session: (optional) A session object capable of making HTTP requests
        while persisting some parameters across requests.
        Defaults to `google.auth.transport.requests.AuthorizedSession <https://google-auth.readthedocs.io/en/latest/reference/google.auth.transport.requests.html#google.auth.transport.requests.AuthorizedSession>`_.

    >>> c = gspread.Client(auth=OAuthCredentialObject)
    """

    def __init__(self, auth, session=None):
        if auth is not None:
            self.auth = convert_credentials(auth)
            self.session = session or AuthorizedSession(self.auth)
        else:
            self.session = session

        self.timeout = None

    def login(self):
        from google.auth.transport.requests import Request

        self.auth.refresh(Request(self.session))

        self.session.headers.update({"Authorization": "Bearer %s" % self.auth.token})

    def set_timeout(self, timeout):
        self.timeout = timeout

    def request(
        self,
        method,
        endpoint,
        params=None,
        data=None,
        json=None,
        files=None,
        headers=None,
    ):
        response = getattr(self.session, method)(
            endpoint,
            json=json,
            params=params,
            data=data,
            files=files,
            headers=headers,
            timeout=self.timeout,
        )

        if response.ok:
            return response
        else:
            raise APIError(response)

    def list_spreadsheet_files(self, title=None, folder_id=None):
        files = []
        page_token = ""
        url = DRIVE_FILES_API_V3_URL

        q = 'mimeType="application/vnd.google-apps.spreadsheet"'
        if title:
            q += ' and name = "{}"'.format(title)
        if folder_id:
            q += ' and parents in "{}"'.format(folder_id)

        params = {
            "q": q,
            "pageSize": 1000,
            "supportsAllDrives": True,
            "includeItemsFromAllDrives": True,
            "fields": "kind,nextPageToken,files(id,name,createdTime,modifiedTime)",
        }

        while page_token is not None:
            if page_token:
                params["pageToken"] = page_token

            res = self.request("get", url, params=params).json()
            files.extend(res["files"])
            page_token = res.get("nextPageToken", None)

        return files

    def open(self, title, folder_id=None):
        """Opens a spreadsheet.

        :param str title: A title of a spreadsheet.
        :param str folder_id: (optional) If specified can be used to filter
            spreadsheets by parent folder ID.
        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        If there's more than one spreadsheet with same title the first one
        will be opened.

        :raises gspread.SpreadsheetNotFound: if no spreadsheet with
                                             specified `title` is found.

        >>> gc.open('My fancy spreadsheet')
        """
        try:
            properties = finditem(
                lambda x: x["name"] == title,
                self.list_spreadsheet_files(title, folder_id),
            )

            # Drive uses different terminology
            properties["title"] = properties["name"]

            return Spreadsheet(self, properties)
        except StopIteration:
            raise SpreadsheetNotFound

    def open_by_key(self, key):
        """Opens a spreadsheet specified by `key` (a.k.a Spreadsheet ID).

        :param str key: A key of a spreadsheet as it appears in a URL in a browser.
        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        >>> gc.open_by_key('0BmgG6nO_6dprdS1MN3d3MkdPa142WFRrdnRRUWl1UFE')
        """
        return Spreadsheet(self, {"id": key})

    def open_by_url(self, url):
        """Opens a spreadsheet specified by `url`.

        :param str url: URL of a spreadsheet as it appears in a browser.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        :raises gspread.SpreadsheetNotFound: if no spreadsheet with
                                             specified `url` is found.

        >>> gc.open_by_url('https://docs.google.com/spreadsheet/ccc?key=0Bm...FE&hl')
        """
        return self.open_by_key(extract_id_from_url(url))

    def openall(self, title=None):
        """Opens all available spreadsheets.

        :param str title: (optional) If specified can be used to filter
            spreadsheets by title.

        :returns: a list of :class:`~gspread.models.Spreadsheet` instances.
        """
        spreadsheet_files = self.list_spreadsheet_files(title)

        if title:
            spreadsheet_files = [
                spread for spread in spreadsheet_files if title == spread["name"]
            ]

        return [
            Spreadsheet(self, dict(title=x["name"], **x)) for x in spreadsheet_files
        ]

    def create(self, title, folder_id=None):
        """Creates a new spreadsheet.

        :param str title: A title of a new spreadsheet.

        :param str folder_id: Id of the folder where we want to save
            the spreadsheet.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        """
        payload = {
            "name": title,
            "mimeType": "application/vnd.google-apps.spreadsheet",
        }

        params = {
            "supportsAllDrives": True,
        }

        if folder_id is not None:
            payload["parents"] = [folder_id]

        r = self.request("post", DRIVE_FILES_API_V3_URL, json=payload, params=params)
        spreadsheet_id = r.json()["id"]
        return self.open_by_key(spreadsheet_id)

    def copy(
        self,
        file_id,
        title=None,
        copy_permissions=False,
        folder_id=None,
        copy_comments=True,
    ):
        """Copies a spreadsheet.

        :param str file_id: A key of a spreadsheet to copy.
        :param str title: (optional) A title for the new spreadsheet.

        :param bool copy_permissions: (optional) If True, copy permissions from
            the original spreadsheet to the new spreadsheet.

        :param str folder_id: Id of the folder where we want to save
            the spreadsheet.

        :param bool copy_comments: (optional) If True, copy the comments from
            the original spreadsheet to the new spreadsheet.

        :returns: a :class:`~gspread.models.Spreadsheet` instance.

        .. versionadded:: 3.1.0

        .. note::

           If you're using custom credentials without the Drive scope, you need to add
           ``https://www.googleapis.com/auth/drive`` to your OAuth scope in order to use
           this method.

           Example::

              scope = [
                  'https://www.googleapis.com/auth/spreadsheets',
                  'https://www.googleapis.com/auth/drive'
              ]

           Otherwise, you will get an ``Insufficient Permission`` error
           when you try to copy a spreadsheet.

        """
        url = "{}/{}/copy".format(DRIVE_FILES_API_V2_URL, file_id)

        payload = {
            "title": title,
            "mimeType": "application/vnd.google-apps.spreadsheet",
        }

        if folder_id is not None:
            payload["parents"] = [{"id": folder_id}]

        params = {"supportsAllDrives": True}
        r = self.request("post", url, json=payload, params=params)
        spreadsheet_id = r.json()["id"]

        new_spreadsheet = self.open_by_key(spreadsheet_id)

        if copy_permissions is True:
            original = self.open_by_key(file_id)

            permissions = original.list_permissions()
            for p in permissions:
                if p.get("deleted"):
                    continue
                try:
                    new_spreadsheet.share(
                        value=p["emailAddress"],
                        perm_type=p["type"],
                        role=p["role"],
                        notify=False,
                    )
                except Exception:
                    pass

        if copy_comments is True:
            source_url = DRIVE_FILES_API_V3_COMMENTS_URL % (file_id)
            page_token = ""
            comments = []
            params = {
                "fields": "comments/content,comments/anchor,nextPageToken",
                "includeDeleted": False,
                "pageSize": 100,  # API limit to maximum 100
            }

            while page_token is not None:
                params["pageToken"] = page_token
                res = self.request("get", source_url, params=params).json()

                comments.extend(res["comments"])
                page_token = res.get("nextPageToken", None)

            destination_url = DRIVE_FILES_API_V3_COMMENTS_URL % (new_spreadsheet.id)
            # requesting some fields in the response is mandatory from the API.
            # choose 'id' randomly out of all the fields, but no need to use it for now.
            params = {"fields": "id"}
            for comment in comments:
                self.request("post", destination_url, json=comment, params=params)

        return new_spreadsheet

    def del_spreadsheet(self, file_id):
        """Deletes a spreadsheet.

        :param str file_id: a spreadsheet ID (a.k.a file ID).
        """
        url = "{}/{}".format(DRIVE_FILES_API_V3_URL, file_id)

        params = {"supportsAllDrives": True}
        self.request("delete", url, params=params)

    def import_csv(self, file_id, data):
        """Imports data into the first page of the spreadsheet.

        :param str data: A CSV string of data.

        Example:

        .. code::

            # Read CSV file contents
            content = open('file_to_import.csv', 'r').read()

            gc.import_csv(spreadsheet.id, content)

        .. note::

           This method removes all other worksheets and then entirely
           replaces the contents of the first worksheet.

        """
        headers = {"Content-Type": "text/csv"}
        url = "{}/{}".format(DRIVE_FILES_UPLOAD_API_V2_URL, file_id)

        self.request(
            "put",
            url,
            data=data,
            params={
                "uploadType": "media",
                "convert": True,
                "supportsAllDrives": True,
            },
            headers=headers,
        )

    def list_permissions(self, file_id):
        """Retrieve a list of permissions for a file.

        :param str file_id: a spreadsheet ID (aka file ID).
        """
        url = "{}/{}/permissions".format(DRIVE_FILES_API_V2_URL, file_id)

        params = {"supportsAllDrives": True}
        r = self.request("get", url, params=params)

        return r.json()["items"]

    def insert_permission(
        self,
        file_id,
        value,
        perm_type,
        role,
        notify=True,
        email_message=None,
        with_link=False,
    ):
        """Creates a new permission for a file.

        :param str file_id: a spreadsheet ID (aka file ID).
        :param value: user or group e-mail address, domain name
            or None for 'default' type.
        :type value: str, None
        :param str perm_type: (optional) The account type.
            Allowed values are: ``user``, ``group``, ``domain``, ``anyone``
        :param str role: (optional) The primary role for this user.
            Allowed values are: ``owner``, ``writer``, ``reader``
        :param str notify: (optional) Whether to send an email to the target
            user/domain.
        :param str email_message: (optional) An email message to be sent
            if ``notify=True``.
        :param bool with_link: (optional) Whether the link is required for this
            permission to be active.

        Examples::

            # Give write permissions to otto@example.com

            gc.insert_permission(
                '0BmgG6nO_6dprnRRUWl1UFE',
                'otto@example.org',
                perm_type='user',
                role='writer'
            )

            # Make the spreadsheet publicly readable

            gc.insert_permission(
                '0BmgG6nO_6dprnRRUWl1UFE',
                None,
                perm_type='anyone',
                role='reader'
            )

        """

        url = "{}/{}/permissions".format(DRIVE_FILES_API_V2_URL, file_id)

        payload = {
            "value": value,
            "type": perm_type,
            "role": role,
            "withLink": with_link,
        }

        params = {
            "sendNotificationEmails": notify,
            "emailMessage": email_message,
            "supportsAllDrives": "true",
        }

        self.request("post", url, json=payload, params=params)

    def remove_permission(self, file_id, permission_id):
        """Deletes a permission from a file.

        :param str file_id: a spreadsheet ID (aka file ID.)
        :param str permission_id: an ID for the permission.
        """
        url = "{}/{}/permissions/{}".format(
            DRIVE_FILES_API_V2_URL, file_id, permission_id
        )

        params = {"supportsAllDrives": True}
        self.request("delete", url, params=params)
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations

#
# This scripts re-tags images from one branch to another. Since we keep
# images "per-branch" we sometimes need to "clone" the current
# images to provide a starting cache image to build images in the
# new branch. This can be useful in a few situations:
#
# * when starting new release branch (for example `v2-1-test`)
# * when renaming a branch
#
import subprocess

import rich_click as click

PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]

GHCR_IO_PREFIX = "ghcr.io"

GHCR_IO_IMAGES = [
    "{prefix}/{repo}/{branch}/ci/python{python}:latest",
    "{prefix}/{repo}/{branch}/prod/python{python}:latest",
]

# noinspection StrFormat
def pull_push_all_images(
    source_prefix: str,
    target_prefix: str,
    images: list[str],
    source_branch: str,
    source_repo: str,
    target_branch: str,
    target_repo: str,
):
    for python in PYTHON_VERSIONS:
        for image in images:
            source_image = image.format(
                prefix=source_prefix, branch=source_branch, repo=source_repo, python=python
            )
            target_image = image.format(
                prefix=target_prefix, branch=target_branch, repo=target_repo, python=python
            )
            print(f"Copying image: {source_image} -> {target_image}")
            subprocess.run(
                ["regctl", "image", "copy", "--force-recursive", "--digest-tags", source_image, target_image],
                check=True,
            )

@click.group(invoke_without_command=True)
@click.option("--source-branch", type=str, default="main", help="Source branch name [main]")
@click.option("--target-branch", type=str, default="main", help="Target branch name [main]")
@click.option("--source-repo", type=str, default="apache/airflow", help="Source repo")
@click.option("--target-repo", type=str, default="apache/airflow", help="Target repo")
def main(
    source_branch: str,
    target_branch: str,
    source_repo: str,
    target_repo: str,
):
    pull_push_all_images(
        GHCR_IO_PREFIX, GHCR_IO_PREFIX, GHCR_IO_IMAGES, source_branch, source_repo, target_branch, target_repo
    )

if __name__ == "__main__":
    main()
<commit_msg>Use virtual env python in the script shebang of retag_docker_images.py (#27857)<commit_after>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations

#
# This scripts re-tags images from one branch to another. Since we keep
# images "per-branch" we sometimes need to "clone" the current
# images to provide a starting cache image to build images in the
# new branch. This can be useful in a few situations:
#
# * when starting new release branch (for example `v2-1-test`)
# * when renaming a branch
#
import subprocess

import rich_click as click

PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]

GHCR_IO_PREFIX = "ghcr.io"

GHCR_IO_IMAGES = [
    "{prefix}/{repo}/{branch}/ci/python{python}:latest",
    "{prefix}/{repo}/{branch}/prod/python{python}:latest",
]

# noinspection StrFormat
def pull_push_all_images(
    source_prefix: str,
    target_prefix: str,
    images: list[str],
    source_branch: str,
    source_repo: str,
    target_branch: str,
    target_repo: str,
):
    for python in PYTHON_VERSIONS:
        for image in images:
            source_image = image.format(
                prefix=source_prefix, branch=source_branch, repo=source_repo, python=python
            )
            target_image = image.format(
                prefix=target_prefix, branch=target_branch, repo=target_repo, python=python
            )
            print(f"Copying image: {source_image} -> {target_image}")
            subprocess.run(
                ["regctl", "image", "copy", "--force-recursive", "--digest-tags", source_image, target_image],
                check=True,
            )

@click.group(invoke_without_command=True)
@click.option("--source-branch", type=str, default="main", help="Source branch name [main]")
@click.option("--target-branch", type=str, default="main", help="Target branch name [main]")
@click.option("--source-repo", type=str, default="apache/airflow", help="Source repo")
@click.option("--target-repo", type=str, default="apache/airflow", help="Target repo")
def main(
    source_branch: str,
    target_branch: str,
    source_repo: str,
    target_repo: str,
):
    pull_push_all_images(
        GHCR_IO_PREFIX, GHCR_IO_PREFIX, GHCR_IO_IMAGES, source_branch, source_repo, target_branch, target_repo
    )

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Output functions for parsed zs2 chunks."""

# Author: Chris Petrich
# Copyright: Copyright 2015, Chris Petrich
# License: MIT

def chunks_to_XML(chunks, with_address=False):
    """Produces an XML representation of the chunks."""
    # The implementation of this function is not pretty but
    #   requires no dependencies.

    def _xml_attr_escape(string):
        """Escape characters in string"""
        string=string.replace('&','&')
        string=string.replace('>','>')
        string=string.replace('<','<')
        string=string.replace('\\"','"')
        string=string.replace("\\'",''')
        return string

    data_types = [chunk[2] for chunk in chunks]
    if data_types.count('DD') != data_types.count('end'):
        raise ValueError('Cannot generate XML file since section start and end do not balance. Output as text file instead to debug.')

    out=[]
    out.append('<?xml version="1.0" encoding="UTF-8"?>')
    section_names = []
    level = 0
    for chunk in chunks:
        address, name, data_type, data = chunk
        if data_type == 'end': level-=1

        _space = '  '*level
        show_address= "address='%0.6x' "%address if with_address else ''
        if data_type != 'end':
            # here we enclose the value in quotation marks...
            if isinstance(data,int) or isinstance(data,float) or isinstance(data,list):
                # note that 'bool' is sderived from 'int'
                display=repr(str(data))
            else:
                display = repr(data) # introduces character escapes in Python 2
                if display.startswith('u'): # only for Python 2
                    display=display[1:] # granted, this is an ugly way of doing it, but it produces human readable and valid XML
            # escape XML entities in attributes
            value = _xml_attr_escape(display)
            line = "%s<%s %stype='%s' value=%s %s>" % (_space, name,
                                                         show_address,
                                                         data_type, value,
                                                         '/' if data_type != 'DD' else '')
        else:
            line = '%s</%s>' % (_space, section_names[-1])

        if data_type == 'end': section_names.pop()
        elif data_type == 'DD':
            section_names.append(name)
            level+=1
        out.append(line)
    return u'\n'.join(out)

def chunks_to_text_dump(chunks):
    """Produces a string representation."""
    out=[]
    DD_names = []
    level = 0

    data_types = [chunk[2] for chunk in chunks]
    if data_types.count('DD') != data_types.count('end'):
        # do not indent since there's obviously something wrong
        indent = None
    else:
        indent = '  '

    for chunk in chunks:
        address, name, data_type, data = chunk
        if data_type == 'end': level-=1

        _space = indent*level if indent is not None else ''

        comment = '' if data_type != 'end' else '<-'.join(DD_names[-1::-1])
        if data_type == 'end': DD_names.pop()

        line = u' '.join([u'%.6x:'%address, _space+name, '[%s]'%data_type, repr(data), comment])

        if data_type == 'DD':
            DD_names.append(name)
            level+=1
        out.append(line)
    return u'\n'.join(out)
<commit_msg>refactor xml encoder to use python xml<commit_after>"""Output functions for parsed zs2 chunks."""
from xml.dom import minidom
import zs2decode.parser as parser
# Author: Chris Petrich
# Copyright: Copyright 2015-2017, Chris Petrich
# License: MIT

def _add_xml_element(doc, current, name, attributes):
    """Add XML element to tree. Uses 'current' to keep track of nesting."""
    if attributes['type'] != 'end':
        elem = doc.createElement(name)
        for attr in attributes:
            # ugly but works with Py2 and Py3:
            elem.setAttribute(attr, attributes[attr])
        if len(current) == 0:
            doc.appendChild(elem)
        else:
            current[-1].appendChild(elem)
        if attributes['type']=='DD':
            current.append(elem)
    else:
        current.pop(-1)

def chunks_to_XML(chunks, with_address=False):
    """Produces an XML representation of the chunks."""
    if chunks[0][2] != 'DD':
        raise ValueError('First chunk is not of data type 0xDD: %r' % chunks[0])

    data_types = [chunk[2] for chunk in chunks]
    if data_types.count('DD') != data_types.count('end'):
        raise ValueError('Cannot generate XML file since section start and end do not balance. Output as text file instead to debug.')

    doc = minidom.Document()
    current = []
    for chunk in chunks:
        address, name, data_type, data = chunk
        if isinstance(data,int) or isinstance(data,float) or isinstance(data,list):
            # note that 'bool' is derived from 'int'
            display=repr(str(data))
        else:
            display = repr(data) # introduces character escapes in Python 2
            if display.startswith('u'): # only for Python 2
                display=display[1:] # granted, this is an ugly way of doing it, but it produces human readable and valid XML
        if display[0] in ("'",'"'):
            display = display[1:-1]
        attrib = {}
        if with_address:
            attrib['address']='%0.6x' % address
        attrib['type'] = data_type
        attrib['value'] = display

        _add_xml_element(doc, current, name, attrib)

    return parser._to_string(doc.toprettyxml(indent="  ",encoding='UTF-8'))

def chunks_to_text_dump(chunks):
    """Produces a string representation."""
    out=[]
    DD_names = []
    level = 0

    data_types = [chunk[2] for chunk in chunks]
    if data_types.count('DD') != data_types.count('end'):
        # do not indent since there's obviously something wrong
        indent = None
    else:
        indent = '  '

    for chunk in chunks:
        address, name, data_type, data = chunk
        if data_type == 'end': level-=1

        _space = indent*level if indent is not None else ''

        comment = '' if data_type != 'end' else '<-'.join(DD_names[-1::-1])
        if data_type == 'end': DD_names.pop()

        line = u' '.join([u'%.6x:'%address, _space+name, '[%s]'%data_type, repr(data), comment])

        if data_type == 'DD':
            DD_names.append(name)
            level+=1
        out.append(line)
    return u'\n'.join(out)
<|endoftext|>


<commit_before>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled first.
Navigate to carbon-budget/emissions/cpp_util.
Compile the C++ file calc_emissions: c++ calc_emissions_v3.cpp -o calc_emissions_v3.exe -lgdal
calc_emissions_v3.exe should appear in the directory.
Return to carbon-budget/emissions using cd ..
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
Through the magic of C++, calc_emissions_v3.cpp reads them just fine without the user compiling them.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import calculate_gross_emissions
import utilities
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# tile_list = uu.tile_list(cn.AGC_emis_year_dir)
# tile_list = ['50N_020E'] # test tiles
tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_list
print "There are {} tiles to process".format(str(len(tile_list)))

# For downloading all tiles in the folders.
# This takes about 40 minutes.
download_list = [
                # cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                #  cn.peat_mask_dir, cn.ifl_dir, cn.planted_forest_type_unmasked_dir, cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                #  cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                 cn.plant_pre_2000_raw_dir,
                 cn.loss_dir]

for input in download_list:
    uu.s3_folder_download(input, './cpp_util/')

# # For copying individual tiles to s3 for testing
# for tile in tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_dir, tile, cn.pattern_ifl), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
#     except:
#         print "No plantations in", tile
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
#     except:
#         print "No pre-2000 plantations in", tile
#     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')

# print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
# # Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
# # Pre-2000 plantations have not previously been masked, so that is done here.
# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(count/2)
# pool.map(utilities.mask_loss_pre_2000_plantation, tile_list)

# For single processor use
for tile in tile_list:

      utilities.mask_loss_pre_2000_plantation(tile)

# The C++ code expects a plantations tile for every input 10x10.
# However, not all Hansen tiles have plantations.
# This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
# That way, the C++ script gets all the necessary input files
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count-10)
pool.map(uu.make_blank_tile, tile_list)

# # For single processor use
# for tile in tile_list:
#
#     uu.make_blank_tile(tile, cn.pattern_planted_forest_type_unmasked)

# 6.68 GB for four tiles simultaenously
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count-10)
pool.map(calculate_gross_emissions.calc_emissions, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       calculate_gross_emissions.calc_emissions(tile)

uu.upload_final_set(cn.gross_emis_commod_dir, cn.pattern_gross_emis_commod)
uu.upload_final_set(cn.gross_emis_shifting_ag_dir, cn.pattern_gross_emis_shifting_ag)
uu.upload_final_set(cn.gross_emis_forestry_dir, cn.pattern_gross_emis_forestry)
uu.upload_final_set(cn.gross_emis_wildfire_dir, cn.pattern_gross_emis_wildfire)
uu.upload_final_set(cn.gross_emis_urban_dir, cn.pattern_gross_emis_urban)
uu.upload_final_set(cn.gross_emis_no_driver_dir, cn.pattern_gross_emis_no_driver)
uu.upload_final_set(cn.gross_emis_nodes_dir, cn.pattern_gross_emis_nodes)
uu.upload_final_set(cn.gross_emis_all_drivers_dir, cn.pattern_gross_emis_all_drivers)

<commit_msg>I was downloading the raw pre-2000 plantation zip file instead of the pre-2000 plantation tiles. Fixed that.<commit_after>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled first.
Navigate to carbon-budget/emissions/cpp_util.
Compile the C++ file calc_emissions: c++ calc_emissions_v3.cpp -o calc_emissions_v3.exe -lgdal
calc_emissions_v3.exe should appear in the directory.
Return to carbon-budget/emissions using cd ..
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
Through the magic of C++, calc_emissions_v3.cpp reads them just fine without the user compiling them.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import calculate_gross_emissions
import utilities
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# tile_list = uu.tile_list(cn.AGC_emis_year_dir)
# tile_list = ['50N_020E'] # test tiles
tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_list
print "There are {} tiles to process".format(str(len(tile_list)))

# For downloading all tiles in the folders.
# This takes about 40 minutes.
download_list = [
                # cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                #  cn.peat_mask_dir, cn.ifl_dir, cn.planted_forest_type_unmasked_dir, cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                #  cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                 cn.plant_pre_2000_processed_dir,
                 cn.loss_dir]

for input in download_list:
    uu.s3_folder_download(input, './cpp_util/')

# # For copying individual tiles to s3 for testing
# for tile in tile_list:
#
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_dir, tile, cn.pattern_ifl), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
#     except:
#         print "No plantations in", tile
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
#     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
#     try:
#         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
#     except:
#         print "No pre-2000 plantations in", tile
#     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')

print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
# Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
# Pre-2000 plantations have not previously been masked, so that is done here.
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count/2)
pool.map(utilities.mask_loss_pre_2000_plantation, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       utilities.mask_loss_pre_2000_plantation(tile)

# The C++ code expects a plantations tile for every input 10x10.
# However, not all Hansen tiles have plantations.
# This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
# That way, the C++ script gets all the necessary input files
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count-10)
pool.map(uu.make_blank_tile, tile_list)

# # For single processor use
# for tile in tile_list:
#
#     uu.make_blank_tile(tile, cn.pattern_planted_forest_type_unmasked)

# 6.68 GB for four tiles simultaenously
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count-10)
pool.map(calculate_gross_emissions.calc_emissions, tile_list)

# # For single processor use
# for tile in tile_list:
#
#       calculate_gross_emissions.calc_emissions(tile)

uu.upload_final_set(cn.gross_emis_commod_dir, cn.pattern_gross_emis_commod)
uu.upload_final_set(cn.gross_emis_shifting_ag_dir, cn.pattern_gross_emis_shifting_ag)
uu.upload_final_set(cn.gross_emis_forestry_dir, cn.pattern_gross_emis_forestry)
uu.upload_final_set(cn.gross_emis_wildfire_dir, cn.pattern_gross_emis_wildfire)
uu.upload_final_set(cn.gross_emis_urban_dir, cn.pattern_gross_emis_urban)
uu.upload_final_set(cn.gross_emis_no_driver_dir, cn.pattern_gross_emis_no_driver)
uu.upload_final_set(cn.gross_emis_nodes_dir, cn.pattern_gross_emis_nodes)
uu.upload_final_set(cn.gross_emis_all_drivers_dir, cn.pattern_gross_emis_all_drivers)

<|endoftext|>