<commit_before>import itertools
import json
import os.path
import posixpath

from shavar.exceptions import ParseError
from shavar.types import Chunk, ChunkList, Downloads, DownloadsListInfo

def parse_downloads(request):
    parsed = Downloads()

    limit = request.registry.settings.get("shavar.max_downloads_chunks",
                                          10000)

    for lineno, line in enumerate(request.body_file):
        line = line.strip()

        if not line or line.isspace():
            continue

        line = line.decode('utf-8')
        # Did client provide max size preference?
        if line.startswith("s;"):
            if lineno != 0:
                return ParseError("Size request can only be the first line!")
            req_size = line.split(";", 1)[1]
            # Almost certainly redundant due to stripping the line above
            req_size = req_size.strip()
            try:
                req_size = int(req_size)
            except ValueError:
                raise ParseError("Invalid requested size")
            parsed.req_size = req_size
            continue

        if ";" not in line:
            return ParseError("Bad downloads request: no semi-colon")

        lname, chunklist = line.split(";", 1)
        if not lname or '-' not in lname:
            raise ParseError("Invalid list name: \"%s\"" % lname)
        info = DownloadsListInfo(lname, limit=limit)

        chunks = chunklist.split(":")
        # Check for MAC
        if len(chunks) >= 1 and chunks[-1] == "mac":
            if request.GET.get('pver') == '3.0':
                raise ParseError('MAC not supported in protocol version 3')
            info.wants_mac = True
            chunks.pop(-1)
        # Client claims to have chunks for this list
        if not chunks or (len(chunks) == 1 and not chunks[0]):
            parsed.append(info)
            continue
        # Uneven number of chunks should only occur if 'mac' was specified
        if len(chunks) % 2 != 0:
            raise ParseError("Invalid LISTINFO for %s" % lname)

        while chunks:
            ctype = chunks.pop(0)
            if ctype not in ('a', 's'):
                raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" % (ctype,
                                                                      lname))

            list_of_chunks = chunks.pop(0)
            for chunk in list_of_chunks.split(','):
                try:
                    chunk = int(chunk)
                except ValueError:
                    if chunk.find('-'):
                        low, high = chunk.split('-', 1)
                        # FIXME should probably be stricter about testing for
                        #       pure integers only on the input
                        try:
                            low = int(low)
                            high = int(high)
                        except ValueError:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))
                        if low >= high:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))

                        info.add_range_claim(ctype, low, high)
                else:  # Resist temptation to indent!  It's a try/except/else
                    info.add_claim(ctype, chunk)
        parsed.append(info)
    return parsed

def parse_gethash(request):
    parsed = []

    # Early check to be sure we have something within the limits of a
    # reasonably sized header.  Reasonable size defined as an arbitrary max
    # of 2**8 bytes and a minimum of 3("4:4", a single prefix).  256 is
    # probably waaaaaaaaaaay too large for a gethash request header.
    eoh = request.body.find('\n')
    if eoh < 3 or eoh >= 256:
        raise ParseError("Improbably small or large gethash header size: %d"
                         % eoh)

    body_file = request.body_file

    # determine size of individual prefixes and length of payload
    header = body_file.readline()
    try:
        prefix_len, payload_len = [int(x) for x in header.split(':', 1)]
    except ValueError:
        raise ParseError('Invalid prefix or payload size: "%s"' % header)
    if ((payload_len % prefix_len != 0)
            or (payload_len < prefix_len)):
        raise ParseError("Payload length invalid: \"%d\"" % payload_len)

    prefix_total = payload_len / prefix_len
    prefixes_read = 0
    total_read = 0
    while prefixes_read < prefix_total:
        prefix = body_file.read(prefix_len)
        if len(prefix) < prefix_len:
            break
        total_read += len(prefix)
        prefixes_read += 1
        parsed.append(prefix)

    if prefixes_read != prefix_total:
        raise ParseError("Hash read mismatch: client claimed %d, read %d" %
                         (prefix_total, prefixes_read))

    if total_read != payload_len:
        raise ParseError("Mismatch on gethash parse: client: %d, actual: %d" %
                         (payload_len, total_read))

    return set(parsed)  # unique-ify

def parse_file_source(handle):
    """
    Parses a chunk list formatted file
    """
    # We should almost certainly* find the end of the first newline within the
    # first 32 bytes of the file.  It consists of a colon delimited string
    # with the following members:
    #
    #  - type of chunk: 'a' or 's' == 1
    #  - chunk number:  assuming len(2**32) == max of 10
    #  - number of bytes in the hash prefix size: 4 bytes for shavar or
    #                                             32 digest256 == max of 2
    #  - length of the raw data following in octets: len(2**32) == max of 10
    #
    #  These total 23 plus 3 bytes for colons plus one byte for the newline
    #  bring the grand total for likely maximum length to 27 with a minimum
    #  of 8 bytes("1:1:4:1\n").
    #
    #  So 32 byte read should be more than sufficient.
    #
    # * If 64 bit ints get involved, there are other issues to address

    parsed = ChunkList()
    while True:
        blob = handle.read(32)

        # Consume any unnecessary newlines in front of chunks
        blob = blob.lstrip('\n')

        if not blob:
            break

        if len(blob) < 8:
            raise ParseError("Incomplete chunk file? Could only read %d "
                             "bytes of header." % len(blob))

        eol = blob.find('\n')
        if eol < 8:
            raise ParseError('Impossibly short chunk header: "%s"' % eol)
        header = blob[:eol]

        if header.count(':') != 3:
            raise ParseError('Incorrect number of fields in chunk header: '
                             '"%s"' % header)

        add_sub, chunk_num, hash_len, read_len = header.split(':', 3)

        if len(add_sub) != 1:
            raise ParseError('Chunk type is too long: "%s"' % header)
        if add_sub not in ('a', 's'):
            raise ParseError('Invalid chunk type: "%s"' % header)

        try:
            chunk_num = int(chunk_num)
            hash_len = int(hash_len)
            read_len = int(read_len)
        except ValueError:
            raise ParseError('Non-integer chunk values: "%s"' % header)

        if read_len % hash_len != 0:
            raise ParseError('Chunk data length not a multiple of prefix '
                             'size: "%s"' % header)

        blob = blob[eol + 1:]
        blob += handle.read(read_len - len(blob))
        if blob is None or len(blob) < read_len:
            raise ParseError('Chunk data truncated for chunk %d' % chunk_num)

        hashes = []
        pos = 0
        while pos < read_len:
            hashes.append(blob[pos:pos + hash_len])
            pos += hash_len

        parsed.insert_chunk(Chunk(chunk_type=add_sub, number=chunk_num,
                                  hashes=hashes))

    return parsed

def parse_dir_source(handle, exists_cb=os.path.exists, open_cb=open):
    """
    Expects a file alike object with the contents of a JSON formatted index
    file that has the following structure:

    {
        "name": "mozpub-tracking-digest256",
        "basedir": "mozpub-tracking-digest256",
        "chunks": {
            "1": {
                "path": "mozpub-tracking-digest256/1",
                "hashes(optional)": [ "", "" ],
                "prefixes(optional)": [ "", "" ]
            },
            "2": {
                "path": "mozpub-tracking-digest256/2",
                "hashes": [ "", "" ],
                "prefixes": [ "", "" ]
            }
        }
    }

    The basedir, hashes, and prefixes entries are optional.  The chunks to be
    served will be parsed with parse_file_source().  If hashes and prefixes are
    provided, they will be verified against the data provided in the given
    chunk file.
    """
    try:
        index = json.load(handle)
    except ValueError as e:
        raise ParseError("Could not parse index file: %s" % e)

    if 'name' not in index:
        raise ParseError("Incorrectly formatted index: missing list name")

    if 'chunks' not in index:
        raise ParseError("Incorrectly formatted index: missing chunks")

    if 'basedir' in index:
        basedir = posixpath.join(os.path.dirname(handle.name),
                                 index['basedir'])
    else:
        basedir = os.path.dirname(handle.name)

    parsed = ChunkList()
    for key in index['chunks'].keys():
        # A little massaging to make the data structure a little cleaner
        try:
            index['chunks'][int(key)] = index['chunks'][key]
            del index['chunks'][key]
        except KeyError:
            raise ParseError("Some weird behaviour with the list of chunks "
                             "in \"%s\"" % handle.filename)

        chunk_file = posixpath.join(basedir, key)

        if not exists_cb(chunk_file):
            raise ParseError("Invalid chunk filename: \"%s\"" % chunk_file)

        with open_cb(chunk_file, 'rb') as f:
            chunk_list = parse_file_source(f)

        # Only one chunk per file
        if len(chunk_list) > 1:
            raise ParseError("More than one chunk in chunk file \"%s\""
                             % chunk_file)

        for chunk in itertools.chain(iter(chunk_list.adds.values()),
                                     iter(chunk_list.subs.values())):
            parsed.insert_chunk(chunk)

    return parsed
<commit_msg>Parse gethash using explicit byte and string object When checking for valid request body use byte object. Then use string object for parsing the header.<commit_after>import itertools
import json
import os.path
import posixpath

from shavar.exceptions import ParseError
from shavar.types import Chunk, ChunkList, Downloads, DownloadsListInfo

def parse_downloads(request):
    parsed = Downloads()

    limit = request.registry.settings.get("shavar.max_downloads_chunks",
                                          10000)

    for lineno, line in enumerate(request.body_file):
        line = line.strip()

        if not line or line.isspace():
            continue

        line = line.decode('utf-8')
        # Did client provide max size preference?
        if line.startswith("s;"):
            if lineno != 0:
                return ParseError("Size request can only be the first line!")
            req_size = line.split(";", 1)[1]
            # Almost certainly redundant due to stripping the line above
            req_size = req_size.strip()
            try:
                req_size = int(req_size)
            except ValueError:
                raise ParseError("Invalid requested size")
            parsed.req_size = req_size
            continue

        if ";" not in line:
            return ParseError("Bad downloads request: no semi-colon")

        lname, chunklist = line.split(";", 1)
        if not lname or '-' not in lname:
            raise ParseError("Invalid list name: \"%s\"" % lname)
        info = DownloadsListInfo(lname, limit=limit)

        chunks = chunklist.split(":")
        # Check for MAC
        if len(chunks) >= 1 and chunks[-1] == "mac":
            if request.GET.get('pver') == '3.0':
                raise ParseError('MAC not supported in protocol version 3')
            info.wants_mac = True
            chunks.pop(-1)
        # Client claims to have chunks for this list
        if not chunks or (len(chunks) == 1 and not chunks[0]):
            parsed.append(info)
            continue
        # Uneven number of chunks should only occur if 'mac' was specified
        if len(chunks) % 2 != 0:
            raise ParseError("Invalid LISTINFO for %s" % lname)

        while chunks:
            ctype = chunks.pop(0)
            if ctype not in ('a', 's'):
                raise ParseError("Invalid CHUNKTYPE \"%s\" for %s" % (ctype,
                                                                      lname))

            list_of_chunks = chunks.pop(0)
            for chunk in list_of_chunks.split(','):
                try:
                    chunk = int(chunk)
                except ValueError:
                    if chunk.find('-'):
                        low, high = chunk.split('-', 1)
                        # FIXME should probably be stricter about testing for
                        #       pure integers only on the input
                        try:
                            low = int(low)
                            high = int(high)
                        except ValueError:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))
                        if low >= high:
                            raise ParseError("Invalid RANGE \"%s\" for %s" %
                                             (chunk, lname))

                        info.add_range_claim(ctype, low, high)
                else:  # Resist temptation to indent!  It's a try/except/else
                    info.add_claim(ctype, chunk)
        parsed.append(info)
    return parsed

def parse_gethash(request):
    parsed = []

    # Early check to be sure we have something within the limits of a
    # reasonably sized header.  Reasonable size defined as an arbitrary max
    # of 2**8 bytes and a minimum of 3("4:4", a single prefix).  256 is
    # probably waaaaaaaaaaay too large for a gethash request header.
    eoh = request.body.find(b'\n')
    if eoh < 3 or eoh >= 256:
        raise ParseError("Improbably small or large gethash header size: %d"
                         % eoh)

    body_file = request.body_file

    # determine size of individual prefixes and length of payload
    header = body_file.readline().decode()
    try:
        prefix_len, payload_len = [int(x) for x in header.split(':', 1)]
    except ValueError:
        raise ParseError('Invalid prefix or payload size: "%s"' % header)
    if ((payload_len % prefix_len != 0)
            or (payload_len < prefix_len)):
        raise ParseError("Payload length invalid: \"%d\"" % payload_len)

    prefix_total = payload_len / prefix_len
    prefixes_read = 0
    total_read = 0
    while prefixes_read < prefix_total:
        prefix = body_file.read(prefix_len)
        if len(prefix) < prefix_len:
            break
        total_read += len(prefix)
        prefixes_read += 1
        parsed.append(prefix)

    if prefixes_read != prefix_total:
        raise ParseError("Hash read mismatch: client claimed %d, read %d" %
                         (prefix_total, prefixes_read))

    if total_read != payload_len:
        raise ParseError("Mismatch on gethash parse: client: %d, actual: %d" %
                         (payload_len, total_read))

    return set(parsed)  # unique-ify

def parse_file_source(handle):
    """
    Parses a chunk list formatted file
    """
    # We should almost certainly* find the end of the first newline within the
    # first 32 bytes of the file.  It consists of a colon delimited string
    # with the following members:
    #
    #  - type of chunk: 'a' or 's' == 1
    #  - chunk number:  assuming len(2**32) == max of 10
    #  - number of bytes in the hash prefix size: 4 bytes for shavar or
    #                                             32 digest256 == max of 2
    #  - length of the raw data following in octets: len(2**32) == max of 10
    #
    #  These total 23 plus 3 bytes for colons plus one byte for the newline
    #  bring the grand total for likely maximum length to 27 with a minimum
    #  of 8 bytes("1:1:4:1\n").
    #
    #  So 32 byte read should be more than sufficient.
    #
    # * If 64 bit ints get involved, there are other issues to address

    parsed = ChunkList()
    while True:
        blob = handle.read(32)

        # Consume any unnecessary newlines in front of chunks
        blob = blob.lstrip('\n')

        if not blob:
            break

        if len(blob) < 8:
            raise ParseError("Incomplete chunk file? Could only read %d "
                             "bytes of header." % len(blob))

        eol = blob.find('\n')
        if eol < 8:
            raise ParseError('Impossibly short chunk header: "%s"' % eol)
        header = blob[:eol]

        if header.count(':') != 3:
            raise ParseError('Incorrect number of fields in chunk header: '
                             '"%s"' % header)

        add_sub, chunk_num, hash_len, read_len = header.split(':', 3)

        if len(add_sub) != 1:
            raise ParseError('Chunk type is too long: "%s"' % header)
        if add_sub not in ('a', 's'):
            raise ParseError('Invalid chunk type: "%s"' % header)

        try:
            chunk_num = int(chunk_num)
            hash_len = int(hash_len)
            read_len = int(read_len)
        except ValueError:
            raise ParseError('Non-integer chunk values: "%s"' % header)

        if read_len % hash_len != 0:
            raise ParseError('Chunk data length not a multiple of prefix '
                             'size: "%s"' % header)

        blob = blob[eol + 1:]
        blob += handle.read(read_len - len(blob))
        if blob is None or len(blob) < read_len:
            raise ParseError('Chunk data truncated for chunk %d' % chunk_num)

        hashes = []
        pos = 0
        while pos < read_len:
            hashes.append(blob[pos:pos + hash_len])
            pos += hash_len

        parsed.insert_chunk(Chunk(chunk_type=add_sub, number=chunk_num,
                                  hashes=hashes))

    return parsed

def parse_dir_source(handle, exists_cb=os.path.exists, open_cb=open):
    """
    Expects a file alike object with the contents of a JSON formatted index
    file that has the following structure:

    {
        "name": "mozpub-tracking-digest256",
        "basedir": "mozpub-tracking-digest256",
        "chunks": {
            "1": {
                "path": "mozpub-tracking-digest256/1",
                "hashes(optional)": [ "", "" ],
                "prefixes(optional)": [ "", "" ]
            },
            "2": {
                "path": "mozpub-tracking-digest256/2",
                "hashes": [ "", "" ],
                "prefixes": [ "", "" ]
            }
        }
    }

    The basedir, hashes, and prefixes entries are optional.  The chunks to be
    served will be parsed with parse_file_source().  If hashes and prefixes are
    provided, they will be verified against the data provided in the given
    chunk file.
    """
    try:
        index = json.load(handle)
    except ValueError as e:
        raise ParseError("Could not parse index file: %s" % e)

    if 'name' not in index:
        raise ParseError("Incorrectly formatted index: missing list name")

    if 'chunks' not in index:
        raise ParseError("Incorrectly formatted index: missing chunks")

    if 'basedir' in index:
        basedir = posixpath.join(os.path.dirname(handle.name),
                                 index['basedir'])
    else:
        basedir = os.path.dirname(handle.name)

    parsed = ChunkList()
    for key in index['chunks'].keys():
        # A little massaging to make the data structure a little cleaner
        try:
            index['chunks'][int(key)] = index['chunks'][key]
            del index['chunks'][key]
        except KeyError:
            raise ParseError("Some weird behaviour with the list of chunks "
                             "in \"%s\"" % handle.filename)

        chunk_file = posixpath.join(basedir, key)

        if not exists_cb(chunk_file):
            raise ParseError("Invalid chunk filename: \"%s\"" % chunk_file)

        with open_cb(chunk_file, 'rb') as f:
            chunk_list = parse_file_source(f)

        # Only one chunk per file
        if len(chunk_list) > 1:
            raise ParseError("More than one chunk in chunk file \"%s\""
                             % chunk_file)

        for chunk in itertools.chain(iter(chunk_list.adds.values()),
                                     iter(chunk_list.subs.values())):
            parsed.insert_chunk(chunk)

    return parsed
<|endoftext|>


<commit_before># findfiles.py
# ------------
#
# Re-implementation of findfiles.ksh
# Re-implementation of findfiles.pro

#------------------------
#     findfiles.ksh
#------------------------

def getSatellite(satellite):
    # Parameters:
    #   * satellite: String
    #       - Satellite name
    # Returns:
    #   * input_dir: String
    #       - input directory of files
    input_dir = ''
    if satellite == 'NOAA16_GHRR':
        input_dir = 'gpnl'
    elif satellite == 'NOAA17_GHRR':
        input_dir = 'gpnm'
    elif satellite == 'NOAA18_GHRR':
        input_dir = 'gpnn'
    elif satellite == 'NOAA19_GHRR':
        input_dir = 'gpnp'
    elif satellite == 'METOPA_GHRR':
        input_dir = 'gpm2'
    elif satellite == 'METOPA_FRAC':
        input_dir = 'frac'
    return input_dir

#------------------------
#     findfiles.pro
#------------------------

def findFiles(input_dir, satellite, mmdd):
    # Parameters:
    #   * input_dir: String
    #       - input directory of files
    #   * satellite: String
    #       - Name of satellite/data type
    #   * mmdd: String
    #       - Month/Day of file
    pass
<commit_msg>idl/findfiles.py: Intermediate work on implementing findFiles function<commit_after># findfiles.py
# ------------
#
# Re-implementation of findfiles.ksh
# Re-implementation of findfiles.pro

import numpy as np

#------------------------
#     findfiles.ksh
#------------------------

def getSatellite(satellite):
    # Parameters:
    #   * satellite: String
    #       - Satellite name
    # Returns:
    #   * input_dir: String
    #       - input directory of files
    input_dir = ''
    if satellite == 'NOAA16_GHRR':
        input_dir = 'gpnl'
    elif satellite == 'NOAA17_GHRR':
        input_dir = 'gpnm'
    elif satellite == 'NOAA18_GHRR':
        input_dir = 'gpnn'
    elif satellite == 'NOAA19_GHRR':
        input_dir = 'gpnp'
    elif satellite == 'METOPA_GHRR':
        input_dir = 'gpm2'
    elif satellite == 'METOPA_FRAC':
        input_dir = 'frac'
    return input_dir

#------------------------
#     findfiles.pro
#------------------------

def findFiles(input_dir, satellite, mmdd):
    # Parameters:
    #   * input_dir: String
    #       - input directory of files
    #   * satellite: String
    #       - Name of satellite/data type
    #   * mmdd: String
    #       - Month/Day of file

    # Number of special regions
    numRegions = 5
    openDay = np.zeros(numRegions)
    openNight = np.zeros(numRegions)
    iunDay = np.arange(numRegions)
    iunDay = iunDay + 10
    iunNight = iunDay[numRegions - 1] + iunDay + 1

<|endoftext|>


<commit_before>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _is_namespaceable(cls):
    """Partial function for filter."""
    return isinstance(cls, Namespaceable)

def _mro_to_chained(mro, dct):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, dct.path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, dct),
            filter(_is_namespaceable, mro)) if
        Namespace.namespace_exists(dct.path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, Namespaceable):
            return _mro_to_chained(instance.__mro__, dct)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    mro = mro[mro.index(dct.parent_object):]
    return _mro_to_chained(mro, dct)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

def _get(a_map, name):
    """Return a _DescriptorInspector around the attribute, or None."""
    try:
        value = a_map[name]
    except KeyError:
        return None
    else:
        return _DescriptorInspector(value)

def _delete(dct, name):
    """Attempt to delete `name` from `dct`. Raise AttributeError if missing."""
    try:
        del dct[name]
    except KeyError:
        raise AttributeError(name)

def _has_get(value):
    """Return whether the value is a wrapped getter descriptor."""
    return value is not None and value.has_get

def _is_data(value):
    """Return whether the value is a wrapped data descriptor."""
    return value is not None and value.is_data

def _get_data(dct, name):
    """Return the data descriptor associated with `name` in `dct`, if any."""
    value = _get(dct, name)
    if _is_data(value):
        return value

class _NamespaceProxy:

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            return dct[name]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = _get(instance_map, name)
        mro_value = _get(mro_map, name)
        if _is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and _has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif _has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            dct[name] = value
            return
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = _get_data(mro_map, name)
        if target_value is not None:
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            _delete(dct, name)
            return
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            _delete(real_map, name)
            return
        value = _get_data(real_map, name)
        if value is not None:
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        _delete(instance_map, name)

    def __enter__(self):
        dct, _, _ = _PROXY_INFOS[self]
        return dct.__enter__()

    def __exit__(self, exc_type, exc_value, traceback):
        dct, _, _ = _PROXY_INFOS[self]
        return dct.__exit__(exc_type, exc_value, traceback)

class Namespace(dict):

    """Namespace."""

    __slots__ = 'name', 'scope', 'parent', 'active', 'parent_object'

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _NamespaceProxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    def __setitem__(self, key, value):
        if (
                self.parent_object is not None and
                isinstance(value, Namespace) and value.name != key):
            value.push(key, self.scope)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = target.__namespaces
            except AttributeError:
                namespaces = {}
                target.__namespaces = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, dct, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in dct.path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        path, namespaces = self.__get_helper(target, self.path)
        res = namespaces.setdefault(path, self)
        if res is self:
            self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def push(self, name, scope):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', scope.dicts[0])
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            raise ValueError('Cannot reuse namespace')
        if scope.dicts[0] is not self.parent:
            raise ValueError('Cannot reparent namespace')
        self.scope.namespaces.append(self)
        self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.scope is not None and not self.active:
            if self.scope.dicts[0] is not self.parent:
                raise ValueError('Cannot reparent namespace')
            self.active = True
            self.scope.dicts.insert(0, self)

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for Namespaceables."""

    __slots__ = 'dicts', 'namespaces'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = _NamespaceProxy(value, None, None)
        return value

    def __setitem__(self, key, value):
        dct = self.dicts[0]
        if isinstance(value, _NamespaceProxy):
            value, _, _ = _PROXY_INFOS[value]
        if isinstance(value, Namespace) and value.name != key:
            value.push(key, self)
        dct[key] = value

    def __delitem__(self, key):
        del self.dicts[0][key]

    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class Namespaceable(type):

    """Metaclass for classes that can contain namespaces."""

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if isinstance(value, Namespace) and value.name != name:
            value.push(name, _NAMESPACE_SCOPES[cls])
            value.add(cls)
        if issubclass(cls, Namespaceable):
            super().__setattr__(cls, name, value)
        else:
            super().__setattr__(name, value)
<commit_msg>Eliminate a filter.<commit_after>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _mro_to_chained(mro, dct):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, dct.path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, dct),
            (cls for cls in mro if isinstance(cls, Namespaceable))) if
        Namespace.namespace_exists(dct.path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, Namespaceable):
            return _mro_to_chained(instance.__mro__, dct)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    mro = mro[mro.index(dct.parent_object):]
    return _mro_to_chained(mro, dct)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

def _get(a_map, name):
    """Return a _DescriptorInspector around the attribute, or None."""
    try:
        value = a_map[name]
    except KeyError:
        return None
    else:
        return _DescriptorInspector(value)

def _delete(dct, name):
    """Attempt to delete `name` from `dct`. Raise AttributeError if missing."""
    try:
        del dct[name]
    except KeyError:
        raise AttributeError(name)

def _has_get(value):
    """Return whether the value is a wrapped getter descriptor."""
    return value is not None and value.has_get

def _is_data(value):
    """Return whether the value is a wrapped data descriptor."""
    return value is not None and value.is_data

def _get_data(dct, name):
    """Return the data descriptor associated with `name` in `dct`, if any."""
    value = _get(dct, name)
    if _is_data(value):
        return value

class _NamespaceProxy:

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            return dct[name]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = _get(instance_map, name)
        mro_value = _get(mro_map, name)
        if _is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and _has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif _has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            dct[name] = value
            return
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = _get_data(mro_map, name)
        if target_value is not None:
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if owner is None:
            _delete(dct, name)
            return
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            _delete(real_map, name)
            return
        value = _get_data(real_map, name)
        if value is not None:
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        _delete(instance_map, name)

    def __enter__(self):
        dct, _, _ = _PROXY_INFOS[self]
        return dct.__enter__()

    def __exit__(self, exc_type, exc_value, traceback):
        dct, _, _ = _PROXY_INFOS[self]
        return dct.__exit__(exc_type, exc_value, traceback)

class Namespace(dict):

    """Namespace."""

    __slots__ = 'name', 'scope', 'parent', 'active', 'parent_object'

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _NamespaceProxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    def __setitem__(self, key, value):
        if (
                self.parent_object is not None and
                isinstance(value, Namespace) and value.name != key):
            value.push(key, self.scope)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = target.__namespaces
            except AttributeError:
                namespaces = {}
                target.__namespaces = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, dct, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in dct.path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        path, namespaces = self.__get_helper(target, self.path)
        res = namespaces.setdefault(path, self)
        if res is self:
            self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def push(self, name, scope):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', scope.dicts[0])
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            raise ValueError('Cannot reuse namespace')
        if scope.dicts[0] is not self.parent:
            raise ValueError('Cannot reparent namespace')
        self.scope.namespaces.append(self)
        self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.scope is not None and not self.active:
            if self.scope.dicts[0] is not self.parent:
                raise ValueError('Cannot reparent namespace')
            self.active = True
            self.scope.dicts.insert(0, self)

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for Namespaceables."""

    __slots__ = 'dicts', 'namespaces'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = _NamespaceProxy(value, None, None)
        return value

    def __setitem__(self, key, value):
        dct = self.dicts[0]
        if isinstance(value, _NamespaceProxy):
            value, _, _ = _PROXY_INFOS[value]
        if isinstance(value, Namespace) and value.name != key:
            value.push(key, self)
        dct[key] = value

    def __delitem__(self, key):
        del self.dicts[0][key]

    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class Namespaceable(type):

    """Metaclass for classes that can contain namespaces."""

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if isinstance(value, Namespace) and value.name != name:
            value.push(name, _NAMESPACE_SCOPES[cls])
            value.add(cls)
        if issubclass(cls, Namespaceable):
            super().__setattr__(cls, name, value)
        else:
            super().__setattr__(name, value)
<|endoftext|>


<commit_before># This is the PigeonScript interpreter

from pgn_parser import parse

# Take user input and parse it

code = input("Input your program:\n")

parse(code)
<commit_msg>Updated things<commit_after># This is the PigeonScript interpreter

from pgn_parser import parse

# Take user input and parse it

code = input("Input your program:\n")

instructions = parse(code)

'''
I'm just laying out my thoughts here.

A loop will iterate through the list of parsed instrucions, and executes each one as it goes along.
Functions will be executed in a function that will run recursively, so that loops within the program
will be able to run within that single function.
'''

def execute(instruction):
    if instruction
<|endoftext|>


<commit_before># prefer setuptools over distutils
from setuptools import setup, find_packages

# use a consistent encoding
from codecs import open
from os import path
import json

here = path.abspath(path.dirname(__file__))
root = path.dirname(here)

readme = path.join(here, 'README.md')
package_json = path.join(here, 'package.json')

# a workaround when installing locally from git repository with pip install -e .
if not path.isfile(package_json):
    package_json = path.join(root, 'package.json')

# long description from README file
with open(readme, encoding='utf-8') as f:
    long_description = f.read()

# version number and all other params from package.json
with open(package_json, encoding='utf-8') as f:
    package = json.load(f)

setup(

    name=package['name'],
    version=package['version'],

    description=package['description'],
    long_description=long_description,

    # will switch from rst to md shortly
    # long_description_content_type='text/markdown',

    url=package['homepage'],

    author=package['author']['name'],
    author_email=package['author']['email'],

    license=package['license'],

    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Financial and Insurance Industry',
        'Intended Audience :: Information Technology',
        'Topic :: Software Development :: Build Tools',
        'Topic :: Office/Business :: Financial :: Investment',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: JavaScript',
        'Programming Language :: PHP',
        'Operating System :: OS Independent',
        'Environment :: Console'
    ],

    keywords=package['keywords'],
    packages=find_packages(),

    install_requires=[
        'setuptools>=38.5.1',
        'certifi>=2018.1.18',
        'requests>=2.18.4',
        'cryptography>=2.6.1'
    ],

    extras_require={
        ':python_version>="3.5.2"': [
            'aiohttp>=3.0.1',
            'aiodns==1.1.1',
            'ccxt>=1.22.58',
            'yarl==1.1.0',
        ],
        'qa': [
            'flake8==3.5.0'
        ],
        'doc': [
            'Sphinx==1.7.0'
        ]
    }
)
<commit_msg>setup.py include LICENSE.txt<commit_after># prefer setuptools over distutils
from setuptools import setup, find_packages

# use a consistent encoding
from codecs import open
from os import path
import json

here = path.abspath(path.dirname(__file__))
root = path.dirname(here)

readme = path.join(here, 'README.md')
package_json = path.join(here, 'package.json')

# a workaround when installing locally from git repository with pip install -e .
if not path.isfile(package_json):
    package_json = path.join(root, 'package.json')

# long description from README file
with open(readme, encoding='utf-8') as f:
    long_description = f.read()

# version number and all other params from package.json
with open(package_json, encoding='utf-8') as f:
    package = json.load(f)

setup(

    name=package['name'],
    version=package['version'],

    description=package['description'],
    long_description=long_description,

    # will switch from rst to md shortly
    # long_description_content_type='text/markdown',

    url=package['homepage'],

    author=package['author']['name'],
    author_email=package['author']['email'],

    license=package['license'],

    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Financial and Insurance Industry',
        'Intended Audience :: Information Technology',
        'Topic :: Software Development :: Build Tools',
        'Topic :: Office/Business :: Financial :: Investment',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: JavaScript',
        'Programming Language :: PHP',
        'Operating System :: OS Independent',
        'Environment :: Console'
    ],

    keywords=package['keywords'],
    packages=find_packages(),
    data_files=[('', ['LICENSE.txt'])],

    install_requires=[
        'setuptools>=38.5.1',
        'certifi>=2018.1.18',
        'requests>=2.18.4',
        'cryptography>=2.6.1'
    ],

    extras_require={
        ':python_version>="3.5.2"': [
            'aiohttp>=3.0.1',
            'aiodns==1.1.1',
            'ccxt>=1.22.58',
            'yarl==1.1.0',
        ],
        'qa': [
            'flake8==3.5.0'
        ],
        'doc': [
            'Sphinx==1.7.0'
        ]
    }
)
<|endoftext|>


<commit_before># findfiles.py
# ------------
#
# Re-implementation of findfiles.ksh
# Re-implementation of findfiles.pro

import yaml
import netCDF4
import numpy as np
from os import listdir
from os.path import isfile, join
from subprocess import Popen, PIPE

numRegions = 5
openDay = np.zeros(numRegions)
openNight = np.zeros(numRegions)
iunDay = np.arange(numRegions)
iunDay = iunDay + 10
iunNight = iunDay[numRegions - 1] + iunDay + 1

# List containing all special regions
regionName = ['MR', 'ER', 'SR', 'WN', 'BS']

# List of day granules
dayGranules = {}

# List of night granules
nightGranules = {}

# Path of files to look into
# path = input_dir + '_' +  satellite + '_' +  mmdd + '.hdf'
# Need to get a regex to sort files inside the right directory
# TODO: FIND REGEX
# filterFiles(path)
path = "/Users/DrkSephy/Dropbox/granules/"
fileList = [ f for f in listdir(path) if isfile(join(path, f)) ]
# print fileList

specialRegions = {
    'MR': {'lonmin': 98, 'lonmax': -82, 'latmin': 18, 'latmax': 30, 'master': 'MASTER_MR_1KM.hdf'  },
    'ER': {'lonmin': -80, 'lonmax': -61, 'latmin': 30, 'latmax': 47, 'master': 'MASTER_ER_1KM.hdf' },
    'SR': {'lonmin': -89, 'lonmax': -72, 'latmin': 22, 'latmax': 38, 'master': 'MASTER_SR_1KM.hdf' },
    'WN': {'lonmin': -136, 'lonmax': -121, 'latmin': 39, 'latmax': 51, 'master': 'MASTER_WN_1KM.hdf'},
    'BS': {'lonmin': 27, 'lonmax': 42, 'latmin': 40, 'latmax': 48, 'master': 'MASTER_BS_1KM.hdf'}
}

# Create global data structure which has a list for each
data = { name: [] for name in regionName }
daynightData = { }
daynightData['day'] = { name: [] for name in regionName }
daynightData['night'] = { name: [] for name in regionName }
# print data

# Loop over all granules inside fileList
for f in fileList:
    ds = netCDF4.Dataset(path + f)
    # Select l2p_flags variable
    flags = np.array(ds.variables['l2p_flags'])
    # Read day/night bit (bit 10: 0=night, 1=day)
    bit10 = np.array(np.ones(flags.shape)*(2**9), dtype='uint16')
    # Create array of all day pixels
    dayMask = np.squeeze(np.bitwise_and(flags, bit10) > 0)
    # Create array of all night pixels
    nightMask = np.bitwise_not(dayMask)

    # Grab the longitude
    lon = np.array(ds.variables['lon'])

    # Grab the latitude
    lat = np.array(ds.variables['lat'])

    # Iterate over each special region
    # Get the maximum number of pixels bounded in the region
    maxRegion = 0
    # Which region had the most bounded pixels?
    maxRegionName = ''

    for reg in regionName:

        # Create an array of records in the file which are
        # within the special region

        # NOTE: Need to figure out how to do this properly with np.where.
        #       We need to use this array to get the number of day pixels
        region_arr = (np.logical_and(np.logical_and(lon >= specialRegions[reg]['lonmin'], lon <= specialRegions[reg]['lonmax']),
                        np.logical_and(lat >= specialRegions[reg]['latmin'], lat <= specialRegions[reg]['latmax']))).nonzero()

        dayCount = len(dayMask[region_arr].nonzero()[0])
        nightCount = len(nightMask[region_arr].nonzero()[0])
        if dayCount > 0:
            daynightData['day'][reg].append((f, specialRegions[reg]['master'], dayCount, 'day'))
        if nightCount > 0:
            daynightData['night'][reg].append((f, specialRegions[reg]['master'], nightCount, 'night'))

granuleList = yaml.dump(daynightData)
granuleData = yaml.load(granuleList)
# print granuleData

# Iterate over day/night granule entries
for d in granuleData:
    # Iterate over each region for day/night
    for region in granuleData[d]:
        if granuleData[d][region] != []:
            cmd = 'cwregister' + ' ' + granuleData[d][region][0][1] + ' ' + granuleData[d][region][0][0] + ' ' + 'REG' + granuleData[d][region][0][0] + '.hdf'
            pid = Popen(cmd, shell=True)
            pid.wait()
            #print granuleData[d][region][0][0]
            #print granuleData[d][region][0][1]
            #print granuleData[d][region][0][2]
            #print granuleData[d][region][0][3]

# cwregister MASTER_BS_1KM.hdf 20141122065952-STAR-L2P_GHRSST-SST1m-GHRR_METOPA-v02.0-fv01.0.nc newestblacksea.hdf
# Calling command line functions

# Option 1
# from subprocess import Popen, PIPE
# cmd = "ls foobar"
# pid = Popen(cmd, shell=True)
# pid.wait()

# Option 2
# from subprocess import Popen, PIPE
# cmd = "ls foobar"
# pid = Popen(cmd, stderr=PIPE, stdout=PIPE, shell=True)
# stdout, stderr = pid.communicate()

# NEXT STEPS
# Read yaml file
# We have a bunch of granules
#   Outer loop is region
#       day, night (process the granule with the highest counts)
#

<commit_msg>findfiles.py: Intermediate work on generating commands<commit_after># findfiles.py
# ------------
#
# Re-implementation of findfiles.ksh
# Re-implementation of findfiles.pro

import yaml
import netCDF4
import numpy as np
from os import listdir
from os.path import isfile, join
from subprocess import Popen, PIPE

numRegions = 5
openDay = np.zeros(numRegions)
openNight = np.zeros(numRegions)
iunDay = np.arange(numRegions)
iunDay = iunDay + 10
iunNight = iunDay[numRegions - 1] + iunDay + 1

# List containing all special regions
regionName = ['MR', 'ER', 'SR', 'WN', 'BS']

# List of day granules
dayGranules = {}

# List of night granules
nightGranules = {}

# Path of files to look into
# path = input_dir + '_' +  satellite + '_' +  mmdd + '.hdf'
# Need to get a regex to sort files inside the right directory
# TODO: FIND REGEX
# filterFiles(path)
path = "/Users/DrkSephy/Desktop/January/"
fileList = [ f for f in listdir(path) if isfile(join(path, f)) ]
masterPath = "/home/DrkSephy/special_regions/testing/master_files/"
# print fileList

specialRegions = {
    'MR': {'lonmin': 98, 'lonmax': -82, 'latmin': 18, 'latmax': 30, 'master': masterPath + 'MASTER_MR_1KM.hdf'  },
    'ER': {'lonmin': -80, 'lonmax': -61, 'latmin': 30, 'latmax': 47, 'master': masterPath + 'MASTER_ER_1KM.hdf' },
    'SR': {'lonmin': -89, 'lonmax': -72, 'latmin': 22, 'latmax': 38, 'master': masterPath + 'MASTER_SR_1KM.hdf' },
    'WN': {'lonmin': -136, 'lonmax': -121, 'latmin': 39, 'latmax': 51, 'master': masterPath + 'MASTER_WN_1KM.hdf'},
    'BS': {'lonmin': 27, 'lonmax': 42, 'latmin': 40, 'latmax': 48, 'master': masterPath + 'MASTER_BS_1KM.hdf'}
}

# Create global data structure which has a list for each
data = { name: [] for name in regionName }
daynightData = { }
daynightData['day'] = { name: [] for name in regionName }
daynightData['night'] = { name: [] for name in regionName }
# print data

# Loop over all granules inside fileList
for f in fileList:
    ds = netCDF4.Dataset(path + f)
    # Select l2p_flags variable
    flags = np.array(ds.variables['l2p_flags'])
    # Read day/night bit (bit 10: 0=night, 1=day)
    bit10 = np.array(np.ones(flags.shape)*(2**9), dtype='uint16')
    # Create array of all day pixels
    dayMask = np.squeeze(np.bitwise_and(flags, bit10) > 0)
    # Create array of all night pixels
    nightMask = np.bitwise_not(dayMask)

    # Grab the longitude
    lon = np.array(ds.variables['lon'])

    # Grab the latitude
    lat = np.array(ds.variables['lat'])

    # Iterate over each special region
    # Get the maximum number of pixels bounded in the region
    maxRegion = 0
    # Which region had the most bounded pixels?
    maxRegionName = ''

    for reg in regionName:

        # Create an array of records in the file which are
        # within the special region

        # NOTE: Need to figure out how to do this properly with np.where.
        #       We need to use this array to get the number of day pixels
        region_arr = (np.logical_and(np.logical_and(lon >= specialRegions[reg]['lonmin'], lon <= specialRegions[reg]['lonmax']),
                        np.logical_and(lat >= specialRegions[reg]['latmin'], lat <= specialRegions[reg]['latmax']))).nonzero()

        dayCount = len(dayMask[region_arr].nonzero()[0])
        nightCount = len(nightMask[region_arr].nonzero()[0])
        if dayCount > 0:
            daynightData['day'][reg].append((f, specialRegions[reg]['master'], dayCount, 'day'))
        if nightCount > 0:
            daynightData['night'][reg].append((f, specialRegions[reg]['master'], nightCount, 'night'))

# print daynightData['day']
granuleList = yaml.dump(daynightData)
granuleData = yaml.load(granuleList)
# print granuleData
# print granuleList

# Iterate over day/night granule entries
for d in granuleData:
    for region in granuleData[d]:
        if granuleData[d][region] != []:
            for val in granuleData[d][region]:
                cmd = 'cwregister' + ' ' + val[1] + ' ' + val[0] + ' ' + 'REG' + val[0][:-3] + '.hdf'
                print cmd

            #print granuleData[d][region]
            #cmd = 'cwregister' + ' ' + granuleData[d][region][0][1] + ' ' + granuleData[d][region][0][0] + ' ' + 'REG' + granuleData[d][region][0][0] + '.hdf'
            #print cmd

    """
    # Iterate over each region for day/night
    for region in granuleData[d]:
        if granuleData[d][region] != []:
            #cmd = 'cwregister' + ' ' + granuleData[d][region][0][1] + ' ' + granuleData[d][region][0][0] + ' ' + 'REG' + granuleData[d][region][0][0] + '.hdf'
            #print cmd
            #pid = Popen(cmd, shell=True)
            #pid.wait()
            print granuleData[d][region][0][0]
            print granuleData[d][region][0][1]
            print granuleData[d][region][0][2]
            print granuleData[d][region][0][3]
    """

#print granuleData['day']

# cwregister MASTER_BS_1KM.hdf 20141122065952-STAR-L2P_GHRSST-SST1m-GHRR_METOPA-v02.0-fv01.0.nc newestblacksea.hdf
# Calling command line functions

# NEXT STEPS
# Read yaml file
# We have a bunch of granules
#   Outer loop is region
#       day, night (process the granule with the highest counts)
#

<|endoftext|>


<commit_before># Copyright (c) 2014 Stefan C. Mueller

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import os
import ctypes.util
import ipaddress
import collections

import ifaddr._shared as shared
#from ifaddr._shared import sockaddr, Interface, sockaddr_to_ip, ipv6_prefixlength

class ifaddrs(ctypes.Structure):
    pass
ifaddrs._fields_ = [('ifa_next', ctypes.POINTER(ifaddrs)),
                    ('ifa_name', ctypes.c_char_p),
                    ('ifa_flags', ctypes.c_uint),
                    ('ifa_addr', ctypes.POINTER(shared.sockaddr)),
                    ('ifa_netmask', ctypes.POINTER(shared.sockaddr))]

libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)

def get_adapters():

    addr = ctypes.POINTER(ifaddrs)()
    retval = libc.getifaddrs(ctypes.byref(addr))
    if retval != 0:
        eno = ctypes.get_errno()
        raise OSError(eno, os.strerror(eno))

    ips = collections.OrderedDict()

    def add_ip(adapter_name, ip):
        if not adapter_name in ips:
            ips[adapter_name] = shared.Adapter(adapter_name, adapter_name, [])
        ips[adapter_name].ips.append(ip)

    while addr:
        name = addr[0].ifa_name
        ip = shared.sockaddr_to_ip(addr[0].ifa_addr)
        if ip:
            netmask = shared.sockaddr_to_ip(addr[0].ifa_netmask)
            if isinstance(netmask, tuple):
                netmask = netmask[0]
                prefixlen = shared.ipv6_prefixlength(ipaddress.IPv6Address(unicode(netmask)))
            else:
                prefixlen = ipaddress.IPv4Network(unicode('0.0.0.0/' + netmask)).prefixlen
            ip = shared.IP(ip, prefixlen, name)
            add_ip(name, ip)
        addr = addr[0].ifa_next

    return ips.values()<commit_msg>GH-1 workaround cygwin netmask family bug<commit_after># Copyright (c) 2014 Stefan C. Mueller

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import os
import ctypes.util
import ipaddress
import collections

import ifaddr._shared as shared
#from ifaddr._shared import sockaddr, Interface, sockaddr_to_ip, ipv6_prefixlength

class ifaddrs(ctypes.Structure):
    pass
ifaddrs._fields_ = [('ifa_next', ctypes.POINTER(ifaddrs)),
                    ('ifa_name', ctypes.c_char_p),
                    ('ifa_flags', ctypes.c_uint),
                    ('ifa_addr', ctypes.POINTER(shared.sockaddr)),
                    ('ifa_netmask', ctypes.POINTER(shared.sockaddr))]

libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)

def get_adapters():

    addr = ctypes.POINTER(ifaddrs)()
    retval = libc.getifaddrs(ctypes.byref(addr))
    if retval != 0:
        eno = ctypes.get_errno()
        raise OSError(eno, os.strerror(eno))

    ips = collections.OrderedDict()

    def add_ip(adapter_name, ip):
        if not adapter_name in ips:
            ips[adapter_name] = shared.Adapter(adapter_name, adapter_name, [])
        ips[adapter_name].ips.append(ip)

    while addr:
        name = addr[0].ifa_name
        ip = shared.sockaddr_to_ip(addr[0].ifa_addr)
        if ip:
            if addr[0].ifa_netmask and not addr[0].ifa_netmask[0].sa_familiy:
                addr[0].ifa_netmask[0].sa_familiy = addr[0].ifa_addr[0].sa_familiy
            netmask = shared.sockaddr_to_ip(addr[0].ifa_netmask)
            if isinstance(netmask, tuple):
                netmask = netmask[0]
                prefixlen = shared.ipv6_prefixlength(ipaddress.IPv6Address(unicode(netmask)))
            else:
                prefixlen = ipaddress.IPv4Network(unicode('0.0.0.0/' + netmask)).prefixlen
            ip = shared.IP(ip, prefixlen, name)
            add_ip(name, ip)
        addr = addr[0].ifa_next

    return ips.values()<|endoftext|>


<commit_before>import cv2
import numpy as np
from matplotlib import pyplot as plt

def pixel_differences(img, template):
    difference = 0.0
    for img_row, template_row in zip(img, template):
        for img_col, template_col in zip(img_row, template_row):
            normalised_template_alpha = template_col[3] / 255.0
            pixel_difference = abs(int(img_col[0]) - int(template_col[0]))
            pixel_difference += abs(int(img_col[1]) - int(template_col[1]))
            pixel_difference += abs(int(img_col[2]) - int(template_col[2]))
            # transparent parts of the tempalte aren't weighted as heavily
            pixel_difference *= normalised_template_alpha
            difference += pixel_difference
    return difference

img = cv2.imread('img/FishOnScreen.png', cv2.IMREAD_UNCHANGED)

img2 = img.copy()
template = cv2.imread('img/Fish.png', cv2.IMREAD_UNCHANGED)

img_width, img_height, img_depth = img.shape
template_width, template_height, template_depth  = template.shape

min_pixel_difference = pixel_differences(img, template)
min_difference_top_left = (0, 0)
row = 0
col = 1
while(row + template_height < img_height):
    while(col + template_width < img_width):
        pixel_difference = pixel_differences(img[row:, col:], template)
        if pixel_difference < min_pixel_difference:
            min_pixel_difference = pixel_difference
            min_difference_top_left = (col, row)
        col += 1
        print("process col")
    row += 1

img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGBA)
template= cv2.cvtColor(template, cv2.COLOR_BGRA2RGBA)

plt.imshow(img)
plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis
plt.show()

# All the 6 methods for comparison in a list
# methods = ['cv2.TM_CCOEFF', 'cv2.TM_CCOEFF_NORMED', 'cv2.TM_CCORR',
#             'cv2.TM_CCORR_NORMED', 'cv2.TM_SQDIFF', 'cv2.TM_SQDIFF_NORMED']
#
# for meth in methods:
#     img = img2.copy()
#     method = eval(meth)
#
#     # Apply template Matching
#     res = cv2.matchTemplate(img,template,method)
#     min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
#
#     # If the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum
#     if method in [cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED]:
#         top_left = min_loc
#     else:
#         top_left = max_loc
#     bottom_right = (top_left[0] + w, top_left[1] + h)
#
#     cv2.rectangle(img,top_left, bottom_right, 255, 2)
#
#     plt.subplot(121),plt.imshow(res,cmap = 'gray')
#     plt.title('Matching Result'), plt.xticks([]), plt.yticks([])
#     plt.subplot(122),plt.imshow(img,cmap = 'gray')
#     plt.title('Detected Point'), plt.xticks([]), plt.yticks([])
#     plt.suptitle(meth)
#
#     plt.show()
<commit_msg>Restore OpenCV template match functionality test<commit_after>import cv2
import numpy as np
from matplotlib import pyplot as plt

def pixel_differences(img, template):
    difference = 0.0
    for img_row, template_row in zip(img, template):
        for img_col, template_col in zip(img_row, template_row):
            normalised_template_alpha = template_col[3] / 255.0
            pixel_difference = abs(int(img_col[0]) - int(template_col[0]))
            pixel_difference += abs(int(img_col[1]) - int(template_col[1]))
            pixel_difference += abs(int(img_col[2]) - int(template_col[2]))
            # transparent parts of the tempalte aren't weighted as heavily
            pixel_difference *= normalised_template_alpha
            difference += pixel_difference
    return difference

#img = cv2.imread('img/FishOnScreen.png', cv2.IMREAD_UNCHANGED)
img = cv2.imread('img/FishOnScreen.png')

img2 = img.copy()
# template = cv2.imread('img/Fish.png', cv2.IMREAD_UNCHANGED)
template = cv2.imread('img/Fish.png')

img_width, img_height, img_depth = img.shape
template_width, template_height, template_depth  = template.shape
#
# min_pixel_difference = pixel_differences(img, template)
# min_difference_top_left = (0, 0)
# row = 0
# col = 1
# while(row + template_height < img_height):
#     while(col + template_width < img_width):
#         pixel_difference = pixel_differences(img[row:, col:], template)
#         if pixel_difference < min_pixel_difference:
#             min_pixel_difference = pixel_difference
#             min_difference_top_left = (col, row)
#         col += 1
#         print("process col")
#     row += 1
#
#
# img = cv2.cvtColor(img, cv2.COLOR_BGRA2RGBA)
# template= cv2.cvtColor(template, cv2.COLOR_BGRA2RGBA)
#
# plt.imshow(img)
# plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis
# plt.show()

# All the 6 methods for comparison in a list
methods = ['cv2.TM_CCOEFF', 'cv2.TM_CCOEFF_NORMED', 'cv2.TM_CCORR',
            'cv2.TM_CCORR_NORMED', 'cv2.TM_SQDIFF', 'cv2.TM_SQDIFF_NORMED']

for meth in methods:
    img = img2.copy()
    method = eval(meth)

    # Apply template Matching
    res = cv2.matchTemplate(img,template,method)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)

    # If the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum
    if method in [cv2.TM_SQDIFF, cv2.TM_SQDIFF_NORMED]:
        top_left = min_loc
    else:
        top_left = max_loc
    bottom_right = (top_left[0] + template_width, top_left[1] + template_height)

    cv2.rectangle(img,top_left, bottom_right, 255, 2)

    plt.subplot(121),plt.imshow(res,cmap = 'gray')
    plt.title('Matching Result'), plt.xticks([]), plt.yticks([])
    plt.subplot(122),plt.imshow(img,cmap = 'gray')
    plt.title('Detected Point'), plt.xticks([]), plt.yticks([])
    plt.suptitle(meth)

    plt.show()
<|endoftext|>


<commit_before>"""ファイル・フォルダ関連"""
import pathlib

def delete_file(file_path):
    """ファイル削除"""
    file_path = pathlib.Path(file_path)
    if file_path.is_file():
        file_path.unlink()

def read_all_lines(file_path, mode='r', encoding='utf-8'):
    """ファイルの全行を読み込み。戻り値は改行無しの配列。.NETのSystem.IO.File.ReadAllLine()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        return [l.rstrip('\n') for l in f.readlines()]

def write_all_lines(file_path, lines, mode='w', encoding='utf-8'):
    """ファイルの全行を書き込み。linesは改行無しの文字列の配列。.NETのSystem.IO.File.WriteAllLine()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        f.writelines([l + '\n' for l in lines])

def read_all_text(file_path, mode='r', encoding='utf-8'):
    """ファイルの全行を読み込み。戻り値は改行無しの配列。.NETのSystem.IO.File.ReadAllText()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        return f.read()

def write_all_text(file_path, text, mode='w', encoding='utf-8'):
    """ファイルの全行を書き込み。linesは改行無しの文字列の配列。.NETのSystem.IO.File.WriteAllText()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        f.write(text)
<commit_msg>リトライのutilityを追加<commit_after>"""ファイル・フォルダ関連"""
import pathlib
import time

def delete_file(file_path):
    """ファイル削除"""
    file_path = pathlib.Path(file_path)
    if file_path.is_file():
        file_path.unlink()

def read_all_lines(file_path, mode='r', encoding='utf-8'):
    """ファイルの全行を読み込み。戻り値は改行無しの配列。.NETのSystem.IO.File.ReadAllLine()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        return [l.rstrip('\n') for l in f.readlines()]

def write_all_lines(file_path, lines, mode='w', encoding='utf-8'):
    """ファイルの全行を書き込み。linesは改行無しの文字列の配列。.NETのSystem.IO.File.WriteAllLine()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        f.writelines([l + '\n' for l in lines])

def read_all_text(file_path, mode='r', encoding='utf-8'):
    """ファイルの全行を読み込み。戻り値は改行無しの配列。.NETのSystem.IO.File.ReadAllText()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        return f.read()

def write_all_text(file_path, text, mode='w', encoding='utf-8'):
    """ファイルの全行を書き込み。linesは改行無しの文字列の配列。.NETのSystem.IO.File.WriteAllText()風。"""
    with pathlib.Path(file_path).open(mode, encoding=encoding) as f:
        f.write(text)

def do_retry(func, count=10, sleep_seconds=1.0):
    """リトライ処理。"""
    retry = 0
    while True:
        try:
            return func()
        except:
            if retry >= count:
                break
            retry += 1
            time.sleep(sleep_seconds)
<|endoftext|>


<commit_before>def setGalleryImg(*tuplePhoto):

	return tuple((count, tuplePhoto[count]) for count in range(len(tuplePhoto)))

def setTemaData(*tupleData):

	return dict({"name":tupleData})

def getContent():

	volodink = {"name": "Константин Володин",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc35/KO9Y64lP6io.jpg",
				"job_title": "Научный руководитель",
				"comment": ""}

	kulikovo = {"name": "Олег Куликов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Технический консультант",
				"comment": ""}

	garkaeva = {"name": "Александр Гарькаев",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	stramovd = {"name": "Дмитрий Страмов",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc4b/slIt41ns4jY.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	vavilova = {"name": "Андрей Вавилов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	caplinm = {"name": "Михаил Цаплин",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	baikova = {"name": "Антон Байков",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	volckovs = {"name": "Сергей Волчков",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	kopylovv = {"name": "Вадим Копылов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	kolginp = {"name": "Паша Колодкин",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	volkova = {"name": "Александр Волков",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc85/PNEW2q7TdbQ.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	kedelidzeo = {"name": "Кеделидзе Олег",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc44/5OsS6C-RmQ8.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	mesherinovae =  {"name": "Евгения Мещеринова",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc8c/d83ROGuf4Pc.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	main = {"title": "itstime4science | Главная",
			"head_photo": "https://pp.vk.me/c637528/v637528262/1976f/rx5wxLsuZyQ.jpg",
			"head_title": "ВВиВС",
			"head_footer": "Добро пожаловать на интернет-портал лаборатории ВВиВС (ПензГТУ)!",
			"gallery": setGalleryImg("https://pp.vk.me/c636323/v636323235/44666/CYzELG7-Lf0.jpg",
					 		  "https://pp.vk.me/c636323/v636323235/449d1/zsmPgBHN2yM.jpg",
					          "https://pp.vk.me/c636323/v636323235/449ec/EJ6mhheOnXQ.jpg"),
			"team_photo": {"team_row_title": "Сотрудники лаборатории ВВиВС",
							"people": (volodink,
										kulikovo,
										garkaeva,
										kedelidzeo,
										stramovd,
										mesherinovae,
										vavilova,
										caplinm,
										baikova,
										volckovs,
										kopylovv,
										kolginp,
										volkova)}
			}

	copter = {"title": "itstime4science | Коптер",
			  "head_photo": "https://pp.vk.me/c636728/v636728996/52d16/2oWW37OQqEI.jpg",
			  "head_title": "Коптер",
			  "head_footer": "Тут будет текст",
			  "gallery": setGalleryImg("https://pp.vk.me/c636323/v636323235/43d6f/HfCDSO1iE_w.jpg",
								"https://pp.vk.me/c836227/v836227262/1db5d/Oao-irsb8Sc.jpg",
								"https://pp.vk.me/c636323/v636323235/44708/oaaCdr02RSU.jpg"),
			  "team_photo": {"team_row_title": 'Команда "Копер"',
							"people": (kulikovo,
										garkaeva,
										stramovd,
										vavilova,
										baikova)}
			}

	satellite = {"title": "itstime4science | Студенческая лига",
				 "head_photo": "http://www.roscosmos.ru/media/gallery/big/20544/744950758.jpg",
				 "head_title": "Студенческая лига",
				 "head_footer": "Тут будет текст",
				 "gallery": setGalleryImg("https://i.ytimg.com/vi/SXW5eAjPD2U/maxresdefault.jpg",
				 				   "http://i.ytimg.com/vi/teArassP_E4/maxresdefault.jpg"),
				 "team_photo": {"team_row_title": 'Команда "Спутник"',
							"people": (kulikovo,
										garkaeva,
										kedelidzeo,
										caplinm,
										volckovs,
										kopylovv,
										kolginp)}
				}

	return {"main": main,
			"copter": copter,
			"satellite": satellite}

if __name__ == "__main__":

	lal = getContent()

	for i in lal["main"]["gallery"]:
		print(i)
<commit_msg>Update content.py<commit_after>def setGalleryImg(*tuplePhoto):

	return tuple((count, tuplePhoto[count]) for count in range(len(tuplePhoto)))

def setTemaData(*tupleData):

	return dict({"name":tupleData})

def getContent():

	volodink = {"name": "Константин Володин",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc35/KO9Y64lP6io.jpg",
				"job_title": "Научный руководитель",
				"comment": ""}

	kulikovo = {"name": "Олег Куликов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Технический консультант",
				"comment": ""}

	garkaeva = {"name": "Александр Гарькаев",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	stramovd = {"name": "Дмитрий Страмов",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc4b/slIt41ns4jY.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	vavilova = {"name": "Андрей Вавилов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	caplinm = {"name": "Михаил Цаплин",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	baikova = {"name": "Антон Байков",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	volckovs = {"name": "Сергей Волчков",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	kopylovv = {"name": "Вадим Копылов",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	kolginp = {"name": "Паша Колодкин",
				"link_to_photo": "http://placehold.it/200x200",
				"job_title": "Лаборант",
				"comment": ""}

	volkova = {"name": "Александр Волков",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc85/PNEW2q7TdbQ.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	kedelidzeo = {"name": "Кеделидзе Олег",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc44/5OsS6C-RmQ8.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	mesherinovae =  {"name": "Евгения Мещеринова",
				"link_to_photo": "https://pp.vk.me/c638023/v638023262/1bc8c/d83ROGuf4Pc.jpg",
				"job_title": "Лаборант",
				"comment": ""}

	main = {"title": "itstime4science | Главная",
			"head_photo": "https://pp.vk.me/c637528/v637528262/1976f/rx5wxLsuZyQ.jpg",
			"head_title": "ВВиВС",
			"head_footer": "Добро пожаловать на интернет-портал лаборатории ВВиВС (ПензГТУ)!",
			"gallery": setGalleryImg("https://pp.vk.me/c636323/v636323235/44666/CYzELG7-Lf0.jpg",
					 		  "https://pp.vk.me/c636323/v636323235/449d1/zsmPgBHN2yM.jpg",
					          "https://pp.vk.me/c636323/v636323235/449ec/EJ6mhheOnXQ.jpg"),
			"team_photo": {"team_row_title": "Сотрудники лаборатории ВВиВС",
							"people": (volodink,
										kulikovo,
										garkaeva,
										kedelidzeo,
										stramovd,
										mesherinovae,
										vavilova,
										caplinm,
										baikova,
										volckovs,
										kopylovv,
										kolginp,
										volkova)}
			}

	copter = {"title": "itstime4science | Коптер",
			  "head_photo": "https://pp.vk.me/c636728/v636728996/52d16/2oWW37OQqEI.jpg",
			  "head_title": "Коптер",
			  "head_footer": "Тут будет текст",
			  "gallery": setGalleryImg("https://pp.vk.me/c636323/v636323235/43d6f/HfCDSO1iE_w.jpg",
								"https://pp.vk.me/c836227/v836227262/1db5d/Oao-irsb8Sc.jpg",
								"https://pp.vk.me/c636323/v636323235/44708/oaaCdr02RSU.jpg"),
			  "team_photo": {"team_row_title": 'Команда "Коптер"',
							"people": (kulikovo,
										garkaeva,
										stramovd,
										vavilova,
										baikova)}
			}

	satellite = {"title": "itstime4science | Студенческая лига",
				 "head_photo": "http://www.roscosmos.ru/media/gallery/big/20544/744950758.jpg",
				 "head_title": "Студенческая лига",
				 "head_footer": "Тут будет текст",
				 "gallery": setGalleryImg("https://i.ytimg.com/vi/SXW5eAjPD2U/maxresdefault.jpg",
				 				   "http://i.ytimg.com/vi/teArassP_E4/maxresdefault.jpg"),
				 "team_photo": {"team_row_title": 'Команда "Спутник"',
							"people": (kulikovo,
										garkaeva,
										kedelidzeo,
										caplinm,
										volckovs,
										kopylovv,
										kolginp)}
				}

	return {"main": main,
			"copter": copter,
			"satellite": satellite}

if __name__ == "__main__":

	lal = getContent()

	for i in lal["main"]["gallery"]:
		print(i)
<|endoftext|>