<commit_before># -*- coding: utf-8 -*-

#    Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from __future__ import print_function

try:
    from collections import OrderedDict  # noqa
except ImportError:
    from ordereddict import OrderedDict  # noqa

import collections
import contextlib
import json
import os
import shutil
import sys
import tempfile
import traceback

from distutils import version as dist_version

from pip import req as pip_req

import argparse
import requests
import six
from six.moves import urllib

try:
    from pip import util as pip_util  # noqa
except ImportError:
    from pip import utils as pip_util  # noqa

_FINDER_URL_TPL = 'http://pypi.python.org/pypi/%s/json'

# Egg info cache and url fetch caches...
_EGGS_DETAILED = {}
_FINDER_LOOKUPS = {}
_EGGS_FAILED_DETAILED = {}
_MAX_PRIOR_VERSIONS = -1

class RequirementException(Exception):
    pass

class NotFoundException(Exception):
    pass

_MatchedRelease = collections.namedtuple('_MatchedRelease',
                                         ['string_version',
                                          'parsed_version',
                                          'origin_url',
                                          'origin_filename',
                                          'origin_size'])

def req_key(req):
    return req.req.key

@contextlib.contextmanager
def tempdir(**kwargs):
    # This seems like it was only added in python 3.2
    # Make it since its useful...
    # See: http://bugs.python.org/file12970/tempdir.patch
    tdir = tempfile.mkdtemp(**kwargs)
    try:
        yield tdir
    finally:
        shutil.rmtree(tdir)

def get_directory_details(path):
    if not os.path.isdir(path):
        raise IOError("Can not detail non-existent directory %s" % (path))
    req = pip_req.InstallRequirement.from_line(path)
    req.source_dir = path
    req.run_egg_info()
    dependencies = []
    for d in req.requirements():
        if not d.startswith("-e") and d.find("#"):
            d = d.split("#")[0]
        d = d.strip()
        if d:
            dependencies.append(d)
    details = {
        'req': req.req,
        'dependencies': dependencies,
        'name': req.name,
        'pkg_info': req.pkg_info(),
        'dependency_links': req.dependency_links,
        'version': req.installed_version,
    }
    return details

def get_archive_details(filename, filesize, options, prefix=""):
    if not os.path.isfile(filename):
        raise IOError("Can not detail non-existent file %s" % (filename))
    cache_key = "f:%s:%s" % (os.path.basename(filename), filesize)
    if cache_key in _EGGS_FAILED_DETAILED:
        exc_type, exc_value, exc_traceback = _EGGS_FAILED_DETAILED[cache_key]
        six.reraise(exc_type, exc_value, exc_traceback)
    try:
        return _EGGS_DETAILED[cache_key]
    except KeyError:
        if options.verbose:
            print("%s: Extracting egg-info from '%s'"
                  % (prefix, os.path.basename(filename)))
        with tempdir() as a_dir:
            arch_filename = os.path.join(a_dir, os.path.basename(filename))
            shutil.copyfile(filename, arch_filename)
            extract_to = os.path.join(a_dir, 'build')
            os.makedirs(extract_to)
            pip_util.unpack_file(arch_filename, extract_to,
                                 content_type='', link='')
            try:
                details = get_directory_details(extract_to)
            except Exception:
                # Don't bother saving the traceback (we don't care about it...)
                exc_type, exc_value, exc_traceback = sys.exc_info()
                _EGGS_FAILED_DETAILED[cache_key] = (exc_type, exc_value, None)
                raise
            else:
                _EGGS_DETAILED[cache_key] = details
                return details

def create_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-r", "--requirement",
        dest="requirements",
        nargs="+",
        default=[],
        metavar="<file>",
        help="Analyze all the packages listed in the given requirements file")
    parser.add_argument(
        "-s", "--scratch",
        dest="scratch",
        default=os.getcwd(),
        metavar="<path>",
        help="Scratch path (used for caching downloaded data)"
             " [default: %s]" % (os.getcwd()))
    parser.add_argument(
        "--no-verbose",
        dest="verbose",
        action='store_false',
        default=True,
        help="Disable verbose output")
    return parser

def download_url_to(url, save_path, options, size=None, prefix=""):
    if size is not None:
        kb_size = size // 1024
        if options.verbose:
            print("%s: Downloading '%s' (%skB) -> '%s'" % (prefix, url,
                                                           kb_size, save_path))
    else:
        if options.verbose:
            print("%s: Downloading '%s' -> '%s'" % (prefix, url, save_path))
    resp = requests.get(url)
    with open(save_path, 'wb') as fh:
        fh.write(resp.content)
    return resp.content

def parse_requirements(options):
    requirements = OrderedDict()
    for filename in options.requirements:
        try:
            for req in pip_req.parse_requirements(filename):
                if req_key(req) in requirements:
                    raise ValueError("Currently only one requirement for '%s'"
                                     " is allowed, merging is not currently"
                                     " supported" % req_key(req))
                requirements[req_key(req)] = req
        except Exception as ex:
            raise IOError("Cannot parse '%s': %s" % (filename, ex))
    return requirements

def find_versions(pkg_name, options, prefix=""):
    def sorter(r1, r2):
        return cmp(r1[1], r2[1])
    url = _FINDER_URL_TPL % (urllib.parse.quote(pkg_name))
    if url in _FINDER_LOOKUPS:
        return _FINDER_LOOKUPS[url]
    version_path = os.path.join(options.scratch,
                                ".versions", "%s.json" % pkg_name)
    if os.path.exists(version_path):
        with open(version_path, 'rb') as fh:
            resp_data = json.loads(fh.read())
    else:
        resp_data = json.loads(download_url_to(url,
                                               version_path,
                                               options, prefix=prefix))
    releases = []
    for v, release_infos in six.iteritems(resp_data['releases']):
        rel = rel_fn = rel_size = None
        for r in release_infos:
            if r['packagetype'] == 'sdist':
                rel = r['url']
                rel_fn = r['filename']
                rel_size = r['size']
        if not all([rel, rel_fn, rel_size]):
            print("ERROR: no sdist found for '%s==%s'"
                  % (pkg_name, v), file=sys.stderr)
            continue
        try:
            m_rel = _MatchedRelease(
                v, dist_version.LooseVersion(v),
                rel, rel_fn, rel_size)
            releases.append(m_rel)
        except ValueError:
            print("ERROR: failed parsing '%s==%s'"
                  % (pkg_name, v), file=sys.stderr)
    _FINDER_LOOKUPS[url] = sorted(releases, cmp=sorter)
    return _FINDER_LOOKUPS[url]

def fetch_details(req, options, prefix=""):
    origin_filename = req.origin_filename
    origin_url = req.origin_url
    download_path = os.path.join(options.scratch,
                                 '.download', origin_filename)
    if not os.path.exists(download_path):
        download_url_to(origin_url, download_path, options,
                        size=req.origin_size, prefix=prefix)
    return get_archive_details(download_path, req.origin_size, options,
                               prefix=prefix)

def match_available(req, available, options, prefix=""):
    looked_in = []
    useables = []
    for a in reversed(available):
        v = a.string_version
        if v in req:
            line = "%s==%s" % (req.key, v)
            m_req = pip_req.InstallRequirement.from_line(line)
            if options.verbose:
                print("%s: Found '%s' as able to satisfy '%s'"
                      % (prefix, m_req, req))
            m_req.origin_url = a.origin_url
            m_req.origin_filename = a.origin_filename
            m_req.origin_size = a.origin_size
            useables.append(m_req)
            if len(useables) == _MAX_PRIOR_VERSIONS:
                break
        else:
            looked_in.append(v)
    if not useables:
        raise NotFoundException("No requirement found that"
                                " matches '%s' (tried %s)" % (req, looked_in))
    else:
        return useables

def check_is_compatible_alongside(pkg_req, gathered,
                                  options, prefix=""):
    if options.verbose:
        print("%s: Checking if '%s' is compatible along-side:" % (prefix,
                                                                  pkg_req))
        for req_name, other_req in six.iteritems(gathered):
            if req_key(pkg_req) == req_name:
                continue
            print("%s: - %s==%s" % (prefix, req_name,
                                    other_req.details['version']))
            for other_dep in other_req.details['dependencies']:
                print("%s:  + %s" % (prefix, other_dep))
    # If we conflict with the currently gathered requirements, give up...
    for req_name, other_req in six.iteritems(gathered):
        if req_key(pkg_req) == req_name:
            if pkg_req.details['version'] not in other_req.req:
                raise RequirementException("'%s==%s' not in '%s'"
                                           % (pkg_req.details['name'],
                                              pkg_req.details['version'],
                                              other_req))

def probe(requirements, gathered, options, levels):
    if not requirements:
        return gathered
    # Pick one of the requirements, get a version that works with the
    # current known siblings (other requirements that are requested along
    # side this requirement) and then recurse trying to get another
    # requirement that will work, if this is not possible, backtrack and
    # try a different version instead (and repeat)...
    chapters = []
    for c in levels:
        if not chapters:
            chapters.append([c, 1])
        else:
            if c == chapters[-1][0]:
                chapters[-1][1] += 1
            else:
                chapters.append([c, 1])
    prefix = ' %s' % (".".join(str(v) for (_c, v) in chapters))
    gathered = gathered.copy()
    requirements = requirements.copy()
    pkg_name, pkg_req = requirements.popitem()
    print("%s: Probing for valid match for %s" % (prefix, pkg_req))
    possibles = match_available(pkg_req.req,
                                find_versions(pkg_name, options,
                                              prefix=prefix),
                                options,
                                prefix=prefix)
    for m in possibles:
        if not hasattr(m, 'details'):
            try:
                m.details = fetch_details(m, options, prefix=prefix)
            except Exception as e:
                print("ERROR: failed detailing '%s'"
                      % (m), file=sys.stderr)
                e_blob = str(e)
                for line in e_blob.splitlines():
                    print("%s" % (line), file=sys.stderr)
        if not hasattr(m, 'details'):
            continue
        if pkg_name in gathered:
            if m.details['version'] not in gathered[pkg_name].req:
                continue
        gathered[pkg_name] = m
        try:
            check_is_compatible_alongside(m, gathered, options,
                                          prefix=prefix)
            # Search the versions of this package which will work and now
            # deeply expand there dependencies to see if any of those
            # cause issues...
            if m.details['dependencies']:
                deep_requirements = OrderedDict()
                print("%s: Checking if '%s' dependencies are compatible..."
                      % (prefix, m))
                # NOTE: not including the active requirements in this
                # list does limit the scan-space, and may discount
                # various combinations (especially on failures) but this
                # should be ok enough...
                for dep in m.details['dependencies']:
                    d_req = pip_req.InstallRequirement.from_line(dep)
                    deep_requirements[req_key(d_req)] = d_req
                levels.append('d')
                try:
                    probe(deep_requirements, gathered, options, levels)
                finally:
                    levels.pop()
            levels.append('p')
            try:
                result = probe(requirements, gathered, options, levels)
            finally:
                levels.pop()
        except RequirementException as e:
            if options.verbose:
                print("%s: Undoing decision to select '%s'"
                      " due to %s" % (prefix, m, e))
            gathered.pop(pkg_name)
        else:
            gathered.update(result)
            return gathered
    raise RequirementException("No working requirement found for '%s'"
                               % pkg_req)

def main():
    def req_cmp(a, b):
        return cmp(req_key(a), req_key(b))
    parser = create_parser()
    options = parser.parse_args()
    if not options.requirements:
        parser.error("At least one requirement file must be provided")
    initial = parse_requirements(options)
    for d in ['.download', '.versions']:
        scratch_path = os.path.join(options.scratch, d)
        if not os.path.isdir(scratch_path):
            os.makedirs(scratch_path)
    print("+ Initial package set:")
    for r in sorted(list(six.itervalues(initial)), cmp=req_cmp):
        print(" - %s" % r)
    print("+ Probing for a valid set...")
    matches = OrderedDict()
    levels = ['p']
    try:
        matches = probe(initial, matches, options, levels)
    except Exception:
        traceback.print_exc(file=sys.stdout)
    else:
        print("+ Expanded package set:")
        for r in sorted(list(six.itervalues(matches)), cmp=req_cmp):
            print(" - %s" % r)

if __name__ == "__main__":
    main()
<commit_msg>Adjust due to cause print statement<commit_after># -*- coding: utf-8 -*-

#    Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from __future__ import print_function

try:
    from collections import OrderedDict  # noqa
except ImportError:
    from ordereddict import OrderedDict  # noqa

import collections
import contextlib
import json
import os
import shutil
import sys
import tempfile
import traceback

from distutils import version as dist_version

from pip import req as pip_req

import argparse
import requests
import six
from six.moves import urllib

try:
    from pip import util as pip_util  # noqa
except ImportError:
    from pip import utils as pip_util  # noqa

_FINDER_URL_TPL = 'http://pypi.python.org/pypi/%s/json'

# Egg info cache and url fetch caches...
_EGGS_DETAILED = {}
_FINDER_LOOKUPS = {}
_EGGS_FAILED_DETAILED = {}
_MAX_PRIOR_VERSIONS = -1

class RequirementException(Exception):
    pass

class NotFoundException(Exception):
    pass

_MatchedRelease = collections.namedtuple('_MatchedRelease',
                                         ['string_version',
                                          'parsed_version',
                                          'origin_url',
                                          'origin_filename',
                                          'origin_size'])

def req_key(req):
    return req.req.key

@contextlib.contextmanager
def tempdir(**kwargs):
    # This seems like it was only added in python 3.2
    # Make it since its useful...
    # See: http://bugs.python.org/file12970/tempdir.patch
    tdir = tempfile.mkdtemp(**kwargs)
    try:
        yield tdir
    finally:
        shutil.rmtree(tdir)

def get_directory_details(path):
    if not os.path.isdir(path):
        raise IOError("Can not detail non-existent directory %s" % (path))
    req = pip_req.InstallRequirement.from_line(path)
    req.source_dir = path
    req.run_egg_info()
    dependencies = []
    for d in req.requirements():
        if not d.startswith("-e") and d.find("#"):
            d = d.split("#")[0]
        d = d.strip()
        if d:
            dependencies.append(d)
    details = {
        'req': req.req,
        'dependencies': dependencies,
        'name': req.name,
        'pkg_info': req.pkg_info(),
        'dependency_links': req.dependency_links,
        'version': req.installed_version,
    }
    return details

def get_archive_details(filename, filesize, options, prefix=""):
    if not os.path.isfile(filename):
        raise IOError("Can not detail non-existent file %s" % (filename))
    cache_key = "f:%s:%s" % (os.path.basename(filename), filesize)
    if cache_key in _EGGS_FAILED_DETAILED:
        exc_type, exc_value, exc_traceback = _EGGS_FAILED_DETAILED[cache_key]
        six.reraise(exc_type, exc_value, exc_traceback)
    try:
        return _EGGS_DETAILED[cache_key]
    except KeyError:
        if options.verbose:
            print("%s: Extracting egg-info from '%s'"
                  % (prefix, os.path.basename(filename)))
        with tempdir() as a_dir:
            arch_filename = os.path.join(a_dir, os.path.basename(filename))
            shutil.copyfile(filename, arch_filename)
            extract_to = os.path.join(a_dir, 'build')
            os.makedirs(extract_to)
            pip_util.unpack_file(arch_filename, extract_to,
                                 content_type='', link='')
            try:
                details = get_directory_details(extract_to)
            except Exception:
                # Don't bother saving the traceback (we don't care about it...)
                exc_type, exc_value, exc_traceback = sys.exc_info()
                _EGGS_FAILED_DETAILED[cache_key] = (exc_type, exc_value, None)
                raise
            else:
                _EGGS_DETAILED[cache_key] = details
                return details

def create_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-r", "--requirement",
        dest="requirements",
        nargs="+",
        default=[],
        metavar="<file>",
        help="Analyze all the packages listed in the given requirements file")
    parser.add_argument(
        "-s", "--scratch",
        dest="scratch",
        default=os.getcwd(),
        metavar="<path>",
        help="Scratch path (used for caching downloaded data)"
             " [default: %s]" % (os.getcwd()))
    parser.add_argument(
        "--no-verbose",
        dest="verbose",
        action='store_false',
        default=True,
        help="Disable verbose output")
    return parser

def download_url_to(url, save_path, options, size=None, prefix=""):
    if size is not None:
        kb_size = size // 1024
        if options.verbose:
            print("%s: Downloading '%s' (%skB) -> '%s'" % (prefix, url,
                                                           kb_size, save_path))
    else:
        if options.verbose:
            print("%s: Downloading '%s' -> '%s'" % (prefix, url, save_path))
    resp = requests.get(url)
    with open(save_path, 'wb') as fh:
        fh.write(resp.content)
    return resp.content

def parse_requirements(options):
    requirements = OrderedDict()
    for filename in options.requirements:
        try:
            for req in pip_req.parse_requirements(filename):
                if req_key(req) in requirements:
                    raise ValueError("Currently only one requirement for '%s'"
                                     " is allowed, merging is not currently"
                                     " supported" % req_key(req))
                requirements[req_key(req)] = req
        except Exception as ex:
            raise IOError("Cannot parse '%s': %s" % (filename, ex))
    return requirements

def find_versions(pkg_name, options, prefix=""):
    def sorter(r1, r2):
        return cmp(r1[1], r2[1])
    url = _FINDER_URL_TPL % (urllib.parse.quote(pkg_name))
    if url in _FINDER_LOOKUPS:
        return _FINDER_LOOKUPS[url]
    version_path = os.path.join(options.scratch,
                                ".versions", "%s.json" % pkg_name)
    if os.path.exists(version_path):
        with open(version_path, 'rb') as fh:
            resp_data = json.loads(fh.read())
    else:
        resp_data = json.loads(download_url_to(url,
                                               version_path,
                                               options, prefix=prefix))
    releases = []
    for v, release_infos in six.iteritems(resp_data['releases']):
        rel = rel_fn = rel_size = None
        for r in release_infos:
            if r['packagetype'] == 'sdist':
                rel = r['url']
                rel_fn = r['filename']
                rel_size = r['size']
        if not all([rel, rel_fn, rel_size]):
            print("ERROR: no sdist found for '%s==%s'"
                  % (pkg_name, v), file=sys.stderr)
            continue
        try:
            m_rel = _MatchedRelease(
                v, dist_version.LooseVersion(v),
                rel, rel_fn, rel_size)
            releases.append(m_rel)
        except ValueError:
            print("ERROR: failed parsing '%s==%s'"
                  % (pkg_name, v), file=sys.stderr)
    _FINDER_LOOKUPS[url] = sorted(releases, cmp=sorter)
    return _FINDER_LOOKUPS[url]

def fetch_details(req, options, prefix=""):
    origin_filename = req.origin_filename
    origin_url = req.origin_url
    download_path = os.path.join(options.scratch,
                                 '.download', origin_filename)
    if not os.path.exists(download_path):
        download_url_to(origin_url, download_path, options,
                        size=req.origin_size, prefix=prefix)
    return get_archive_details(download_path, req.origin_size, options,
                               prefix=prefix)

def match_available(req, available, options, prefix=""):
    looked_in = []
    useables = []
    for a in reversed(available):
        v = a.string_version
        if v in req:
            line = "%s==%s" % (req.key, v)
            m_req = pip_req.InstallRequirement.from_line(line)
            if options.verbose:
                print("%s: Found '%s' as able to satisfy '%s'"
                      % (prefix, m_req, req))
            m_req.origin_url = a.origin_url
            m_req.origin_filename = a.origin_filename
            m_req.origin_size = a.origin_size
            useables.append(m_req)
            if len(useables) == _MAX_PRIOR_VERSIONS:
                break
        else:
            looked_in.append(v)
    if not useables:
        raise NotFoundException("No requirement found that"
                                " matches '%s' (tried %s)" % (req, looked_in))
    else:
        return useables

def check_is_compatible_alongside(pkg_req, gathered,
                                  options, prefix=""):
    if options.verbose:
        print("%s: Checking if '%s' is compatible along-side:" % (prefix,
                                                                  pkg_req))
        for req_name, other_req in six.iteritems(gathered):
            if req_key(pkg_req) == req_name:
                continue
            print("%s: - %s==%s" % (prefix, req_name,
                                    other_req.details['version']))
            for other_dep in other_req.details['dependencies']:
                print("%s:  + %s" % (prefix, other_dep))
    # If we conflict with the currently gathered requirements, give up...
    for req_name, other_req in six.iteritems(gathered):
        if req_key(pkg_req) == req_name:
            if pkg_req.details['version'] not in other_req.req:
                raise RequirementException("'%s==%s' not in '%s'"
                                           % (pkg_req.details['name'],
                                              pkg_req.details['version'],
                                              other_req))

def probe(requirements, gathered, options, levels):
    if not requirements:
        return gathered
    # Pick one of the requirements, get a version that works with the
    # current known siblings (other requirements that are requested along
    # side this requirement) and then recurse trying to get another
    # requirement that will work, if this is not possible, backtrack and
    # try a different version instead (and repeat)...
    chapters = []
    for c in levels:
        if not chapters:
            chapters.append([c, 1])
        else:
            if c == chapters[-1][0]:
                chapters[-1][1] += 1
            else:
                chapters.append([c, 1])
    prefix = ' %s' % (".".join(str(v) for (_c, v) in chapters))
    gathered = gathered.copy()
    requirements = requirements.copy()
    pkg_name, pkg_req = requirements.popitem()
    print("%s: Probing for valid match for %s" % (prefix, pkg_req))
    possibles = match_available(pkg_req.req,
                                find_versions(pkg_name, options,
                                              prefix=prefix),
                                options,
                                prefix=prefix)
    for m in possibles:
        if not hasattr(m, 'details'):
            try:
                m.details = fetch_details(m, options, prefix=prefix)
            except Exception as e:
                print("ERROR: failed detailing '%s'"
                      % (m), file=sys.stderr)
                e_blob = str(e)
                for line in e_blob.splitlines():
                    print("%s" % (line), file=sys.stderr)
        if not hasattr(m, 'details'):
            continue
        if pkg_name in gathered:
            if m.details['version'] not in gathered[pkg_name].req:
                continue
        gathered[pkg_name] = m
        try:
            check_is_compatible_alongside(m, gathered, options,
                                          prefix=prefix)
            # Search the versions of this package which will work and now
            # deeply expand there dependencies to see if any of those
            # cause issues...
            if m.details['dependencies']:
                deep_requirements = OrderedDict()
                print("%s: Checking if '%s' dependencies are compatible..."
                      % (prefix, m))
                # NOTE: not including the active requirements in this
                # list does limit the scan-space, and may discount
                # various combinations (especially on failures) but this
                # should be ok enough...
                for dep in m.details['dependencies']:
                    d_req = pip_req.InstallRequirement.from_line(dep)
                    deep_requirements[req_key(d_req)] = d_req
                levels.append('d')
                try:
                    probe(deep_requirements, gathered, options, levels)
                finally:
                    levels.pop()
            levels.append('p')
            try:
                result = probe(requirements, gathered, options, levels)
            finally:
                levels.pop()
        except RequirementException as e:
            if options.verbose:
                print("%s: Undoing decision to select '%s'"
                      " due to: %s" % (prefix, m, e))
            gathered.pop(pkg_name)
        else:
            gathered.update(result)
            return gathered
    raise RequirementException("No working requirement found for '%s'"
                               % pkg_req)

def main():
    def req_cmp(a, b):
        return cmp(req_key(a), req_key(b))
    parser = create_parser()
    options = parser.parse_args()
    if not options.requirements:
        parser.error("At least one requirement file must be provided")
    initial = parse_requirements(options)
    for d in ['.download', '.versions']:
        scratch_path = os.path.join(options.scratch, d)
        if not os.path.isdir(scratch_path):
            os.makedirs(scratch_path)
    print("+ Initial package set:")
    for r in sorted(list(six.itervalues(initial)), cmp=req_cmp):
        print(" - %s" % r)
    print("+ Probing for a valid set...")
    matches = OrderedDict()
    levels = ['p']
    try:
        matches = probe(initial, matches, options, levels)
    except Exception:
        traceback.print_exc(file=sys.stdout)
    else:
        print("+ Expanded package set:")
        for r in sorted(list(six.itervalues(matches)), cmp=req_cmp):
            print(" - %s" % r)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import rethinkdb as r
from typing import Dict

class TrafficData:

    def __init__(self):
        pass

    def insert_json_data(self, data: Dict) -> None:
        """
        inputs: data: Dict(json dictionary)

        The function ...

        For documentation of the file format, refer to
        http://traffic.cit.api.here.com/traffic/6.0/xsd/flow.xsd?app_id=F8aPRXcW3MmyUvQ8Z3J9&app_code=IVp1_zoGHdLdz0GvD_Eqsw

        return: None
        """
        ## insert the data into original_data table
        result = r.db('Traffic').table('original_data').insert(data).run()
        unique_id = result['generated_keys'][0]

        ##

        for RWS_item in data['RWS']:
            for RW_item in RWS_item['RW']:
                # will use RW_item['DE']
                for FIS_item in RW_item['FIS']:
                    for FI_item in FIS_item['FI']:
                        # our CUSTOM attributes
                        FI_item['CUSTOM'] = {
                            'parent_DE': RW_item['DE'],
                            'original_data_id': unique_id
                        }
                        r.db('Traffic').table('road_data').insert(FI_item).run()

    def read_traffic_data(url: str) -> Dict:
        pass

    def connect(self):
        r.connect('localhost', 28015).repl()

<commit_msg>tested insert_json_data() with rethinkdb and add helper_create_tables()<commit_after>import rethinkdb as r
from typing import Dict, List
import copy

class TrafficData:

    def __init__(self):
        pass

    def insert_json_data(self, data: Dict) -> None:
        """
        inputs: data: Dict(json dictionary)

        The function ...

        For documentation of the file format, refer to
        http://traffic.cit.api.here.com/traffic/6.0/xsd/flow.xsd?app_id=F8aPRXcW3MmyUvQ8Z3J9&app_code=IVp1_zoGHdLdz0GvD_Eqsw

        return: None
        """
        ## insert the data into original_data table
        result = r.db('Traffic').table('original_data').insert(data).run()
        original_data_id = result['generated_keys'][0]

        ## start parsing the data
        for RWS_item in data['RWS']:
            for RW_item in RWS_item['RW']:
                for FIS_item in RW_item['FIS']:
                    for FI_item in FIS_item['FI']:
                        # our CUSTOM attributes, record
                        # parent_DE: parent location description
                        # original_data_id: the original json document that we have downloaded
                        FI_item['CUSTOM'] = {
                            'parent_DE': RW_item['DE'],
                            'original_data_id': original_data_id
                        }
                        SHP_list = copy.deepcopy(FI_item['SHP'])
                        FI_item['SHP'] = "See table road_data"
                        result = r.db('Traffic').table('flow_data').insert(FI_item).run()
                        flow_data_id = result['generated_keys'][0]

                        for SHP_item in SHP_list:
                            SHP_item['flow_data_id'] = flow_data_id
                            try:
                                SHP_item['geometry'] = r.line(r.args(self.parse_SHP_values(SHP_item['value']))).run()
                                r.db('Traffic').table('road_data').insert(SHP_item).run()
                            except:
                                print('exception in parsing SHP values')
                                raise

    def parse_SHP_values(self, value: List) -> List:
        """
        inputs: value: List(the ['SHP']['values'])
        example inputs:
        ["34.9495,-82.43912 34.94999,-82.4392 34.95139,-82.4394 "], where each point is

        This function convert the values into a list that conforms line specification of RethinkDB.
        More specifically, in RethinkDB every point has the **opposite** parameters: (long, lat) while our
        data has point (lat, long). For more details,
        refer to https://rethinkdb.com/api/python/line/ and https://rethinkdb.com/api/javascript/point/

        example outputs:
        [[-82.43912, 34.9495], [-82.4392, 34.94999], [-82.4394, 34.95139]]
        """
        temp = value[0].split()
        result = []
        for item in temp:
            item = item.split(',')
            item = [float(i) for i in item]
             # we need to reverse the [lat, long] to [long, lat] to comply with
             # RethinkDB specifications at https://rethinkdb.com/api/javascript/point/
            item.reverse()
            result += [item]

        return result

    def read_traffic_data(url: str) -> Dict:
        pass

    def connect(self):
        r.connect('localhost', 28015).repl()

    def helper_create_tables(self):
        """
        This is a helper function to create tables in rethinkDB
        """
        r.db('Traffic').table_create('original_data').run()
        r.db('Traffic').table_create('road_data').run()
        r.db('Traffic').table_create('flow_data').run()
        r.db('Traffic').table('road_data').index_create('geometry', geo=True).run()

<|endoftext|>


<commit_before>#  File:       kindergarten_garden.py
#  Purpose:    Write a program that, given a diagram, can tell you which plants each child in the kindergarten class is responsible for.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Thursday 10th September 2016, 10:51 PM

class Garden(object):
    plant_names = {"C": "Clover", "G": "Grass", "R": "Radishes", "V": "Violets"}
    def __init__(self, blueprint,child):
         self.child=("Alice Bob Charlie David Eve Fred Ginny Harriet Ileana Joseph Kincaid Larry"
         self.child=sorted(child.split())
         self.cup_row=blueprint.split()

<commit_msg>Add plant methord<commit_after>#  File:       kindergarten_garden.py
#  Purpose:    Write a program that, given a diagram, can tell you which plants each child in the kindergarten class is responsible for.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Thursday 10th September 2016, 10:51 PM

class Garden(object):
    plant_names = {"C": "Clover", "G": "Grass", "R": "Radishes", "V": "Violets"}
    def __init__(self, blueprint, child):
         self.child = "Alice Bob Charlie David Eve Fred Ginny Harriet Ileana Joseph Kincaid Larry"
         self.child = sorted(child.split())
         self.cup_row = blueprint.split()
    def plant(self, child):
        self.cup_start = self.child.index(child) * 2
        cup = slice(cup_start,cup_start + 2)
        for item in (cup_row[0][cup] + cup_row[1][cup])
        return self.plant_names[item]
<|endoftext|>


<commit_before>import pprint
from utils import getHandPower

class Player:
    VERSION = "Default Python folding player"

    def get_hand(self, game_state):
        player = [elem['hole_cards'] for elem in game_state['players'] if elem['version'] == self.VERSION]
        return player

    def get_community_card(self, game_state):
        return game_state["community_cards"]

    def betRequest(self, game_state):
        print("#######################################")
        print("#######################################")
        print("                 GAME STATE                       ")
        pprint.pprint(game_state, width=1)
        print("#######################################")
        print("#######################################")
        hand = self.get_hand(game_state)
        community_cards = self.get_community_card(game_state)
        hand_power = getHandPower(hand)
        if hand_power >= 25:
            bet = 9999
        elif hand_power>= 20:
            bet = 105
        elif hand_power > 10:
            bet = 30
        else:
            bet = 0
        print("#######################################")
        print("                 OUR HAND                       ")
        print(hand)
        print("#######################################")
        print("#######################################")
        print("                 COMMUNITY CARDS                ")
        print(community_cards)
        print("#######################################")
        print("#######################################")
        print("                 HAND POWER                      ")
        print(hand_power)
        print("#######################################")
        print("#######################################")
        print("                 OUR BET               ")
        print(bet)
        return bet

    def get_winner_stats(self, game_state):
        winner = [elem for elem in game_state['players'] if elem['status'] == 'active'][0]
        winner_hand = winner['hole_cards']
        winner_hand_power = getHandPower(winner_hand)
        return {'winner': winner['name'],
                'hand': winner_hand,
                'hand_power': winner_hand_power}

    def showdown(self, game_state):
        winner_stats = self.get_winner_stats(game_state)
        print("#######################################")
        print("#######################################")
        print("                 PLAYERS ON SHOWDOWN            ")
        pprint.pprint(game_state['players'])
        print("#######################################")
        print("#######################################")
        print("            COMMUNITY CARDS IN SHOWDOWN         ")
        print(self.get_community_card(game_state))
        print("#######################################")
        print("#######################################")
        print("                WINNER STATS                          ")
        pprint.pprint(winner_stats, width=1)
        print("#######################################")
        print("#######################################")

<commit_msg>strategy<commit_after>import pprint
from utils import getHandPower

class Player:
    VERSION = "Default Python folding player"

    def get_hand(self, game_state):
        player = [elem['hole_cards'] for elem in game_state['players'] if elem['version'] == self.VERSION]
        return player

    def get_community_card(self, game_state):
        return game_state["community_cards"]

    def betRequest(self, game_state):
        print("#######################################")
        print("#######################################")
        print("                 GAME STATE                       ")
        pprint.pprint(game_state, width=1)
        print("#######################################")
        print("#######################################")
        hand = self.get_hand(game_state)
        community_cards = self.get_community_card(game_state)
        hand_power = getHandPower(hand)

        if hand_power >= 23:
            bet = 9999
        elif hand_power >= 21:
            bet = 300
        elif hand_power >= 19:
            bet = 200
        elif hand_power > 10:
            bet = 30
        else:
            bet = 0
        print("#######################################")
        print("                 OUR HAND                       ")
        print(hand)
        print("#######################################")
        print("#######################################")
        print("                 COMMUNITY CARDS                ")
        print(community_cards)
        print("#######################################")
        print("#######################################")
        print("                 HAND POWER                      ")
        print(hand_power)
        print("#######################################")
        print("#######################################")
        print("                 OUR BET               ")
        print(bet)
        return bet

    def get_winner_stats(self, game_state):
        winner = [elem for elem in game_state['players'] if elem['status'] == 'active'][0]
        winner_hand = winner['hole_cards']
        winner_hand_power = getHandPower(winner_hand)
        return {'winner': winner['name'],
                'hand': winner_hand,
                'hand_power': winner_hand_power}

    def showdown(self, game_state):
        winner_stats = self.get_winner_stats(game_state)
        print("#######################################")
        print("#######################################")
        print("                 PLAYERS ON SHOWDOWN            ")
        pprint.pprint(game_state['players'])
        print("#######################################")
        print("#######################################")
        print("            COMMUNITY CARDS IN SHOWDOWN         ")
        print(self.get_community_card(game_state))
        print("#######################################")
        print("#######################################")
        print("                WINNER STATS                          ")
        pprint.pprint(winner_stats, width=1)
        print("#######################################")
        print("#######################################")

<|endoftext|>


<commit_before>#coding: utf8

import logging
log = logging.getLogger(__name__)
from optparse import make_option
import os, os.path
import sys
import random

from zipfile import ZipFile
from tempfile import mkdtemp

from django.conf import settings
from django.contrib.gis.gdal import CoordTransform, DataSource, OGRGeometry, OGRGeomType
from django.core.management.base import BaseCommand
from django.db import connections, DEFAULT_DB_ALIAS, transaction
from django.template.defaultfilters import slugify

import boundaries
from boundaries.models import BoundarySet, Boundary, app_settings

GEOMETRY_COLUMN = 'shape'

class Command(BaseCommand):
    help = 'Import boundaries described by shapefiles.'
    option_list = BaseCommand.option_list + (
        make_option('-r', '--reload', action='store_true', dest='reload',
            help='Reload BoundarySets that have already been imported.'),
        make_option('-d', '--data-dir', action='store', dest='data_dir',
            default=app_settings.SHAPEFILES_DIR,
            help='Load shapefiles from this directory'),
        make_option('-e', '--except', action='store', dest='except',
            default=False, help='Don\'t load these kinds of Areas, comma-delimited.'),
        make_option('-o', '--only', action='store', dest='only',
            default=False, help='Only load these kinds of Areas, comma-delimited.'),
        make_option('-u', '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS, help='Specify a database to load shape data into.'),
        make_option('-c', '--color', action='store_true', dest='color',
            default=False, help='Automatically set colors to the boundaries.'),
    )

    def get_version(self):
        return '0.1'

    def handle(self, *args, **options):
        # Load configuration
        boundaries.autodiscover(options['data_dir'])

        all_sources = boundaries.registry

        if options['only']:
            only = options['only'].split(',')
            # TODO: stripping whitespace here because optparse doesn't handle it correctly
            sources = [s for s in all_sources if s.replace(' ', '') in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in all_sources if s.replace(' ', '') not in exceptions]
        else:
            sources = [s for s in all_sources]

        for kind, config in all_sources.items():
            if kind not in sources:
                log.debug('Skipping %s.' % kind)
                continue

            if (not options['reload']) and BoundarySet.objects.filter(slug=kind).exists():
                log.info('Already loaded %s, skipping.' % kind)
                continue

            self.load_set(kind, config, options)

    @transaction.commit_on_success
    def load_set(self, kind, config, options):
        log.info('Processing %s.' % kind)

        BoundarySet.objects.filter(slug=kind).delete()

        path = config['file']
        datasources = create_datasources(path)

        layer = datasources[0][0]

        # Add some default values
        if 'singular' not in config and kind.endswith('s'):
            config['singular'] = kind[:-1]
        if 'id_func' not in config:
            config['id_func'] = lambda f: ''
        if 'slug_func' not in config:
            config['slug_func'] = config['name_func']

        # Create BoundarySet
        bset = BoundarySet.objects.create(
            slug=kind,
            name=config.get('name', kind),
            singular=config['singular'],
            authority=config.get('authority', ''),
            domain=config.get('domain', ''),
            last_updated=config.get('last_updated'),
            source_url=config.get('source_url', ''),
            notes=config.get('notes', ''),
            licence_url=config.get('licence_url', ''),
        )

        for datasource in datasources:
            log.info("Loading %s from %s" % (kind, datasource.name))
            # Assume only a single-layer in shapefile
            if datasource.layer_count > 1:
                log.warn('%s shapefile [%s] has multiple layers, using first.' % (datasource.name, kind))
            layer = datasource[0]
            self.add_boundaries_for_layer(config, layer, bset, options['database'])

        if options["color"]:
            self.assign_colors(bset)

        log.info('%s count: %i' % (kind, Boundary.objects.filter(set=bset).count()))

    @staticmethod
    def polygon_to_multipolygon(geom):
        """
        Convert polygons to multipolygons so all features are homogenous in the database.
        """
        if geom.__class__.__name__ == 'Polygon':
            g = OGRGeometry(OGRGeomType('MultiPolygon'))
            g.add(geom)
            return g
        elif geom.__class__.__name__ == 'MultiPolygon':
            return geom
        else:
            raise ValueError('Geom is neither Polygon nor MultiPolygon.')

    def add_boundaries_for_layer(self, config, layer, bset, database):
        # Get spatial reference system for the postgis geometry field
        geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
        SpatialRefSys = connections[database].ops.spatial_ref_sys()
        db_srs = SpatialRefSys.objects.using(database).get(srid=geometry_field.srid).srs

        if 'srid' in config and config['srid']:
            layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
        else:
            layer_srs = layer.srs

        # Create a convertor to turn the source data into
        transformer = CoordTransform(layer_srs, db_srs)

        for feature in layer:
            geometry = feature.geom

            feature = UnicodeFeature(feature, encoding=config.get('encoding', 'ascii'))

            if config.get('is_valid_func', lambda feature : True)(feature) == False:
                continue

            # Transform the geometry to the correct SRS
            geometry = self.polygon_to_multipolygon(geometry)
            geometry.transform(transformer)

            # Create simplified geometry field by collapsing points within 1/1000th of a degree.
            # Since Chicago is at approx. 42 degrees latitude this works out to an margin of
            # roughly 80 meters latitude and 112 meters longitude.
            # Preserve topology prevents a shape from ever crossing over itself.
            simple_geometry = geometry.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True)

            # Conversion may force multipolygons back to being polygons
            simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)

            # Extract metadata into a dictionary
            metadata = dict(
                ( (field, feature.get(field)) for field in layer.fields )
            )

            external_id = str(config['id_func'](feature))
            feature_name = config['name_func'](feature)
            feature_slug = unicode(slugify(config['slug_func'](feature)).replace(u'—', '-'))

            log.info('%s...' % feature_slug)

            Boundary.objects.create(
                set=bset,
                set_name=bset.singular,
                external_id=external_id,
                name=feature_name,
                slug=feature_slug,
                metadata=metadata,
                shape=geometry.wkt,
                simple_shape=simple_geometry.wkt,
                centroid=geometry.geos.centroid,
                label_point=config.get("label_point_func", lambda x : None)(feature),
                color=config.get("color_func", lambda x : None)(feature),
                )

    @staticmethod
    def assign_colors(bset):
        # For each boundary in the set, assign a color such that it does not have the same
        # color as any other boundary it touches. Use the main colors from the Brewer spectrum,
        # based on http://colorbrewer2.org. This is done in a pretty dumb way: loop through
        # each boundary, query for each boundary it touches, look for a remaining color, and
        # then continue. In principle only four colors should be needed (the Four Color Theorem),
        # but finding a coloring that only uses four colors is algorithmically difficult. In practice,
        # around 8 is enough, and if we get stuck we just reuse a neighboring color --- oh well.
        color_choices = [ (44,162,95), (136,86,167), (67,162,202), (255, 237, 160), (240,59,32), (153,216,201), (158,188,218), (253,187,132), (166,189,219), (201,148,199) ]
        bset.boundaries.all().update(color=None)
        for bdry in bset.boundaries.all().only("shape"):
            used_colors = set()

            # What shapes are neighbors? 'Touches' is the right operator, but to be flexible
            # we use intersects, which will allow some overlap for poorly defined geometry.
            # Sometimes __intersects throws an error ("django.db.utils.DatabaseError: GEOS
            # intersects() threw an error!") and we'll just try to pass over those.
            try:
                # Looping over the polygons w/in the multipolygon isn't necessary.
                for part in (bdry.shape if bdry.shape.geom_type == "MultiPolygon" else [bdry.shape]):
                    qs = bset.boundaries.filter(shape__intersects=part).exclude(color=None).only("name", "color")
                    for b2 in qs:
                        used_colors.add(tuple(b2.color))
            except:
                print '%s had a problem looking for intersecting boundaries...' % bdry.slug
                bdry.color = random.choice(color_choices)
                bdry.save()
                continue

            # Choose the first available color. We prefer not to randomize so that a) this process
            # is relatively stable from run to run, and b) we can prioritize the colors we'd rather
            # use. The colors above are roughly from stronger to weaker.
            for c in color_choices:
                if c not in used_colors:
                    bdry.color = c
                    break
            else:
                # We ran out of colors. Just choose one at random.
                bdry.color = random.choice(color_choices)
            bdry.save()

def create_datasources(path):
    if path.endswith('.zip'):
        path = temp_shapefile_from_zip(path)

    if path.endswith('.shp'):
        return [DataSource(path)]

    # assume it's a directory...
    sources = []
    for fn in os.listdir(path):
        fn = os.path.join(path,fn)
        if fn.endswith('.zip'):
            fn = temp_shapefile_from_zip(fn)
        if fn.endswith('.shp'):
            sources.append(DataSource(fn))

    return sources

class UnicodeFeature(object):

    def __init__(self, feature, encoding='ascii'):
        self.feature = feature
        self.encoding = encoding

    def get(self, field):
        val = self.feature.get(field)
        if isinstance(val, str):
            return val.decode(self.encoding)
        return val

def temp_shapefile_from_zip(zip_path):
    """Given a path to a ZIP file, unpack it into a temp dir and return the path
       to the shapefile that was in there.  Doesn't clean up after itself unless
       there was an error.

       If you want to cleanup later, you can derive the temp dir from this path.
    """
    zf = ZipFile(zip_path)
    tempdir = mkdtemp()
    shape_path = None
    # Copy the zipped files to a temporary directory, preserving names.
    for name in zf.namelist():
        if name.endswith("/"): continue
        data = zf.read(name)
        outfile = os.path.join(tempdir, os.path.basename(name))
        if name.endswith('.shp'):
            shape_path = outfile
        f = open(outfile, 'w')
        f.write(data)
        f.close()

    if shape_path is None:
        log.warn("No shapefile, cleaning up")
        # Clean up after ourselves.
        for file in os.listdir(tempdir):
            os.unlink(os.path.join(tempdir, file))
        os.rmdir(tempdir)
        raise ValueError("No shapefile found in zip")

    return shape_path
<commit_msg>clean up (delete) temporary directories after loadshapefiles is done<commit_after>#coding: utf8

import logging
log = logging.getLogger(__name__)
from optparse import make_option
import os, os.path
import sys
import random

from zipfile import ZipFile
from tempfile import mkdtemp
from shutil import rmtree

from django.conf import settings
from django.contrib.gis.gdal import CoordTransform, DataSource, OGRGeometry, OGRGeomType
from django.core.management.base import BaseCommand
from django.db import connections, DEFAULT_DB_ALIAS, transaction
from django.template.defaultfilters import slugify

import boundaries
from boundaries.models import BoundarySet, Boundary, app_settings

GEOMETRY_COLUMN = 'shape'

class Command(BaseCommand):
    help = 'Import boundaries described by shapefiles.'
    option_list = BaseCommand.option_list + (
        make_option('-r', '--reload', action='store_true', dest='reload',
            help='Reload BoundarySets that have already been imported.'),
        make_option('-d', '--data-dir', action='store', dest='data_dir',
            default=app_settings.SHAPEFILES_DIR,
            help='Load shapefiles from this directory'),
        make_option('-e', '--except', action='store', dest='except',
            default=False, help='Don\'t load these kinds of Areas, comma-delimited.'),
        make_option('-o', '--only', action='store', dest='only',
            default=False, help='Only load these kinds of Areas, comma-delimited.'),
        make_option('-u', '--database', action='store', dest='database',
            default=DEFAULT_DB_ALIAS, help='Specify a database to load shape data into.'),
        make_option('-c', '--color', action='store_true', dest='color',
            default=False, help='Automatically set colors to the boundaries.'),
    )

    def get_version(self):
        return '0.1'

    def handle(self, *args, **options):
        # Load configuration
        boundaries.autodiscover(options['data_dir'])

        all_sources = boundaries.registry

        if options['only']:
            only = options['only'].split(',')
            # TODO: stripping whitespace here because optparse doesn't handle it correctly
            sources = [s for s in all_sources if s.replace(' ', '') in only]
        elif options['except']:
            exceptions = options['except'].upper().split(',')
            # See above
            sources = [s for s in all_sources if s.replace(' ', '') not in exceptions]
        else:
            sources = [s for s in all_sources]

        for kind, config in all_sources.items():
            if kind not in sources:
                log.debug('Skipping %s.' % kind)
                continue

            if (not options['reload']) and BoundarySet.objects.filter(slug=kind).exists():
                log.info('Already loaded %s, skipping.' % kind)
                continue

            self.load_set(kind, config, options)

    @transaction.commit_on_success
    def load_set(self, kind, config, options):
        log.info('Processing %s.' % kind)

        BoundarySet.objects.filter(slug=kind).delete()

        path = config['file']
        datasources, tmpdirs = create_datasources(path)

        try:
            self.load_set_2(kind, config, options, datasources)
        finally:
            for path in tmpdirs:
                rmtree(path)

    def load_set_2(self, kind, config, options, datasources):
        layer = datasources[0][0]

        # Add some default values
        if 'singular' not in config and kind.endswith('s'):
            config['singular'] = kind[:-1]
        if 'id_func' not in config:
            config['id_func'] = lambda f: ''
        if 'slug_func' not in config:
            config['slug_func'] = config['name_func']

        # Create BoundarySet
        bset = BoundarySet.objects.create(
            slug=kind,
            name=config.get('name', kind),
            singular=config['singular'],
            authority=config.get('authority', ''),
            domain=config.get('domain', ''),
            last_updated=config.get('last_updated'),
            source_url=config.get('source_url', ''),
            notes=config.get('notes', ''),
            licence_url=config.get('licence_url', ''),
        )

        for datasource in datasources:
            log.info("Loading %s from %s" % (kind, datasource.name))
            # Assume only a single-layer in shapefile
            if datasource.layer_count > 1:
                log.warn('%s shapefile [%s] has multiple layers, using first.' % (datasource.name, kind))
            layer = datasource[0]
            self.add_boundaries_for_layer(config, layer, bset, options['database'])

        if options["color"]:
            self.assign_colors(bset)

        log.info('%s count: %i' % (kind, Boundary.objects.filter(set=bset).count()))

    @staticmethod
    def polygon_to_multipolygon(geom):
        """
        Convert polygons to multipolygons so all features are homogenous in the database.
        """
        if geom.__class__.__name__ == 'Polygon':
            g = OGRGeometry(OGRGeomType('MultiPolygon'))
            g.add(geom)
            return g
        elif geom.__class__.__name__ == 'MultiPolygon':
            return geom
        else:
            raise ValueError('Geom is neither Polygon nor MultiPolygon.')

    def add_boundaries_for_layer(self, config, layer, bset, database):
        # Get spatial reference system for the postgis geometry field
        geometry_field = Boundary._meta.get_field_by_name(GEOMETRY_COLUMN)[0]
        SpatialRefSys = connections[database].ops.spatial_ref_sys()
        db_srs = SpatialRefSys.objects.using(database).get(srid=geometry_field.srid).srs

        if 'srid' in config and config['srid']:
            layer_srs = SpatialRefSys.objects.get(srid=config['srid']).srs
        else:
            layer_srs = layer.srs

        # Create a convertor to turn the source data into
        transformer = CoordTransform(layer_srs, db_srs)

        for feature in layer:
            geometry = feature.geom

            feature = UnicodeFeature(feature, encoding=config.get('encoding', 'ascii'))

            if config.get('is_valid_func', lambda feature : True)(feature) == False:
                continue

            # Transform the geometry to the correct SRS
            geometry = self.polygon_to_multipolygon(geometry)
            geometry.transform(transformer)

            # Create simplified geometry field by collapsing points within 1/1000th of a degree.
            # Since Chicago is at approx. 42 degrees latitude this works out to an margin of
            # roughly 80 meters latitude and 112 meters longitude.
            # Preserve topology prevents a shape from ever crossing over itself.
            simple_geometry = geometry.geos.simplify(app_settings.SIMPLE_SHAPE_TOLERANCE, preserve_topology=True)

            # Conversion may force multipolygons back to being polygons
            simple_geometry = self.polygon_to_multipolygon(simple_geometry.ogr)

            # Extract metadata into a dictionary
            metadata = dict(
                ( (field, feature.get(field)) for field in layer.fields )
            )

            external_id = str(config['id_func'](feature))
            feature_name = config['name_func'](feature)
            feature_slug = unicode(slugify(config['slug_func'](feature)).replace(u'—', '-'))

            log.info('%s...' % feature_slug)

            Boundary.objects.create(
                set=bset,
                set_name=bset.singular,
                external_id=external_id,
                name=feature_name,
                slug=feature_slug,
                metadata=metadata,
                shape=geometry.wkt,
                simple_shape=simple_geometry.wkt,
                centroid=geometry.geos.centroid,
                label_point=config.get("label_point_func", lambda x : None)(feature),
                color=config.get("color_func", lambda x : None)(feature),
                )

    @staticmethod
    def assign_colors(bset):
        # For each boundary in the set, assign a color such that it does not have the same
        # color as any other boundary it touches. Use the main colors from the Brewer spectrum,
        # based on http://colorbrewer2.org. This is done in a pretty dumb way: loop through
        # each boundary, query for each boundary it touches, look for a remaining color, and
        # then continue. In principle only four colors should be needed (the Four Color Theorem),
        # but finding a coloring that only uses four colors is algorithmically difficult. In practice,
        # around 8 is enough, and if we get stuck we just reuse a neighboring color --- oh well.
        color_choices = [ (44,162,95), (136,86,167), (67,162,202), (255, 237, 160), (240,59,32), (153,216,201), (158,188,218), (253,187,132), (166,189,219), (201,148,199) ]
        bset.boundaries.all().update(color=None)
        for bdry in bset.boundaries.all().only("shape"):
            used_colors = set()

            # What shapes are neighbors? 'Touches' is the right operator, but to be flexible
            # we use intersects, which will allow some overlap for poorly defined geometry.
            # Sometimes __intersects throws an error ("django.db.utils.DatabaseError: GEOS
            # intersects() threw an error!") and we'll just try to pass over those.
            try:
                # Looping over the polygons w/in the multipolygon isn't necessary.
                for part in (bdry.shape if bdry.shape.geom_type == "MultiPolygon" else [bdry.shape]):
                    qs = bset.boundaries.filter(shape__intersects=part).exclude(color=None).only("name", "color")
                    for b2 in qs:
                        used_colors.add(tuple(b2.color))
            except:
                print '%s had a problem looking for intersecting boundaries...' % bdry.slug
                bdry.color = random.choice(color_choices)
                bdry.save()
                continue

            # Choose the first available color. We prefer not to randomize so that a) this process
            # is relatively stable from run to run, and b) we can prioritize the colors we'd rather
            # use. The colors above are roughly from stronger to weaker.
            for c in color_choices:
                if c not in used_colors:
                    bdry.color = c
                    break
            else:
                # We ran out of colors. Just choose one at random.
                bdry.color = random.choice(color_choices)
            bdry.save()

def create_datasources(path):
    tmpdirs = []

    if path.endswith('.zip'):
        tmpdir, path = temp_shapefile_from_zip(path)
        tmpdirs.append(tmpdir)
        if not path: return

    if path.endswith('.shp'):
        return [DataSource(path)]

    # assume it's a directory...
    sources = []
    for fn in os.listdir(path):
        fn = os.path.join(path,fn)
        if fn.endswith('.zip'):
            tmpdir, fn = temp_shapefile_from_zip(fn)
            tmpdirs.append(tmpdir)
        if fn and fn.endswith('.shp'):
            sources.append(DataSource(fn))

    return sources, tmpdirs

class UnicodeFeature(object):

    def __init__(self, feature, encoding='ascii'):
        self.feature = feature
        self.encoding = encoding

    def get(self, field):
        val = self.feature.get(field)
        if isinstance(val, str):
            return val.decode(self.encoding)
        return val

def temp_shapefile_from_zip(zip_path):
    """Given a path to a ZIP file, unpack it into a temp dir and return the path
       to the shapefile that was in there.  Doesn't clean up after itself unless
       there was an error.

       If you want to cleanup later, you can derive the temp dir from this path.
    """
    zf = ZipFile(zip_path)
    tempdir = mkdtemp()
    shape_path = None
    # Copy the zipped files to a temporary directory, preserving names.
    for name in zf.namelist():
        if name.endswith("/"): continue
        data = zf.read(name)
        outfile = os.path.join(tempdir, os.path.basename(name))
        if name.endswith('.shp'):
            shape_path = outfile
        f = open(outfile, 'w')
        f.write(data)
        f.close()

    return tempdir, shape_path
<|endoftext|>


<commit_before>import os
import math
from cerberus import Validator

import virtool.utils
import virtool.job
from virtool.handlers.utils import json_response, bad_request, not_found, invalid_query, \
    compose_regex_query, no_content

async def find(req):
    """
    Return a list of job documents.

    """
    db = req.app["db"]

    # Validator for URL query.
    v = Validator({
        "term": {"type": "string", "default": "", "coerce": str},
        "page": {"type": "integer", "coerce": int, "default": 1, "min": 1},
        "per_page": {"type": "integer", "coerce": int, "default": 15, "min": 1, "max": 100}
    })

    if not v(dict(req.query)):
        return invalid_query(v.errors)

    query = v.document

    page = query["page"]
    per_page = query["per_page"]

    db_query = dict()

    if query["term"]:
        db_query.update(compose_regex_query(query["term"], ["task", "user_id"]))

    total_count = await db.jobs.count()

    cursor = db.jobs.find(
        db_query,
        virtool.job.LIST_PROJECTION,
        sort=[("name", 1)]
    )

    found_count = await cursor.count()

    if page > 1:
        cursor.skip((page - 1) * per_page)

    documents = [virtool.job.dispatch_processor(d) for d in await cursor.to_list(per_page)]

    return json_response({
        "documents": documents,
        "total_count": total_count,
        "found_count": found_count,
        "page": page,
        "per_page": per_page,
        "page_count": int(math.ceil(found_count / per_page))
    })

async def get(req):
    """
    Return the complete document for a given job.

    """
    job_id = req.match_info["job_id"]

    document = await req.app["db"].jobs.find_one(job_id)

    if not document:
        return not_found()

    return json_response(virtool.utils.base_processor(document))

async def cancel(req):
    """
    Cancel a job.

    """
    db = req.app["db"]

    job_id = req.match_info["job_id"]

    document = await db.jobs.find_one(job_id, ["status"])

    if not document:
        return not_found()

    if not document["status"][-1]["state"] in ["waiting", "running"]:
        return bad_request("Not cancellable")

    await req.app["job_manager"].cancel(job_id)

    document = await db.jobs.find_one(job_id)

    return json_response(virtool.utils.base_processor(document))

async def remove(req):
    """
    Remove a job.

    """
    db = req.app["db"]

    job_id = req.match_info["job_id"]

    document = await db.jobs.find_one(job_id)

    if not document:
        return not_found()

    latest_state = document["status"][-1]["state"]

    if latest_state in ["running", "waiting"]:
        return json_response({
            "id": "conflict",
            "message": "Job is running or waiting and cannot be removed"
        }, status=409)

    # Removed the documents associated with the job ids from the database.
    await db.jobs.delete_one({"_id": job_id})

    try:
        # Calculate the log path and remove the log file. If it exists, return True.
        path = os.path.join(req.app["settings"].get("data_path"), "logs", "jobs", job_id + ".log")
        await virtool.utils.rm(path)
    except OSError:
        pass

    return no_content()

async def clear(req):
    db = req.app["db"]

    query = dict()

    if req.path == "/api/jobs/complete":
        query["status.state"] = "complete"

    if req.path == "/api/jobs/failed":
        query["$or"] = [
            {"status.state": "error"},
            {"status.state": "cancelled"}
        ]

    if req.path == "/api/jobs/finished":
        query["$or"] = [
            {"status.state": "error"},
            {"status.state": "cancelled"},
            {"status.state": "complete"}
        ]

    removed = await db.jobs.find(query).distinct("_id")

    await db.jobs.delete_many(query)

    return json_response({
        "removed": removed
    })

async def dummy_job(req):
    """
    Submit a dummy job

    """
    data = await req.json()

    task_args = {key: data.get(key, False) for key in ["generate_python_error", "generate_process_error", "long"]}

    task_args["message"] = "hello world"
    task_args["long"] = True
    task_args["use_executor"] = True

    document = await req.app["job_manager"].new(
        "dummy",
        task_args,
        req["session"].user_id or "test"
    )

    return json_response(document)
<commit_msg>Fix job clear endpoint<commit_after>import os
import math
from cerberus import Validator

import virtool.utils
import virtool.job
from virtool.handlers.utils import json_response, bad_request, not_found, invalid_query, \
    compose_regex_query, no_content

async def find(req):
    """
    Return a list of job documents.

    """
    db = req.app["db"]

    # Validator for URL query.
    v = Validator({
        "term": {"type": "string", "default": "", "coerce": str},
        "page": {"type": "integer", "coerce": int, "default": 1, "min": 1},
        "per_page": {"type": "integer", "coerce": int, "default": 15, "min": 1, "max": 100}
    })

    if not v(dict(req.query)):
        return invalid_query(v.errors)

    query = v.document

    page = query["page"]
    per_page = query["per_page"]

    db_query = dict()

    if query["term"]:
        db_query.update(compose_regex_query(query["term"], ["task", "user_id"]))

    total_count = await db.jobs.count()

    cursor = db.jobs.find(
        db_query,
        virtool.job.LIST_PROJECTION,
        sort=[("name", 1)]
    )

    found_count = await cursor.count()

    if page > 1:
        cursor.skip((page - 1) * per_page)

    documents = [virtool.job.dispatch_processor(d) for d in await cursor.to_list(per_page)]

    return json_response({
        "documents": documents,
        "total_count": total_count,
        "found_count": found_count,
        "page": page,
        "per_page": per_page,
        "page_count": int(math.ceil(found_count / per_page))
    })

async def get(req):
    """
    Return the complete document for a given job.

    """
    job_id = req.match_info["job_id"]

    document = await req.app["db"].jobs.find_one(job_id)

    if not document:
        return not_found()

    return json_response(virtool.utils.base_processor(document))

async def cancel(req):
    """
    Cancel a job.

    """
    db = req.app["db"]

    job_id = req.match_info["job_id"]

    document = await db.jobs.find_one(job_id, ["status"])

    if not document:
        return not_found()

    if not document["status"][-1]["state"] in ["waiting", "running"]:
        return bad_request("Not cancellable")

    await req.app["job_manager"].cancel(job_id)

    document = await db.jobs.find_one(job_id)

    return json_response(virtool.utils.base_processor(document))

async def remove(req):
    """
    Remove a job.

    """
    db = req.app["db"]

    job_id = req.match_info["job_id"]

    document = await db.jobs.find_one(job_id)

    if not document:
        return not_found()

    latest_state = document["status"][-1]["state"]

    if latest_state in ["running", "waiting"]:
        return json_response({
            "id": "conflict",
            "message": "Job is running or waiting and cannot be removed"
        }, status=409)

    # Removed the documents associated with the job ids from the database.
    await db.jobs.delete_one({"_id": job_id})

    try:
        # Calculate the log path and remove the log file. If it exists, return True.
        path = os.path.join(req.app["settings"].get("data_path"), "logs", "jobs", job_id + ".log")
        await virtool.utils.rm(path)
    except OSError:
        pass

    return no_content()

async def clear(req):
    db = req.app["db"]

    query = None

    if req.path == "/api/jobs/complete":
        query = {
            "status.state": "complete"
        }

    if req.path == "/api/jobs/failed":
        query = {
            "$or": [
                {"status.state": "error"},
                {"status.state": "cancelled"}
            ]
        }

    if req.path == "/api/jobs/finished":
        query = {
            "$or": [
                {"status.state": "error"},
                {"status.state": "cancelled"},
                {"status.state": "complete"}
            ]
        }

    removed = list()

    if query is not None:
        removed = await db.jobs.find(query).distinct("_id")
        await db.jobs.delete_many(query)

    return json_response({
        "removed": removed
    })

async def dummy_job(req):
    """
    Submit a dummy job

    """
    data = await req.json()

    task_args = {key: data.get(key, False) for key in ["generate_python_error", "generate_process_error", "long"]}

    task_args["message"] = "hello world"
    task_args["long"] = True
    task_args["use_executor"] = True

    document = await req.app["job_manager"].new(
        "dummy",
        task_args,
        req["session"].user_id or "test"
    )

    return json_response(document)
<|endoftext|>


<commit_before>from vyapp.app import root
from vyapp.areavi import AreaVi
from Tkinter import Frame

def go_left_area(area):
    map   = area.master.master.panes()
    count = map.index(str(area.master))
    count = count - 1
    wid   = area.nametowidget(map[count])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_right_area(area):
    map   = area.master.master.panes()
    count = map.index(str(area.master))
    count = (count + 1) % len(map)
    wid   = area.nametowidget(map[count])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_down_area(area):
    map   = area.master.master.panes()
    index = map.index(str(area.master))

    map   = area.master.master.master.panes()
    count = map.index(str(area.master.master))
    count = (count + 1) % len(map)

    wid   = area.nametowidget(map[count])
    size  = len(wid.panes())
    wid   = area.nametowidget(wid.panes()[index if index < size else (size - 1)])

    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_up_area(area):
    map   = area.master.master.panes()
    index = map.index(str(area.master))

    map   = area.master.master.master.panes()
    count = map.index(str(area.master.master))
    count = count - 1

    wid   = area.nametowidget(map[count])
    size  = len(wid.panes())
    wid   = area.nametowidget(wid.panes()[index if index < size else (size - 1)])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

INSTALL =  [(-1, '<Shift-F9>', lambda event: root.note.scroll_left()),
            (-1, '<Shift-F10>', lambda event: root.note.scroll_right()),
            (-1, '<Left>', lambda event: go_left_area(event.widget)),
            (-1, '<Right>', lambda event: go_right_area(event.widget)),
            (-1, '<Up>', lambda event: go_up_area(event.widget)),
            (-1, '<Down>', lambda event: go_down_area(event.widget)),

            (-1, '<F9>', lambda event: root.note.select(root.note.index(root.note.select()) - 1)),
            (-1, '<F10>', lambda event: root.note.select(root.note.index(root.note.select()) + 1))]

def install(area):
    area.install(*INSTALL)

<commit_msg>Fixing browser key scheme.<commit_after>from vyapp.app import root
from vyapp.areavi import AreaVi
from Tkinter import Frame

def go_left_area(area):
    map   = area.master.master.panes()
    count = map.index(str(area.master))
    count = count - 1
    wid   = area.nametowidget(map[count])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_right_area(area):
    map   = area.master.master.panes()
    count = map.index(str(area.master))
    count = (count + 1) % len(map)
    wid   = area.nametowidget(map[count])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_down_area(area):
    map   = area.master.master.panes()
    index = map.index(str(area.master))

    map   = area.master.master.master.panes()
    count = map.index(str(area.master.master))
    count = (count + 1) % len(map)

    wid   = area.nametowidget(map[count])
    size  = len(wid.panes())
    wid   = area.nametowidget(wid.panes()[index if index < size else (size - 1)])

    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

def go_up_area(area):
    map   = area.master.master.panes()
    index = map.index(str(area.master))

    map   = area.master.master.master.panes()
    count = map.index(str(area.master.master))
    count = count - 1

    wid   = area.nametowidget(map[count])
    size  = len(wid.panes())
    wid   = area.nametowidget(wid.panes()[index if index < size else (size - 1)])
    wid   = filter(lambda ind: isinstance(ind, AreaVi), wid.winfo_children())

    # as there is only one.
    wid[0].focus_set()

INSTALL =  [(-1, '<Shift-F9>', lambda event: root.note.scroll_left()),
            (-1, '<Shift-F10>', lambda event: root.note.scroll_right()),
            (-1, '<F9>', lambda event: go_left_area(event.widget)),
            (-1, '<F10>', lambda event: go_right_area(event.widget)),
            (-1, '<F11>', lambda event: go_up_area(event.widget)),
            (-1, '<F12>', lambda event: go_down_area(event.widget)),

            (-1, '<Shift-F11>', lambda event: root.note.select(root.note.index(root.note.select()) - 1)),
            (-1, '<Shift-F12>', lambda event: root.note.select(root.note.index(root.note.select()) + 1))]

def install(area):
    area.install(*INSTALL)

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Start listener, just for testing
import os, sys
#import logging
from gettext import gettext as _
import cherrypy
import cfg, plugin_mount
from util import *
from logger import Logger
#from modules.auth import AuthController, require, member_of, name_is

def error_page(status, dynamic_msg, stock_msg):
    return page_template(template="err", title=status, main="
%s
%s" % (dynamic_msg, stock_msg))

def error_page_404(status, message, traceback, version):
    return error_page(status, message, """
If you believe this missing page should exist, please file a
    bug with either the Plinth project or the people responsible for
    the module you are trying to access.

Sorry for the mistake.

    """)

def error_page_500(status, message, traceback, version):
    cfg.log.error("500 Internal Server Error.  Trackback is above.")
    more="""
This is an internal error and not something you caused or can fix.  You are welcome to report it.
    TODO: add link to bug tracker.
"""
    return error_page(status, message, "
%s

%s
" % (more, "\n".join(traceback.split("\n"))))

class Root(plugin_mount.PagePlugin):
    @cherrypy.expose
    def index(self):
        if cherrypy.session.get(cfg.session_key, None):
            raise cherrypy.InternalRedirect('/router')
        else:
            raise cherrypy.InternalRedirect('/help/about')

def load_modules():
    """Import all the symlinked .py files in the modules directory and
    all the .py files in directories linked in the modules directory
    (but don't dive deeper than that).  Also, ignore the installed
    directory."""
    for name in os.listdir("modules"):
        if name.endswith(".py") and not name.startswith('.'):
            cfg.log.info("importing modules/%s" % name)
            try:
                exec("import modules.%s" % (name[:-3]))
            except ImportError, e:
                cfg.log.error(_("Couldn't import modules/%s: %s") % (name, e))
        else:
            cfg.log("skipping %s" % name)

def setup():
    os.chdir(cfg.file_root)
    cherrypy.config.update({'error_page.404': error_page_404})
    cherrypy.config.update({'error_page.500': error_page_500})
    cfg.log = Logger()
    load_modules()
    cfg.html_root = Root()
    cfg.page_plugins = plugin_mount.PagePlugin.get_plugins()
    cfg.log("Loaded %d page plugins" % len(cfg.page_plugins))
    cfg.users = plugin_mount.UserStoreModule.get_plugins()[0]
    cfg.forms = plugin_mount.FormPlugin.get_plugins()
def main():
    setup()
    cherrypy.quickstart(cfg.html_root, script_name='/', config="cherrypy.config")

if __name__ == '__main__':
    main()
<commit_msg>added link to bug tracker<commit_after>#!/usr/bin/env python

# Start listener, just for testing
import os, sys
#import logging
from gettext import gettext as _
import cherrypy
import cfg, plugin_mount
from util import *
from logger import Logger
#from modules.auth import AuthController, require, member_of, name_is

def error_page(status, dynamic_msg, stock_msg):
    return page_template(template="err", title=status, main="
%s
%s" % (dynamic_msg, stock_msg))

def error_page_404(status, message, traceback, version):
    return error_page(status, message, """
If you believe this missing page should exist, please file a
    bug with either the Plinth project or the people responsible for
    the module you are trying to access.

Sorry for the mistake.

    """)

def error_page_500(status, message, traceback, version):
    cfg.log.error("500 Internal Server Error.  Trackback is above.")
    more="""
This is an internal error and not something you caused
    or can fix.  Please report the error on the bug tracker so
    we can fix it.
"""
    return error_page(status, message, "
%s

%s
" % (more, "\n".join(traceback.split("\n"))))

class Root(plugin_mount.PagePlugin):
    @cherrypy.expose
    def index(self):
        if cherrypy.session.get(cfg.session_key, None):
            raise cherrypy.InternalRedirect('/router')
        else:
            raise cherrypy.InternalRedirect('/help/about')

def load_modules():
    """Import all the symlinked .py files in the modules directory and
    all the .py files in directories linked in the modules directory
    (but don't dive deeper than that).  Also, ignore the installed
    directory."""
    for name in os.listdir("modules"):
        if name.endswith(".py") and not name.startswith('.'):
            cfg.log.info("importing modules/%s" % name)
            try:
                exec("import modules.%s" % (name[:-3]))
            except ImportError, e:
                cfg.log.error(_("Couldn't import modules/%s: %s") % (name, e))
        else:
            cfg.log("skipping %s" % name)

def setup():
    os.chdir(cfg.file_root)
    cherrypy.config.update({'error_page.404': error_page_404})
    cherrypy.config.update({'error_page.500': error_page_500})
    cfg.log = Logger()
    load_modules()
    cfg.html_root = Root()
    cfg.page_plugins = plugin_mount.PagePlugin.get_plugins()
    cfg.log("Loaded %d page plugins" % len(cfg.page_plugins))
    cfg.users = plugin_mount.UserStoreModule.get_plugins()[0]
    cfg.forms = plugin_mount.FormPlugin.get_plugins()
def main():
    setup()
    cherrypy.quickstart(cfg.html_root, script_name='/', config="cherrypy.config")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import unittest
import sys

from sos_notebook.test_utils import sos_kernel
from ipykernel.tests.utils import execute, wait_for_idle, assemble_output

@unittest.skipIf(sys.platform == 'win32', 'octave does not exist on win32')
class TestSoSKernel(unittest.TestCase):
    def testKernel(self):
        with sos_kernel() as kc:
            execute(kc=kc, code='a = 1')
            stdout, stderr = assemble_output(kc.iopub_channel)
            self.assertEqual(stdout.strip(), '', f'Stdout is not empty, "{stdout}" received')
            self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received')
            execute(kc=kc, code='%use Julia\n%get a\nprint(a)')
            stdout, stderr = assemble_output(kc.iopub_channel)
            self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received')
            self.assertEqual(stdout.strip(), '1', f'Stdout should be 1, "{stdout}" received')

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix a typo<commit_after>import unittest
import sys

from sos_notebook.test_utils import sos_kernel
from ipykernel.tests.utils import execute, wait_for_idle, assemble_output

@unittest.skipIf(sys.platform == 'win32', 'julia does not exist on win32')
class TestSoSKernel(unittest.TestCase):
    def testKernel(self):
        with sos_kernel() as kc:
            execute(kc=kc, code='a = 1')
            stdout, stderr = assemble_output(kc.iopub_channel)
            self.assertEqual(stdout.strip(), '', f'Stdout is not empty, "{stdout}" received')
            self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received')
            execute(kc=kc, code='%use Julia\n%get a\nprint(a)')
            stdout, stderr = assemble_output(kc.iopub_channel)
            self.assertEqual(stderr.strip(), '', f'Stderr is not empty, "{stderr}" received')
            self.assertEqual(stdout.strip(), '1', f'Stdout should be 1, "{stdout}" received')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main driver logic for managing accounts on GCE instances."""

import logging
import os
import time

LOCKFILE = '/var/lock/manage-accounts.lock'

class AccountsManager(object):
  """Create accounts on a machine."""

  def __init__(self, accounts_module, desired_accounts, system, lock_file,
               lock_fname, single_pass=true):
    """Construct an AccountsFromMetadata with the given module injections."""
    if not lock_fname:
      lock_fname = LOCKFILE
    self.accounts = accounts_module
    self.desired_accounts = desired_accounts
    self.lock_file = lock_file
    self.lock_fname = lock_fname
    self.system = system
    self.single_pass = single_pass

  def Main(self):
    logging.debug('AccountsManager main loop')
    # Run this once per interval forever.
    while True:
      # If this is a one-shot execution, then this can be run normally.
      # Otherwise, run the actual operations in a subprocess so that any
      # errors don't kill the long-lived process.
      if self.single_pass:
        self.RegenerateKeysAndCreateAccounts()
        break
      else:
        # Fork and run the key regeneration and account creation while the
        # parent waits for the subprocess to finish before continuing.
        pid = os.fork()
        if pid:
          os.waitpid(pid, 0)
        else:
          self.RegenerateKeysAndCreateAccounts()
          # The use of os._exit here is recommended for subprocesses spawned
          # by forking to avoid issues with running the cleanup tasks that
          # sys.exit() runs by preventing issues from the cleanup being run
          # once by the subprocess and once by the parent process.
          os._exit(0)

  def RegenerateKeysAndCreateAccounts(self):
    """Regenerate the keys and create accounts as needed."""
    logging.debug('RegenerateKeysAndCreateAccounts')
    if self.system.IsExecutable('/usr/share/google/first-boot'):
      self.system.RunCommand('/usr/share/google/first-boot')

    self.lock_file.RunExclusively(self.lock_fname, self.CreateAccounts)

  def CreateAccounts(self):
    """Create all accounts that should be present."""
    desired_accounts = self.desired_accounts.GetDesiredAccounts()
    if not desired_accounts:
      return

    for username, ssh_keys in desired_accounts.iteritems():
      if not username:
        continue

      self.accounts.CreateUser(username, ssh_keys)
<commit_msg>Fixing typo<commit_after># Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main driver logic for managing accounts on GCE instances."""

import logging
import os
import time

LOCKFILE = '/var/lock/manage-accounts.lock'

class AccountsManager(object):
  """Create accounts on a machine."""

  def __init__(self, accounts_module, desired_accounts, system, lock_file,
               lock_fname, single_pass=True):
    """Construct an AccountsFromMetadata with the given module injections."""
    if not lock_fname:
      lock_fname = LOCKFILE
    self.accounts = accounts_module
    self.desired_accounts = desired_accounts
    self.lock_file = lock_file
    self.lock_fname = lock_fname
    self.system = system
    self.single_pass = single_pass

  def Main(self):
    logging.debug('AccountsManager main loop')
    # Run this once per interval forever.
    while True:
      # If this is a one-shot execution, then this can be run normally.
      # Otherwise, run the actual operations in a subprocess so that any
      # errors don't kill the long-lived process.
      if self.single_pass:
        self.RegenerateKeysAndCreateAccounts()
        break
      else:
        # Fork and run the key regeneration and account creation while the
        # parent waits for the subprocess to finish before continuing.
        pid = os.fork()
        if pid:
          os.waitpid(pid, 0)
        else:
          self.RegenerateKeysAndCreateAccounts()
          # The use of os._exit here is recommended for subprocesses spawned
          # by forking to avoid issues with running the cleanup tasks that
          # sys.exit() runs by preventing issues from the cleanup being run
          # once by the subprocess and once by the parent process.
          os._exit(0)

  def RegenerateKeysAndCreateAccounts(self):
    """Regenerate the keys and create accounts as needed."""
    logging.debug('RegenerateKeysAndCreateAccounts')
    if self.system.IsExecutable('/usr/share/google/first-boot'):
      self.system.RunCommand('/usr/share/google/first-boot')

    self.lock_file.RunExclusively(self.lock_fname, self.CreateAccounts)

  def CreateAccounts(self):
    """Create all accounts that should be present."""
    desired_accounts = self.desired_accounts.GetDesiredAccounts()
    if not desired_accounts:
      return

    for username, ssh_keys in desired_accounts.iteritems():
      if not username:
        continue

      self.accounts.CreateUser(username, ssh_keys)
<|endoftext|>