<commit_before>" Helpers for the package "

import zipfile
import simplejson

from base.shortcuts import get_object_with_related_or_404
from jetpack.models import Package, PackageRevision
from jetpack.errors import FilenameExistException, ManifestNotValid

def get_package_revision(id_name, type_id,
                         revision_number=None,
                         version_name=None, latest=False):
    """
    Return revision of the package
    """
    if not (revision_number or version_name):
        # get default revision - one linked via Package:version
        package = get_object_with_related_or_404(Package, id_number=id_name,
                                                 type=type_id)
        package_revision = package.latest if latest else package.version

    elif revision_number:
        # get version given by revision number
        package_revision = get_object_with_related_or_404(PackageRevision,
                            package__id_number=id_name, package__type=type_id,
                            revision_number=revision_number)
    elif version_name:
        # get version given by version name
        package_revision = get_object_with_related_or_404(PackageRevision,
                            package__id_number=id_name, package__type=type_id,
                            version_name=version_name)
    return package_revision

def create_from_archive(path, author, package_type='a'):
    """
    Create new package from the PKZip archive.
    Call Package.create_revision_from_archive.

    Args:
       path (str): direct, full path of the archive

    Returns:
        Package object

    Raises:
        ManifestNotValid, BadZipFile, LargeZipFile
    """
    # read package.json
    packed = zipfile.ZipFile(path, 'r')
    try:
        manifest = packed.open('package.json', 'r')
    except Exception, err:
        raise ManifestNotValid("Problem with opening package.json.\n"
                "Error: %s" % str(err))
    try:
        manifest = simplejson.loads(manifest.read())
    except Exception, err:
        raise ManifestNotValid("Problem with reading manifest's data.\n"
                "Error: %s" % str(err))
    if not ('name' in manifest and \
            'license' in manifest and \
            'version' in manifest):
        raise ManifestNotValid("Manifest is not valid.\n"
                "name, license and version fields are obsolete")

    if 'fullName' not in manifest:
        manifest['fullName'] = manifest['name']

    # * optional - check for jid - might be a problem with private key
    # create Package
    obj = Package(
        author=author,
        full_name=manifest['fullName'],
        name=manifest['name'],
        type=package_type,
        license=manifest['license'],
        description=manifest['description'] \
                if 'description' in manifest else ''
    )
    obj.save()
    obj.latest.set_version('empty.uploaded')
    obj.create_revision_from_archive(path, manifest)

    return obj

<commit_msg>pass archive<commit_after>" Helpers for the package "

import zipfile
import simplejson

from base.shortcuts import get_object_with_related_or_404
from jetpack.models import Package, PackageRevision
from jetpack.errors import FilenameExistException, ManifestNotValid

def get_package_revision(id_name, type_id,
                         revision_number=None,
                         version_name=None, latest=False):
    """
    Return revision of the package
    """
    if not (revision_number or version_name):
        # get default revision - one linked via Package:version
        package = get_object_with_related_or_404(Package, id_number=id_name,
                                                 type=type_id)
        package_revision = package.latest if latest else package.version

    elif revision_number:
        # get version given by revision number
        package_revision = get_object_with_related_or_404(PackageRevision,
                            package__id_number=id_name, package__type=type_id,
                            revision_number=revision_number)
    elif version_name:
        # get version given by version name
        package_revision = get_object_with_related_or_404(PackageRevision,
                            package__id_number=id_name, package__type=type_id,
                            version_name=version_name)
    return package_revision

def create_from_archive(path, author, package_type='a'):
    """
    Create new package from the PKZip archive.
    Call Package.create_revision_from_archive.

    Args:
       path (str): direct, full path of the archive

    Returns:
        Package object

    Raises:
        ManifestNotValid, BadZipFile, LargeZipFile
    """
    # read package.json
    packed = zipfile.ZipFile(path, 'r')
    try:
        manifest = simplejson.loads(manifest.read())
    except Exception, err:
        raise ManifestNotValid("Problem with reading manifest's data.\n"
                "Error: %s" % str(err))
    if not ('name' in manifest and \
            'license' in manifest and \
            'version' in manifest):
        raise ManifestNotValid("Manifest is not valid.\n"
                "name, license and version fields are obsolete")

    if 'fullName' not in manifest:
        manifest['fullName'] = manifest['name']

    # * optional - check for jid - might be a problem with private key
    # create Package
    obj = Package(
        author=author,
        full_name=manifest['fullName'],
        name=manifest['name'],
        type=package_type,
        license=manifest['license'],
        description=manifest['description'] \
                if 'description' in manifest else ''
    )
    obj.save()
    obj.latest.set_version('empty.uploaded')
    obj.create_revision_from_archive(packed, manifest)

    return obj

<|endoftext|>


<commit_before># Gitless - a version control system built on top of Git.
# Copyright (c) 2013  Santiago Perez De Rosso.
# Licensed under GNU GPL, version 2.

"""gl branch - Create, edit, delete or switch branches."""

from gitless.core import branch as branch_lib
from gitless.core import sync as sync_lib

from . import pprint

def parser(subparsers):
  """Adds the branch parser to the given subparsers object."""
  branch_parser = subparsers.add_parser(
      'branch', help='list, create, edit, delete or switch branches')
  branch_parser.add_argument(
      'branch', nargs='?',
      help='switch to branch (will be created if it doesn\'t exist yet)')
  branch_parser.add_argument(
      'divergent_point', nargs='?',
      help='the commit from where to \'branch out\' (only relevant if a new '
      'branch is created; defaults to HEAD)', default='HEAD')
  branch_parser.add_argument(
      '-d', '--delete', nargs='+', help='delete branch(es)', dest='delete_b')
  branch_parser.add_argument(
      '-su', '--set-upstream', help='set the upstream branch',
      dest='upstream_b')
  branch_parser.set_defaults(func=main)

def main(args):
  ret = True
  if args.branch:
    b_st = branch_lib.status(args.branch)
    if b_st and b_st.is_current:
      pprint.err(
          'You are already in branch %s. No need to switch.' % args.branch)
      pprint.err_exp('to list existing branches do gl branch')
      return False

    if sync_lib.rebase_in_progress():
      pprint.err(
          'You can\'t switch branches when a rebase is in progress (yet '
          '-- this will be implemented in the future)')
      return False
    elif sync_lib.merge_in_progress():
      pprint.err(
          'You can\'t switch branches when merge is in progress (yet '
          '-- this will be implemented in the future)')
      return False

    if not b_st and not _do_create(args.branch, args.divergent_point):
      return False

    branch_lib.switch(args.branch)
    pprint.msg('Switched to branch %s' % args.branch)
  elif args.delete_b:
    ret = _do_delete(args.delete_b)
  elif args.upstream_b:
    ret = _do_set_upstream(args.upstream_b)
  else:
    _do_list()

  return ret

def _do_create(branch_name, divergent_point):
  errors_found = False

  ret = branch_lib.create(branch_name, dp=divergent_point)
  if ret == branch_lib.INVALID_NAME:
    pprint.err('Invalid branch name')
    errors_found = True
  elif ret == branch_lib.INVALID_DP:
    pprint.msg('Invalid divergent point {0}'.format(divergent_point))
    errors_found = True
  elif ret == branch_lib.SUCCESS:
    pprint.msg('Created new branch %s' % branch_name)
  else:
    raise Exception('Unrecognized ret code %s' % ret)
  return not errors_found

def _do_list():
  pprint.msg('List of branches:')
  pprint.exp('do gl branch <b> to create or switch to branch b')
  pprint.exp('do gl branch -d <b> to delete branch b')
  pprint.exp(
      'do gl branch -su <upstream> to set an upstream for the current branch')
  pprint.exp('* = current branch')
  pprint.blank()
  for name, is_current, upstream, upstream_exists in branch_lib.status_all():
    current_str = '*' if is_current else ' '
    upstream_str = ''
    if upstream:
      np_str = ' --not present in remote yet' if not upstream_exists else ''
      upstream_str = '(upstream is %s%s)' % (upstream, np_str)
    pprint.item('%s %s %s' % (current_str, name, upstream_str))

def _do_delete(delete_b):
  errors_found = False

  for b in delete_b:
    b_st = branch_lib.status(b)
    if not b_st:
      pprint.err('Can\'t remove non-existent branch %s' % b)
      pprint.err_exp('do gl branch to list existing branches')
      errors_found = True
    elif b_st and b_st.is_current:
      pprint.err('Can\'t remove current branch %s' % b)
      pprint.err_exp(
          'do gl branch <b> to create or switch to another branch b and then '
          'gl branch -d %s to remove branch %s' % (b, b))
      errors_found = True
    elif not pprint.conf_dialog('Branch %s will be removed' % b):
      pprint.msg('Aborted: removal of branch %s' % b)
    else:
      branch_lib.delete(b)
      pprint.msg('Branch %s removed successfully' % b)

  return not errors_found

def _do_set_upstream(upstream):
  if '/' not in upstream:
    pprint.err(
        'Invalid upstream branch. It must be in the format remote/branch')
    return True

  ret = branch_lib.set_upstream(upstream)

  errors_found = False
  upstream_remote, upstream_branch = upstream.split('/')
  if ret is branch_lib.REMOTE_NOT_FOUND:
    pprint.err('Remote %s not found' % upstream_remote)
    pprint.err_exp('do gl remote to list all existing remotes')
    pprint.err_exp(
        'do gl remote %s <r_url> to add a new remote %s mapping to '
        'r_url' % (upstream_remote, upstream_remote))
    errors_found = True
  elif ret is branch_lib.SUCCESS:
    pprint.msg('Current branch %s set to track %s/%s' % (
        branch_lib.current(), upstream_remote, upstream_branch))

  return not errors_found
<commit_msg>unset upstream (cli changes)<commit_after># Gitless - a version control system built on top of Git.
# Copyright (c) 2013  Santiago Perez De Rosso.
# Licensed under GNU GPL, version 2.

"""gl branch - Create, edit, delete or switch branches."""

from gitless.core import branch as branch_lib
from gitless.core import sync as sync_lib

from . import pprint

def parser(subparsers):
  """Adds the branch parser to the given subparsers object."""
  branch_parser = subparsers.add_parser(
      'branch', help='list, create, edit, delete or switch branches')
  branch_parser.add_argument(
      'branch', nargs='?',
      help='switch to branch (will be created if it doesn\'t exist yet)')
  branch_parser.add_argument(
      'divergent_point', nargs='?',
      help='the commit from where to \'branch out\' (only relevant if a new '
      'branch is created; defaults to HEAD)', default='HEAD')
  branch_parser.add_argument(
      '-d', '--delete', nargs='+', help='delete branch(es)', dest='delete_b')
  branch_parser.add_argument(
      '-su', '--set-upstream', help='set the upstream branch',
      dest='upstream_b')
  branch_parser.add_argument(
      '-uu', '--unset-upstream', help='unset the upstream branch',
      action='store_true')
  branch_parser.set_defaults(func=main)

def main(args):
  ret = True
  if args.branch:
    b_st = branch_lib.status(args.branch)
    if b_st and b_st.is_current:
      pprint.err(
          'You are already in branch %s. No need to switch.' % args.branch)
      pprint.err_exp('to list existing branches do gl branch')
      return False

    if sync_lib.rebase_in_progress():
      pprint.err(
          'You can\'t switch branches when a rebase is in progress (yet '
          '-- this will be implemented in the future)')
      return False
    elif sync_lib.merge_in_progress():
      pprint.err(
          'You can\'t switch branches when merge is in progress (yet '
          '-- this will be implemented in the future)')
      return False

    if not b_st and not _do_create(args.branch, args.divergent_point):
      return False

    branch_lib.switch(args.branch)
    pprint.msg('Switched to branch %s' % args.branch)
  elif args.delete_b:
    ret = _do_delete(args.delete_b)
  elif args.upstream_b:
    ret = _do_set_upstream(args.upstream_b)
  elif args.unset_upstream:
    ret = _do_unset_upstream()
  else:
    _do_list()

  return ret

def _do_create(branch_name, divergent_point):
  errors_found = False

  ret = branch_lib.create(branch_name, dp=divergent_point)
  if ret == branch_lib.INVALID_NAME:
    pprint.err('Invalid branch name')
    errors_found = True
  elif ret == branch_lib.INVALID_DP:
    pprint.msg('Invalid divergent point {0}'.format(divergent_point))
    errors_found = True
  elif ret == branch_lib.SUCCESS:
    pprint.msg('Created new branch %s' % branch_name)
  else:
    raise Exception('Unrecognized ret code %s' % ret)
  return not errors_found

def _do_list():
  pprint.msg('List of branches:')
  pprint.exp('do gl branch <b> to create or switch to branch b')
  pprint.exp('do gl branch -d <b> to delete branch b')
  pprint.exp(
      'do gl branch -su <upstream> to set an upstream for the current branch')
  pprint.exp('* = current branch')
  pprint.blank()
  for name, is_current, upstream, upstream_exists in branch_lib.status_all():
    current_str = '*' if is_current else ' '
    upstream_str = ''
    if upstream:
      np_str = ' --not present in remote yet' if not upstream_exists else ''
      upstream_str = '(upstream is %s%s)' % (upstream, np_str)
    pprint.item('%s %s %s' % (current_str, name, upstream_str))

def _do_delete(delete_b):
  errors_found = False

  for b in delete_b:
    b_st = branch_lib.status(b)
    if not b_st:
      pprint.err('Can\'t remove non-existent branch %s' % b)
      pprint.err_exp('do gl branch to list existing branches')
      errors_found = True
    elif b_st and b_st.is_current:
      pprint.err('Can\'t remove current branch %s' % b)
      pprint.err_exp(
          'do gl branch <b> to create or switch to another branch b and then '
          'gl branch -d %s to remove branch %s' % (b, b))
      errors_found = True
    elif not pprint.conf_dialog('Branch %s will be removed' % b):
      pprint.msg('Aborted: removal of branch %s' % b)
    else:
      branch_lib.delete(b)
      pprint.msg('Branch %s removed successfully' % b)

  return not errors_found

def _do_set_upstream(upstream):
  if '/' not in upstream:
    pprint.err(
        'Invalid upstream branch. It must be in the format remote/branch')
    return True

  ret = branch_lib.set_upstream(upstream)

  errors_found = False
  upstream_remote, upstream_branch = upstream.split('/')
  if ret is branch_lib.REMOTE_NOT_FOUND:
    pprint.err('Remote %s not found' % upstream_remote)
    pprint.err_exp('do gl remote to list all existing remotes')
    pprint.err_exp(
        'do gl remote %s <r_url> to add a new remote %s mapping to '
        'r_url' % (upstream_remote, upstream_remote))
    errors_found = True
  elif ret is branch_lib.SUCCESS:
    pprint.msg('Current branch %s set to track %s/%s' % (
        branch_lib.current(), upstream_remote, upstream_branch))

  return not errors_found

def _do_unset_upstream():
  ret = branch_lib.unset_upstream()

  errors_found = False
  if ret is branch_lib.UPSTREAM_NOT_SET:
    pprint.err('Current branch has no upstream set')
    pprint.err_exp(
        'do gl branch to list all existing branches -- if a branch has an '
        'upstream set it will be shown')
    pprint.err_exp(
      'do gl branch -su <upstream> to set an upstream for the current branch')
    errors_found = True
  elif ret is branch_lib.SUCCESS:
    pprint.msg('Upstream unset for current branch')

  return not errors_found
<|endoftext|>


<commit_before>import pytest

from thinglang.runner import run

def test_simple_conditionals():
    assert run("""
thing Program
    does start
        if "dog" eq "dog"
            Output.write("dog is dog")

        if "dog" eq "cat"
            Output.write("dog is cat")

    """).output == """dog is dog"""

def test_boolean_constant():
    assert run("""
thing Program
    does start
        if true
            Output.write("true is truthy")

        if false
            Output.write("false is truthy")

    """).output == """true is truthy"""

def test_unconditional_else():
    assert run("""
thing Program
    does start
        if "dog" eq "dog"
            Output.write("dog is dog")
        otherwise
            Output.write("dog is not dog")

        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise
            Output.write("dog is not cat")
    """).output == """dog is dog\ndog is not cat"""

def test_conditional_else():
    assert run("""
thing Program
    does start
        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise if "dog" eq "dog"
            Output.write("dog is dog")
        otherwise if "dog" eq "dog"
            Output.write("dog is still dog")
        otherwise
            Output.write("dog is not dog and not cat")

        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise if "dog" eq "Dog"
            Output.write("dog is Dog")
        otherwise if "dog" eq "mouse"
            Output.write("dog is mouse")
        otherwise
            Output.write("dog is not cat and not mouse and not Dog")
    """).output == """dog is dog\ndog is not cat and not mouse and not Dog"""
<commit_msg>Updated conditional integration test<commit_after>import pytest

from thinglang.runner import run

def test_simple_conditionals():
    assert run("""
thing Program
    does start
        if "dog" eq "dog"
            Output.write("dog is dog")

        if "dog" eq "cat"
            Output.write("dog is cat")

    """).output == """dog is dog"""

def test_boolean_constant():
    assert run("""
thing Program
    does start
        if true
            Output.write("true is truthy")

        if false
            Output.write("false is truthy")

    """).output == """true is truthy"""

def test_unconditional_else():
    assert run("""
thing Program
    does start
        if "dog" eq "dog"
            Output.write("dog is dog")
        otherwise
            Output.write("dog is not dog")

        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise
            Output.write("dog is not cat")
    """).output == """dog is dog\ndog is not cat"""

def test_conditional_else():
    assert run("""
thing Program
    does start
        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise if "dog" eq "dog"
            Output.write("dog is dog")
        otherwise if "dog" eq "dog"
            Output.write("dog is still dog")
        otherwise if true
            Output.write("true is truthy")
        otherwise
            Output.write("dog is not dog and not cat")

        if "dog" eq "cat"
            Output.write("dog is cat")
        otherwise if "dog" eq "Dog"
            Output.write("dog is Dog")
        otherwise if "dog" eq "mouse"
            Output.write("dog is mouse")
        otherwise
            Output.write("dog is not cat and not mouse and not Dog")
    """).output == """dog is dog\ndog is not cat and not mouse and not Dog"""
<|endoftext|>


<commit_before>from numpy import asarray, atleast_2d, concatenate, dot, kron, ones, stack, zeros
from glimix_core.util import vec, unvec
from optimix import Func, Vector

class KronMean(Func):
    r""" Kronecker mean function.

    Let
    - n be the number of samples;
    - p the number of traits;
    - c the number of covariates.

    The mathematical representation is

        f(𝐀,𝐅)=(𝐀⊗𝐅)vec(𝐁)

    where 𝐀 is a p×p trait design matrix of fixed effects and 𝐅 is a n×c sample design
    matrix of fixed effects. 𝐁 is a c×p matrix of fixed-effect sizes.
    """

    def __init__(self, c, p):
        vecB = zeros((c, p)).ravel()
        self._c = c
        self._p = p
        self._A = None
        self._F = None
        self._vecB = Vector(vecB)
        Func.__init__(self, "KronMean", vecB=self._vecB)

    # def _set_data(self):
    #     if self._A is None or self._F is None:
    #         return
    #     item = concatenate((self._A.ravel(), self._F.ravel()))
    #     c = self._c
    #     p = self._p
    #     self._A = item[: p * p].reshape((p, p))
    #     self._F = item[p * p :].reshape((-1, c))
    #     assert self._A.base is item
    #     assert self._F.base is item
    #     self.set_data(item)

    @property
    def A(self):
        return self._A

    @A.setter
    def A(self, A):
        self._A = A

    @property
    def F(self):
        return self._F

    @F.setter
    def F(self, F):
        self._F = F

    @property
    def AF(self):
        r""" A ⊗ F. """
        return kron(self.A, self.F)

    # def _value(self, A, F):
    #     """ reshape((A ⊗ F) vec(B), n, p) """
    #     b = self._vecB.value
    #     n = F.shape[0]
    #     p = A.shape[0]
    #     return dot(kron(A, F), b).reshape((n, p))

    def value(self):
        """
        Kronecker mean function.
        """
        return self.AF @ self._vecB.value

    def gradient(self):
        r"""Gradient of the linear mean function.

        Returns
        -------
        dict
            Dictionary having the `effsizes` key for :math:`\mathbf x`.
        """
        return {"vecB": self.AF}
        # from numpy import diag

        # x = atleast_2d(asarray(x, float))
        # p = self._p
        # c = self._c
        # n = (len(x[0]) - p * p) // c
        # As = [i[: p * p].reshape((p, p)) for i in x]
        # Fs = [i[p * p :].reshape((n, c)) for i in x]
        # one = diag(ones(p * c))
        # AF1s = [dot(kron(A, F), one).reshape((n, p, p * c)) for A, F in zip(As, Fs)]
        # r = stack(AF1s)
        # return dict(vecB=r)

    @property
    def B(self):
        """
        Effect-sizes parameter.
        """
        return unvec(self._vecB.value, (self._c, self._p))

    @B.setter
    def B(self, v):
        self._vecB.value = vec(asarray(v, float))

    def __str__(self):
        tname = type(self).__name__
        p = self._p
        c = self._c
        msg = "{}(c={},p={})".format(tname, c, p)
        if self.name is not None:
            msg += ": {}".format(self.name)
        msg += "\n"
        mat = format(self.B)
        msg += "  B: " + "\n     ".join(mat.split("\n"))
        return msg

# def _compact_form(x):
#     d = x.shape[0] * x.shape[2]
#     return x.transpose((2, 0, 3, 1)).reshape(d, d)

# def _compact_form_grad(x):
#     breakpoint()
#     assert x.shape[0] == 1
#     x = x[0, ...]
#     nparams = x.shape[2]
#     return x.ravel(order="F").reshape((-1, nparams), )
#     # x = x.transpose([2, 0, 1])
#     # mats = []
#     # for i in x:
#     #     mats.append(_compact_form(i))
#     # return stack(mats, axis=0)
<commit_msg>clean mean kron<commit_after>from numpy import asarray, atleast_2d, concatenate, dot, kron, ones, stack, zeros
from glimix_core.util import vec, unvec
from optimix import Func, Vector

class KronMean(Func):
    r""" Kronecker mean function.

    Let
    - n be the number of samples;
    - p the number of traits;
    - c the number of covariates.

    The mathematical representation is

        f(𝐀,𝐅)=(𝐀⊗𝐅)vec(𝐁)

    where 𝐀 is a p×p trait design matrix of fixed effects and 𝐅 is a n×c sample design
    matrix of fixed effects. 𝐁 is a c×p matrix of fixed-effect sizes.
    """

    def __init__(self, c, p):
        vecB = zeros((c, p)).ravel()
        self._c = c
        self._p = p
        self._A = None
        self._F = None
        self._vecB = Vector(vecB)
        Func.__init__(self, "KronMean", vecB=self._vecB)

    # def _set_data(self):
    #     if self._A is None or self._F is None:
    #         return
    #     item = concatenate((self._A.ravel(), self._F.ravel()))
    #     c = self._c
    #     p = self._p
    #     self._A = item[: p * p].reshape((p, p))
    #     self._F = item[p * p :].reshape((-1, c))
    #     assert self._A.base is item
    #     assert self._F.base is item
    #     self.set_data(item)

    @property
    def A(self):
        return self._A

    @A.setter
    def A(self, A):
        self._A = A

    @property
    def F(self):
        return self._F

    @F.setter
    def F(self, F):
        self._F = F

    @property
    def AF(self):
        r""" A ⊗ F. """
        return kron(self.A, self.F)

    def value(self):
        """
        Kronecker mean function.
        """
        return self.AF @ self._vecB.value

    def gradient(self):
        r"""Gradient of the linear mean function.

        Returns
        -------
        dict
            Dictionary having the `effsizes` key for :math:`\mathbf x`.
        """
        return {"vecB": self.AF}

    @property
    def B(self):
        """
        Effect-sizes parameter.
        """
        return unvec(self._vecB.value, (self._c, self._p))

    @B.setter
    def B(self, v):
        self._vecB.value = vec(asarray(v, float))

    def __str__(self):
        tname = type(self).__name__
        p = self._p
        c = self._c
        msg = "{}(c={},p={})".format(tname, c, p)
        if self.name is not None:
            msg += ": {}".format(self.name)
        msg += "\n"
        mat = format(self.B)
        msg += "  B: " + "\n     ".join(mat.split("\n"))
        return msg
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from sys import exit
import errno
import json
import os
import shutil
import socket
import xml.etree.ElementTree as ET
import requests
from ipolutils.read_text_file import read_commented_text_file

import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../ipol_demo/modules/"))
from dispatcher.demorunnerinfo import DemoRunnerInfo
from dispatcher.policy import Policy

HOST = socket.gethostbyname(socket.gethostname())
user = 'ipol'
compilation_path = os.path.join("/", "home", user, "ipolDevel", "ci_tests", "compilation_folder")
ignored_ids_file_path = sys.argv[1] if len(sys.argv) > 1 else ''

def main():
    """
    Main
    """
    all_success = True
    demorunners = read_demorunners()

    id_list = []
    if os.path.exists(ignored_ids_file_path):
        id_list = [int(id) for id in read_commented_text_file(ignored_ids_file_path)]

    for demo_id in get_published_demos():
        json_ddl = json.loads(get_ddl(demo_id))
        build_section = json_ddl.get('build')
        requirements = json_ddl.get('general').get('requirements')

        if build_section is None:
            print(f"Demo #{demo_id} doesn't have a build section")
            all_success = False
            continue

        if demo_id in id_list:
            continue

        if not build(demo_id, build_section, requirements, demorunners):
            all_success = False

    if all_success:
        exit(0)
    else:
        exit(-1)

def get_compilation_path(demo_id):
    """
    Get compilation path
    """
    return os.path.join(compilation_path, str(demo_id))

def create_dir(path):
    """
    Implement the UNIX shell command "mkdir -p"
    with given path as parameter.
    """
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get_published_demos():
    """
    Get demos
    """
    resp = post(f'http://{HOST}/api/demoinfo/demo_list')
    if resp == None:
        print("Can't get the list of demos!")
        return []

    response = resp.json()
    demos = []
    if response['status'] != 'OK':
        print("ERROR: demo_list returned KO")
        return []
    for demo in response['demo_list']:
        if demo.get('state') == "published":
            demos.append(demo['editorsdemoid'])
    return demos

def get_ddl(demo_id):
    """
    Read the DDL of the demo
    """
    resp = post(f'http://{HOST}/api/demoinfo/get_ddl', params={"demo_id": demo_id})
    if resp == None:
        print(f"Can't get the DDL of demo #{demo_id}!")
        return ""

    response = resp.json()
    if response['status'] != 'OK':
        print(f"ERROR: get_ddl returned KO for demo #{demo_id}")
    last_demodescription = response.get('last_demodescription')
    return last_demodescription.get('ddl')

def build(demo_id, build_section, requirements, demorunners):
    """
    Execute the build section.
    Returns False if any build fails. True is everything's fine.
    """
    suitable_demorunners = Policy.get_suitable_demorunners(requirements, demorunners)
    for demorunner in suitable_demorunners:
        demorunner_host = demorunner.server
        params = {'ddl_build': json.dumps(build_section), 'compilation_path': get_compilation_path(demo_id)}
        response = post(f'{demorunner_host}api/demorunner/{demorunner.name}/test_compilation', params)

        if response == None or response.status_code == None:
            print(f"Bad response from DR={demorunner.name} when trying to build demo #{demo_id}: {str(response)}")
            return False

        if response.status_code == 200:
            json_response = response.json()
            if json_response.get('status') == 'OK':
                continue
            else:
                print(f"Couldn't build demo #{demo_id} in DR={demorunner.name} ({json_response}).")
                return False
        else:
            msg = f"Bad HTTP response status_code from DR={demorunner.name} when trying to build demo #{demo_id}."
            if response.status_code == 504:
                msg += " HTTP error 504 (gateway timeout)"
            elif response.status_code == 404:
                msg += f" HTTP error 404 (not found)"
            else:
                msg += f" HTTP error {response.status_code}"
            print(msg)
            return False

    # No DR failed or KO: compilation test passed
    return True

def read_demorunners():
    """
    Read demorunners xml
    """
    demorunners = []
    tree = ET.parse(os.path.join("/", "home", user, "ipolDevel", "ipol_demo", "modules",
                                 "config_common", "demorunners.xml"))
    root = tree.getroot()
    for demorunner in root.findall('demorunner'):
        capabilities = []

        for capability in demorunner.findall('capability'):
            capabilities.append(capability.text)

        demorunner = DemoRunnerInfo(
            demorunner.find('server').text,
            demorunner.get('name'),
            demorunner.find('serverSSH').text,
            capabilities
        )

        demorunners.append(demorunner)

    return demorunners

def post(url, params=None):
    """
    Post
    """
    try:
        return requests.post(url, params=params)
    except Exception as ex:
        return None

main()
<commit_msg>[ci_tests] ignore docker demos<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from sys import exit
import errno
import json
import os
import shutil
import socket
import xml.etree.ElementTree as ET
import requests
from ipolutils.read_text_file import read_commented_text_file

import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../ipol_demo/modules/"))
from dispatcher.demorunnerinfo import DemoRunnerInfo
from dispatcher.policy import Policy

HOST = socket.gethostbyname(socket.gethostname())
user = 'ipol'
compilation_path = os.path.join("/", "home", user, "ipolDevel", "ci_tests", "compilation_folder")
ignored_ids_file_path = sys.argv[1] if len(sys.argv) > 1 else ''

def main():
    """
    Main
    """
    all_success = True
    demorunners = read_demorunners()

    id_list = []
    if os.path.exists(ignored_ids_file_path):
        id_list = [int(id) for id in read_commented_text_file(ignored_ids_file_path)]

    for demo_id in get_published_demos():
        json_ddl = json.loads(get_ddl(demo_id))
        build_section = json_ddl.get('build')
        requirements = json_ddl.get('general').get('requirements')

        if build_section is None:
            print(f"Demo #{demo_id} doesn't have a build section")
            all_success = False
            continue

        if demo_id in id_list:
            continue

        if 'docker' in requirements.split(','):
            # don't test the compilation of docker demos since we keep images
            continue

        if not build(demo_id, build_section, requirements, demorunners):
            all_success = False

    if all_success:
        exit(0)
    else:
        exit(-1)

def get_compilation_path(demo_id):
    """
    Get compilation path
    """
    return os.path.join(compilation_path, str(demo_id))

def create_dir(path):
    """
    Implement the UNIX shell command "mkdir -p"
    with given path as parameter.
    """
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get_published_demos():
    """
    Get demos
    """
    resp = post(f'http://{HOST}/api/demoinfo/demo_list')
    if resp == None:
        print("Can't get the list of demos!")
        return []

    response = resp.json()
    demos = []
    if response['status'] != 'OK':
        print("ERROR: demo_list returned KO")
        return []
    for demo in response['demo_list']:
        if demo.get('state') == "published":
            demos.append(demo['editorsdemoid'])
    return demos

def get_ddl(demo_id):
    """
    Read the DDL of the demo
    """
    resp = post(f'http://{HOST}/api/demoinfo/get_ddl', params={"demo_id": demo_id})
    if resp == None:
        print(f"Can't get the DDL of demo #{demo_id}!")
        return ""

    response = resp.json()
    if response['status'] != 'OK':
        print(f"ERROR: get_ddl returned KO for demo #{demo_id}")
    last_demodescription = response.get('last_demodescription')
    return last_demodescription.get('ddl')

def build(demo_id, build_section, requirements, demorunners):
    """
    Execute the build section.
    Returns False if any build fails. True is everything's fine.
    """
    suitable_demorunners = Policy.get_suitable_demorunners(requirements, demorunners)
    for demorunner in suitable_demorunners:
        demorunner_host = demorunner.server
        params = {'ddl_build': json.dumps(build_section), 'compilation_path': get_compilation_path(demo_id)}
        response = post(f'{demorunner_host}api/demorunner/{demorunner.name}/test_compilation', params)

        if response == None or response.status_code == None:
            print(f"Bad response from DR={demorunner.name} when trying to build demo #{demo_id}: {str(response)}")
            return False

        if response.status_code == 200:
            json_response = response.json()
            if json_response.get('status') == 'OK':
                continue
            else:
                print(f"Couldn't build demo #{demo_id} in DR={demorunner.name} ({json_response}).")
                return False
        else:
            msg = f"Bad HTTP response status_code from DR={demorunner.name} when trying to build demo #{demo_id}."
            if response.status_code == 504:
                msg += " HTTP error 504 (gateway timeout)"
            elif response.status_code == 404:
                msg += f" HTTP error 404 (not found)"
            else:
                msg += f" HTTP error {response.status_code}"
            print(msg)
            return False

    # No DR failed or KO: compilation test passed
    return True

def read_demorunners():
    """
    Read demorunners xml
    """
    demorunners = []
    tree = ET.parse(os.path.join("/", "home", user, "ipolDevel", "ipol_demo", "modules",
                                 "config_common", "demorunners.xml"))
    root = tree.getroot()
    for demorunner in root.findall('demorunner'):
        capabilities = []

        for capability in demorunner.findall('capability'):
            capabilities.append(capability.text)

        demorunner = DemoRunnerInfo(
            demorunner.find('server').text,
            demorunner.get('name'),
            demorunner.find('serverSSH').text,
            capabilities
        )

        demorunners.append(demorunner)

    return demorunners

def post(url, params=None):
    """
    Post
    """
    try:
        return requests.post(url, params=params)
    except Exception as ex:
        return None

main()
<|endoftext|>


<commit_before>from unittest import TestCase
from mock import patch

from dusty import constants
from dusty.compiler.compose import (get_compose_dict, _composed_app_dict, _composed_service_dict,
                                    _get_ports_list, _compile_docker_command, _get_compose_volumes)
from ..test_test_cases import all_test_configs

def repo_path(name):
    return "/Users/gc/{}".format(name.split('/')[-1])

@patch('dusty.compiler.compose.repo_path', side_effect=repo_path)
class TestComposeCompiler(TestCase):
    def test_composed_volumes(self, *args):
        specs = {
            'apps': {
                'app1': {
                    'repo': '/cool/repo/app1',
                    'depends': {
                        'libs': ['lib1', 'lib2']
                    }
                }
            },
            'libs': {
                'lib1': {
                    'repo': '/cool/repo/lib1',
                },
                'lib2': {
                    'repo': '/cool/repo/lib2',
                },
                'lib3': {
                    'repo': '/cool/repo/lib3',
                }
            }
        }
        expected_volumes = [
            '/Users/gc/app1:/gc/app1',
            '/Users/gc/lib1:/gc/lib1',
            '/Users/gc/lib2:/gc/lib2'
        ]
        returned_volumes = _get_compose_volumes('app1', specs)
        self.assertEqual(expected_volumes, returned_volumes)

    def test_compile_command_with_once(self, *args):
        app_spec = {
            'repo': 'cool/repo/app1',
            'commands': {
                'once': "one_time.sh",
                'always': "always.sh"
            }
        }
        expected_command_list = ["sh -c \"export PATH=$PATH:/gc/app1",
                                 " if [ ! -f /var/run/dusty/docker_first_time_started ]",
                                 " then touch /var/run/dusty/docker_first_time_started",
                                 " one_time.sh",
                                 " fi",
                                 " always.sh\""]
        returned_command = _compile_docker_command(app_spec).split(";")
        self.assertEqual(expected_command_list, returned_command)

    def test_compile_command_without_once(self, *args):
        app_spec = {
            'repo': 'cool/repo/app1',
            'commands': {
                'always': "always.sh"
            }
        }
        expected_command_list = ["sh -c \"export PATH=$PATH:/gc/app1",
                                 " if [ ! -f /var/run/dusty/docker_first_time_started ]",
                                 " then touch /var/run/dusty/docker_first_time_started",
                                 " fi",
                                 " always.sh\""]
        returned_command = _compile_docker_command(app_spec).split(";")
        self.assertEqual(expected_command_list, returned_command)

    def test_ports_list(self, *args):
        port_spec = {
            'docker_compose': {
                'app1': [
                    { 'mapped_host_port': 8000, 'in_container_port': 1},
                    { 'mapped_host_port': 8005, 'in_container_port': 90}
                ],
                'app2': []
            }
        }
        expected_port_lists = {
            'app1': [
                '8000:1',
                '8005:90'
            ],
            'app2': []
        }
        for app in ['app1', 'app2']:
            self.assertEqual(expected_port_lists[app], _get_ports_list(app, port_spec))

    @patch('dusty.compiler.compose._compile_docker_command', return_value="what command?")
    def test_composed_app(self, *args):
        specs = {
            'apps': {
                'app1': {
                    'repo': '/cool/repo/app1',
                    'depends': {
                        'libs': ['lib1', 'lib2'],
                        'services': ['service1', 'service2'],
                        'apps': ['app2']
                    },
                    'image': 'awesomeGCimage'
                    'mount': '/gc/app1'
                }
            },
            'libs': {
                'lib1': {
                    'repo': '/cool/repo/lib1',
                },
                'lib2': {
                    'repo': '/cool/repo/lib2',
                },
                'lib3': {
                    'repo': '/cool/repo/lib3',
                }
            },
            'services':{
                'service1': {

                },
                'service2': {

                }
            }
        }
        port_spec = {
            'docker_compose': {
                'app1': [
                    { 'mapped_host_port': 8000, 'in_container_port': 1},
                    { 'mapped_host_port': 8005, 'in_container_port': 90}
                ],
                'app2': []
            }
        }
        expected_app_config = {
            'image': 'awesomeGCimage',
            'command': 'what command?',
            'links': [
                'service1',
                'service2',
                'app2'
            ],
            'volumes': [
                '/Users/gc/app1:/gc/app1',
                '/Users/gc/lib1:/gc/lib1',
                '/Users/gc/lib2:/gc/lib2'
            ],
            'ports': [
                '8000:1',
                '8005:90'
            ]
        }
        retured_config = _composed_app_dict('app1', specs, port_spec)
        self.assertEqual(expected_app_config, retured_config)
<commit_msg>fix random stuff<commit_after>from unittest import TestCase
from mock import patch

from dusty import constants
from dusty.compiler.compose import (get_compose_dict, _composed_app_dict, _composed_service_dict,
                                    _get_ports_list, _compile_docker_command, _get_compose_volumes)
from ..test_test_cases import all_test_configs

def repo_path(name):
    return "/Users/gc/{}".format(name.split('/')[-1])

@patch('dusty.compiler.compose.repo_path', side_effect=repo_path)
class TestComposeCompiler(TestCase):
    def test_composed_volumes(self, *args):
        specs = {
            'apps': {
                'app1': {
                    'repo': '/cool/repo/app1',
                    'mount': '/gc/app1',
                    'depends': {
                        'libs': ['lib1', 'lib2']
                    }
                }
            },
            'libs': {
                'lib1': {
                    'repo': '/cool/repo/lib1',
                    'mount': '/gc/lib1'
                },
                'lib2': {
                    'repo': '/cool/repo/lib2',
                    'mount': '/gc/lib2'
                },
                'lib3': {
                    'repo': '/cool/repo/lib3',
                }
            }
        }
        expected_volumes = [
            '/Users/gc/app1:/gc/app1',
            '/Users/gc/lib1:/gc/lib1',
            '/Users/gc/lib2:/gc/lib2'
        ]
        returned_volumes = _get_compose_volumes('app1', specs)
        self.assertEqual(expected_volumes, returned_volumes)

    def test_compile_command_with_once(self, *args):
        app_spec = {
            'repo': '/cool/repo/app1',
            'mount': '/gc/app1',
            'commands': {
                'once': "one_time.sh",
                'always': "always.sh"
            }
        }
        expected_command_list = ["sh -c \"export PATH=$PATH:/gc/app1",
                                 " if [ ! -f /var/run/dusty/docker_first_time_started ]",
                                 " then touch /var/run/dusty/docker_first_time_started",
                                 " one_time.sh",
                                 " fi",
                                 " always.sh\""]
        returned_command = _compile_docker_command(app_spec).split(";")
        self.assertEqual(expected_command_list, returned_command)

    def test_compile_command_without_once(self, *args):
        app_spec = {
            'repo': '/cool/repo/app1',
            'mount': '/gc/app1',
            'commands': {
                'always': "always.sh"
            }
        }
        expected_command_list = ["sh -c \"export PATH=$PATH:/gc/app1",
                                 " if [ ! -f /var/run/dusty/docker_first_time_started ]",
                                 " then touch /var/run/dusty/docker_first_time_started",
                                 " fi",
                                 " always.sh\""]
        returned_command = _compile_docker_command(app_spec).split(";")
        self.assertEqual(expected_command_list, returned_command)

    def test_ports_list(self, *args):
        port_spec = {
            'docker_compose': {
                'app1': [
                    { 'mapped_host_port': 8000, 'in_container_port': 1},
                    { 'mapped_host_port': 8005, 'in_container_port': 90}
                ],
                'app2': []
            }
        }
        expected_port_lists = {
            'app1': [
                '8000:1',
                '8005:90'
            ],
            'app2': []
        }
        for app in ['app1', 'app2']:
            self.assertEqual(expected_port_lists[app], _get_ports_list(app, port_spec))

    @patch('dusty.compiler.compose._compile_docker_command', return_value="what command?")
    def test_composed_app(self, *args):
        specs = {
            'apps': {
                'app1': {
                    'repo': '/cool/repo/app1',
                    'depends': {
                        'libs': ['lib1', 'lib2'],
                        'services': ['service1', 'service2'],
                        'apps': ['app2']
                    },
                    'image': 'awesomeGCimage',
                    'mount': '/gc/app1'
                }
            },
            'libs': {
                'lib1': {
                    'repo': '/cool/repo/lib1',
                    'mount': '/gc/lib1'
                },
                'lib2': {
                    'repo': '/cool/repo/lib2',
                    'mount': '/gc/lib2'
                },
                'lib3': {
                    'repo': '/cool/repo/lib3',
                    'mount': '/gc/lib3'
                }
            },
            'services':{
                'service1': {

                },
                'service2': {

                }
            }
        }
        port_spec = {
            'docker_compose': {
                'app1': [
                    { 'mapped_host_port': 8000, 'in_container_port': 1},
                    { 'mapped_host_port': 8005, 'in_container_port': 90}
                ],
                'app2': []
            }
        }
        expected_app_config = {
            'image': 'awesomeGCimage',
            'command': 'what command?',
            'links': [
                'service1',
                'service2',
                'app2'
            ],
            'volumes': [
                '/Users/gc/app1:/gc/app1',
                '/Users/gc/lib1:/gc/lib1',
                '/Users/gc/lib2:/gc/lib2'
            ],
            'ports': [
                '8000:1',
                '8005:90'
            ]
        }
        retured_config = _composed_app_dict('app1', specs, port_spec)
        self.assertEqual(expected_app_config, retured_config)
<|endoftext|>


<commit_before>"""
<Program Name>
  install.py

<Date Started>
  January 27, 2009

<Author>
  Carter Butaud

<Purpose>
  Installs seattle on a Windows Mobile device.
"""

import os
import platform
import windows_api

STARTER_SCRIPT_NAME = "start_seattle.py"

def output(text):
    print text

def get_startup_folder(version):
    # Returns the startup folder if it can be found,
    # takes version either "XP" or "Vista"
    try:
        try:
            # See if the installer executable found the startup folder
            # in the registry
            startup_file = open("startup.dat")
            startup_path = ""
            for line in startup_file:
                if line:
                    startup_path = line
            if startup_path and os.path.exists(startup_path):
                return startup_path
            else:
                raise Exception

        except:
            # Try locating the startup folder by looking at the registry key
            key_handle = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders")
            (startup_path, data_type) = _winreg.QueryValueEx(key_handle, "Startup")
            if startup_path and os.path.exists(startup_path):
                return startup_path
            else:
                raise Exception

    except:
        # If that fails, look in a couple obvious places, based on OS version
        if version == "Vista":
            # Look in probable Vista places
            startup_path = os.environ.get("HOMEDRIVE") + os.environ.get("HOMEPATH") + "\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup"
            if os.path.exists(startup_path):
                return startup_path
        elif version == "XP":
            # Look in probable XP places
            startup_path = os.environ.get("HOMEDRIVE") + os.environ.get("HOMEPATH") + "\\Start Menu\\Programs\\Startup"
            if os.path.exists(startup_path):
                return startup_path
        return ""

def main():
    global STARTER_SCRIPT_NAME
    added_to_startup = 0
    prog_path = os.getcwd()
    output("Checking OS...")
    if platform.system() != "Windows":
        # If it's not Windows, this isn't the right installer
        output("This installer is designed for use on Windows XP or Vista only.")
        output("Please download the correct installer for your operating system and try again.")
    else:
        if not (platform.release() == "Vista" or platform.release() == "XP"):
            # If it's not XP or Vista, display error and fail
            output("Unsupported Windows Version: " + platform.system + " " + platform.release)
        else:
            # Correct version of Windows, go ahead with the install
            output("Done.")
            output("Locating startup folder...")
            startup = get_startup_folder(platform.release())
            if (startup):
                # If we did find the location of the startup folder
                output("Done.")
                added_to_startup = 1
                if os.path.exists(startup + "\\" + STARTER_SCRIPT_NAME):
                    # If the starter script is already in the User's startup folder, assume that
                    # it is already installed and quit
                    output("Seattle is already installed on this computer.")
                else:
                    # Assume seattle is not installed, and move ahead
                    # Generate a script to run from the startup folder
                    output("Generating startup script...")
                    script_lines = []
                    script_lines.append("import sys\n")
                    script_lines.append("\n")
                    script_lines.append("SEATTLE_DIR = r\"" + prog_path + "\"\n")
                    script_lines.append("\n")
                    script_lines.append("def main():\n")
                    script_lines.append("  sys.path.append(SEATTLE_DIR)\n")
                    script_lines.append("  import windows_api\n")
                    script_lines.append("  windows_api.launchPythonScript(INSTALL_DIR + \"\\start_seattle.py\")\n")
                    script_lines.append("\n")
                    script_lines.append("if __name__ == \"__main__\":\n")
                    script_lines.append("  main()\n")
                    starter_f = open(startup + "\\" + STARTER_SCRIPT_NAME, "w")
                    starter_f.writelines(script_lines)
                    starter_f.close()
                    output("Done.")

                    # Generate an uninstaller, which will just delete the startup script
                    output("Generating uninstaller script...")
                    uninstaller_f = open("uninstall.py", "w")
                    script_lines = []
                    script_lines.append("import os\n")
                    script_lines.append("\n")
                    script_lines.append("STARTER_SCRIPT = r\"" + startup + "\\" + STARTER_SCRIPT_NAME + "\"\n")
                    script_lines.append("\n")
                    script_lines.append("def main():\n")
                    script_lines.append("  os.remove(STARTER_SCRIPT)\n")
                    script_lines.append("  print \"seattle has been uninstalled. You may now remove the installation " +
                                        "files from your computer, if you wish.\"\n")
                    script_lines.append("\n")
                    script_lines.append("if __name__ == \"__main__\":\n")
                    script_lines.append("  main()\n")
                    uninstaller_f.writelines(script_lines)
                    uninstaller_f.close()
                    output("Done.")

                    post_startup_steps(prog_path, added_to_startup)

            else:
                # We couldn't find the startup folder, note that we couldn't,
                # but move on with the install
                # TODO: It should be relatively simple to allow the user to
                # manually enter the location of their startup folder, if they
                # so desire
                output("Failed.")
                added_to_startup = 0
                post_startup_steps(prog_path, added_to_startup)

def post_startup_steps(prog_path, added_to_startup):
    # Next, generate keys for the node by calling the appropriate script
    output("Generating identity key (may take a few minutes)...")
    windows_api.launchPythonScript("createnodekeys.py")
    output("Done.")

    # Set the program's path in repyconstants.py
    const_f = open(prog_path + "/repy_constants.py")
    lines = []
    for line in const_f:
        to_append = line
        if line.startswith("PATH_PYTHON_INSTALL"):
            to_append = 'PATH_PYTHON_INSTALL = "' + prog_path + '"\n'
        if line.startswith("PATH_SEATTLE_INSTALL"):
            to_append = 'PATH_SEATTLE_INSTALL = "' + prog_path + '"\n'
        lines.append(to_append)
    const_f.close()
    const_f = open(prog_path + "/repy_constants.py", "w")
    const_f.writelines(lines)
    const_f.close()

    # Finally, call the script that starts the node manager and software
    # updater
    windows_api.launchPythonScript("start_seattle.py")
    if added_to_startup == 1:
        # If everything went smoothly, display basic success messages
        output("Seattle was successfully installed on your computer.")
        output("If you would like to uninstall Seattle at any time, run the uninstall.bat script located in this directory.")
    else:
        # If we weren't able to locate the startup folder, inform the user
        # about this, and give some alternative instructions
        output("We were unable to find the startup folder on your computer.")
        output("Seattle has been started, but it will not automatically run when you start your computer.")
        output("You can manually start it at any time, just run the start_seattle.bat script in this directory.")
        output("This will only start Seattle if it is not already running, so you don't have to worry about multiple instances taking up too much memory.")

if __name__ == "__main__":
    main()
<commit_msg>Fixed how installer finds startup folder<commit_after>"""
<Program Name>
  install.py

<Date Started>
  January 27, 2009

<Author>
  Carter Butaud

<Purpose>
  Installs seattle on a Windows Mobile device.
"""

import os
import platform
import windows_api
import nonportable

STARTER_SCRIPT_NAME = "start_seattle.py"

def output(text):
    print text

def get_startup_folder(version):
    # Returns the startup folder if it can be found,
    # takes version either "XP" or "Vista"
    try:
        try:
            # See if the installer executable found the startup folder
            # in the registry
            startup_file = open("startup.dat")
            startup_path = ""
            for line in startup_file:
                if line:
                    startup_path = line
            if startup_path and os.path.exists(startup_path):
                return startup_path
            else:
                raise Exception
        except:
            # Try locating the startup folder by looking at the registry key
            key_handle = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders")
            (startup_path, data_type) = _winreg.QueryValueEx(key_handle, "Startup")
            if startup_path and os.path.exists(startup_path):
                return startup_path
            else:
                raise Exception

    except:
        # If that fails, look in a couple obvious places, based on OS version
        if version == "Vista":
            # Look in probable Vista places
            startup_path = os.environ.get("HOMEDRIVE") + os.environ.get("HOMEPATH") + "\\AppData\\Roaming\\Microsoft\\Windows\\Start Menu\\Programs\\Startup"
            if os.path.exists(startup_path):
                return startup_path
        elif version == "XP":
            # Look in probable XP places
            startup_path = os.environ.get("HOMEDRIVE") + os.environ.get("HOMEPATH") + "\\Start Menu\\Programs\\Startup"
            if os.path.exists(startup_path):
                return startup_path
        elif version == "WindowsCE":
            # Look in probable Mobile places
            startup_path = "\\Windows\\Startup"
        return ""

def main():
    global STARTER_SCRIPT_NAME
    added_to_startup = 0
    prog_path = os.getcwd()
    output("Checking OS...")
    if platform.system() != "Windows":
        # If it's not Windows, this isn't the right installer
        output("This installer is designed for use on Windows XP or Vista only.")
        output("Please download the correct installer for your operating system and try again.")
    else:
        if not (nonportable.ostype == "WindowsCE"):
            # If it's not Windows Mobile, display error and fail
            output("Unsupported Windows Version: " + platform.system + " " + platform.release)
        else:
            # Correct version of Windows, go ahead with the install
            output("Done.")
            output("Locating startup folder...")
            startup = get_startup_folder(platform.release())
            if (startup):
                # If we did find the location of the startup folder
                output("Done.")
                added_to_startup = 1
                if os.path.exists(startup + "\\" + STARTER_SCRIPT_NAME):
                    # If the starter script is already in the User's startup folder, assume that
                    # it is already installed and quit
                    output("Seattle is already installed on this computer.")
                else:
                    # Assume seattle is not installed, and move ahead
                    # Generate a script to run from the startup folder
                    output("Generating startup script...")
                    script_lines = []
                    script_lines.append("import sys\n")
                    script_lines.append("\n")
                    script_lines.append("SEATTLE_DIR = r\"" + prog_path + "\"\n")
                    script_lines.append("\n")
                    script_lines.append("def main():\n")
                    script_lines.append("  sys.path.append(SEATTLE_DIR)\n")
                    script_lines.append("  import windows_api\n")
                    script_lines.append("  windows_api.launchPythonScript(INSTALL_DIR + \"\\start_seattle.py\")\n")
                    script_lines.append("\n")
                    script_lines.append("if __name__ == \"__main__\":\n")
                    script_lines.append("  main()\n")
                    starter_f = open(startup + "\\" + STARTER_SCRIPT_NAME, "w")
                    starter_f.writelines(script_lines)
                    starter_f.close()
                    output("Done.")

                    # Generate an uninstaller, which will just delete the startup script
                    output("Generating uninstaller script...")
                    uninstaller_f = open("uninstall.py", "w")
                    script_lines = []
                    script_lines.append("import os\n")
                    script_lines.append("\n")
                    script_lines.append("STARTER_SCRIPT = r\"" + startup + "\\" + STARTER_SCRIPT_NAME + "\"\n")
                    script_lines.append("\n")
                    script_lines.append("def main():\n")
                    script_lines.append("  os.remove(STARTER_SCRIPT)\n")
                    script_lines.append("  print \"seattle has been uninstalled. You may now remove the installation " +
                                        "files from your computer, if you wish.\"\n")
                    script_lines.append("\n")
                    script_lines.append("if __name__ == \"__main__\":\n")
                    script_lines.append("  main()\n")
                    uninstaller_f.writelines(script_lines)
                    uninstaller_f.close()
                    output("Done.")

                    post_startup_steps(prog_path, added_to_startup)

            else:
                # We couldn't find the startup folder, note that we couldn't,
                # but move on with the install
                # TODO: It should be relatively simple to allow the user to
                # manually enter the location of their startup folder, if they
                # so desire
                output("Failed.")
                added_to_startup = 0
                post_startup_steps(prog_path, added_to_startup)

def post_startup_steps(prog_path, added_to_startup):
    # Next, generate keys for the node by calling the appropriate script
    output("Generating identity key (may take a few minutes)...")
    windows_api.launchPythonScript("createnodekeys.py")
    output("Done.")

    # Set the program's path in repyconstants.py
    const_f = open(prog_path + "/repy_constants.py")
    lines = []
    for line in const_f:
        to_append = line
        if line.startswith("PATH_PYTHON_INSTALL"):
            to_append = 'PATH_PYTHON_INSTALL = "' + prog_path + '"\n'
        if line.startswith("PATH_SEATTLE_INSTALL"):
            to_append = 'PATH_SEATTLE_INSTALL = "' + prog_path + '"\n'
        lines.append(to_append)
    const_f.close()
    const_f = open(prog_path + "/repy_constants.py", "w")
    const_f.writelines(lines)
    const_f.close()

    # Finally, call the script that starts the node manager and software
    # updater
    windows_api.launchPythonScript("start_seattle.py")
    if added_to_startup == 1:
        # If everything went smoothly, display basic success messages
        output("Seattle was successfully installed on your computer.")
        output("If you would like to uninstall Seattle at any time, run the uninstall.bat script located in this directory.")
    else:
        # If we weren't able to locate the startup folder, inform the user
        # about this, and give some alternative instructions
        output("We were unable to find the startup folder on your computer.")
        output("Seattle has been started, but it will not automatically run when you start your computer.")
        output("You can manually start it at any time, just run the start_seattle.bat script in this directory.")
        output("This will only start Seattle if it is not already running, so you don't have to worry about multiple instances taking up too much memory.")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import re

from google.appengine.api import users

import jinja2
import webapp2

import api

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    )

class MainPage(webapp2.RequestHandler):

    def get(self):
        path = self.request.path

        if path.startswith('/login'):
            return self.redirect(users.create_login_url('/'))
        else:
            user = users.get_current_user()
            template = JINJA_ENVIRONMENT.get_template('index.html')
            env = {
                'user': user,
                'users': users,
            }
            self.response.write(template.render(env))
            # if user:
            #     greeting = ('Welcome, %s! (sign out)' %
            #                 (user.nickname(), users.create_logout_url('/')))
            # else:
            #     greeting = ('Sign in or register.' %
            #                 users.create_login_url('/'))

            # self.response.out.write('%s' % greeting)

class ApiHandler(webapp2.RequestHandler):

    def get(self):
        #api.get_oauth_user()  # just for authentication
        path = self.request.path
        if path.startswith('/api/card/'):
            # retrieve an individual card
            response = api.card_view(self)
        elif path == '/api/cards':
            # query for many cards
            # '/api/cards' -> returns all cards, ordered by date added desc
            # '/api/cards?tag=tag1,tag2' -> as above, but with tag filtering
            # '/api/cards?user=<user_key> -> returns cards for a single user
            # '/api/cards?user=user_key&tags=tag1,tag2' -> obvious
            # '/api/cards?user=<user_key>&review=1' -> all cards for the
            #      user but sorted by next_review
            response = api.card_query(self)
        elif path == '/api/user':
            response = api.user_view_current(self)
        elif path.startswith('/api/user/'):
            # retrieve an individual user
            response = api.user_view(self)

        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def post(self):
        #api.get_oauth_user()  # just for authentication
        path = self.request.path
        if path == '/api/card':
            # add a new card
            response = api.card_add(self)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def put(self):
        path = self.request.path
        if re.match('/api/card/.+/import', path):
            # import a card to the current user's feed
            response = api.card_import(self)
        if re.match('/api/card/.+/review', path):
            # record a review done on a card
            response = api.card_update(self, review=True)
        elif path.startswith('/api/card/'):
            # update an individual card
            response = api.card_update(self)
        elif path.startswith('/api/user/'):
            # update an individual user
            response = api.user_update(self)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def delete(self):
        path = self.request.path
        if path.startswith('/api/card/'):
            # delete an individual card
            response = api.card_update(self, delete=True)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

application = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/api/.*', ApiHandler),
    ], debug=True)
<commit_msg>better login/logout handlers<commit_after>import os
import re

from google.appengine.api import users

import jinja2
import webapp2

import api

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.dirname(__file__)),
    extensions=['jinja2.ext.autoescape'],
    )

class MainPage(webapp2.RequestHandler):

    def get(self):
        path = self.request.path
        user = users.get_current_user()
        template = JINJA_ENVIRONMENT.get_template('index.html')
        env = {
            'user': user,
            'users': users,
        }
        self.response.write(template.render(env))
        # if user:
        #     greeting = ('Welcome, %s! (sign out)' %
        #                 (user.nickname(), users.create_logout_url('/')))
        # else:
        #     greeting = ('Sign in or register.' %
        #                 users.create_login_url('/'))

        # self.response.out.write('%s' % greeting)

class LoginHandler(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if self.request.path.startswith('/login'):
            return self.redirect(users.create_login_url('/'))
        if self.request.path.startswith('/logout'):
            return self.redirect(users.create_logout_url('/'))

class ApiHandler(webapp2.RequestHandler):

    def get(self):
        #api.get_oauth_user()  # just for authentication
        path = self.request.path
        if path.startswith('/api/card/'):
            # retrieve an individual card
            response = api.card_view(self)
        elif path == '/api/cards':
            # query for many cards
            # '/api/cards' -> returns all cards, ordered by date added desc
            # '/api/cards?tag=tag1,tag2' -> as above, but with tag filtering
            # '/api/cards?user=<user_key> -> returns cards for a single user
            # '/api/cards?user=user_key&tags=tag1,tag2' -> obvious
            # '/api/cards?user=<user_key>&review=1' -> all cards for the
            #      user but sorted by next_review
            response = api.card_query(self)
        elif path == '/api/user':
            response = api.user_view_current(self)
        elif path.startswith('/api/user/'):
            # retrieve an individual user
            response = api.user_view(self)

        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def post(self):
        #api.get_oauth_user()  # just for authentication
        path = self.request.path
        if path == '/api/card':
            # add a new card
            response = api.card_add(self)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def put(self):
        path = self.request.path
        if re.match('/api/card/.+/import', path):
            # import a card to the current user's feed
            response = api.card_import(self)
        if re.match('/api/card/.+/review', path):
            # record a review done on a card
            response = api.card_update(self, review=True)
        elif path.startswith('/api/card/'):
            # update an individual card
            response = api.card_update(self)
        elif path.startswith('/api/user/'):
            # update an individual user
            response = api.user_update(self)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

    def delete(self):
        path = self.request.path
        if path.startswith('/api/card/'):
            # delete an individual card
            response = api.card_update(self, delete=True)
        else:
            raise Exception("Unsupported API path: %s" % path)

        self.response.out.write(response)

application = webapp2.WSGIApplication([
    ('/', MainPage),
    ('/login', )
    ('/api/.*', ApiHandler),
    ], debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for `lyricsmaster` package."""

from collections import Iterable
import pytest
from click.testing import CliRunner

from lyricsmaster import lyricsmaster
from lyricsmaster import cli

@pytest.fixture(scope="module")
def songs():
    songs = [lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None),
             lyricsmaster.Song('Ragged and dirty', 'Bad news is coming', 'Luther Alison', None),
             lyricsmaster.Song('Red rooster', 'Bad news is coming', 'Luther Alison', None),
             lyricsmaster.Song('Life is bitch', 'Bad news is coming', 'Luther Alison', None)]
    return songs

class TestSongs:
    """Tests for Song Class."""
    song = lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None)

    def test_song(self):
        assert self.song.__repr__() == 'Song: Bad Love'

class TestAlbums:
    """Tests for Album Class."""

    songs = songs()
    album = lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)

    def test_album(self):
        assert self.album.idx == 0
        assert self.album.title == 'Bad news is coming'
        assert self.album.author == 'Luther Alison'

    def test_album_isiter(self):
        assert self.album.songs is Iterable
        assert len(self.album) == 4
        assert [elmt for elmt in self.album] == songs

class TestDiscography:
    """Tests for Album Class."""

    albums = [lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs),
                       lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)]
    discography = lyricsmaster.Discography('Luther Allison', albums)

    def test_album_isiter(self):
        assert self.discography.albums is Iterable
        assert len(self.discography) == 2
        assert [elmt for elmt in self.discography] == self.albums

def test_command_line_interface():
    """Test the CLI."""
    runner = CliRunner()
    result = runner.invoke(cli.main)
    assert result.exit_code == 0
    assert 'lyricsmaster.cli.main' in result.output
    help_result = runner.invoke(cli.main, ['--help'])
    assert help_result.exit_code == 0
    assert '--help  Show this message and exit.' in help_result.output
<commit_msg>tweaked tests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for `lyricsmaster` package."""

from collections import Iterable
import pytest
from click.testing import CliRunner

from lyricsmaster import lyricsmaster
from lyricsmaster import cli

@pytest.fixture(scope="module")
def songs():
    songs = [lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None),
             lyricsmaster.Song('Ragged and dirty', 'Bad news is coming', 'Luther Alison', None),
             lyricsmaster.Song('Red rooster', 'Bad news is coming', 'Luther Alison', None),
             lyricsmaster.Song('Life is bitch', 'Bad news is coming', 'Luther Alison', None)]
    return songs

class TestSongs:
    """Tests for Song Class."""
    song = lyricsmaster.Song('Bad Love', 'Bad news is coming','Luther Alison', None)

    def test_song(self):
        assert self.song.__repr__() == 'Song: Bad Love'

class TestAlbums:
    """Tests for Album Class."""

    songs = songs()
    album = lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)

    def test_album(self):
        assert self.album.idx == 0
        assert self.album.title == 'Bad news is coming'
        assert self.album.author == 'Luther Alison'

    def test_album_isiter(self):
        assert len(self.album) == 4
        assert [elmt for elmt in self.album] == songs

class TestDiscography:
    """Tests for Album Class."""

    albums = [lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs),
                       lyricsmaster.Album('Bad news is coming', 'Luther Alison', songs)]
    discography = lyricsmaster.Discography('Luther Allison', albums)

    def test_album_isiter(self):
        assert len(self.discography) == 2
        assert [elmt for elmt in self.discography] == self.albums

def test_command_line_interface():
    """Test the CLI."""
    runner = CliRunner()
    result = runner.invoke(cli.main)
    assert result.exit_code == 0
    assert 'lyricsmaster.cli.main' in result.output
    help_result = runner.invoke(cli.main, ['--help'])
    assert help_result.exit_code == 0
    assert '--help  Show this message and exit.' in help_result.output
<|endoftext|>


<commit_before>"""
conway.server
~~~~~~~~~~~~~~~
This module contains the web server logic.
"""
import os

from game import Board
from images import draw_frames, create_gif
from twitter import twitter_client

import redis
r = redis.from_url(os.environ.get("REDIS_URL"))

def grab_tweets():
    tweet = None
    max_id = r.get('max_id')
    while not tweet:
        tweets = twitter_client.search('@tweetgameoflife') #, max_id)
        try:
            tweet = next(tweet for tweet in tweets if '://' not in tweet.text and \
                                                      not tweet.text.startswith('RT'))
            r.set('max_id', tweets[0].id_str)
            return tweet
        except StopIteration:
            return None

def reply_with_conway():
    original_tweet = grab_tweets()
    if original_tweet:
        pattern = create_pattern_from_tweet(original_tweet)
        gif_name = gif_from_pattern(pattern)
        user_name = original_tweet.user.screen_name
        message = "Hi @{}, look at the complexity of your tweet!".format(user_name)
        twitter_client.update_with_media(filename=gif_name,
                                  status=message,
                                  in_reply_to_status_id=original_tweet.id_str)
    else:
        print("No one is tweeting, try again later :(")

def get_word_bin(string):
    return ''.join(format(ord(x), 'b') for x in string)

def create_pattern_from_tweet(original_tweet):
    words = original_tweet.text.split(' ')
    return '\n'.join([get_word_bin(word) for word in words])

def gif_from_pattern(pattern):
    """
    Main application logic: given a conway pattern
    from the lexicon, it returns a link to the image on imgur
    """
    draw_frames(Board(pattern))
    name = create_gif()
    return name

if __name__ == '__main__':
    reply_with_conway()
<commit_msg>only reply to ones not seen previously<commit_after>"""
conway.server
~~~~~~~~~~~~~~~
This module contains the web server logic.
"""
import os

from game import Board
from images import draw_frames, create_gif
from twitter import twitter_client

import redis
r = redis.from_url(os.environ.get("REDIS_URL"))

def grab_tweets():
    tweet = None
    max_id = r.get('max_id')
    while not tweet:
        tweets = twitter_client.search('@tweetgameoflife', max_id)
        try:
            tweet = next(tweet for tweet in tweets if '://' not in tweet.text and \
                                                      not tweet.text.startswith('RT'))
            r.set('max_id', tweets[0].id_str)
            return tweet
        except StopIteration:
            return None

def reply_with_conway():
    original_tweet = grab_tweets()
    if original_tweet:
        pattern = create_pattern_from_tweet(original_tweet)
        gif_name = gif_from_pattern(pattern)
        user_name = original_tweet.user.screen_name
        message = "Hi @{}, look at the complexity of your tweet!".format(user_name)
        twitter_client.update_with_media(filename=gif_name,
                                  status=message,
                                  in_reply_to_status_id=original_tweet.id_str)
    else:
        print("No one is tweeting, try again later :(")

def get_word_bin(string):
    return ''.join(format(ord(x), 'b') for x in string)

def create_pattern_from_tweet(original_tweet):
    words = original_tweet.text.split(' ')
    return '\n'.join([get_word_bin(word) for word in words])

def gif_from_pattern(pattern):
    """
    Main application logic: given a conway pattern
    from the lexicon, it returns a link to the image on imgur
    """
    draw_frames(Board(pattern))
    name = create_gif()
    return name

if __name__ == '__main__':
    reply_with_conway()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Build an ansible inventory list suitable for use by -i by finding the active
Auto Scaling Group in an Elastic Load Balancer.

If multiple ASGs are active in the ELB, no inventory is returned.

Assuming a single active ASG is found, a single machine is returned.  This inventory
is generally used to target a single machine in a cluster to run a cmomand.

Typical reponse:

10.2.42.79,

Typical use

ansible -i $(active_instances_in_asg.py --asg stage-edx-edxapp) -m shell -a 'management command'

"""

from __future__ import print_function
import argparse
import botocore.session
import botocore.exceptions
import sys
from collections import defaultdict
from os import environ
from itertools import chain

class ActiveInventory():

    profile = None

    def __init__(self, profile, region):
        self.profile = profile
        self.region  = region

    def run(self,asg_name):
        session = botocore.session.Session(profile=self.profile)
        asg = session.create_client('autoscaling',self.region)
        ec2 = session.create_client('ec2',self.region)

        groups = asg.describe_auto_scaling_groups()
        matching_groups = [g for g in groups['AutoScalingGroups'] for t in g['Tags'] if t['Key'] == 'Name' and t['Value'] == asg_name]

        groups_to_instances = {group['AutoScalingGroupName']: [instance['InstanceId'] for instance in group['Instances']] for group in matching_groups}
        instances_to_groups = {instance['InstanceId']: group['AutoScalingGroupName'] for group in matching_groups for instance in group['Instances'] }

        # We only need to check for ASGs in an ELB if we have more than 1.
        # If a cluster is running with an ASG out of the ELB, then there are larger problems.
        active_groups = defaultdict(dict)
        if len(matching_groups) > 1:
            elb = session.create_client('elb',self.region)
            for group in matching_groups:
                for load_balancer_name in group['LoadBalancerNames']:
                    instances = elb.describe_instance_health(LoadBalancerName=load_balancer_name)
                    active_instances = [instance['InstanceId'] for instance in instances['InstanceStates'] if instance['State'] == 'InService']
                    for instance_id in active_instances:
                        active_groups[instances_to_groups[instance_id]] = 1
            if len(active_groups) > 1:
                # When we have more than a single active ASG, we need to bail out as we don't know what ASG to pick an instance from
                print("Multiple active ASGs - unable to choose an instance", file=sys.stderr)
                return
            # If we found no active groups, because there are no ELBs (edxapp workers normally)
            # print a sensible reason why we failed
            elbs = list(chain.from_iterable([group['LoadBalancerNames'] for group in matching_groups]))
            if not (active_groups or elbs):
                print("Multiple ASGs and no ELB - unable to choose an instance", file=sys.stderr)
                return
        else:
            active_groups = { g['AutoScalingGroupName']: 1 for g in matching_groups }

        for group in active_groups.keys():
            for group_instance in groups_to_instances[group]:
                instance = ec2.describe_instances(InstanceIds=[group_instance])['Reservations'][0]['Instances'][0]
                if 'PrivateIpAddress' in instance:
                    print("{},".format(instance['PrivateIpAddress']))
                    return # We only want a single IP

if __name__=="__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--profile', help='The aws profile to use when connecting.')
    parser.add_argument('-l', '--list', help='Ansible passes this, we ignore it.', action='store_true', default=True)
    parser.add_argument('--asg',help='Name of the ASG we want active instances from.', required=True)
    args = parser.parse_args()

    region = environ.get('AWS_REGION','us-east-1')

    ActiveInventory(args.profile,region).run(args.asg)
<commit_msg>Handle multiple worker ASGs<commit_after>#!/usr/bin/env python

"""
Build an ansible inventory list suitable for use by -i by finding the active
Auto Scaling Group in an Elastic Load Balancer.

If multiple ASGs are active in the ELB, no inventory is returned.

Assuming a single active ASG is found, a single machine is returned.  This inventory
is generally used to target a single machine in a cluster to run a cmomand.

Typical reponse:

10.2.42.79,

Typical use

ansible -i $(active_instances_in_asg.py --asg stage-edx-edxapp) -m shell -a 'management command'

"""

from __future__ import print_function
import argparse
import botocore.session
import botocore.exceptions
import sys
from collections import defaultdict
from os import environ
from itertools import chain

class ActiveInventory():

    profile = None

    def __init__(self, profile, region):
        self.profile = profile
        self.region  = region

    def run(self,asg_name):
        session = botocore.session.Session(profile=self.profile)
        asg = session.create_client('autoscaling',self.region)
        ec2 = session.create_client('ec2',self.region)

        groups = asg.describe_auto_scaling_groups()
        matching_groups = [g for g in groups['AutoScalingGroups'] for t in g['Tags'] if t['Key'] == 'Name' and t['Value'] == asg_name]

        groups_to_instances = {group['AutoScalingGroupName']: [instance['InstanceId'] for instance in group['Instances']] for group in matching_groups}
        instances_to_groups = {instance['InstanceId']: group['AutoScalingGroupName'] for group in matching_groups for instance in group['Instances'] }

        # We only need to check for ASGs in an ELB if we have more than 1.
        # If a cluster is running with an ASG out of the ELB, then there are larger problems.
        active_groups = defaultdict(dict)
        if len(matching_groups) > 1:
            elb = session.create_client('elb',self.region)
            for group in matching_groups:
                for load_balancer_name in group['LoadBalancerNames']:
                    instances = elb.describe_instance_health(LoadBalancerName=load_balancer_name)
                    active_instances = [instance['InstanceId'] for instance in instances['InstanceStates'] if instance['State'] == 'InService']
                    for instance_id in active_instances:
                        active_groups[instances_to_groups[instance_id]] = 1

            # If we found no active groups, because there are no ELBs (edxapp workers normally)
            elbs = list(chain.from_iterable([group['LoadBalancerNames'] for group in matching_groups]))
            if not (active_groups or elbs):
                # This implies we're in a worker cluster since we have no ELB and we didn't find an active group above
                for group in matching_groups:
                    # Asgard marks a deleting ASG with SuspendedProcesses
                    # If the ASG doesn't have those, then it's "Active" and a worker since there was no ELB above
                    if not {'Launch','AddToLoadBalancer'} <= {i['ProcessName'] for i in group['SuspendedProcesses']}:
                        active_groups[group['AutoScalingGroupName']] = 1

            if len(active_groups) > 1:
                # When we have more than a single active ASG, we need to bail out as we don't know what ASG to pick an instance from
                print("Multiple active ASGs - unable to choose an instance", file=sys.stderr)
                return
        else:
            active_groups = { g['AutoScalingGroupName']: 1 for g in matching_groups }

        for group in active_groups.keys():
            for group_instance in groups_to_instances[group]:
                instance = ec2.describe_instances(InstanceIds=[group_instance])['Reservations'][0]['Instances'][0]
                if 'PrivateIpAddress' in instance:
                    print("{},".format(instance['PrivateIpAddress']))
                    return # We only want a single IP

if __name__=="__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--profile', help='The aws profile to use when connecting.')
    parser.add_argument('-l', '--list', help='Ansible passes this, we ignore it.', action='store_true', default=True)
    parser.add_argument('--asg',help='Name of the ASG we want active instances from.', required=True)
    args = parser.parse_args()

    region = environ.get('AWS_REGION','us-east-1')

    ActiveInventory(args.profile,region).run(args.asg)
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User
from django.contrib.admin.util import quote
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import smart_unicode
from django.utils.safestring import mark_safe

ADDITION = 1
CHANGE = 2
DELETION = 3

class LogEntryManager(models.Manager):
    def log_action(self, user_id, content_type_id, object_id, object_repr, action_flag, change_message=''):
        e = self.model(None, None, user_id, content_type_id, smart_unicode(object_id), object_repr[:200], action_flag, change_message)
        e.save()

class LogEntry(models.Model):
    action_time = models.DateTimeField(_('action time'), auto_now=True)
    user = models.ForeignKey(User)
    content_type = models.ForeignKey(ContentType, blank=True, null=True)
    object_id = models.TextField(_('object id'), blank=True, null=True)
    object_repr = models.CharField(_('object repr'), max_length=200)
    action_flag = models.PositiveSmallIntegerField(_('action flag'))
    change_message = models.TextField(_('change message'), blank=True)
    objects = LogEntryManager()
    class Meta:
        verbose_name = _('log entry')
        verbose_name_plural = _('log entries')
        db_table = 'django_admin_log'
        ordering = ('-action_time',)

    def __repr__(self):
        return smart_unicode(self.action_time)

    def __unicode__(self):

        if self.action_flag == ADDITION:
            return _('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return _('Changed "%(object)s" - %(changes)s') % {'object': self.object_repr, 'changes': self.change_message}
        elif self.action_flag == DELETION:
            return _('Deleted "%(object)s."') % {'object': self.object_repr}

        return _('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

    def get_edited_object(self):
        "Returns the edited object represented by this log entry"
        return self.content_type.get_object_for_this_type(pk=self.object_id)

    def get_admin_url(self):
        """
        Returns the admin URL to edit the object represented by this log entry.
        This is relative to the Django admin index page.
        """
        if self.content_type and self.object_id:
            return mark_safe(u"%s/%s/%s/" % (self.content_type.app_label, self.content_type.model, quote(self.object_id)))
        return None
<commit_msg>Fixed #16438 -- Corrected whitespace in admin models module. Thanks, philipkimmey.<commit_after>from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User
from django.contrib.admin.util import quote
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import smart_unicode
from django.utils.safestring import mark_safe

ADDITION = 1
CHANGE = 2
DELETION = 3

class LogEntryManager(models.Manager):
    def log_action(self, user_id, content_type_id, object_id, object_repr, action_flag, change_message=''):
        e = self.model(None, None, user_id, content_type_id, smart_unicode(object_id), object_repr[:200], action_flag, change_message)
        e.save()

class LogEntry(models.Model):
    action_time = models.DateTimeField(_('action time'), auto_now=True)
    user = models.ForeignKey(User)
    content_type = models.ForeignKey(ContentType, blank=True, null=True)
    object_id = models.TextField(_('object id'), blank=True, null=True)
    object_repr = models.CharField(_('object repr'), max_length=200)
    action_flag = models.PositiveSmallIntegerField(_('action flag'))
    change_message = models.TextField(_('change message'), blank=True)

    objects = LogEntryManager()

    class Meta:
        verbose_name = _('log entry')
        verbose_name_plural = _('log entries')
        db_table = 'django_admin_log'
        ordering = ('-action_time',)

    def __repr__(self):
        return smart_unicode(self.action_time)

    def __unicode__(self):
        if self.action_flag == ADDITION:
            return _('Added "%(object)s".') % {'object': self.object_repr}
        elif self.action_flag == CHANGE:
            return _('Changed "%(object)s" - %(changes)s') % {'object': self.object_repr, 'changes': self.change_message}
        elif self.action_flag == DELETION:
            return _('Deleted "%(object)s."') % {'object': self.object_repr}

        return _('LogEntry Object')

    def is_addition(self):
        return self.action_flag == ADDITION

    def is_change(self):
        return self.action_flag == CHANGE

    def is_deletion(self):
        return self.action_flag == DELETION

    def get_edited_object(self):
        "Returns the edited object represented by this log entry"
        return self.content_type.get_object_for_this_type(pk=self.object_id)

    def get_admin_url(self):
        """
        Returns the admin URL to edit the object represented by this log entry.
        This is relative to the Django admin index page.
        """
        if self.content_type and self.object_id:
            return mark_safe(u"%s/%s/%s/" % (self.content_type.app_label, self.content_type.model, quote(self.object_id)))
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import subprocess
import sys

skslc = sys.argv[1]
clangFormat = sys.argv[2]
fetchClangFormat = sys.argv[3]
processors = sys.argv[4:]

exeSuffix = '.exe' if sys.platform.startswith('win') else '';

for p in processors:
    print("Recompiling " + p + "...")
    try:
        if not os.path.isfile(clangFormat + exeSuffix):
            subprocess.check_call([sys.executable, fetchClangFormat]);

        noExt, _ = os.path.splitext(p)
        head, tail = os.path.split(noExt)
        targetDir = os.path.join(head, "generated")
        if not os.path.isdir(targetDir):
            os.mkdir(targetDir)
        target = os.path.join(targetDir, tail)
        subprocess.check_output([skslc, p, target + ".h"])
        subprocess.check_call(clangFormat + " --sort-includes=false -i \"" +
                              target + ".h\"", shell=True)
        subprocess.check_output([skslc, p, target + ".cpp"])
        subprocess.check_call(clangFormat + " --sort-includes=false -i \"" +
                              target + ".cpp\"", shell=True)
    except subprocess.CalledProcessError as err:
        print("### Error compiling " + p + ":")
        print(err.output)
        exit(1)
<commit_msg>Remove "Recompiling..." output when building .fp files<commit_after>#!/usr/bin/env python
#
# Copyright 2017 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import subprocess
import sys

skslc = sys.argv[1]
clangFormat = sys.argv[2]
fetchClangFormat = sys.argv[3]
processors = sys.argv[4:]

exeSuffix = '.exe' if sys.platform.startswith('win') else '';

for p in processors:
    try:
        if not os.path.isfile(clangFormat + exeSuffix):
            subprocess.check_call([sys.executable, fetchClangFormat]);

        noExt, _ = os.path.splitext(p)
        head, tail = os.path.split(noExt)
        targetDir = os.path.join(head, "generated")
        if not os.path.isdir(targetDir):
            os.mkdir(targetDir)
        target = os.path.join(targetDir, tail)
        subprocess.check_output([skslc, p, target + ".h"])
        subprocess.check_call(clangFormat + " --sort-includes=false -i \"" +
                              target + ".h\"", shell=True)
        subprocess.check_output([skslc, p, target + ".cpp"])
        subprocess.check_call(clangFormat + " --sort-includes=false -i \"" +
                              target + ".cpp\"", shell=True)
    except subprocess.CalledProcessError as err:
        print("### Error compiling " + p + ":")
        print(err.output)
        exit(1)
<|endoftext|>