<commit_before>
import os

from collections import defaultdict
from twitter.common.collections import OrderedSet
from twitter.pants import get_buildroot, JvmTarget, TaskError, JarLibrary, InternalTarget, JarDependency

class JvmDependencyAnalyzer(object):
  def __init__(self, context, check_missing_deps, check_missing_direct_deps, check_unnecessary_deps):
    self._context = context
    self._context.products.require('classes')
    self._context.products.require('ivy_jar_products')

    self._check_missing_deps = check_missing_deps
    self._check_missing_direct_deps = check_missing_direct_deps
    self._check_unnecessary_deps = check_unnecessary_deps

    # Memoized map from absolute path of source, class or jar file to an OrderedSet of targets.
    #
    # The value is usually a singleton, because a source or class file belongs to a single target.
    # However a single jar may be provided (transitively or intransitively) by multiple JarLibrary targets.
    # But if there is a JarLibrary target that depends on a jar directly, then that "canonical" target
    # will be the first one in the list of targets.
    #
    # Don't access directly. Call self.get_targets_by_file() instead.
    self._targets_by_file = None

    # Memoized map from target to all its transitive dep targets.
    # Don't access directly. Call self.get_transitive_deps_by_target() instead.
    self._transitive_deps_by_target = None

  def get_targets_by_file(self):
    """Compute the targets_by_file mapping for all targets in-play for this pants run.

    Memoizes for efficiency. Should only be called after codegen, so that all synthetic targets
    and injected deps are taken into account.
    """
    if self._targets_by_file is None:
      return self._compute_targets_by_file()  # Will memoize if allowed.
    else:
      return self._targets_by_file

  def _compute_targets_by_file(self):
    targets_by_file = defaultdict(OrderedSet)
    jarlibs_by_id = defaultdict(set)  # Multiple JarLibrary targets can provide the same (org, name).
    # Compute src -> target.
    buildroot = get_buildroot()
    # Look at all targets in-play for this pants run. Does not include synthetic targets,
    for target in self._context.targets():
      if isinstance(target, JvmTarget):
        for src in target.sources:
          targets_by_file[os.path.join(buildroot, target.target_base, src)].add(target)
      elif isinstance(target, JarLibrary):
        for jardep in target.dependencies:
          if isinstance(jardep, JarDependency):
            jarlibs_by_id[(jardep.org, jardep.name)].add(target)

    # Compute class -> target for classes in previous compile groups.
    genmap = self._context.products.get('classes')
    for tgt, products in genmap.itermappings():
      # tgt could also be a string, as we (ab)use the same genmap for the relsrc->classes mapping.
      if isinstance(tgt, JvmTarget):
        for basedir, classes in products.items():
          for cls in classes:
            targets_by_file[os.path.join(basedir, cls)].add(tgt)

    # Compute jar -> target.
    ivy_products = self._context.products.get_data('ivy_jar_products')
    if ivy_products:
      for ivyinfo in ivy_products.values():
        for ref in ivyinfo.modules_by_ref:
          target_key = (ref.org, ref.name)
          if target_key in jarlibs_by_id:
            jarlib_targets = jarlibs_by_id[target_key]
            for jar in ivyinfo.modules_by_ref[ref].artifacts:
              for jarlib_target in jarlib_targets:
                targets_by_file[jar.path].add(jarlib_target)
            # Map all indirect, transitive deps of the jar to this target as well, since we allow deps on them.
            for dep in ivyinfo.deps_by_caller.get(ref, []):
              depmodule = ivyinfo.modules_by_ref.get(dep, None)
              if depmodule:
                for depjar in depmodule.artifacts:
                  for jarlib_target in jarlib_targets:
                    targets_by_file[depjar.path].add(jarlib_target)
      # Only memoize once ivy_jar_products are available.
      self._targets_by_file = targets_by_file
    return targets_by_file

  def get_transitive_deps_by_target(self):
    """Compute the transitive_deps_by_target mapping from all targets in-play for this pants run.

    Memoizes for efficiency. Should only be called after codegen, so that all synthetic targets
    and injected deps are taken into account.
    """
    if self._transitive_deps_by_target is None:
      return self._compute_transitive_deps_by_target()
    else:
      return self._transitive_deps_by_target

  def _compute_transitive_deps_by_target(self):
    # Sort from least to most dependent.
    sorted_targets = reversed(InternalTarget.sort_targets(self._context.targets()))
    transitive_deps_by_target = defaultdict(set)
    # Iterate in dep order, to accumulate the transitive deps for each target.
    for target in sorted_targets:
      transitive_deps = set()
      if hasattr(target, 'dependencies'):
        for dep in target.dependencies:
          transitive_deps.update(transitive_deps_by_target.get(dep, []))
          transitive_deps.add(dep)
        transitive_deps_by_target[target] = transitive_deps
    self._transitive_deps_by_target = transitive_deps_by_target  # Memoize.
    return transitive_deps_by_target

  def check(self, srcs, actual_deps):
    """Check for missing deps.

    See docstring for _compute_missing_deps for details.
    """
    if self._check_missing_deps or self._check_missing_direct_deps or self._check_unnecessary_deps:
      missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps = \
        self._compute_missing_deps(srcs, actual_deps)

      buildroot = get_buildroot()
      def shorten(path):  # Make the output easier to read.
        for prefix in [buildroot, self._context.ivy_home]:
          if path.startswith(prefix):
            return os.path.relpath(path, prefix)
        return path

      if self._check_missing_deps and (missing_file_deps or missing_tgt_deps):
        for (tgt_pair, evidence) in missing_tgt_deps:
          evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1])) for e in evidence])
          self._context.log.error('Missing BUILD dependency %s -> %s because:\n%s' %
                                  (tgt_pair[0].address.reference(), tgt_pair[1].address.reference(), evidence_str))
        for (src_tgt, dep) in missing_file_deps:
          self._context.log.error('Missing BUILD dependency %s -> %s' % (src_tgt.address.reference(), shorten(dep)))
        if self._check_missing_deps == 'fatal':
          raise TaskError('Missing deps.')

      if self._check_missing_direct_deps:
        for (tgt_pair, evidence) in missing_direct_tgt_deps:
          evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1])) for e in evidence])
          self._context.log.warn('Missing direct BUILD dependency %s -> %s because:\n%s' %
                                  (tgt_pair[0].address, tgt_pair[1].address, evidence_str))
        if self._check_missing_direct_deps == 'fatal':
          raise TaskError('Missing direct deps.')

      if self._check_unnecessary_deps:
        raise TaskError('Unnecessary dep warnings not implemented yet.')

  def _compute_missing_deps(self, srcs, actual_deps):
    """Computes deps that are used by the compiler but not specified in a BUILD file.

    These deps are bugs waiting to happen: the code may happen to compile because the dep was brought
    in some other way (e.g., by some other root target), but that is obviously fragile.

    Note that in practice we're OK with reliance on indirect deps that are only brought in transitively.
    E.g., in Scala type inference can bring in such a dep subtly. Fortunately these cases aren't as fragile
    as a completely missing dependency. It's still a good idea to have explicit direct deps where relevant,
    so we optionally warn about indirect deps, to make them easy to find and reason about.

    - actual_deps: a map src -> list of actual deps (source, class or jar file) as noted by the compiler.

    Returns a triple (missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps) where:

    - missing_file_deps: a list of pairs (src_tgt, dep_file) where src_tgt requires dep_file, and we're
      unable to map to a target (because its target isn't in the total set of targets in play, and we
      don't want to parse every BUILD file in the workspace just to find it).

    - missing_tgt_deps: a list of pairs (src_tgt, dep_tgt) where src_tgt is missing a necessary
                        transitive dependency on dep_tgt.

    - missing_direct_tgt_deps: a list of pairs (src_tgt, dep_tgt) where src_tgt is missing a direct
                               dependency on dep_tgt but has a transitive dep on it.

    All paths in the input and output are absolute.
    """
    def must_be_explicit_dep(dep):
      # We don't require explicit deps on the java runtime, so we shouldn't consider that
      # a missing dep.
      return not dep.startswith(self._context.java_home)

    targets_by_file = self.get_targets_by_file()
    transitive_deps_by_target = self.get_transitive_deps_by_target()

    # Find deps that are actual but not specified.
    missing_file_deps = []  # (src, src).
    missing_tgt_deps_map = defaultdict(list)  # (tgt, tgt) -> a list of (src, src) as evidence.
    missing_direct_tgt_deps_map = defaultdict(list)  # The same, but for direct deps.

    buildroot = get_buildroot()
    abs_srcs = [os.path.join(buildroot, src) for src in srcs]
    for src in abs_srcs:
      src_tgt = next(iter(targets_by_file.get(src)))
      if src_tgt is not None:
        for actual_dep in filter(must_be_explicit_dep, actual_deps.get(src, [])):
          actual_dep_tgts = targets_by_file.get(actual_dep)
          canonical_actual_dep_tgt = next(iter(actual_dep_tgts))
          # actual_dep_tgts is usually a singleton. If it's not, we only need one of these
          # to be in our declared deps to be OK.
          if actual_dep_tgts is None:
            missing_file_deps.append((src_tgt, actual_dep))
          elif src_tgt not in actual_dep_tgts:  # Obviously intra-target deps are fine.
            if actual_dep_tgts.isdisjoint(transitive_deps_by_target.get(src_tgt, [])):
              missing_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append((src, actual_dep))
            elif canonical_actual_dep_tgt not in src_tgt.dependencies:
              # The canonical dep is the only one a direct dependency makes sense on.
              missing_direct_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append((src, actual_dep))
      else:
        raise TaskError('Requested dep info for unknown source file: %s' % src)

    return missing_file_deps, missing_tgt_deps_map.items(), missing_direct_tgt_deps_map.items()
<commit_msg>Fix dep analyzer bug.<commit_after>
import os

from collections import defaultdict
from twitter.common.collections import OrderedSet
from twitter.pants import get_buildroot, JvmTarget, TaskError, JarLibrary, InternalTarget, JarDependency

class JvmDependencyAnalyzer(object):
  def __init__(self, context, check_missing_deps, check_missing_direct_deps, check_unnecessary_deps):
    self._context = context
    self._context.products.require('classes')
    self._context.products.require('ivy_jar_products')

    self._check_missing_deps = check_missing_deps
    self._check_missing_direct_deps = check_missing_direct_deps
    self._check_unnecessary_deps = check_unnecessary_deps

    # Memoized map from absolute path of source, class or jar file to an OrderedSet of targets.
    #
    # The value is usually a singleton, because a source or class file belongs to a single target.
    # However a single jar may be provided (transitively or intransitively) by multiple JarLibrary targets.
    # But if there is a JarLibrary target that depends on a jar directly, then that "canonical" target
    # will be the first one in the list of targets.
    #
    # Don't access directly. Call self.get_targets_by_file() instead.
    self._targets_by_file = None

    # Memoized map from target to all its transitive dep targets.
    # Don't access directly. Call self.get_transitive_deps_by_target() instead.
    self._transitive_deps_by_target = None

  def get_targets_by_file(self):
    """Compute the targets_by_file mapping for all targets in-play for this pants run.

    Memoizes for efficiency. Should only be called after codegen, so that all synthetic targets
    and injected deps are taken into account.
    """
    if self._targets_by_file is None:
      return self._compute_targets_by_file()  # Will memoize if allowed.
    else:
      return self._targets_by_file

  def _compute_targets_by_file(self):
    targets_by_file = defaultdict(OrderedSet)
    jarlibs_by_id = defaultdict(set)  # Multiple JarLibrary targets can provide the same (org, name).
    # Compute src -> target.
    buildroot = get_buildroot()
    # Look at all targets in-play for this pants run. Does not include synthetic targets,
    for target in self._context.targets():
      if isinstance(target, JvmTarget):
        for src in target.sources:
          targets_by_file[os.path.join(buildroot, target.target_base, src)].add(target)
      elif isinstance(target, JarLibrary):
        for jardep in target.dependencies:
          if isinstance(jardep, JarDependency):
            jarlibs_by_id[(jardep.org, jardep.name)].add(target)

    # Compute class -> target for classes in previous compile groups.
    genmap = self._context.products.get('classes')
    for tgt, products in genmap.itermappings():
      # tgt could also be a string, as we (ab)use the same genmap for the relsrc->classes mapping.
      if isinstance(tgt, JvmTarget):
        for basedir, classes in products.items():
          for cls in classes:
            targets_by_file[os.path.join(basedir, cls)].add(tgt)

    # Compute jar -> target.
    ivy_products = self._context.products.get_data('ivy_jar_products')
    if ivy_products:
      for ivyinfo in ivy_products.values():
        for ref in ivyinfo.modules_by_ref:
          target_key = (ref.org, ref.name)
          if target_key in jarlibs_by_id:
            jarlib_targets = jarlibs_by_id[target_key]
            for jar in ivyinfo.modules_by_ref[ref].artifacts:
              for jarlib_target in jarlib_targets:
                targets_by_file[jar.path].add(jarlib_target)
            # Map all indirect, transitive deps of the jar to this target as well, since we allow deps on them.
            for dep in ivyinfo.deps_by_caller.get(ref, []):
              depmodule = ivyinfo.modules_by_ref.get(dep, None)
              if depmodule:
                for depjar in depmodule.artifacts:
                  for jarlib_target in jarlib_targets:
                    targets_by_file[depjar.path].add(jarlib_target)
      # Only memoize once ivy_jar_products are available.
      self._targets_by_file = targets_by_file
    return targets_by_file

  def get_transitive_deps_by_target(self):
    """Compute the transitive_deps_by_target mapping from all targets in-play for this pants run.

    Memoizes for efficiency. Should only be called after codegen, so that all synthetic targets
    and injected deps are taken into account.
    """
    if self._transitive_deps_by_target is None:
      return self._compute_transitive_deps_by_target()
    else:
      return self._transitive_deps_by_target

  def _compute_transitive_deps_by_target(self):
    # Sort from least to most dependent.
    sorted_targets = reversed(InternalTarget.sort_targets(self._context.targets()))
    transitive_deps_by_target = defaultdict(set)
    # Iterate in dep order, to accumulate the transitive deps for each target.
    for target in sorted_targets:
      transitive_deps = set()
      if hasattr(target, 'dependencies'):
        for dep in target.dependencies:
          transitive_deps.update(transitive_deps_by_target.get(dep, []))
          transitive_deps.add(dep)
        transitive_deps_by_target[target] = transitive_deps
    self._transitive_deps_by_target = transitive_deps_by_target  # Memoize.
    return transitive_deps_by_target

  def check(self, srcs, actual_deps):
    """Check for missing deps.

    See docstring for _compute_missing_deps for details.
    """
    if self._check_missing_deps or self._check_missing_direct_deps or self._check_unnecessary_deps:
      missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps = \
        self._compute_missing_deps(srcs, actual_deps)

      buildroot = get_buildroot()
      def shorten(path):  # Make the output easier to read.
        for prefix in [buildroot, self._context.ivy_home]:
          if path.startswith(prefix):
            return os.path.relpath(path, prefix)
        return path

      if self._check_missing_deps and (missing_file_deps or missing_tgt_deps):
        for (tgt_pair, evidence) in missing_tgt_deps:
          evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1])) for e in evidence])
          self._context.log.error('Missing BUILD dependency %s -> %s because:\n%s' %
                                  (tgt_pair[0].address.reference(), tgt_pair[1].address.reference(), evidence_str))
        for (src_tgt, dep) in missing_file_deps:
          self._context.log.error('Missing BUILD dependency %s -> %s' % (src_tgt.address.reference(), shorten(dep)))
        if self._check_missing_deps == 'fatal':
          raise TaskError('Missing deps.')

      if self._check_missing_direct_deps:
        for (tgt_pair, evidence) in missing_direct_tgt_deps:
          evidence_str = '\n'.join(['    %s uses %s' % (shorten(e[0]), shorten(e[1])) for e in evidence])
          self._context.log.warn('Missing direct BUILD dependency %s -> %s because:\n%s' %
                                  (tgt_pair[0].address, tgt_pair[1].address, evidence_str))
        if self._check_missing_direct_deps == 'fatal':
          raise TaskError('Missing direct deps.')

      if self._check_unnecessary_deps:
        raise TaskError('Unnecessary dep warnings not implemented yet.')

  def _compute_missing_deps(self, srcs, actual_deps):
    """Computes deps that are used by the compiler but not specified in a BUILD file.

    These deps are bugs waiting to happen: the code may happen to compile because the dep was brought
    in some other way (e.g., by some other root target), but that is obviously fragile.

    Note that in practice we're OK with reliance on indirect deps that are only brought in transitively.
    E.g., in Scala type inference can bring in such a dep subtly. Fortunately these cases aren't as fragile
    as a completely missing dependency. It's still a good idea to have explicit direct deps where relevant,
    so we optionally warn about indirect deps, to make them easy to find and reason about.

    - actual_deps: a map src -> list of actual deps (source, class or jar file) as noted by the compiler.

    Returns a triple (missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps) where:

    - missing_file_deps: a list of pairs (src_tgt, dep_file) where src_tgt requires dep_file, and we're
      unable to map to a target (because its target isn't in the total set of targets in play, and we
      don't want to parse every BUILD file in the workspace just to find it).

    - missing_tgt_deps: a list of pairs (src_tgt, dep_tgt) where src_tgt is missing a necessary
                        transitive dependency on dep_tgt.

    - missing_direct_tgt_deps: a list of pairs (src_tgt, dep_tgt) where src_tgt is missing a direct
                               dependency on dep_tgt but has a transitive dep on it.

    All paths in the input and output are absolute.
    """
    def must_be_explicit_dep(dep):
      # We don't require explicit deps on the java runtime, so we shouldn't consider that
      # a missing dep.
      return not dep.startswith(self._context.java_home)

    targets_by_file = self.get_targets_by_file()
    transitive_deps_by_target = self.get_transitive_deps_by_target()

    # Find deps that are actual but not specified.
    missing_file_deps = []  # (src, src).
    missing_tgt_deps_map = defaultdict(list)  # (tgt, tgt) -> a list of (src, src) as evidence.
    missing_direct_tgt_deps_map = defaultdict(list)  # The same, but for direct deps.

    buildroot = get_buildroot()
    abs_srcs = [os.path.join(buildroot, src) for src in srcs]
    for src in abs_srcs:
      src_tgt = next(iter(targets_by_file.get(src)))
      if src_tgt is not None:
        for actual_dep in filter(must_be_explicit_dep, actual_deps.get(src, [])):
          actual_dep_tgts = targets_by_file.get(actual_dep)
          # actual_dep_tgts is usually a singleton. If it's not, we only need one of these
          # to be in our declared deps to be OK.
          if actual_dep_tgts is None:
            missing_file_deps.append((src_tgt, actual_dep))
          elif src_tgt not in actual_dep_tgts:  # Obviously intra-target deps are fine.
            canonical_actual_dep_tgt = next(iter(actual_dep_tgts))
            if actual_dep_tgts.isdisjoint(transitive_deps_by_target.get(src_tgt, [])):
              missing_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append((src, actual_dep))
            elif canonical_actual_dep_tgt not in src_tgt.dependencies:
              # The canonical dep is the only one a direct dependency makes sense on.
              missing_direct_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append((src, actual_dep))
      else:
        raise TaskError('Requested dep info for unknown source file: %s' % src)

    return missing_file_deps, missing_tgt_deps_map.items(), missing_direct_tgt_deps_map.items()
<|endoftext|>


<commit_before>"""extract_shape_msa.py

Output one shapefile per MSA containing all the blockgroups it contains
"""
import os
import csv
import fiona

#
# Import MSA to blockgroup crosswalk
#
msa_to_bg = {}
with open('data/crosswalks/msa_blockgroup.csv', 'r') as source:
    reader = csv.reader(source, delimiter='\t')
    reader.next()
    for rows in reader:
        msa = rows[0]
        bg = rows[1]
        if msa not in msa_to_bg:
            msa_to_bg[msa] = []
        msa_to_bg[msa].append(bg)

#
# Perform the extraction
#
for msa in msa_to_bg:
    states = list(set([b[:2] for b in msa_to_bg[msa]]))

    ## Get all blockgroups
    all_bg = {}
    for st in states:
        with fiona.open('data/shp/state/%s/blockgroups.shp'%st, 'r',
                'ESRI Shapefile') as source:
            source_crs = source.crs
            for f in source:
                all_bg[f['properties']['BKGPIDFP00']] = f['geometry']

    ## blockgroups within cbsa
    msa_bg = {bg: all_bg[bg] for bg in msa_to_bg[msa]}

    ## Save
    if not os.path.isdir('data/shp/msa/%s'%msa):
        os.mkdir('data/shp/msa/%s'%msa)

    schema = {'geometry': 'Polygon',
              'properties': {'BKGPIDFP00': 'str'}}
    with fiona.open('data/shp/msa/%s/blockgroups.shp'%msa, 'w',
            'ESRI Shapefile',
            crs = source_rs
            schema = schema) as output:
        for bg in msa_bg:
            rec = {'geometry':msa_bg[bg], 'properties':{'BKGPIDFP00':bg}}
            output.write(rec)
<commit_msg>Correct bugs<commit_after>"""extract_shape_msa.py

Output one shapefile per MSA containing all the blockgroups it contains
"""
import os
import csv
import fiona

#
# Import MSA to blockgroup crosswalk
#
msa_to_bg = {}
with open('data/crosswalks/msa_blockgroup.csv', 'r') as source:
    reader = csv.reader(source, delimiter='\t')
    reader.next()
    for rows in reader:
        msa = rows[0]
        bg = rows[1]
        if msa not in msa_to_bg:
            msa_to_bg[msa] = []
        msa_to_bg[msa].append(bg)

#
# Perform the extraction
#
for msa in msa_to_bg:
    states = list(set([b[:2] for b in msa_to_bg[msa]]))

    ## Get all blockgroups
    all_bg = {}
    for st in states:
        with fiona.open('data/shp/state/%s/blockgroups.shp'%st, 'r',
                'ESRI Shapefile') as source:
            source_crs = source.crs
            for f in source:
                all_bg[f['properties']['BKGPIDFP00']] = f['geometry']

    ## blockgroups within cbsa
    msa_bg = {bg: all_bg[bg] for bg in msa_to_bg[msa]}

    ## Save
    if not os.path.isdir('data/shp/msa/%s'%msa):
        os.mkdir('data/shp/msa/%s'%msa)

    schema = {'geometry': 'Polygon',
              'properties': {'BKGPIDFP00': 'str'}}
    with fiona.open('data/shp/msa/%s/blockgroups.shp'%msa, 'w',
            'ESRI Shapefile',
            crs = source_crs,
            schema = schema) as output:
        for bg in msa_bg:
            rec = {'geometry':msa_bg[bg], 'properties':{'BKGPIDFP00':bg}}
            output.write(rec)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import os
import subprocess
import textwrap

import click

DEFAULT_BASE_IMAGES = ['python:2.7-onbuild']

@click.command()
@click.version_option()
@click.option('-b', '--base-images',
              default=None,
              help='Base docker images (comma-separated list) - e.g.: '
                   '"python:2.7-onbuild,python:3.4-onbuild". Conflicts with '
                   '--python-versions.')
@click.option('-c', '--cmd',
              help='Command (CMD) to set in image. Conflicts with --procfile')
@click.option('-e', '--entrypoint',
              help='Entry point (ENTRYPOINT) to set in image')
@click.option('-p', '--python-versions',
              default=None,
              help='Python versions (comma-separated list) - e.g.: '
                   '"2.7,3.4". Conflicts with --base-images.')
@click.option('--procfile', type=click.File(),
              help='Procfile to get command from. Conflicts with --cmd.')
@click.option('-t', '--tag',
              help='Repository name (and optionally a tag) to be applied to '
                   'the resulting image in case of success')
@click.argument('requirements_file', type=click.Path(exists=True),
                default='requirements.txt')
@click.pass_context
def pydockerize(ctx, requirements_file, tag, cmd, entrypoint, procfile,
                base_images=None, python_versions=None):
    """Create Docker images for Python apps"""

    if base_images is not None and python_versions is not None:
        raise Exception(
            'Cannot specify both --base-images and --python_versions')

    if base_images is not None:
        base_images = base_images.split(',')
    else:
        base_images = DEFAULT_BASE_IMAGES

    if python_versions is not None:
        python_versions = python_versions.split(',')
        base_images = get_base_images_from_python_versions(python_versions)

    if cmd is not None and procfile is not None:
        raise Exception('Cannot specify both --cmd and --procfile')

    if cmd is None and procfile is None and os.path.exists('Procfile'):
        procfile = open('Procfile')

    if procfile:
        cmd = get_cmd_from_procfile(procfile)

    for base_image in base_images:
        filename = write_dockerfile(base_image, requirements_file,
                                    cmd, entrypoint)
        invoke_docker_build(tag, base_image, filename)

    if tag:
        print('\nShowing Docker images for %s:\n' % tag)
        show_docker_images(tag)

def get_base_images_from_python_versions(python_versions):
    return ['python:%s-onbuild' % python_version
            for python_version in python_versions]

def get_cmd_from_procfile(procfile):
    lines = procfile.readlines()
    if len(lines) > 1:
        raise Exception(
            'Procfile with multiple lines not supported')
    return lines[0].split(':')[1].strip()

def write_dockerfile(base_image, requirements_file, cmd, entrypoint):
    print('write_dockerfile: base_image = %r' % base_image)
    print('write_dockerfile: requirements_file = %r' % requirements_file)
    print('write_dockerfile: cmd = %r' % cmd)
    print('write_dockerfile: entrypoint = %r' % entrypoint)

    filename = 'Dockerfile-' + base_image
    print('write_dockerfile: Writing %s' % filename)

    with open(filename, 'w+') as f:
        f.write(textwrap.dedent("""\
            # This Docker image takes care of doing:
            #
            #     pip install -r requirements.txt
            #
            # For more details on this Docker image, see:
            # https://registry.hub.docker.com/_/python/
            FROM {base_image}

            # This is so one can mount a volume from the host to give the
            # container access to the host's current working directory.
            #
            # E.g.:
            #
            #   - `docker run -v $(pwd):/host` from command-line
            #         or
            #   - `volumes: [".:/host"]` in fig.yml
            WORKDIR /host
        """.format(base_image=base_image)))
        if entrypoint:
            f.write(textwrap.dedent("""\

                ENTRYPOINT {entrypoint}
            """.format(entrypoint=entrypoint)))
        if cmd:
            f.write(textwrap.dedent("""\

                CMD {cmd}
            """.format(cmd=cmd)))

    return filename

def invoke_docker_build(repo_and_tag, base_image, filename):
    print('invoke_docker_build: repo_and_tag = %r' % repo_and_tag)
    cmd = ['docker', 'build']
    if repo_and_tag:
        if ':' in repo_and_tag:
            raise Exception("':' in tag not supported yet")
        tag = get_tag_from_base_image(base_image)
        cmd.append('--tag')
        cmd.append(repo_and_tag + ':' + tag)
    cmd.append('--file')
    cmd.append(filename)
    cmd.append('.')
    print('invoke_docker_build: Calling subprocess with cmd = %r\n'
          % ' '.join(cmd))
    status = subprocess.call(cmd)
    if status == 0:
        print('Docker build succeeded.')
    else:
        print('Docker build failed with %d' % status)
        raise click.Abort()

def show_docker_images(repo):
    cmd = ['docker', 'images', repo]
    return subprocess.call(cmd)

def get_tag_from_base_image(base_image):
    tag = base_image
    replacements = {'python:': 'py', '-onbuild': ''}

    for old, new in replacements.items():
        tag = tag.replace(old, new)

    return tag

if __name__ == '__main__':
    pydockerize()
<commit_msg>write_dockerfile: Simpler write - CMD & ENTRYPOINT<commit_after>#!/usr/bin/env python

from __future__ import print_function

import os
import subprocess
import textwrap

import click

DEFAULT_BASE_IMAGES = ['python:2.7-onbuild']

@click.command()
@click.version_option()
@click.option('-b', '--base-images',
              default=None,
              help='Base docker images (comma-separated list) - e.g.: '
                   '"python:2.7-onbuild,python:3.4-onbuild". Conflicts with '
                   '--python-versions.')
@click.option('-c', '--cmd',
              help='Command (CMD) to set in image. Conflicts with --procfile')
@click.option('-e', '--entrypoint',
              help='Entry point (ENTRYPOINT) to set in image')
@click.option('-p', '--python-versions',
              default=None,
              help='Python versions (comma-separated list) - e.g.: '
                   '"2.7,3.4". Conflicts with --base-images.')
@click.option('--procfile', type=click.File(),
              help='Procfile to get command from. Conflicts with --cmd.')
@click.option('-t', '--tag',
              help='Repository name (and optionally a tag) to be applied to '
                   'the resulting image in case of success')
@click.argument('requirements_file', type=click.Path(exists=True),
                default='requirements.txt')
@click.pass_context
def pydockerize(ctx, requirements_file, tag, cmd, entrypoint, procfile,
                base_images=None, python_versions=None):
    """Create Docker images for Python apps"""

    if base_images is not None and python_versions is not None:
        raise Exception(
            'Cannot specify both --base-images and --python_versions')

    if base_images is not None:
        base_images = base_images.split(',')
    else:
        base_images = DEFAULT_BASE_IMAGES

    if python_versions is not None:
        python_versions = python_versions.split(',')
        base_images = get_base_images_from_python_versions(python_versions)

    if cmd is not None and procfile is not None:
        raise Exception('Cannot specify both --cmd and --procfile')

    if cmd is None and procfile is None and os.path.exists('Procfile'):
        procfile = open('Procfile')

    if procfile:
        cmd = get_cmd_from_procfile(procfile)

    for base_image in base_images:
        filename = write_dockerfile(base_image, requirements_file,
                                    cmd, entrypoint)
        invoke_docker_build(tag, base_image, filename)

    if tag:
        print('\nShowing Docker images for %s:\n' % tag)
        show_docker_images(tag)

def get_base_images_from_python_versions(python_versions):
    return ['python:%s-onbuild' % python_version
            for python_version in python_versions]

def get_cmd_from_procfile(procfile):
    lines = procfile.readlines()
    if len(lines) > 1:
        raise Exception(
            'Procfile with multiple lines not supported')
    return lines[0].split(':')[1].strip()

def write_dockerfile(base_image, requirements_file, cmd, entrypoint):
    print('write_dockerfile: base_image = %r' % base_image)
    print('write_dockerfile: requirements_file = %r' % requirements_file)
    print('write_dockerfile: cmd = %r' % cmd)
    print('write_dockerfile: entrypoint = %r' % entrypoint)

    filename = 'Dockerfile-' + base_image
    print('write_dockerfile: Writing %s' % filename)

    with open(filename, 'w+') as f:
        f.write(textwrap.dedent("""\
            # This Docker image takes care of doing:
            #
            #     pip install -r requirements.txt
            #
            # For more details on this Docker image, see:
            # https://registry.hub.docker.com/_/python/
            FROM {base_image}

            # This is so one can mount a volume from the host to give the
            # container access to the host's current working directory.
            #
            # E.g.:
            #
            #   - `docker run -v $(pwd):/host` from command-line
            #         or
            #   - `volumes: [".:/host"]` in fig.yml
            WORKDIR /host
        """.format(base_image=base_image)))
        if entrypoint:
            f.write("\nENTRYPOINT " + entrypoint)
        if cmd:
            f.write("\nCMD " + cmd)

    return filename

def invoke_docker_build(repo_and_tag, base_image, filename):
    print('invoke_docker_build: repo_and_tag = %r' % repo_and_tag)
    cmd = ['docker', 'build']
    if repo_and_tag:
        if ':' in repo_and_tag:
            raise Exception("':' in tag not supported yet")
        tag = get_tag_from_base_image(base_image)
        cmd.append('--tag')
        cmd.append(repo_and_tag + ':' + tag)
    cmd.append('--file')
    cmd.append(filename)
    cmd.append('.')
    print('invoke_docker_build: Calling subprocess with cmd = %r\n'
          % ' '.join(cmd))
    status = subprocess.call(cmd)
    if status == 0:
        print('Docker build succeeded.')
    else:
        print('Docker build failed with %d' % status)
        raise click.Abort()

def show_docker_images(repo):
    cmd = ['docker', 'images', repo]
    return subprocess.call(cmd)

def get_tag_from_base_image(base_image):
    tag = base_image
    replacements = {'python:': 'py', '-onbuild': ''}

    for old, new in replacements.items():
        tag = tag.replace(old, new)

    return tag

if __name__ == '__main__':
    pydockerize()
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import re
import StringIO

from telemetry.core import util
from telemetry.test import tab_test_case

class TabConsoleTest(tab_test_case.TabTestCase):
  def testConsoleOutputStream(self):
    unittest_data_dir = os.path.join(os.path.dirname(__file__),
                                     '..', '..', '..', 'unittest_data')
    self._browser.SetHTTPServerDirectories(unittest_data_dir)

    stream = StringIO.StringIO()
    self._tab.message_output_stream = stream

    self._tab.Navigate(
      self._browser.http_server.UrlOf('page_that_logs_to_console.html'))
    self._tab.WaitForDocumentReadyStateToBeComplete()

    initial = self._tab.EvaluateJavaScript('window.__logCount')
    def GotLog():
      current = self._tab.EvaluateJavaScript('window.__logCount')
      return current > initial
    util.WaitFor(GotLog, 5)

    lines = [l for l in stream.getvalue().split('\n') if len(l)]

    self.assertTrue(len(lines) >= 1)
    for l in lines:
      u_l = 'http://localhost:(\d+)/page_that_logs_to_console.html:9'
      self.assertTrue(re.match('At %s: Hello, world' % u_l, l))

<commit_msg>[telemetry] Fix testConsoleOutputStream failure<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import re
import StringIO

from telemetry.core import util
from telemetry.test import tab_test_case

class TabConsoleTest(tab_test_case.TabTestCase):
  def testConsoleOutputStream(self):
    unittest_data_dir = os.path.join(os.path.dirname(__file__),
                                     '..', '..', '..', 'unittest_data')
    self._browser.SetHTTPServerDirectories(unittest_data_dir)

    stream = StringIO.StringIO()
    self._tab.message_output_stream = stream

    self._tab.Navigate(
      self._browser.http_server.UrlOf('page_that_logs_to_console.html'))
    self._tab.WaitForDocumentReadyStateToBeComplete()

    initial = self._tab.EvaluateJavaScript('window.__logCount')
    def GotLog():
      current = self._tab.EvaluateJavaScript('window.__logCount')
      return current > initial
    util.WaitFor(GotLog, 5)

    lines = [l for l in stream.getvalue().split('\n') if len(l)]

    self.assertTrue(len(lines) >= 1)
    for line in lines:
      prefix = 'http://(.+)/page_that_logs_to_console.html:9'
      expected_line = 'At %s: Hello, world' % prefix
      self.assertTrue(re.match(expected_line, line))

<|endoftext|>


<commit_before>from django.db import models
from django.contrib import auth
from django.db.models import Sum
from django.core.validators import RegexValidator

from colorful import fields

# Create your models here.
class User(auth.models.User):

    class Meta:
        proxy = True

    def total_points(self):
        return self.points.aggregate(Sum('points'))['points__sum']

    def daily_points(self, date):
        return self.points.filter(date=date).aggregate(Sum('points'))['points__sum']

    def from_begin_points(self, date):
        return self.points.filter(date__lt=date).aggregate(Sum('points'))['points__sum']

    def total_points_list(self):
        return self.points.all()

    def daily_points_list(self):
        return self.points.filter(date=date).points.all()

    def from_begin_points_list(self, date):
        return self.points.filter(date__lt=date).points.all()

class Label(models.Model):
    name_validator = RegexValidator(regex='^[A-Za-z0-9_-]+$',
            message="Please enter only characters, number, underscores or hyphens.")
    name = models.CharField(max_length=20, unique=True, validators=[name_validator])
    color = fields.RGBColorField(default="#000000",verbose_name="Background color")
    inverted = models.BooleanField(default=True)

    def total_label_points(self):
        return self.points.aggregate(Sum('points'))['points__sum']

    def label_points_list(self):
        return self.points.all()

    def __str__(self):
        return self.name

class EventPoint(models.Model):
    by = models.ForeignKey(User, related_name='from')
    to = models.ForeignKey(User, related_name='points')
    label = models.ForeignKey(Label, related_name='points')
    date = models.DateTimeField(auto_now_add=True)
    points = models.IntegerField()

    def __str__(self):
        return '%s->%s:%s' % (self.by.username,self.to.username,self.points)

<commit_msg>less or equal bug in models<commit_after>from django.db import models
from django.contrib import auth
from django.db.models import Sum
from django.core.validators import RegexValidator

from colorful import fields

# Create your models here.
class User(auth.models.User):

    class Meta:
        proxy = True

    def total_points(self):
        return self.points.aggregate(Sum('points'))['points__sum']

    def daily_points(self, date):
        return self.points.filter(date=date).aggregate(Sum('points'))['points__sum']

    def from_begin_points(self, date):
        return self.points.filter(date__lte=date).aggregate(Sum('points'))['points__sum']

    def total_points_list(self):
        return self.points.all()

    def daily_points_list(self):
        return self.points.filter(date=date).points.all()

    def from_begin_points_list(self, date):
        return self.points.filter(date__lte=date).points.all()

class Label(models.Model):
    name_validator = RegexValidator(regex='^[A-Za-z0-9_-]+$',
            message="Please enter only characters, number, underscores or hyphens.")
    name = models.CharField(max_length=20, unique=True, validators=[name_validator])
    color = fields.RGBColorField(default="#000000",verbose_name="Background color")
    inverted = models.BooleanField(default=True)

    def total_label_points(self):
        return self.points.aggregate(Sum('points'))['points__sum']

    def label_points_list(self):
        return self.points.all()

    def __str__(self):
        return self.name

class EventPoint(models.Model):
    by = models.ForeignKey(User, related_name='from')
    to = models.ForeignKey(User, related_name='points')
    label = models.ForeignKey(Label, related_name='points')
    date = models.DateTimeField(auto_now_add=True)
    points = models.IntegerField()

    def __str__(self):
        return '%s->%s:%s' % (self.by.username,self.to.username,self.points)

<|endoftext|>


<commit_before>""" pygame module for monitoring time
"""

from pygame._sdl import sdl, ffi
from pygame._error import SDLError

def _get_init():
    return sdl.SDL_WasInit(sdl.SDL_INIT_TIMER)

def _try_init():
    if not _get_init():
        if sdl.SDL_InitSubSystem(sdl.SDL_INIT_TIMER):
            raise SDLError.from_sdl_error()

class Clock(object):
    """ Clock() -> Clock
    create an object to help track time
    """

    def __init__(self):
        _try_init()
        self._last_tick = get_ticks()
        self._rawpassed = 0
        self._fps_count = 0
        self._fps_tick = 0

    def _base(self, framerate=None):
        if framerate:
            endtime = int((1.0 / framerate) * 1000.)
            self._rawpassed = sdl.SDL_GetTicks() - self._last_tick
            delay = endtime - self._rawpassed

            _try_init()

            use_accurate_delay = False
            # XXX
            if use_accurate_delay:
                # delay = accurate_delay (delay);
                if delay == -1:
                    raise SDLError.from_sdl_error()
            else:
                delay = max(delay, 0)
                sdl.SDL_Delay(delay)

        nowtime = sdl.SDL_GetTicks()
        self._timepassed = nowtime - self._last_tick
        self._fps_count += 1
        self._last_tick = nowtime
        if not framerate:
            self._rawpassed = self._timepassed

        if not self._fps_tick:
            self._fps_count = 0
            self._fps_tick = nowtime
        elif self._fps_count >= 10:
            self._fps = (self._fps_count /
                         ((nowtime - self._fps_tick) / 1000.0))
            self._fps_count = 0
            self._fps_tick = nowtime

        return self._timepassed

    def tick(self, framerate=0):
        """ tick(framerate=0) -> milliseconds
        update the clock
        """
        self._base(framerate)

    def get_fps(self):
        """ get_fps() -> float
        compute the clock framerate
        """
        #{ "get_fps", (PyCFunction) clock_get_fps,

    def get_time(self):
        """ get_time() -> milliseconds
        time used in the previous tick
        """
        #{ "get_time", (PyCFunction) clock_get_time, METH_NOARGS,
        #DOC_CLOCKGETTIME },

    def get_rawtime(self):
        """ get_rawtime() -> milliseconds
        actual time used in the previous tick
        """
        #{ "get_rawtime", (PyCFunction) clock_get_rawtime, METH_NOARGS,
        #DOC_CLOCKGETRAWTIME },

    def tick_busy_loop(self):
        """ tick_busy_loop(framerate=0) -> milliseconds
        update the clock
        """
        #{ "tick_busy_loop", clock_tick_busy_loop, METH_VARARGS,
        #DOC_CLOCKTICKBUSYLOOP },

def get_ticks():
    """ get_ticks() -> milliseconds
    get the time in milliseconds
    """
    if not _get_init():
        return 0
    return sdl.SDL_GetTicks()

_event_timers = {}

@ffi.callback("SDL_NewTimerCallback")
def _timer_callback(interval, param):
    if sdl.SDL_WasInit(sdl.SDL_INIT_VIDEO):
        event = ffi.new("SDL_Event")
        event.type = ffi.cast("intptr_t", param)
        sdl.SDL_PushEvent(event)
    return interval

def set_timer(eventid, milliseconds):
    """set_timer(eventid, milliseconds) -> None
    repeatedly create an event on the event queue"
    """
    if eventid <= sdl.SDL_NOEVENT or eventid >= sdl.SDL_NUMEVENTS:
        raise ValueError("Event id must be between NOEVENT(0) and"
                         " NUMEVENTS(32)")

    old_event = _event_timers.pop(eventid, None)
    if old_event:
        sdl.SDL_RemoveTimer(old_event)

    if milliseconds <= 0:
        return

    _try_init()

    handle = ffi.cast("void *", eventid)
    newtimer = sdl.SDL_AddTimer(milliseconds, _timer_callback, handle)
    if not newtimer:
        SDLError.from_sdl_error()

    _event_timers[eventid] = newtimer

def wait(milliseconds):
    """ wait(milliseconds) -> time
    pause the program for an amount of time
    """

def delay(milliseconds):
    """ delay(milliseconds) -> time
    pause the program for an amount of time
    """
<commit_msg>tick should return timepassed<commit_after>""" pygame module for monitoring time
"""

from pygame._sdl import sdl, ffi
from pygame._error import SDLError

def _get_init():
    return sdl.SDL_WasInit(sdl.SDL_INIT_TIMER)

def _try_init():
    if not _get_init():
        if sdl.SDL_InitSubSystem(sdl.SDL_INIT_TIMER):
            raise SDLError.from_sdl_error()

class Clock(object):
    """ Clock() -> Clock
    create an object to help track time
    """

    def __init__(self):
        _try_init()
        self._last_tick = get_ticks()
        self._rawpassed = 0
        self._fps_count = 0
        self._fps_tick = 0

    def _base(self, framerate=None):
        if framerate:
            endtime = int((1.0 / framerate) * 1000.)
            self._rawpassed = sdl.SDL_GetTicks() - self._last_tick
            delay = endtime - self._rawpassed

            _try_init()

            use_accurate_delay = False
            # XXX
            if use_accurate_delay:
                # delay = accurate_delay (delay);
                if delay == -1:
                    raise SDLError.from_sdl_error()
            else:
                delay = max(delay, 0)
                sdl.SDL_Delay(delay)

        nowtime = sdl.SDL_GetTicks()
        self._timepassed = nowtime - self._last_tick
        self._fps_count += 1
        self._last_tick = nowtime
        if not framerate:
            self._rawpassed = self._timepassed

        if not self._fps_tick:
            self._fps_count = 0
            self._fps_tick = nowtime
        elif self._fps_count >= 10:
            self._fps = (self._fps_count /
                         ((nowtime - self._fps_tick) / 1000.0))
            self._fps_count = 0
            self._fps_tick = nowtime

        return self._timepassed

    def tick(self, framerate=0):
        """ tick(framerate=0) -> milliseconds
        update the clock
        """
        return self._base(framerate)

    def get_fps(self):
        """ get_fps() -> float
        compute the clock framerate
        """
        #{ "get_fps", (PyCFunction) clock_get_fps,

    def get_time(self):
        """ get_time() -> milliseconds
        time used in the previous tick
        """
        #{ "get_time", (PyCFunction) clock_get_time, METH_NOARGS,
        #DOC_CLOCKGETTIME },

    def get_rawtime(self):
        """ get_rawtime() -> milliseconds
        actual time used in the previous tick
        """
        #{ "get_rawtime", (PyCFunction) clock_get_rawtime, METH_NOARGS,
        #DOC_CLOCKGETRAWTIME },

    def tick_busy_loop(self):
        """ tick_busy_loop(framerate=0) -> milliseconds
        update the clock
        """
        #{ "tick_busy_loop", clock_tick_busy_loop, METH_VARARGS,
        #DOC_CLOCKTICKBUSYLOOP },

def get_ticks():
    """ get_ticks() -> milliseconds
    get the time in milliseconds
    """
    if not _get_init():
        return 0
    return sdl.SDL_GetTicks()

_event_timers = {}

@ffi.callback("SDL_NewTimerCallback")
def _timer_callback(interval, param):
    if sdl.SDL_WasInit(sdl.SDL_INIT_VIDEO):
        event = ffi.new("SDL_Event")
        event.type = ffi.cast("intptr_t", param)
        sdl.SDL_PushEvent(event)
    return interval

def set_timer(eventid, milliseconds):
    """set_timer(eventid, milliseconds) -> None
    repeatedly create an event on the event queue"
    """
    if eventid <= sdl.SDL_NOEVENT or eventid >= sdl.SDL_NUMEVENTS:
        raise ValueError("Event id must be between NOEVENT(0) and"
                         " NUMEVENTS(32)")

    old_event = _event_timers.pop(eventid, None)
    if old_event:
        sdl.SDL_RemoveTimer(old_event)

    if milliseconds <= 0:
        return

    _try_init()

    handle = ffi.cast("void *", eventid)
    newtimer = sdl.SDL_AddTimer(milliseconds, _timer_callback, handle)
    if not newtimer:
        SDLError.from_sdl_error()

    _event_timers[eventid] = newtimer

def wait(milliseconds):
    """ wait(milliseconds) -> time
    pause the program for an amount of time
    """

def delay(milliseconds):
    """ delay(milliseconds) -> time
    pause the program for an amount of time
    """
<|endoftext|>


<commit_before>#! /usr/bin/env python
'''Simple Service Container
   -- use with wsdl2py generated modules.
'''

import urlparse, types, os, sys, cStringIO as StringIO
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from ZSI import ParseException, FaultFromException
from ZSI import _copyright, _seqtypes, resolvers
from ZSI.parse import ParsedSoap
from ZSI.writer import SoapWriter
from ZSI.dispatch import _ModPythonSendXML, _ModPythonSendFault, _CGISendXML, _CGISendFault
from ZSI.dispatch import SOAPRequestHandler as BaseSOAPRequestHandler

"""
Functions:
    _Dispatch
    AsServer

Classes:
    NoSuchService
    PostNotSpecified
    SOAPActionNotSpecified
    ServiceSOAPBinding
    SOAPRequestHandler
    ServiceContainer
"""

def _Dispatch(ps, server, SendResponse, SendFault, post, action, nsdict={}, **kw):
    '''Send ParsedSoap instance to ServiceContainer, which dispatches to
    appropriate service via post, and method via action.  Response is a
    self-describing pyobj, which is passed to a SoapWriter.

    Call SendResponse or SendFault to send the reply back, appropriately.
        server -- ServiceContainer instance

    '''
    print "_Dispatch POST", post
    try:
        result = server(ps, post, action)
    except Exception, e:
        return SendFault(FaultFromException(e, 0, sys.exc_info()[2]), **kw)
    if result == None:
        return
    reply = StringIO.StringIO()
    try:
        SoapWriter(reply, nsdict=nsdict).serialize(result)
        return SendResponse(reply.getvalue(), **kw)
    except Exception, e:
        return SendFault(FaultFromException(e, 0, sys.exc_info()[2]), **kw)

def AsServer(port=80, services=()):
    '''port --
       services -- list of service instances
    '''
    address = ('', port)
    sc = ServiceContainer(address)
    for service in services:
        path = service.getPost()
        sc.setNode(service, path)
    sc.serve_forever()

class NoSuchService(Exception): pass
class PostNotSpecified(Exception): pass
class SOAPActionNotSpecified(Exception): pass

class ServiceSOAPBinding:
    """Binding defines the set of wsdl:binding operations, it takes as input a
    ParsedSoap instance and parses it into a pyobj.  It returns a response pyobj.

    class variables:
        soapAction -- dictionary of soapAction keys, and operation name values.
           These are specified in the WSDL soap bindings. There must be a
           class method matching the operation name value.

    """
    soapAction = {}

    def __init__(self, post):
        self.post = post

    def __call___(self, action, ps):
        return self.getOperation(action)(ps)

    def getPost(self):
        return self.post

    def getOperation(self, action):
        '''Returns a method of class.
           action -- soapAction value
        '''
        opName = self.getOperationName(action)
        return getattr(self, opName)

    def getOperationName(self, action):
        '''Returns operation name.
           action -- soapAction value
        '''
        if not self.soapAction.has_key(action):
            raise SOAPActionNotSpecified, '%s is NOT in soapAction dictionary' %action
        return self.soapAction[action]

class SOAPRequestHandler(BaseSOAPRequestHandler):
    '''SOAP handler.
    '''
    def do_POST(self):
        '''The POST command.
        '''
        soapAction = self.headers.getheader('SOAPAction').strip('\'"')
        post = self.path
        if not post:
            raise PostNotSpecified, 'HTTP POST not specified in request'
        if not soapAction:
            raise SOAPActionNotSpecified, 'SOAP Action not specified in request'
        soapAction = soapAction.strip('\'"')
        post = post.strip('\'"')
        try:
            ct = self.headers['content-type']
            if ct.startswith('multipart/'):
                cid = resolvers.MIMEResolver(ct, self.rfile)
                xml = cid.GetSOAPPart()
                ps = ParsedSoap(xml, resolver=cid.Resolve)
            else:
                length = int(self.headers['content-length'])
                ps = ParsedSoap(self.rfile.read(length))
        except ParseException, e:
            self.send_fault(FaultFromZSIException(e))
        except Exception, e:
            # Faulted while processing; assume it's in the header.
            self.send_fault(FaultFromException(e, 1, sys.exc_info()[2]))
        else:
            _Dispatch(ps, self.server, self.send_xml, self.send_fault,
                post=post, action=soapAction)

class ServiceContainer(HTTPServer):
    '''HTTPServer that stores service instances according
    to POST values.  An action value is instance specific,
    and specifies an operation (function) of an instance.
    '''
    class NodeTree:
        '''Simple dictionary implementation of a node tree
        '''
        def __init__(self):
            self.__dict = {}
            self.__post_dict = {}

        def __str__(self):
            return str(self.__dict)

        def getNode(self, url):
            node = self.__dict
            scheme,netloc,path,query,fragment = urlparse.urlsplit(url)
            try:
                dir = path.split('/')[1:]
                for i in dir:
                    node = node[i]
            except KeyError, ex:
                raise NoSuchService, 'No service(%s) in ServiceContainer' %path
            return node

        def setNode(self, service, url):
            if not isinstance(service, ServiceSOAPBinding):
               raise TypeError, 'A Service must implement class ServiceSOAPBinding'
            scheme,netloc,path,query,fragment = urlparse.urlsplit(url)
            self.__post_dict[path] = service
            node = self.__dict
            dir = path.split('/')[1:]
            for i in dir[:-1]:
                if not node.has_key(i): node[i] = {}
                node = node[i]
            node[dir[-1]] = service

    def __init__(self, server_address, RequestHandlerClass=SOAPRequestHandler):
        '''server_address --
           RequestHandlerClass --
        '''
        HTTPServer.__init__(self, server_address, RequestHandlerClass)
        self._nodes = self.NodeTree()

    def __call__(self, ps, post, action):
        '''ps -- ParsedSoap representing the request
           post -- HTTP POST --> instance
           action -- Soap Action header --> method
        '''
        return self.getCallBack(post, action)(ps)

    def getNode(self, post):
        '''post -- POST HTTP value
        '''
        return self._nodes.getNode(post)

    def setNode(self, service, post):
        '''service -- service instance
           post -- POST HTTP value
        '''
        self._nodes.setNode(service, post)

    def getCallBack(self, post, action):
        '''post -- POST HTTP value
           action -- SOAP Action value
        '''
        node = self.getNode(post)
        if node is None:
            raise NoSuchFunction
        return node.getOperation(action)

if __name__ == '__main__': print _copyright
<commit_msg>Flattened NodeTree to add removeNode, and added authorization callback support.<commit_after>#! /usr/bin/env python
'''Simple Service Container
   -- use with wsdl2py generated modules.
'''

import urlparse, types, os, sys, cStringIO as StringIO
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from ZSI import ParseException, FaultFromException
from ZSI import _copyright, _seqtypes, resolvers
from ZSI.parse import ParsedSoap
from ZSI.writer import SoapWriter
from ZSI.dispatch import _ModPythonSendXML, _ModPythonSendFault, _CGISendXML, _CGISendFault
from ZSI.dispatch import SOAPRequestHandler as BaseSOAPRequestHandler

"""
Functions:
    _Dispatch
    AsServer

Classes:
    NoSuchService
    PostNotSpecified
    SOAPActionNotSpecified
    ServiceSOAPBinding
    SOAPRequestHandler
    ServiceContainer
"""

def _Dispatch(ps, server, SendResponse, SendFault, post, action, nsdict={}, **kw):
    '''Send ParsedSoap instance to ServiceContainer, which dispatches to
    appropriate service via post, and method via action.  Response is a
    self-describing pyobj, which is passed to a SoapWriter.

    Call SendResponse or SendFault to send the reply back, appropriately.
        server -- ServiceContainer instance

    '''
    print "_Dispatch POST", post
    try:
        result = server(ps, post, action)
    except Exception, e:
        return SendFault(FaultFromException(e, 0, sys.exc_info()[2]), **kw)
    if result == None:
        return
    reply = StringIO.StringIO()
    try:
        SoapWriter(reply, nsdict=nsdict).serialize(result)
        return SendResponse(reply.getvalue(), **kw)
    except Exception, e:
        return SendFault(FaultFromException(e, 0, sys.exc_info()[2]), **kw)

def AsServer(port=80, services=()):
    '''port --
       services -- list of service instances
    '''
    address = ('', port)
    sc = ServiceContainer(address)
    for service in services:
        path = service.getPost()
        sc.setNode(service, path)
    sc.serve_forever()

class NoSuchService(Exception): pass
class NoSuchMethod(Exception): pass
class NotAuthorized(Exception): pass
class ServiceAlreadyPresent(Exception): pass
class PostNotSpecified(Exception): pass
class SOAPActionNotSpecified(Exception): pass

class ServiceSOAPBinding:
    """Binding defines the set of wsdl:binding operations, it takes as input a
    ParsedSoap instance and parses it into a pyobj.  It returns a response pyobj.

    class variables:
        soapAction -- dictionary of soapAction keys, and operation name values.
           These are specified in the WSDL soap bindings. There must be a
           class method matching the operation name value.

    """
    soapAction = {}
    auth_method = None

    def __init__(self, post):
        self.post = post

    def authorize(self, auth_info, post, action):
        if self.auth_method:
            return getattr(self, self.auth_method)(auth_info, post, action)
        else:
            return 1

    def __call___(self, action, ps):
        return self.getOperation(action)(ps)

    def getPost(self):
        return self.post

    def getOperation(self, action):
        '''Returns a method of class.
           action -- soapAction value
        '''
        opName = self.getOperationName(action)
        return getattr(self, opName)

    def getOperationName(self, action):
        '''Returns operation name.
           action -- soapAction value
        '''
        if not self.soapAction.has_key(action):
            raise SOAPActionNotSpecified, '%s is NOT in soapAction dictionary' %action
        return self.soapAction[action]

class SOAPRequestHandler(BaseSOAPRequestHandler):
    '''SOAP handler.
    '''
    def do_POST(self):
        '''The POST command.
        '''
        soapAction = self.headers.getheader('SOAPAction').strip('\'"')
        post = self.path
        if not post:
            raise PostNotSpecified, 'HTTP POST not specified in request'
        if not soapAction:
            raise SOAPActionNotSpecified, 'SOAP Action not specified in request'
        soapAction = soapAction.strip('\'"')
        post = post.strip('\'"')
        try:
            ct = self.headers['content-type']
            if ct.startswith('multipart/'):
                cid = resolvers.MIMEResolver(ct, self.rfile)
                xml = cid.GetSOAPPart()
                ps = ParsedSoap(xml, resolver=cid.Resolve)
            else:
                length = int(self.headers['content-length'])
                ps = ParsedSoap(self.rfile.read(length))
        except ParseException, e:
            self.send_fault(FaultFromZSIException(e))
        except Exception, e:
            # Faulted while processing; assume it's in the header.
            self.send_fault(FaultFromException(e, 1, sys.exc_info()[2]))
        else:
            _Dispatch(ps, self.server, self.send_xml, self.send_fault,
                post=post, action=soapAction)

class ServiceContainer(HTTPServer):
    '''HTTPServer that stores service instances according
    to POST values.  An action value is instance specific,
    and specifies an operation (function) of an instance.
    '''
    class NodeTree:
        '''Simple dictionary implementation of a node tree
        '''
        def __init__(self):
            self.__dict = {}

        def __str__(self):
            return str(self.__dict)

        def getNode(self, url):
            path = urlparse.urlsplit(url)[2]

            if self.__dict.has_key(path):
                return self.__dict[path]
            else:
                raise NoSuchService, 'No service(%s) in ServiceContainer' %path

        def setNode(self, service, url):
            if not isinstance(service, ServiceSOAPBinding):
               raise TypeError, 'A Service must implement class ServiceSOAPBinding'
            path = urlparse.urlsplit(url)[2]

            if self.__dict.has_key(path):
                raise ServiceAlreadyPresent, 'Service(%s) already in ServiceContainer' % path
            else:
                self.__dict[path] = service

        def removeNode(self, url):
            path = urlparse.urlsplit(url)[2]

            if self.__dict.has_key(path):
                node = self.__dict[path]
                del self.__dict[path]
                return node
            else:
                raise NoSuchService, 'No service(%s) in ServiceContainer' %path

    def __init__(self, server_address, RequestHandlerClass=SOAPRequestHandler):
        '''server_address --
           RequestHandlerClass --
        '''
        HTTPServer.__init__(self, server_address, RequestHandlerClass)
        self._nodes = self.NodeTree()

    def __call__(self, ps, post, action):
        '''ps -- ParsedSoap representing the request
           post -- HTTP POST --> instance
           action -- Soap Action header --> method
        '''
        return self.getCallBack(post, action)(ps)

    def getNode(self, post):
        '''post -- POST HTTP value
        '''
        return self._nodes.getNode(post)

    def setNode(self, service, post):
        '''service -- service instance
           post -- POST HTTP value
        '''
        self._nodes.setNode(service, post)

    def removeNode(self, post):
        '''post -- POST HTTP value
        '''
        self._nodes.removeNode(post)

    def getCallBack(self, post, action):
        '''post -- POST HTTP value
           action -- SOAP Action value
        '''
        node = self.getNode(post)

        if node is None:
            raise NoSuchMethod, "Method (%s) not found at path (%s)" % (action,
                                                                        path)
        if node.authorize(None, post, action):
        	return node.getOperation(action)
        else:
            raise NotAuthorized, "Authorization failed for method %s" % action

if __name__ == '__main__': print _copyright
<|endoftext|>


<commit_before>"""
Generalized Minimum Residual Method (GMRES).

GMRES iteratively refines the initial solution guess to the system Ax=b.

This implementation was based mainly on the gmres_mgs from PyAMG, where
modified Gram-Schmidt is used to orthogonalize the Krylov Space and
Givens Rotations are used to provide the residual norm each iteration.

Guidance code:

 - PyAMG library:
      https://github.com/pyamg/pyamg/blob/master/pyamg/krylov/_gmres_mgs.py
"""

import numpy
import scipy
from warnings import warn

from scipy.linalg import get_blas_funcs, get_lapack_funcs
from scipy.sparse.sputils import upcast

from pygbe.matrixfree import gmres_dot

def apply_givens(Q, v, k):
    """
    Apply the first k Givens rotations in Q to the vector v.

    Arguments
    ---------
        Q: list, list of consecutive 2x2 Givens rotations
        v: array, vector to apply the rotations to
        k: int, number of rotations to apply

    Returns
    -------
        v: array, that is changed in place.

    """

    for j in range(k):
        Qloc = Q[j]
        v[j:j+2] = scipy.dot(Qloc, v[j:j+2])

def gmres_mgs(surf_array, field_array, X, b, param, ind0, timing, kernel):
    """
    GMRES solver.

    Arguments
    ----------
    surf_array : array, contains the surface classes of each region on the
                        surface.
    field_array: array, contains the Field classes of each region on the surface.
    X          : array, initial guess.
    b          : array, right hand side.
    param      : class, parameters related to the surface.
    ind0       : class, it contains the indices related to the treecode
                        computation.
    timing     : class, it contains timing information for different parts of
                        the code.
    kernel     : pycuda source module.

    Returns
    --------
    X          : array, an updated guess to the solution.
    iteration  : int, number of outer iterations for convergence

    References
    ----------
    .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
       Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
       http://www-users.cs.umn.edu/~saad/books.html
    .. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html
    """

    # Defining xtype as dtype of the problem, to decide which BLAS functions
    # import.
    xtype = upcast(X.dtype, b.dtype)

    # Get fast access to underlying BLAS routines
    # dotc is the conjugate dot, dotu does no conjugation

    [lartg] = get_lapack_funcs(['lartg'], [X] )
    if numpy.iscomplexobj(numpy.zeros((1,), dtype=xtype)):
        [axpy, dotu, dotc, scal] =\
            get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [X])
    else:
        # real type
        [axpy, dotu, dotc, scal] =\
            get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [X])

    # Make full use of direct access to BLAS by defining own norm
    def norm(z):
        return numpy.sqrt(numpy.real(dotc(z, z)))

    # Defining dimension
    dimen = len(X)

    max_iter = param.max_iter
    R = param.restart
    tol = param.tol

    # Set number of outer and inner iterations

    if R > dimen:
        warn('Setting number of inner iterations (restrt) to maximum\
              allowed, which is A.shape[0] ')
        R = dimen

    max_inner = R

    #max_outer should be max_iter/max_inner but this might not be an integer
    #so we get the ceil of the division.
    #In the inner loop there is a if statement to break in case max_iter is
    #reached.

    max_outer = int(numpy.ceil(max_iter/max_inner))

    # Prep for method
    aux = gmres_dot(X, surf_array, field_array, ind0, param, timing, kernel)
    r = b - aux

    normr = norm(r)

    # Check initial guess ( scaling by b, if b != 0, must account for
    # case when norm(b) is very small)
    normb = norm(b)
    if normb == 0.0:
        normb = 1.0
    if normr < tol*normb:
        return X

    iteration = 0

    # Here start the GMRES
    for outer in range(max_outer):

        # Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space
        # Space required is O(dimen*max_inner).
        # NOTE:  We are dealing with row-major matrices, so we traverse in a
        #        row-major fashion,
        #        i.e., H and V's transpose is what we store.

        Q = []  # Initialzing Givens Rotations
        # Upper Hessenberg matrix, which is then
        # converted to upper triagonal with Givens Rotations

        H = numpy.zeros((max_inner+1, max_inner+1), dtype=xtype)
        V = numpy.zeros((max_inner+1, dimen), dtype=xtype)  # Krylov space

        # vs store the pointers to each column of V.
        # This saves a considerable amount of time.
        vs = []

        # v = r/normr
        V[0, :] = scal(1.0/normr, r)  # scal wrapper of dscal --> x = a*x
        vs.append(V[0, :])

        #Saving initial residual to be used to calculate the rel_resid
        if iteration==0:
            res_0 = normb

        #RHS vector in the Krylov space
        g = numpy.zeros((dimen, ), dtype=xtype)
        g[0] = normr

        for inner in range(max_inner):
            #New search direction
            v= V[inner+1, :]
            v[:] = gmres_dot(vs[-1], surf_array, field_array, ind0, param,
 timing, kernel)
            vs.append(v)

            #Modified Gram Schmidt
            for k in range(inner+1):
                vk = vs[k]
                alpha = dotc(vk, v)
                H[inner, k] = alpha
                v[:] = axpy(vk, v, dimen, -alpha)  # y := a*x + y
                #axpy is a wrapper for daxpy (blas function)

            normv = norm(v)
            H[inner, inner+1] = normv

            #Check for breakdown
            if H[inner, inner+1] != 0.0:
                v[:] = scal(1.0/H[inner, inner+1], v)

            #Apply for Givens rotations to H
            if inner > 0:
                apply_givens(Q, H[inner, :], inner)

            #Calculate and apply next complex-valued Givens rotations

            #If max_inner = dimen, we don't need to calculate, this
            #is unnecessary for the last inner iteration when inner = dimen -1

            if inner != dimen - 1:
                if H[inner, inner+1] != 0:
                    #lartg is a lapack function that computes the parameters
                    #for a Givens rotation
                    [c, s, _] = lartg(H[inner, inner], H[inner, inner+1])
                    Qblock = numpy.array([[c, s], [-numpy.conjugate(s),c]], dtype=xtype)
                    Q.append(Qblock)

                    #Apply Givens Rotations to RHS for the linear system in
                    # the krylov space.
                    g[inner:inner+2] = scipy.dot(Qblock, g[inner:inner+2])

                    #Apply Givens rotations to H
                    H[inner, inner] = dotu(Qblock[0,:], H[inner, inner:inner+2])
                    H[inner, inner+1] = 0.0

            iteration+= 1

            if inner < max_inner-1:
                normr = abs(g[inner+1])
                rel_resid = normr/res_0

                if rel_resid < tol:
                    break

            if iteration%1==0:
                print('Iteration: {}, relative residual: {}'.format(iteration,rel_resid))

            if (inner + 1 == R):
                print('Residual: {}. Restart...'.format(rel_resid))

            if iteration==max_iter:
                print('Warning!!!!'
                'You have reached the maximum number of iterations : {}.'.format(iteration)
                'The run will stop. Check the residual behaviour you might have a bug.'
                'For future runs you might consider changing the tolerance or'
                ' increasing the number of max_iter.')

                break

        # end inner loop, back to outer loop

        # Find best update to X in Krylov Space V.  Solve inner X inner system.
        y = scipy.linalg.solve (H[0:inner+1, 0:inner+1].T, g[0:inner+1])
        update = numpy.ravel(scipy.mat(V[:inner+1, :]).T * y.reshape(-1,1))
        X= X + update
        aux = gmres_dot(X, surf_array, field_array, ind0, param, timing, kernel)
        r = b - aux

        normr = norm(r)
        rel_resid = normr/res_0

        # test for convergence
        if rel_resid < tol:
            print('GMRES solve')
            print('Converged after {} iterations to a residual of {}'.format(iteration,rel_resid))
            print('Time weight vector: {}'.format(timing.time_mass))
            print('Time sort         : {}'.format(timing.time_sort))
            print('Time data transfer: {}'.format(timing.time_trans))
            print('Time P2M          : {}'.format(timing.time_P2M))
            print('Time M2M          : {}'.format(timing.time_M2M))
            print('Time M2P          : {}'.format(timing.time_M2P))
            print('Time P2P          : {}'.format(timing.time_P2P))
            print('\tTime analy: {}'.format(timing.time_an))

            return X, iteration

    #end outer loop

    return X, iteration
<commit_msg>Fix print statement<commit_after>"""
Generalized Minimum Residual Method (GMRES).

GMRES iteratively refines the initial solution guess to the system Ax=b.

This implementation was based mainly on the gmres_mgs from PyAMG, where
modified Gram-Schmidt is used to orthogonalize the Krylov Space and
Givens Rotations are used to provide the residual norm each iteration.

Guidance code:

 - PyAMG library:
      https://github.com/pyamg/pyamg/blob/master/pyamg/krylov/_gmres_mgs.py
"""

import numpy
import scipy
from warnings import warn

from scipy.linalg import get_blas_funcs, get_lapack_funcs
from scipy.sparse.sputils import upcast

from pygbe.matrixfree import gmres_dot

def apply_givens(Q, v, k):
    """
    Apply the first k Givens rotations in Q to the vector v.

    Arguments
    ---------
        Q: list, list of consecutive 2x2 Givens rotations
        v: array, vector to apply the rotations to
        k: int, number of rotations to apply

    Returns
    -------
        v: array, that is changed in place.

    """

    for j in range(k):
        Qloc = Q[j]
        v[j:j+2] = scipy.dot(Qloc, v[j:j+2])

def gmres_mgs(surf_array, field_array, X, b, param, ind0, timing, kernel):
    """
    GMRES solver.

    Arguments
    ----------
    surf_array : array, contains the surface classes of each region on the
                        surface.
    field_array: array, contains the Field classes of each region on the surface.
    X          : array, initial guess.
    b          : array, right hand side.
    param      : class, parameters related to the surface.
    ind0       : class, it contains the indices related to the treecode
                        computation.
    timing     : class, it contains timing information for different parts of
                        the code.
    kernel     : pycuda source module.

    Returns
    --------
    X          : array, an updated guess to the solution.
    iteration  : int, number of outer iterations for convergence

    References
    ----------
    .. [1] Yousef Saad, "Iterative Methods for Sparse Linear Systems,
       Second Edition", SIAM, pp. 151-172, pp. 272-275, 2003
       http://www-users.cs.umn.edu/~saad/books.html
    .. [2] C. T. Kelley, http://www4.ncsu.edu/~ctk/matlab_roots.html
    """

    # Defining xtype as dtype of the problem, to decide which BLAS functions
    # import.
    xtype = upcast(X.dtype, b.dtype)

    # Get fast access to underlying BLAS routines
    # dotc is the conjugate dot, dotu does no conjugation

    [lartg] = get_lapack_funcs(['lartg'], [X] )
    if numpy.iscomplexobj(numpy.zeros((1,), dtype=xtype)):
        [axpy, dotu, dotc, scal] =\
            get_blas_funcs(['axpy', 'dotu', 'dotc', 'scal'], [X])
    else:
        # real type
        [axpy, dotu, dotc, scal] =\
            get_blas_funcs(['axpy', 'dot', 'dot', 'scal'], [X])

    # Make full use of direct access to BLAS by defining own norm
    def norm(z):
        return numpy.sqrt(numpy.real(dotc(z, z)))

    # Defining dimension
    dimen = len(X)

    max_iter = param.max_iter
    R = param.restart
    tol = param.tol

    # Set number of outer and inner iterations

    if R > dimen:
        warn('Setting number of inner iterations (restrt) to maximum\
              allowed, which is A.shape[0] ')
        R = dimen

    max_inner = R

    #max_outer should be max_iter/max_inner but this might not be an integer
    #so we get the ceil of the division.
    #In the inner loop there is a if statement to break in case max_iter is
    #reached.

    max_outer = int(numpy.ceil(max_iter/max_inner))

    # Prep for method
    aux = gmres_dot(X, surf_array, field_array, ind0, param, timing, kernel)
    r = b - aux

    normr = norm(r)

    # Check initial guess ( scaling by b, if b != 0, must account for
    # case when norm(b) is very small)
    normb = norm(b)
    if normb == 0.0:
        normb = 1.0
    if normr < tol*normb:
        return X

    iteration = 0

    # Here start the GMRES
    for outer in range(max_outer):

        # Preallocate for Givens Rotations, Hessenberg matrix and Krylov Space
        # Space required is O(dimen*max_inner).
        # NOTE:  We are dealing with row-major matrices, so we traverse in a
        #        row-major fashion,
        #        i.e., H and V's transpose is what we store.

        Q = []  # Initialzing Givens Rotations
        # Upper Hessenberg matrix, which is then
        # converted to upper triagonal with Givens Rotations

        H = numpy.zeros((max_inner+1, max_inner+1), dtype=xtype)
        V = numpy.zeros((max_inner+1, dimen), dtype=xtype)  # Krylov space

        # vs store the pointers to each column of V.
        # This saves a considerable amount of time.
        vs = []

        # v = r/normr
        V[0, :] = scal(1.0/normr, r)  # scal wrapper of dscal --> x = a*x
        vs.append(V[0, :])

        #Saving initial residual to be used to calculate the rel_resid
        if iteration==0:
            res_0 = normb

        #RHS vector in the Krylov space
        g = numpy.zeros((dimen, ), dtype=xtype)
        g[0] = normr

        for inner in range(max_inner):
            #New search direction
            v= V[inner+1, :]
            v[:] = gmres_dot(vs[-1], surf_array, field_array, ind0, param,
 timing, kernel)
            vs.append(v)

            #Modified Gram Schmidt
            for k in range(inner+1):
                vk = vs[k]
                alpha = dotc(vk, v)
                H[inner, k] = alpha
                v[:] = axpy(vk, v, dimen, -alpha)  # y := a*x + y
                #axpy is a wrapper for daxpy (blas function)

            normv = norm(v)
            H[inner, inner+1] = normv

            #Check for breakdown
            if H[inner, inner+1] != 0.0:
                v[:] = scal(1.0/H[inner, inner+1], v)

            #Apply for Givens rotations to H
            if inner > 0:
                apply_givens(Q, H[inner, :], inner)

            #Calculate and apply next complex-valued Givens rotations

            #If max_inner = dimen, we don't need to calculate, this
            #is unnecessary for the last inner iteration when inner = dimen -1

            if inner != dimen - 1:
                if H[inner, inner+1] != 0:
                    #lartg is a lapack function that computes the parameters
                    #for a Givens rotation
                    [c, s, _] = lartg(H[inner, inner], H[inner, inner+1])
                    Qblock = numpy.array([[c, s], [-numpy.conjugate(s),c]], dtype=xtype)
                    Q.append(Qblock)

                    #Apply Givens Rotations to RHS for the linear system in
                    # the krylov space.
                    g[inner:inner+2] = scipy.dot(Qblock, g[inner:inner+2])

                    #Apply Givens rotations to H
                    H[inner, inner] = dotu(Qblock[0,:], H[inner, inner:inner+2])
                    H[inner, inner+1] = 0.0

            iteration+= 1

            if inner < max_inner-1:
                normr = abs(g[inner+1])
                rel_resid = normr/res_0

                if rel_resid < tol:
                    break

            if iteration%1==0:
                print('Iteration: {}, relative residual: {}'.format(iteration,rel_resid))

            if (inner + 1 == R):
                print('Residual: {}. Restart...'.format(rel_resid))

            if iteration==max_iter:
                print('Warning!!!!'
                'You have reached the maximum number of iterations : {}.'.format(iteration))
                print('The run will stop. Check the residual behaviour you might have a bug.'
                'For future runs you might consider changing the tolerance or'
                ' increasing the number of max_iter.')

                break

        # end inner loop, back to outer loop

        # Find best update to X in Krylov Space V.  Solve inner X inner system.
        y = scipy.linalg.solve (H[0:inner+1, 0:inner+1].T, g[0:inner+1])
        update = numpy.ravel(scipy.mat(V[:inner+1, :]).T * y.reshape(-1,1))
        X= X + update
        aux = gmres_dot(X, surf_array, field_array, ind0, param, timing, kernel)
        r = b - aux

        normr = norm(r)
        rel_resid = normr/res_0

        # test for convergence
        if rel_resid < tol:
            print('GMRES solve')
            print('Converged after {} iterations to a residual of {}'.format(iteration,rel_resid))
            print('Time weight vector: {}'.format(timing.time_mass))
            print('Time sort         : {}'.format(timing.time_sort))
            print('Time data transfer: {}'.format(timing.time_trans))
            print('Time P2M          : {}'.format(timing.time_P2M))
            print('Time M2M          : {}'.format(timing.time_M2M))
            print('Time M2P          : {}'.format(timing.time_M2P))
            print('Time P2P          : {}'.format(timing.time_P2P))
            print('\tTime analy: {}'.format(timing.time_an))

            return X, iteration

    #end outer loop

    return X, iteration
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os

try:
    from configparser import SafeConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser

class Config(object):
    def __init__(self, config_dir=None, config_file=None):
        self.config_dir = config_dir
        self.config_file = config_file

        if config_dir is None:
            self.config_dir = os.path.join(
                os.environ.get('HOME', './'),
                '.pyhn')
        if config_file is None:
            self.config_file = "config"

        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir)

        self.config_path = os.path.join(self.config_dir, self.config_file)

        self.parser = SafeConfigParser()
        self.read()

    def read(self):
        self.parser.read(self.config_path)

        # Keybindings
        if not self.parser.has_section('keybindings'):
            self.parser.add_section('keybindings')

        if not self.parser.has_option('keybindings', 'page_up'):
            self.parser.set('keybindings', 'page_up', 'ctrl u')
        if not self.parser.has_option('keybindings', 'page_down'):
            self.parser.set('keybindings', 'page_down', 'ctrl d')
        if not self.parser.has_option('keybindings', 'first_story'):
            self.parser.set('keybindings', 'first_story', 'g')
        if not self.parser.has_option('keybindings', 'last_story'):
            self.parser.set('keybindings', 'last_story', 'G')
        if not self.parser.has_option('keybindings', 'up'):
            self.parser.set('keybindings', 'up', 'j')
        if not self.parser.has_option('keybindings', 'down'):
            self.parser.set('keybindings', 'down', 'k')
        if not self.parser.has_option('keybindings', 'refresh'):
            self.parser.set('keybindings', 'refresh', 'r')
        if not self.parser.has_option('keybindings', 'show_comments_link'):
            self.parser.set('keybindings', 'show_comments_link', 'c')
        if not self.parser.has_option('keybindings', 'open_comments_link'):
            self.parser.set('keybindings', 'open_comments_link', 'C')
        if not self.parser.has_option('keybindings', 'show_story_link'):
            self.parser.set('keybindings', 'show_story_link', 's')
        if not self.parser.has_option('keybindings', 'open_story_link'):
            self.parser.set('keybindings', 'open_story_link', 'S,enter')
        if not self.parser.has_option('keybindings', 'show_submitter_link'):
            self.parser.set('keybindings', 'show_submitter_link', 'u')
        if not self.parser.has_option('keybindings', 'open_submitter_link'):
            self.parser.set('keybindings', 'open_submitter_link', 'U')
        if not self.parser.has_option('keybindings', 'reload_config'):
            self.parser.set('keybindings', 'reload_config', 'ctrl R')
        if not self.parser.has_option('keybindings', 'newest_stories'):
            self.parser.set('keybindings', 'newest_stories', 'n')
        if not self.parser.has_option('keybindings', 'top_stories'):
            self.parser.set('keybindings', 'top_stories', 't')
        if not self.parser.has_option('keybindings', 'best_stories'):
            self.parser.set('keybindings', 'best_stories', 'b')

        # Paths
        if not self.parser.has_section('settings'):
            self.parser.add_section('settings')

        if not self.parser.has_option('settings', 'extra_page'):
            self.parser.set('settings', 'extra_page', '1')

        if not self.parser.has_option('settings', 'cache'):
            self.parser.set(
                'settings',
                'cache',
                os.path.join(os.environ.get('HOME', './'), '.pyhn', 'cache'))
        if not self.parser.has_option('settings', 'cache_age'):
            self.parser.set('settings', 'cache_age', "5")
        if not self.parser.has_option('settings', 'browser_cmd'):
            self.parser.set('settings', 'browser_cmd', '__default__')

        # Colors
        if not self.parser.has_section('colors'):
            self.parser.add_section('colors')

        if not self.parser.has_option('colors', 'body'):
            self.parser.set('colors', 'body', 'default||standout')
        if not self.parser.has_option('colors', 'focus'):
            self.parser.set('colors', 'focus', 'black||underline')
        if not self.parser.has_option('colors', 'footer'):
            self.parser.set('colors', 'footer', 'black|light gray')
        if not self.parser.has_option('colors', 'footer-error'):
            self.parser.set('colors', 'footer-error', 'dark red,bold|light gray')
        if not self.parser.has_option('colors', 'header'):
            self.parser.set('colors', 'header', 'dark gray,bold|white|')
        if not self.parser.has_option('colors', 'title'):
            self.parser.set('colors', 'title', 'dark red,bold|light gray')
        if not self.parser.has_option('colors', 'help'):
            self.parser.set('colors', 'help', 'black|dark cyan|standout')

        if not os.path.exists(self.config_path):
            self.parser.write(open(self.config_path, 'w'))

    def get_palette(self):
        palette = []
        for item in self.parser.items('colors'):
            name = item[0]
            settings = item[1]
            foreground = ""
            background = ""
            monochrome = ""
            if len(settings.split('|')) == 3:
                foreground = settings.split('|')[0]
                background = settings.split('|')[1]
                monochrome = settings.split('|')[2]
            elif len(settings.split('|')) == 2:
                foreground = settings.split('|')[0]
                background = settings.split('|')[1]
            elif len(settings.split('|')) == 1:
                foreground = settings.split('|')[0]

            palette.append((name, foreground, background, monochrome))
        return palette
<commit_msg>Add Show/Show New/Ask/Job keybindings<commit_after># -*- coding: utf-8 -*-
import os

try:
    from configparser import SafeConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser

class Config(object):
    def __init__(self, config_dir=None, config_file=None):
        self.config_dir = config_dir
        self.config_file = config_file

        if config_dir is None:
            self.config_dir = os.path.join(
                os.environ.get('HOME', './'),
                '.pyhn')
        if config_file is None:
            self.config_file = "config"

        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir)

        self.config_path = os.path.join(self.config_dir, self.config_file)

        self.parser = SafeConfigParser()
        self.read()

    def read(self):
        self.parser.read(self.config_path)

        # Keybindings
        if not self.parser.has_section('keybindings'):
            self.parser.add_section('keybindings')

        if not self.parser.has_option('keybindings', 'page_up'):
            self.parser.set('keybindings', 'page_up', 'ctrl u')
        if not self.parser.has_option('keybindings', 'page_down'):
            self.parser.set('keybindings', 'page_down', 'ctrl d')
        if not self.parser.has_option('keybindings', 'first_story'):
            self.parser.set('keybindings', 'first_story', 'g')
        if not self.parser.has_option('keybindings', 'last_story'):
            self.parser.set('keybindings', 'last_story', 'G')
        if not self.parser.has_option('keybindings', 'up'):
            self.parser.set('keybindings', 'up', 'j')
        if not self.parser.has_option('keybindings', 'down'):
            self.parser.set('keybindings', 'down', 'k')
        if not self.parser.has_option('keybindings', 'refresh'):
            self.parser.set('keybindings', 'refresh', 'r')

        if not self.parser.has_option('keybindings', 'show_comments_link'):
            self.parser.set('keybindings', 'show_comments_link', 'c')
        if not self.parser.has_option('keybindings', 'open_comments_link'):
            self.parser.set('keybindings', 'open_comments_link', 'C')
        if not self.parser.has_option('keybindings', 'show_story_link'):
            self.parser.set('keybindings', 'show_story_link', 's')
        if not self.parser.has_option('keybindings', 'open_story_link'):
            self.parser.set('keybindings', 'open_story_link', 'S,enter')
        if not self.parser.has_option('keybindings', 'show_submitter_link'):
            self.parser.set('keybindings', 'show_submitter_link', 'u')
        if not self.parser.has_option('keybindings', 'open_submitter_link'):
            self.parser.set('keybindings', 'open_submitter_link', 'U')

        if not self.parser.has_option('keybindings', 'reload_config'):
            self.parser.set('keybindings', 'reload_config', 'ctrl R')

        if not self.parser.has_option('keybindings', 'newest_stories'):
            self.parser.set('keybindings', 'newest_stories', 'n')
        if not self.parser.has_option('keybindings', 'top_stories'):
            self.parser.set('keybindings', 'top_stories', 't')
        if not self.parser.has_option('keybindings', 'best_stories'):
            self.parser.set('keybindings', 'best_stories', 'b')
        if not self.parser.has_option('keybindings', 'show_stories'):
            self.parser.set('keybindings', 'show_stories', 'd')
        if not self.parser.has_option('keybindings', 'show_newest_stories'):
            self.parser.set('keybindings', 'show_newest_stories', 'D')
        if not self.parser.has_option('keybindings', 'ask_stories'):
            self.parser.set('keybindings', 'ask_stories', 'a')
        if not self.parser.has_option('keybindings', 'jobs_stories'):
            self.parser.set('keybindings', 'jobs_stories', 'J')
        # Paths
        if not self.parser.has_section('settings'):
            self.parser.add_section('settings')

        if not self.parser.has_option('settings', 'extra_page'):
            self.parser.set('settings', 'extra_page', '1')

        if not self.parser.has_option('settings', 'cache'):
            self.parser.set(
                'settings',
                'cache',
                os.path.join(os.environ.get('HOME', './'), '.pyhn', 'cache'))
        if not self.parser.has_option('settings', 'cache_age'):
            self.parser.set('settings', 'cache_age', "5")
        if not self.parser.has_option('settings', 'browser_cmd'):
            self.parser.set('settings', 'browser_cmd', '__default__')

        # Colors
        if not self.parser.has_section('colors'):
            self.parser.add_section('colors')

        if not self.parser.has_option('colors', 'body'):
            self.parser.set('colors', 'body', 'default||standout')
        if not self.parser.has_option('colors', 'focus'):
            self.parser.set('colors', 'focus', 'black||underline')
        if not self.parser.has_option('colors', 'footer'):
            self.parser.set('colors', 'footer', 'black|light gray')
        if not self.parser.has_option('colors', 'footer-error'):
            self.parser.set('colors', 'footer-error', 'dark red,bold|light gray')
        if not self.parser.has_option('colors', 'header'):
            self.parser.set('colors', 'header', 'dark gray,bold|white|')
        if not self.parser.has_option('colors', 'title'):
            self.parser.set('colors', 'title', 'dark red,bold|light gray')
        if not self.parser.has_option('colors', 'help'):
            self.parser.set('colors', 'help', 'black|dark cyan|standout')

        if not os.path.exists(self.config_path):
            self.parser.write(open(self.config_path, 'w'))

    def get_palette(self):
        palette = []
        for item in self.parser.items('colors'):
            name = item[0]
            settings = item[1]
            foreground = ""
            background = ""
            monochrome = ""
            if len(settings.split('|')) == 3:
                foreground = settings.split('|')[0]
                background = settings.split('|')[1]
                monochrome = settings.split('|')[2]
            elif len(settings.split('|')) == 2:
                foreground = settings.split('|')[0]
                background = settings.split('|')[1]
            elif len(settings.split('|')) == 1:
                foreground = settings.split('|')[0]

            palette.append((name, foreground, background, monochrome))
        return palette
<|endoftext|>