<commit_before>import itertools
import hashlib
import re

import util
import parsers

RE_LINE = re.compile(r'\s*\d+\|')
RE_CALL = re.compile(r'\s*\(call\)\|')

def stuff_in_dict(this_dict, key):
    """
    if the `this_dict` dictionary is not empty then return a new dictionary that
    has one (key, value) pair, where the key is set to `key` and the value is set to `this_dict`

    :param this_dict: dictionary
    :param key: string
    :return: dictionary
    """
    return {key: this_dict} if len(this_dict) > 0 else {}

def command_line_handler(section):
    """
    Extracts the command_line information from section

    :param section: sequence of lines
    :return: yields a dict(('command_line', <command>), ) or {}
    """
    for line in section:
        if line.startswith('Command line:'):
            yield {'command_line': parsers.command_line_parser(line)}

def total_duration_handler(section):
    """
    Parses lines of the following form:

       Total duration: 5248.89s

    :param section: generator of lines
    :return: yields a dict(('total_duration', <seconds>), ) or {}
    """
    for line in section:
        if line.startswith('Total duration:'):
            yield {'total_duration': parsers.total_duration_parser(line)}

def summary_handler(section):
    """
    Extract the information from the PProfile summary section

    :param section: sequence of lines
    :return: dictionary
    """
    summary_dict = dict()
    for section_dict in itertools.chain.from_iterable(parse_section(section, (command_line_handler,
                                                                              total_duration_handler))):
        summary_dict = util.update(summary_dict, section_dict)

    yield {'summary': summary_dict}

def file_summary_handle(section):
    """
    Extract the file summary information for a given file

    :param section: sequence of lines
    :return: dictionary
    """
    fs_dict = dict()
    for line in section:
        if line.startswith('File:'):
            fs_dict = util.update(fs_dict, {'name': parsers.file_name_parser(line)})
        if line.startswith('File duration:'):
            fs_dict = util.update(fs_dict, {'duration': parsers.file_duration_parser(line)})
            fs_dict = util.update(fs_dict, {'percentage': parsers.file_percentage_parser(line)})

    yield stuff_in_dict(fs_dict, 'file_summary')

def line_handler(section):
    """
    Extract line information. This is a tricky thing since the also want to handle the (call)...
    information, which will allows to navigate more easily through the code. However, to parse
    (call) information we need to add information to the last line (non-call) handled...

    :param section: sequence of lines
    :return: dictionary
    """
    lines = list()
    for line in section:
        if RE_LINE.match(line):
            lines.append(parsers.parse_line(line))
        if RE_CALL.match(line):
            # parse the line and add it to the calls if the last line parsed
            call = parsers.parse_call(line)
            lines[-1]['calls'].append(call)

    yield stuff_in_dict(lines, 'lines')

def file_handler(section):
    """
    Extract the information for a single file in the profile

    :param section: sequence of lines
    :return: dictionary
    """
    file_dict = {}
    for file_section in split_sections(section, section_marker='-----'):
        for fdict in itertools.chain.from_iterable(parse_section(file_section, (file_summary_handle,
                                                                                line_handler))):
            file_dict = util.update(file_dict, fdict)

    yield stuff_in_dict(file_dict, hashlib.sha1(section[0]).hexdigest())

def parse_section(section, handlers):
    """
    Applies each handler in the list of `handlers` to the section, which is a
    sequence of lines. Each handler yields one or more dictionaries containing
    the information extracted from the section.

    :param section: sequence of strings representing all the lines in a section
    :param handlers: sequence of handlers

    :return: yields dictionaries containing the information contained within
             each section
    """
    return (handler(section) for handler in handlers)

def split_sections(lines, section_marker='File:'):
    """
    Splits the sequences of `lines` into sections based on the `section_marker`.
    A line that starts with the section_marker signals the start of a new section
    in the sequence of lines.

    :param lines: sequence of lines
    :keyword section_marker: string that demarkates the start of a new section;
             default is 'File:'

    :return: a generator of sections
    """
    section = list()
    for line in lines:
        if line.startswith(section_marker):
            # this is the start of a new section so yield the current section
            yield section
            section = list()
        # add the line to the current section
        section.append(line)

    yield section
<commit_msg>switch to using util.hashkey() function<commit_after>import itertools
import re

import util
import parsers

RE_LINE = re.compile(r'\s*\d+\|')
RE_CALL = re.compile(r'\s*\(call\)\|')

def stuff_in_dict(this_dict, key):
    """
    if the `this_dict` dictionary is not empty then return a new dictionary that
    has one (key, value) pair, where the key is set to `key` and the value is set to `this_dict`

    :param this_dict: dictionary
    :param key: string
    :return: dictionary
    """
    return {key: this_dict} if len(this_dict) > 0 else {}

def command_line_handler(section):
    """
    Extracts the command_line information from section

    :param section: sequence of lines
    :return: yields a dict(('command_line', <command>), ) or {}
    """
    for line in section:
        if line.startswith('Command line:'):
            yield {'command_line': parsers.command_line_parser(line)}

def total_duration_handler(section):
    """
    Parses lines of the following form:

       Total duration: 5248.89s

    :param section: generator of lines
    :return: yields a dict(('total_duration', <seconds>), ) or {}
    """
    for line in section:
        if line.startswith('Total duration:'):
            yield {'total_duration': parsers.total_duration_parser(line)}

def summary_handler(section):
    """
    Extract the information from the PProfile summary section

    :param section: sequence of lines
    :return: dictionary
    """
    summary_dict = dict()
    for section_dict in itertools.chain.from_iterable(parse_section(section, (command_line_handler,
                                                                              total_duration_handler))):
        summary_dict = util.update(summary_dict, section_dict)

    yield {'summary': summary_dict}

def file_summary_handle(section):
    """
    Extract the file summary information for a given file

    :param section: sequence of lines
    :return: dictionary
    """
    fs_dict = dict()
    for line in section:
        if line.startswith('File:'):
            fs_dict = util.update(fs_dict, {'name': parsers.file_name_parser(line)})
        if line.startswith('File duration:'):
            fs_dict = util.update(fs_dict, {'duration': parsers.file_duration_parser(line)})
            fs_dict = util.update(fs_dict, {'percentage': parsers.file_percentage_parser(line)})

    yield stuff_in_dict(fs_dict, 'file_summary')

def line_handler(section):
    """
    Extract line information. This is a tricky thing since the also want to handle the (call)...
    information, which will allows to navigate more easily through the code. However, to parse
    (call) information we need to add information to the last line (non-call) handled...

    :param section: sequence of lines
    :return: dictionary
    """
    lines = list()
    for line in section:
        if RE_LINE.match(line):
            lines.append(parsers.parse_line(line))
        if RE_CALL.match(line):
            # parse the line and add it to the calls if the last line parsed
            call = parsers.parse_call(line)
            lines[-1]['calls'].append(call)

    yield stuff_in_dict(lines, 'lines')

def file_handler(section):
    """
    Extract the information for a single file in the profile

    :param section: sequence of lines
    :return: dictionary
    """
    file_dict = {}
    for file_section in split_sections(section, section_marker='-----'):
        for fdict in itertools.chain.from_iterable(parse_section(file_section, (file_summary_handle,
                                                                                line_handler))):
            file_dict = util.update(file_dict, fdict)

    yield stuff_in_dict(file_dict, util.hashkey(file_dict['file_summary']['name']) if file_dict else '')

def parse_section(section, handlers):
    """
    Applies each handler in the list of `handlers` to the section, which is a
    sequence of lines. Each handler yields one or more dictionaries containing
    the information extracted from the section.

    :param section: sequence of strings representing all the lines in a section
    :param handlers: sequence of handlers

    :return: yields dictionaries containing the information contained within
             each section
    """
    return (handler(section) for handler in handlers)

def split_sections(lines, section_marker='File:'):
    """
    Splits the sequences of `lines` into sections based on the `section_marker`.
    A line that starts with the section_marker signals the start of a new section
    in the sequence of lines.

    :param lines: sequence of lines
    :keyword section_marker: string that demarkates the start of a new section;
             default is 'File:'

    :return: a generator of sections
    """
    section = list()
    for line in lines:
        if line.startswith(section_marker):
            # this is the start of a new section so yield the current section
            yield section
            section = list()
        # add the line to the current section
        section.append(line)

    yield section
<|endoftext|>


<commit_before>import sys, os, string

# JDC: It makes things much simpler for other developers if there is only one class per file, and if the filename and class match.  That way, we don't have to hunt for particular classes in files.  The files are less cluttered that way.
# JDC: Classes should be named starting with a capital letter, such as "IonCalculator".  Maybe 'IonicEnvironmentCalculation' would be more appropriate.
class ionCalculator(object):
    # JDC: This class lacked a docstring, so I created one.
    """A utility object to calculate numbers, concentrations of ions.

    AUTHOR
      Vincent Voelz - primary author
      John Chodera - secondary cleanup, commenting

    REQUIREMENTS
      gromacs must be installed, and GMXLIB, GMXPATH, and MMTOOLSPATH set appropriately.

    EXAMPLES
      # Create a gromacs system from a given PDB file, using the 'ffamber99p' forcefield.
      import mmtools.gromacstools.system as system
      gromacsSystem = system.GromacsSystem('test.pdb' useff='ffamber99p')

      # Add ions to bring up to 150 mM NaCl salt conditions.
      gromacsSystem.setup.setSaltConditions('NaCl', 0.150)

      # Prepare a system, writing TPR and GRO files, and *mdrun* script in the current directory
      import os, sys
      gromacsSystem.prepare(outname='prod', os.path.abspath(os.curdir))
    """

    def __init__(self, useff):
        # JDC: Added documentation on what required argument 'useff' signified.
        """Initialize the ion calculator given the gromacs forcefield of choice.

        REQUIRED ARGUMENTS
          useff - the name of the forcefield to use in calling grompp (e.g. 'ffamber99p')
        """

        # Store the chosen forcefield.
        # JDC: Shouldn't we check here to see if this forcefield is supported, so we can throw an exception of not?
        self.useff = useff

        # All possible ion names that we know about and could occur in gromacs forcefield files.
        # JDC: Should this be moved to static class data?
        self.possibleIons = ['Ca','Cl','Na','Mg', 'K', 'Rb', 'CS', 'Li', 'Zn', 'Sr', 'Ba' ]

        # Create empty dictionary to contain all the ions and ion information provided by the forcefield.
        self.ions = {}    # a dictionary of Ion objects {'Cl', <Ion object> }

        # Read list of ions, their charges, and masses from gromacs forcefield files.
        # JDC: We should through a Exception of some sort if this fails.
        self.getIons()

        return

    # JDC: Is this a private method, or part of the public API?
    # JDC: Should this use arguments/return data, rather than operating on just internal object data?
    # JDC: Does this need to be instantiated separately for each object, or should it be a Singleton, only performed once for each available forcefield upon first class startup?
    def getIons(self):
        """Get atomic mass constants for all ion types listed in self.possibleIons from the gromacs forcefield files, storing them in self.ions.

        """

        # Read the contents of the .rtp (residue topology) file for this forcefield.
        rtpfile = os.path.join(os.environ['GMXLIB'],(self.useff+'.rtp'))
        fin = open(rtpfile,'r')
        rtplines  = fin.readlines()
        fin.close()

        # Build up dictionary of ions present in .rtp file.
        for line in rtplines:
            fields = line.split()
            # JDC: Instead of using fields[0], fields[1], fields[2], the line should be parsed into meaningfully-named tokens.
            # This makes the comprehension of what this codes much more straightforward.
            if len(fields) > 0:
                if (fields[0] in self.possibleIons):
                    # JDC: This idiom of parsing into named variables and then processing is much more readable, if more verbose, to other coders.
                    # Parse fields.
                    # Example format:
                    # Ca     amber99_15   2.00000     1
                    atomName = fields[0]
                    atomType = fields[1]
                    charge = float(fields[2])
                    # Store this ion in object ion dictionary.
                    self.ions[atomName] = Ion(atomName, atomType, charge)

        # Read contents of .atp (atom type masses) file for this forcefield.
        atpfile = os.path.join(os.environ['GMXLIB'],(self.useff+'.atp'))
        fin = open(atpfile,'r')
        atplines  = fin.readlines()
        fin.close()

        # Parse .atp file to determine atomic masses.
        for line in atplines:
            fields = line.split()
            if len(fields) > 0:
                # find the right ion
                for key in self.ions.keys():
                    # JDC: This could be more clear.
                    if self.ions[key].atomType == fields[0]:
                        self.ions[key].atomicMass = float(fields[1])
                        self.ions[key].atomDescription = string.joinfields(fields[3:])

    # JDC: Should use Units class for concentration.  There is a 'Units.M' or 'Units.Molar' unit.
    def molarityToFraction(self, concentration, solvent='water'):
        """Compute the number fraction of ion molecules given a Molar concentration.

        REQUIRED ARGUMENTS
          concentration - the concentration of ions (Molar)

        OPTIONAL ARGUMENTS
          solvent - the name of the solvent ('water' is only solvent currently supported, defaults to 'water')

        RETURNS
          ???????????
        """

        # JDC: Should raise an exception if unrecognized solvent is passed.
        if solvent != 'water':
            print 'Only solvent=\'water\' is supported.  Exiting.'
            sys.exit(1)

        # JDC: Should use Units clas for this.
        waterGramPerMol = 18.016
        waterMolPerGram = 1./waterGramPerMol
        waterGramPerLiter = 1000.0
        waterMolPerLiter = waterMolPerGram * waterGramPerLiter

        # JDC: I scratched my head for a while and couldn't figure out what the quantity in parenthesis represented.  Maybe I'm just tired, but it would be much less confusing if you assigned it to a variable like 'numberOfIonsInBox' and then returned *this* variable instead.
        return (concentration/waterMolPerLiter)

# JDC: This class should be in a separate file.
class Ion(object):
    """A monoatomic ion.

    Stores atom name, forcefield atom type, charge, a textual description, and the atomic mass of a single-atom ion.

    """

    def __init__(self, atomName, atomType, charge):
        """Initialize the data structures.

        REQUIRED ARGUMENTS
          atomName - the ion name
          atomType - the forcefield atom type
          charge - the partial atomic charge of the ion (in units of electron charge)

        """

        # Initialize object data.              # JDC: Every variable should be described the first time it is defined.
        self.atomName = atomName               # the atom name (e.g. 'Cl')
        self.atomType = atomType               # the forcefield atom type (e.g. 'amber99_30')
        self.charge = charge                   # the partial atomic charge for the ion (in units of electron charge)
        self.atomDescription = None            # a textual description of the ion (e.g. "chloride ion")
        self.atomicMass = None                 # the mass of the ion (in amu)

        return

    def __repr__(self):
        """Self-documented string representation of the internal data structures.

        """

        # Construct string representation
        # JDC: Descriptions are much clearer to construct line-by-line.
        tmpstr = '%s:\n' % self.atomName
        tmpstr += '\tatomtype: %s\n' % self.atomType
        tmpstr += '\tcharge %2.1f\n' % self.charge # JDC: Is this a sufficient number of decimal places?
        tmpstr += '\tdescription = %s\n' % self.atomDescription
        tmpstr += '\tatomic mass %2.1f\n' % self.atomicMass # JDC: Is this a sufficint number of decimal places?

        # Return string representation.
        return tmpstr

<commit_msg>Minor additional comment to ioncalc.py. Still have not tested.<commit_after>import sys, os, string

# JDC: It makes things much simpler for other developers if there is only one class per file, and if the filename and class match.  That way, we don't have to hunt for particular classes in files.  The files are less cluttered that way.
# JDC: Classes should be named starting with a capital letter, such as "IonCalculator".  Maybe 'IonicEnvironmentCalculation' would be more appropriate.
class ionCalculator(object):
    # JDC: This class lacked a docstring, so I created one.
    """A utility object to calculate numbers, concentrations of ions.

    AUTHOR
      Vincent Voelz - primary author
      John Chodera - secondary cleanup, commenting

    REQUIREMENTS
      gromacs must be installed, and GMXLIB, GMXPATH, and MMTOOLSPATH set appropriately.

    EXAMPLES
      # Create a gromacs system from a given PDB file, using the 'ffamber99p' forcefield.
      import mmtools.gromacstools.system as system
      gromacsSystem = system.GromacsSystem('test.pdb' useff='ffamber99p')

      # Add ions to bring up to 150 mM NaCl salt conditions.
      gromacsSystem.setup.setSaltConditions('NaCl', 0.150)

      # Prepare a system, writing TPR and GRO files, and *mdrun* script in the current directory
      import os, sys
      gromacsSystem.prepare(outname='prod', os.path.abspath(os.curdir))
    """

    # JDC: 'useff' might be named something more clear, like 'forcefieldName', or perhaps 'ffname'.
    def __init__(self, useff):
        # JDC: Added documentation on what required argument 'useff' signified.
        """Initialize the ion calculator given the gromacs forcefield of choice.

        REQUIRED ARGUMENTS
          useff - the name of the forcefield to use in calling grompp (e.g. 'ffamber99p')
        """

        # Store the chosen forcefield.
        # JDC: Shouldn't we check here to see if this forcefield is supported, so we can throw an exception of not?
        self.useff = useff

        # All possible ion names that we know about and could occur in gromacs forcefield files.
        # JDC: Should this be moved to static class data?
        self.possibleIons = ['Ca','Cl','Na','Mg', 'K', 'Rb', 'CS', 'Li', 'Zn', 'Sr', 'Ba' ]

        # Create empty dictionary to contain all the ions and ion information provided by the forcefield.
        self.ions = {}    # a dictionary of Ion objects {'Cl', <Ion object> }

        # Read list of ions, their charges, and masses from gromacs forcefield files.
        # JDC: We should through a Exception of some sort if this fails.
        self.getIons()

        return

    # JDC: Is this a private method, or part of the public API?
    # JDC: Should this use arguments/return data, rather than operating on just internal object data?
    # JDC: Does this need to be instantiated separately for each object, or should it be a Singleton, only performed once for each available forcefield upon first class startup?
    def getIons(self):
        """Get atomic mass constants for all ion types listed in self.possibleIons from the gromacs forcefield files, storing them in self.ions.

        """

        # Read the contents of the .rtp (residue topology) file for this forcefield.
        rtpfile = os.path.join(os.environ['GMXLIB'],(self.useff+'.rtp'))
        fin = open(rtpfile,'r')
        rtplines  = fin.readlines()
        fin.close()

        # Build up dictionary of ions present in .rtp file.
        for line in rtplines:
            fields = line.split()
            # JDC: Instead of using fields[0], fields[1], fields[2], the line should be parsed into meaningfully-named tokens.
            # This makes the comprehension of what this codes much more straightforward.
            if len(fields) > 0:
                if (fields[0] in self.possibleIons):
                    # JDC: This idiom of parsing into named variables and then processing is much more readable, if more verbose, to other coders.
                    # Parse fields.
                    # Example format:
                    # Ca     amber99_15   2.00000     1
                    atomName = fields[0]
                    atomType = fields[1]
                    charge = float(fields[2])
                    # Store this ion in object ion dictionary.
                    self.ions[atomName] = Ion(atomName, atomType, charge)

        # Read contents of .atp (atom type masses) file for this forcefield.
        atpfile = os.path.join(os.environ['GMXLIB'],(self.useff+'.atp'))
        fin = open(atpfile,'r')
        atplines  = fin.readlines()
        fin.close()

        # Parse .atp file to determine atomic masses.
        for line in atplines:
            fields = line.split()
            if len(fields) > 0:
                # find the right ion
                for key in self.ions.keys():
                    # JDC: This could be more clear.
                    if self.ions[key].atomType == fields[0]:
                        self.ions[key].atomicMass = float(fields[1])
                        self.ions[key].atomDescription = string.joinfields(fields[3:])

    # JDC: Should use Units class for concentration.  There is a 'Units.M' or 'Units.Molar' unit.
    def molarityToFraction(self, concentration, solvent='water'):
        """Compute the number fraction of ion molecules given a Molar concentration.

        REQUIRED ARGUMENTS
          concentration - the concentration of ions (Molar)

        OPTIONAL ARGUMENTS
          solvent - the name of the solvent ('water' is only solvent currently supported, defaults to 'water')

        RETURNS
          ???????????
        """

        # JDC: Should raise an exception if unrecognized solvent is passed.
        if solvent != 'water':
            print 'Only solvent=\'water\' is supported.  Exiting.'
            sys.exit(1)

        # JDC: Should use Units clas for this.
        waterGramPerMol = 18.016
        waterMolPerGram = 1./waterGramPerMol
        waterGramPerLiter = 1000.0
        waterMolPerLiter = waterMolPerGram * waterGramPerLiter

        # JDC: I scratched my head for a while and couldn't figure out what the quantity in parenthesis represented.  Maybe I'm just tired, but it would be much less confusing if you assigned it to a variable like 'numberOfIonsInBox' and then returned *this* variable instead.
        return (concentration/waterMolPerLiter)

# JDC: This class should be in a separate file.
class Ion(object):
    """A monoatomic ion.

    Stores atom name, forcefield atom type, charge, a textual description, and the atomic mass of a single-atom ion.

    """

    def __init__(self, atomName, atomType, charge):
        """Initialize the data structures.

        REQUIRED ARGUMENTS
          atomName - the ion name
          atomType - the forcefield atom type
          charge - the partial atomic charge of the ion (in units of electron charge)

        """

        # Initialize object data.              # JDC: Every variable should be described the first time it is defined.
        self.atomName = atomName               # the atom name (e.g. 'Cl')
        self.atomType = atomType               # the forcefield atom type (e.g. 'amber99_30')
        self.charge = charge                   # the partial atomic charge for the ion (in units of electron charge)
        self.atomDescription = None            # a textual description of the ion (e.g. "chloride ion")
        self.atomicMass = None                 # the mass of the ion (in amu)

        return

    def __repr__(self):
        """Self-documented string representation of the internal data structures.

        """

        # Construct string representation
        # JDC: Descriptions are much clearer to construct line-by-line.
        tmpstr = '%s:\n' % self.atomName
        tmpstr += '\tatomtype: %s\n' % self.atomType
        tmpstr += '\tcharge %2.1f\n' % self.charge # JDC: Is this a sufficient number of decimal places?
        tmpstr += '\tdescription = %s\n' % self.atomDescription
        tmpstr += '\tatomic mass %2.1f\n' % self.atomicMass # JDC: Is this a sufficint number of decimal places?

        # Return string representation.
        return tmpstr

<|endoftext|>


<commit_before>"""
The Builder is responsible for finding and building docker images in
the correct order

Building an image requires building all dependent images, creating the necessary
context, and actually building the current image.
"""

from harpoon.errors import NoSuchImage, BadCommand, FailedImage, UserQuit
from harpoon.ship.progress_stream import ProgressStream, Failure, Unknown
from harpoon.ship.runner import Runner
from harpoon.layers import Layers

from input_algorithms.spec_base import NotSpecified
from contextlib import contextmanager
from itertools import chain
import humanize
import logging
import uuid
import six
import sys
import os

log = logging.getLogger("harpoon.ship.builder")

########################
###   PROGRESS STREAM
########################

class BuildProgressStream(ProgressStream):
    def setup(self):
        self.current_container = None

    def interpret_line(self, line_detail):
        if "stream" in line_detail:
            self.interpret_stream(line_detail["stream"])
        elif "status" in line_detail:
            self.interpret_status(line_detail["status"])
        else:
            self.interpret_unknown(line_detail)

    def interpret_stream(self, line):
        if line.strip().startswith("---> Running in"):
            self.current_container = line[len("---> Running in "):].strip()
        elif line.strip().startswith("Successfully built"):
            self.current_container = line[len("Successfully built"):].strip()

        if line.strip().startswith("---> Running in"):
            self.cached = False
        elif line.strip().startswith("---> Using cache"):
            self.cached = True

        self.add_line(line)

    def interpret_status(self, line):
        if line.startswith("Pulling image"):
            if not line.endswith("\n"):
                line = "{0}\n".format(line)
        else:
            line = "\r{0}".format(line)

        if "already being pulled by another client" in line or "Pulling repository" in line:
            self.cached = False
        self.add_line(line)

########################
###   BUILDER
########################

class Builder(object):
    """Build an image from Image configuration"""

    ########################
    ###   USAGE
    ########################

    def make_image(self, conf, images, chain=None, parent_chain=None, made=None, ignore_deps=False, ignore_parent=False):
        """Make us an image"""
        made = made or {}
        chain = chain or []
        parent_chain = parent_chain or []

        if conf.name in made:
            return

        if conf.name in chain and not ignore_deps:
            raise BadCommand("Recursive dependency images", chain=chain + [conf.name])

        if conf.name in parent_chain and not ignore_parent:
            raise BadCommand("Recursive FROM statements", chain=parent_chain + [conf.name])

        if conf.name not in images:
            raise NoSuchImage(looking_for=conf.name, available=images.keys())

        if not ignore_deps:
            for dependency, image in conf.dependency_images():
                self.make_image(images[dependency], images, chain=chain + [conf.name], made=made, ignore_deps=True)

        if not ignore_parent:
            parent_image = conf.commands.parent_image
            if not isinstance(parent_image, six.string_types):
                self.make_image(parent_image, images, chain, parent_chain + [conf.name], made=made, ignore_deps=True)

        # Should have all our dependencies now
        log.info("Making image for '%s' (%s) - FROM %s", conf.name, conf.image_name, conf.commands.parent_image_name)
        self.build_image(conf)
        made[conf.name] = True

    def build_image(self, conf):
        """Build this image"""
        with self.context(conf) as context:
            try:
                stream = BuildProgressStream(conf.harpoon.silent_build)
                with self.remove_replaced_images(conf):
                    if conf.recursive is NotSpecified:
                        self.do_build(conf, context, stream)
                    else:
                        self.do_recursive_build(conf, context, stream)
            except (KeyboardInterrupt, Exception) as error:
                exc_info = sys.exc_info()
                if stream.current_container:
                    Runner().stage_build_intervention(conf, stream.current_container)

                if isinstance(error, KeyboardInterrupt):
                    raise UserQuit()
                else:
                    six.reraise(*exc_info)

    def layered(self, images, only_pushable=False):
        """Yield layers of images"""
        if only_pushable:
            operate_on = dict((image, instance) for image, instance in images.items() if instance.image_index)
        else:
            operate_on = images

        layers = Layers(operate_on, all_images=images)
        layers.add_all_to_layers()
        return layers.layered

    ########################
    ###   UTILITY
    ########################

    @contextmanager
    def context(self, conf):
        with conf.make_context() as context:
            self.log_context_size(context, conf)
            yield context

    def log_context_size(self, context, conf):
        context_size = humanize.naturalsize(os.stat(context.name).st_size)
        log.info("Building '%s' in '%s' with %s of context", conf.name, conf.context.parent_dir, context_size)

    @contextmanager
    def remove_replaced_images(self, conf):
        current_ids = None
        if not conf.harpoon.keep_replaced:
            images = conf.harpoon.docker_context.images()
            current_ids = [image["Id"] for image in images if "{0}:latest".format(conf.image_name) in image["RepoTags"]]

        yield

        if current_ids:
            images = conf.harpoon.docker_context.images()
            untagged = [image["Id"] for image in images if image["RepoTags"] == ["<none>:<none>"]]
            for image in current_ids:
                if image in untagged:
                    log.info("Deleting replaced image\ttag=%s\told_hash=%s", "{0}:latest".format(conf.image_name), image)
                    try:
                        conf.harpoon.docker_context.remove_image(image)
                    except Exception as error:
                        log.error("Failed to remove replaced image\thash=%s\terror=%s", image, error)

    def do_build(self, conf, context, stream, image_name=None, verbose=False):
        if image_name is None:
            image_name = conf.image_name

        context.close()
        for line in conf.harpoon.docker_context.build(fileobj=context.tmpfile, custom_context=True, tag=image_name, stream=True, rm=True):
            try:
                stream.feed(line)
            except Failure as error:
                raise FailedImage("Failed to build an image", image=conf.name, msg=error)
            except Unknown as error:
                log.warning("Unknown line\tline=%s", error)

            for part in stream.printable():
                sys.stdout.write(part.encode("utf-8", "replace"))
            sys.stdout.flush()

        return stream.cached

    def do_recursive_build(self, conf, context, stream):
        """Do a recursive build!"""
        from harpoon.option_spec.image_objs import Volumes
        from harpoon.ship.runner import Runner
        cached = self.do_build(conf, context, stream, image_name="{0}-tester".format(conf.image_name))
        have_final = "{0}:latest".format(conf.image_name) in chain.from_iterable([image["RepoTags"] for image in conf.harpoon.docker_context.images()])

        provider_name = "{0}-provider".format(conf.image_name)
        provider_conf = conf.clone()
        provider_conf.name = "provider"
        provider_conf.image_name = provider_name
        provider_conf.container_id = None
        provider_conf.container_name = "{0}-intermediate-{1}".format(provider_name, str(uuid.uuid1())).replace("/", "__")

        with self.remove_replaced_images(provider_conf):
            if not have_final:
                log.info("Building first image for recursive image")
                with context.clone_with_new_dockerfile(conf, conf.recursive.make_first_dockerfile(conf.docker_file)) as new_context:
                    self.do_build(provider_conf, new_context, stream, image_name=provider_name)
                conf.harpoon.docker_context.tag(provider_name, conf.image_name, tag="latest", force=True)
            else:
                log.info("Building intermediate provider for recursive image")
                if not cached:
                    with context.clone_with_new_dockerfile(conf, conf.recursive.make_changed_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                        self.log_context_size(provider_context, provider_conf)
                        self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
                else:
                    with conf.make_context(docker_file=conf.recursive.make_provider_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                        self.log_context_size(provider_context, provider_conf)
                        self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
                        conf.from_name = conf.image_name
                        conf.image_name = provider_name
                        return

        builder_name = "{0}-for-commit".format(conf.image_name)
        builder_conf = conf.clone()

        builder_conf.image_name = builder_name
        builder_conf.container_id = None
        builder_conf.container_name = "{0}-intermediate-{1}".format(builder_name, str(uuid.uuid1())).replace("/", "__")
        builder_conf.volumes = Volumes(mount=[], share_with=[provider_conf])
        log.info("Building intermediate builder for recursive image")
        with self.remove_replaced_images(builder_conf):
            with context.clone_with_new_dockerfile(conf, conf.recursive.make_builder_dockerfile(conf.docker_file)) as provider_context:
                self.log_context_size(context, conf)
                self.do_build(builder_conf, provider_context, stream, image_name=builder_name)

        log.info("Running and committing builder container for recursive image")
        with self.remove_replaced_images(conf):
            Runner().run_container(builder_conf, {provider_conf.name:provider_conf, builder_conf.name:builder_conf}, detach=False, dependency=False, tag=conf.image_name)

        log.info("Building final provider of recursive image")
        with self.remove_replaced_images(provider_conf):
            with conf.make_context(docker_file=conf.recursive.make_provider_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                self.log_context_size(provider_context, provider_conf)
                self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
        conf.from_name = conf.image_name
        conf.image_name = provider_name

<commit_msg>Don't need registry name for intermediate containers<commit_after>"""
The Builder is responsible for finding and building docker images in
the correct order

Building an image requires building all dependent images, creating the necessary
context, and actually building the current image.
"""

from harpoon.errors import NoSuchImage, BadCommand, FailedImage, UserQuit
from harpoon.ship.progress_stream import ProgressStream, Failure, Unknown
from harpoon.ship.runner import Runner
from harpoon.layers import Layers

from input_algorithms.spec_base import NotSpecified
from contextlib import contextmanager
from itertools import chain
import humanize
import logging
import uuid
import six
import sys
import os

log = logging.getLogger("harpoon.ship.builder")

########################
###   PROGRESS STREAM
########################

class BuildProgressStream(ProgressStream):
    def setup(self):
        self.current_container = None

    def interpret_line(self, line_detail):
        if "stream" in line_detail:
            self.interpret_stream(line_detail["stream"])
        elif "status" in line_detail:
            self.interpret_status(line_detail["status"])
        else:
            self.interpret_unknown(line_detail)

    def interpret_stream(self, line):
        if line.strip().startswith("---> Running in"):
            self.current_container = line[len("---> Running in "):].strip()
        elif line.strip().startswith("Successfully built"):
            self.current_container = line[len("Successfully built"):].strip()

        if line.strip().startswith("---> Running in"):
            self.cached = False
        elif line.strip().startswith("---> Using cache"):
            self.cached = True

        self.add_line(line)

    def interpret_status(self, line):
        if line.startswith("Pulling image"):
            if not line.endswith("\n"):
                line = "{0}\n".format(line)
        else:
            line = "\r{0}".format(line)

        if "already being pulled by another client" in line or "Pulling repository" in line:
            self.cached = False
        self.add_line(line)

########################
###   BUILDER
########################

class Builder(object):
    """Build an image from Image configuration"""

    ########################
    ###   USAGE
    ########################

    def make_image(self, conf, images, chain=None, parent_chain=None, made=None, ignore_deps=False, ignore_parent=False):
        """Make us an image"""
        made = made or {}
        chain = chain or []
        parent_chain = parent_chain or []

        if conf.name in made:
            return

        if conf.name in chain and not ignore_deps:
            raise BadCommand("Recursive dependency images", chain=chain + [conf.name])

        if conf.name in parent_chain and not ignore_parent:
            raise BadCommand("Recursive FROM statements", chain=parent_chain + [conf.name])

        if conf.name not in images:
            raise NoSuchImage(looking_for=conf.name, available=images.keys())

        if not ignore_deps:
            for dependency, image in conf.dependency_images():
                self.make_image(images[dependency], images, chain=chain + [conf.name], made=made, ignore_deps=True)

        if not ignore_parent:
            parent_image = conf.commands.parent_image
            if not isinstance(parent_image, six.string_types):
                self.make_image(parent_image, images, chain, parent_chain + [conf.name], made=made, ignore_deps=True)

        # Should have all our dependencies now
        log.info("Making image for '%s' (%s) - FROM %s", conf.name, conf.image_name, conf.commands.parent_image_name)
        self.build_image(conf)
        made[conf.name] = True

    def build_image(self, conf):
        """Build this image"""
        with self.context(conf) as context:
            try:
                stream = BuildProgressStream(conf.harpoon.silent_build)
                with self.remove_replaced_images(conf):
                    if conf.recursive is NotSpecified:
                        self.do_build(conf, context, stream)
                    else:
                        self.do_recursive_build(conf, context, stream)
            except (KeyboardInterrupt, Exception) as error:
                exc_info = sys.exc_info()
                if stream.current_container:
                    Runner().stage_build_intervention(conf, stream.current_container)

                if isinstance(error, KeyboardInterrupt):
                    raise UserQuit()
                else:
                    six.reraise(*exc_info)

    def layered(self, images, only_pushable=False):
        """Yield layers of images"""
        if only_pushable:
            operate_on = dict((image, instance) for image, instance in images.items() if instance.image_index)
        else:
            operate_on = images

        layers = Layers(operate_on, all_images=images)
        layers.add_all_to_layers()
        return layers.layered

    ########################
    ###   UTILITY
    ########################

    @contextmanager
    def context(self, conf):
        with conf.make_context() as context:
            self.log_context_size(context, conf)
            yield context

    def log_context_size(self, context, conf):
        context_size = humanize.naturalsize(os.stat(context.name).st_size)
        log.info("Building '%s' in '%s' with %s of context", conf.name, conf.context.parent_dir, context_size)

    @contextmanager
    def remove_replaced_images(self, conf):
        current_ids = None
        if not conf.harpoon.keep_replaced:
            images = conf.harpoon.docker_context.images()
            current_ids = [image["Id"] for image in images if "{0}:latest".format(conf.image_name) in image["RepoTags"]]

        yield

        if current_ids:
            images = conf.harpoon.docker_context.images()
            untagged = [image["Id"] for image in images if image["RepoTags"] == ["<none>:<none>"]]
            for image in current_ids:
                if image in untagged:
                    log.info("Deleting replaced image\ttag=%s\told_hash=%s", "{0}:latest".format(conf.image_name), image)
                    try:
                        conf.harpoon.docker_context.remove_image(image)
                    except Exception as error:
                        log.error("Failed to remove replaced image\thash=%s\terror=%s", image, error)

    def do_build(self, conf, context, stream, image_name=None, verbose=False):
        if image_name is None:
            image_name = conf.image_name

        context.close()
        for line in conf.harpoon.docker_context.build(fileobj=context.tmpfile, custom_context=True, tag=image_name, stream=True, rm=True):
            try:
                stream.feed(line)
            except Failure as error:
                raise FailedImage("Failed to build an image", image=conf.name, msg=error)
            except Unknown as error:
                log.warning("Unknown line\tline=%s", error)

            for part in stream.printable():
                sys.stdout.write(part.encode("utf-8", "replace"))
            sys.stdout.flush()

        return stream.cached

    def do_recursive_build(self, conf, context, stream):
        """Do a recursive build!"""
        from harpoon.option_spec.image_objs import Volumes
        from harpoon.ship.runner import Runner
        conf_image_name = conf.name
        if conf.image_name_prefix is not NotSpecified:
            conf_image_name = "{0}-{1}".format(conf.image_name_prefix, conf.name)

        with self.remove_replaced_images(conf):
            cached = self.do_build(conf, context, stream, image_name="{0}-tester".format(conf_image_name))
        have_final = "{0}:latest".format(conf.image_name) in chain.from_iterable([image["RepoTags"] for image in conf.harpoon.docker_context.images()])

        provider_name = "{0}-provider".format(conf_image_name)
        provider_conf = conf.clone()
        provider_conf.name = "provider"
        provider_conf.image_name = provider_name
        provider_conf.container_id = None
        provider_conf.container_name = "{0}-intermediate-{1}".format(provider_name, str(uuid.uuid1())).replace("/", "__")

        with self.remove_replaced_images(provider_conf):
            if not have_final:
                log.info("Building first image for recursive image")
                with context.clone_with_new_dockerfile(conf, conf.recursive.make_first_dockerfile(conf.docker_file)) as new_context:
                    self.do_build(provider_conf, new_context, stream, image_name=provider_name)
                conf.harpoon.docker_context.tag(provider_name, conf.image_name, tag="latest", force=True)
            else:
                log.info("Building intermediate provider for recursive image")
                if not cached:
                    with context.clone_with_new_dockerfile(conf, conf.recursive.make_changed_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                        self.log_context_size(provider_context, provider_conf)
                        self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
                else:
                    with conf.make_context(docker_file=conf.recursive.make_provider_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                        self.log_context_size(provider_context, provider_conf)
                        self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
                        conf.from_name = conf.image_name
                        conf.image_name = provider_name
                        return

        builder_name = "{0}-for-commit".format(conf_image_name)
        builder_conf = conf.clone()

        builder_conf.image_name = builder_name
        builder_conf.container_id = None
        builder_conf.container_name = "{0}-intermediate-{1}".format(builder_name, str(uuid.uuid1())).replace("/", "__")
        builder_conf.volumes = Volumes(mount=[], share_with=[provider_conf])
        log.info("Building intermediate builder for recursive image")
        with self.remove_replaced_images(builder_conf):
            with context.clone_with_new_dockerfile(conf, conf.recursive.make_builder_dockerfile(conf.docker_file)) as provider_context:
                self.log_context_size(context, conf)
                self.do_build(builder_conf, provider_context, stream, image_name=builder_name)

        log.info("Running and committing builder container for recursive image")
        with self.remove_replaced_images(conf):
            Runner().run_container(builder_conf, {provider_conf.name:provider_conf, builder_conf.name:builder_conf}, detach=False, dependency=False, tag=conf.image_name)

        log.info("Building final provider of recursive image")
        with self.remove_replaced_images(provider_conf):
            with conf.make_context(docker_file=conf.recursive.make_provider_dockerfile(conf.docker_file, conf.image_name)) as provider_context:
                self.log_context_size(provider_context, provider_conf)
                self.do_build(provider_conf, provider_context, stream, image_name=provider_name)
        conf.from_name = conf.image_name
        conf.image_name = provider_name

<|endoftext|>


<commit_before><commit_msg>add running_ status file<commit_after><|endoftext|>


<commit_before>__version__ = "2.2"
<commit_msg>Update version number to 2.3 for release.<commit_after>__version__ = "2.3"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import UserError

class Goods(models.Model):
    """
    继承了core里面定义的goods 模块，并定义了视图和添加字段。
    """
    _inherit = 'goods'

    @api.multi
    def get_tax_rate(self, goods, partner, type):
        """
        获得税率
        如果商品上没有税率，则取商品分类上的税率；
        商品税率和业务伙伴税率做比较：如果都存在，取小的；其中一个存在取该值；都不存在取公司上的进/销项税
        """
        if not goods:
            return
        goods_tax_rate, partner_tax_rate = False, False
        # 如果商品上没有税率，则取商品分类上的税率
        if goods.tax_rate:
            goods_tax_rate = goods.tax_rate
        elif goods.goods_class_id.tax_rate:
            goods_tax_rate = goods.goods_class_id.tax_rate
        # 取业务伙伴税率
        if partner:
            partner_tax_rate = partner.tax_rate

        # 商品税率和业务伙伴税率做比较，并根据情况返回
        if goods_tax_rate and partner_tax_rate:
            if goods_tax_rate >= partner_tax_rate:
                return partner_tax_rate
            else:
                return goods_tax_rate
        elif goods_tax_rate and not partner_tax_rate:
            return goods_tax_rate
        elif not goods_tax_rate and partner_tax_rate:
            return partner_tax_rate
        else:
            if type == 'buy':
                return self.env.user.company_id.import_tax_rate
            elif type == 'sell':
                return self.env.user.company_id.output_tax_rate

    no_stock = fields.Boolean(u'虚拟商品')
    using_batch = fields.Boolean(u'管理批号')
    force_batch_one = fields.Boolean(u'管理序列号')
    attribute_ids = fields.One2many('attribute', 'goods_id', string=u'属性')
    image = fields.Binary(u'图片', attachment=True)
    supplier_id = fields.Many2one('partner',
                                  u'默认供应商',
                                  ondelete='restrict',
                                  domain=[('s_category_id', '!=', False)])
    price = fields.Float(u'零售价')
    barcode = fields.Char(u'条形码')
    note = fields.Text(u'备注')
    goods_class_id = fields.Many2one(
        'goods.class', string=u'商品分类',
        help="Those categories are used to group similar products for point of sale.")

    _sql_constraints = [
        ('barcode_uniq', 'unique(barcode)', u'条形码不能重复'),
    ]

    @api.onchange('uom_id')
    def onchange_uom(self):
        """
        :return: 当选取单位时辅助单位默认和 单位相等。
        """
        self.uos_id = self.uom_id

    @api.onchange('using_batch')
    def onchange_using_batch(self):
        """
        :return: 当将管理批号的勾去掉后，自动将管理序列号的勾去掉
        """
        if not self.using_batch:
            self.force_batch_one = False

    def conversion_unit(self, qty):
        """ 数量 × 转化率 = 辅助数量
        :param qty: 传进来数量计算出辅助数量
        :return: 返回辅助数量
        """
        self.ensure_one()
        return self.conversion * qty

    def anti_conversion_unit(self, qty):
        """ 数量 = 辅助数量 / 转化率
        :param qty: 传入值为辅助数量
        :return: 数量
        """
        self.ensure_one()
        return self.conversion and qty / self.conversion or 0

class Attribute(models.Model):
    _name = 'attribute'
    _description = u'属性'

    @api.one
    @api.depends('value_ids')
    def _compute_name(self):
        self.name = ' '.join(
            [value.category_id.name + ':' + value.value_id.name for value in self.value_ids])

    @api.model
    def name_search(self, name='', args=None, operator='ilike', limit=100):
        '''在many2one字段中支持按条形码搜索'''
        args = args or []
        if name:
            attribute_ids = self.search([('ean', '=', name)])
            if attribute_ids:
                return attribute_ids.name_get()
        return super(Attribute, self).name_search(
            name=name, args=args, operator=operator, limit=limit)

    ean = fields.Char(u'条码')
    name = fields.Char(u'属性', compute='_compute_name',
                       store=True, readonly=True)
    goods_id = fields.Many2one('goods', u'商品', ondelete='cascade')
    value_ids = fields.One2many(
        'attribute.value', 'attribute_id', string=u'属性')
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

    _sql_constraints = [
        ('ean_uniq', 'unique (ean)', u'该条码已存在'),
        ('goods_attribute_uniq', 'unique (goods_id, name)', u'该SKU已存在'),
    ]

    @api.one
    @api.constrains('value_ids')
    def check_value_ids(self):
        att_dict = {}
        for line in self.value_ids:
            if not att_dict.has_key(line.category_id):
                att_dict[line.category_id] = line.category_id
            else:
                raise UserError(u'属性值的类别不能相同')

class AttributeValue(models.Model):
    _name = 'attribute.value'
    _rec_name = 'value_id'
    _description = u'属性明细'

    attribute_id = fields.Many2one('attribute', u'属性', ondelete='cascade')
    category_id = fields.Many2one('core.category', u'属性',
                                  ondelete='cascade',
                                  domain=[('type', '=', 'attribute')],
                                  context={'type': 'attribute'},
                                  required='1')
    value_id = fields.Many2one('attribute.value.value', u'值',
                               ondelete='restrict',
                               domain="[('category_id','=',category_id)]",
                               default=lambda self: self.env.context.get(
                                   'default_category_id'),
                               required='1')
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

class AttributeValueValue(models.Model):
    _name = 'attribute.value.value'
    _description = u'属性值'

    category_id = fields.Many2one('core.category', u'属性',
                                  ondelete='cascade',
                                  domain=[('type', '=', 'attribute')],
                                  context={'type': 'attribute'},
                                  required='1')
    name = fields.Char(u'值', required=True)
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

    _sql_constraints = [
        ('name_category_uniq', 'unique(category_id,name)', u'同一属性的值不能重复')
    ]
<commit_msg>[FIX] 逐级取商品分类上的税率 #1797<commit_after># -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import UserError

class Goods(models.Model):
    """
    继承了core里面定义的goods 模块，并定义了视图和添加字段。
    """
    _inherit = 'goods'

    @api.multi
    def get_parent_tax_rate(self, parent_id):
        # 逐级取商品分类上的税率
        tax_rate = parent_id.tax_rate
        if not tax_rate and parent_id.parent_id:
            tax_rate = self.get_parent_tax_rate(parent_id.parent_id)

        return tax_rate

    @api.multi
    def get_tax_rate(self, goods, partner, type):
        """
        获得税率
        如果商品上没有税率，则逐级取商品分类上的税率；
        商品税率和业务伙伴税率做比较：如果都存在，取小的；其中一个存在取该值；都不存在取公司上的进/销项税
        """
        if not goods:
            return
        goods_tax_rate, partner_tax_rate = False, False
        # 如果商品上没有税率，则取商品分类上的税率
        if goods.tax_rate:
            goods_tax_rate = goods.tax_rate
        elif goods.goods_class_id.tax_rate:
            goods_tax_rate = goods.goods_class_id.tax_rate
        elif goods.goods_class_id.parent_id:  # 逐级取商品分类上的税率
            goods_tax_rate = self.get_parent_tax_rate(goods.goods_class_id.parent_id)

        # 取业务伙伴税率
        if partner:
            partner_tax_rate = partner.tax_rate

        # 商品税率和业务伙伴税率做比较，并根据情况返回
        if goods_tax_rate and partner_tax_rate:
            if goods_tax_rate >= partner_tax_rate:
                return partner_tax_rate
            else:
                return goods_tax_rate
        elif goods_tax_rate and not partner_tax_rate:
            return goods_tax_rate
        elif not goods_tax_rate and partner_tax_rate:
            return partner_tax_rate
        else:
            if type == 'buy':
                return self.env.user.company_id.import_tax_rate
            elif type == 'sell':
                return self.env.user.company_id.output_tax_rate

    no_stock = fields.Boolean(u'虚拟商品')
    using_batch = fields.Boolean(u'管理批号')
    force_batch_one = fields.Boolean(u'管理序列号')
    attribute_ids = fields.One2many('attribute', 'goods_id', string=u'属性')
    image = fields.Binary(u'图片', attachment=True)
    supplier_id = fields.Many2one('partner',
                                  u'默认供应商',
                                  ondelete='restrict',
                                  domain=[('s_category_id', '!=', False)])
    price = fields.Float(u'零售价')
    barcode = fields.Char(u'条形码')
    note = fields.Text(u'备注')
    goods_class_id = fields.Many2one(
        'goods.class', string=u'商品分类',
        help="Those categories are used to group similar products for point of sale.")

    _sql_constraints = [
        ('barcode_uniq', 'unique(barcode)', u'条形码不能重复'),
    ]

    @api.onchange('uom_id')
    def onchange_uom(self):
        """
        :return: 当选取单位时辅助单位默认和 单位相等。
        """
        self.uos_id = self.uom_id

    @api.onchange('using_batch')
    def onchange_using_batch(self):
        """
        :return: 当将管理批号的勾去掉后，自动将管理序列号的勾去掉
        """
        if not self.using_batch:
            self.force_batch_one = False

    def conversion_unit(self, qty):
        """ 数量 × 转化率 = 辅助数量
        :param qty: 传进来数量计算出辅助数量
        :return: 返回辅助数量
        """
        self.ensure_one()
        return self.conversion * qty

    def anti_conversion_unit(self, qty):
        """ 数量 = 辅助数量 / 转化率
        :param qty: 传入值为辅助数量
        :return: 数量
        """
        self.ensure_one()
        return self.conversion and qty / self.conversion or 0

class Attribute(models.Model):
    _name = 'attribute'
    _description = u'属性'

    @api.one
    @api.depends('value_ids')
    def _compute_name(self):
        self.name = ' '.join(
            [value.category_id.name + ':' + value.value_id.name for value in self.value_ids])

    @api.model
    def name_search(self, name='', args=None, operator='ilike', limit=100):
        '''在many2one字段中支持按条形码搜索'''
        args = args or []
        if name:
            attribute_ids = self.search([('ean', '=', name)])
            if attribute_ids:
                return attribute_ids.name_get()
        return super(Attribute, self).name_search(
            name=name, args=args, operator=operator, limit=limit)

    ean = fields.Char(u'条码')
    name = fields.Char(u'属性', compute='_compute_name',
                       store=True, readonly=True)
    goods_id = fields.Many2one('goods', u'商品', ondelete='cascade')
    value_ids = fields.One2many(
        'attribute.value', 'attribute_id', string=u'属性')
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

    _sql_constraints = [
        ('ean_uniq', 'unique (ean)', u'该条码已存在'),
        ('goods_attribute_uniq', 'unique (goods_id, name)', u'该SKU已存在'),
    ]

    @api.one
    @api.constrains('value_ids')
    def check_value_ids(self):
        att_dict = {}
        for line in self.value_ids:
            if not att_dict.has_key(line.category_id):
                att_dict[line.category_id] = line.category_id
            else:
                raise UserError(u'属性值的类别不能相同')

class AttributeValue(models.Model):
    _name = 'attribute.value'
    _rec_name = 'value_id'
    _description = u'属性明细'

    attribute_id = fields.Many2one('attribute', u'属性', ondelete='cascade')
    category_id = fields.Many2one('core.category', u'属性',
                                  ondelete='cascade',
                                  domain=[('type', '=', 'attribute')],
                                  context={'type': 'attribute'},
                                  required='1')
    value_id = fields.Many2one('attribute.value.value', u'值',
                               ondelete='restrict',
                               domain="[('category_id','=',category_id)]",
                               default=lambda self: self.env.context.get(
                                   'default_category_id'),
                               required='1')
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

class AttributeValueValue(models.Model):
    _name = 'attribute.value.value'
    _description = u'属性值'

    category_id = fields.Many2one('core.category', u'属性',
                                  ondelete='cascade',
                                  domain=[('type', '=', 'attribute')],
                                  context={'type': 'attribute'},
                                  required='1')
    name = fields.Char(u'值', required=True)
    company_id = fields.Many2one(
        'res.company',
        string=u'公司',
        change_default=True,
        default=lambda self: self.env['res.company']._company_default_get())

    _sql_constraints = [
        ('name_category_uniq', 'unique(category_id,name)', u'同一属性的值不能重复')
    ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from collections import defaultdict

from django.contrib.admin.utils import NestedObjects
from django.core.management import call_command
from django.template import Context, Template
from django.conf import settings

import json
import os

DENIS_SH_TEMPLATE = """#!/bin/sh

if [ -z $MANAGE ]; then
    MANAGE={{ managepy }}
fi

{% for model in models %}
echo PROCESSING {{ model.name }}
$MANAGE loaddata --database {{ database }} {{ model.fixture }}
{% endfor %}
"""

class Denis(object):

    def __init__(self, qs, outdir, using, **kwargs):
        self.qs = qs
        self.output_dir = outdir
        self.using = using

        self.models = []
        self.objects = defaultdict(list)

        self.models_blacklist = kwargs.get('blacklist')
        if not self.models_blacklist:
            self.models_blacklist = []

        # dumpdata options
        self.indent = kwargs.get('indent')
        if not self.indent:
            self.indent = 2
        self.format = kwargs.get('format')
        if not self.format:
            self.format = 'json'

        # a cache of m2m fields per model
        self.m2m_fields = {}

    def collect(self):
        collector = NestedObjects(using=self.using)
        collector.collect(self.qs)
        objs = collector.nested()

        def model_name(obj):
            return "%s.%s" % (obj._meta.app_label, obj._meta.model_name)

        def get_m2m_fields(model):
            return [
                f.name for f in model._meta.get_fields()
                    if (f.many_to_many and not f.auto_created)
            ]

        def collect_objects(obj):
            try:
                iter(obj)
                for o in obj:
                    collect_objects(o)
            except TypeError:
                name = model_name(obj)
                if name not in self.models and name not in self.models_blacklist:
                    self.models.append(name)

                    m2m_fields = get_m2m_fields(obj)
                    self.m2m_fields[name] = m2m_fields
                self.objects[name].append(str(obj.pk))

        for obj in objs:
            collect_objects(obj)

    def dump(self, database=None):
        outdir = os.path.abspath(self.output_dir)
        try:
            os.mkdir(outdir)
        except OSError:
            pass
        except:
            raise

        if not database:
            database = 'default'

        def model_fixture(outdir, model, format):
            return os.path.join(outdir, '%s.%s' % (model, self.format))

        script = os.path.join(outdir, 'denis.sh')
        with open(script, 'w+') as f:
            self.collect()

            models = []
            for model in self.models:
                pks = ','.join(self.objects[model])
                output = model_fixture(outdir, model, self.format)
                if os.path.exists(output):
                    os.unlink(output)

                opts = {
                    'primary_keys': pks,
                    'output': output,
                    'format': self.format,
                    'database': self.using,
                    'indent': self.indent,
                    'exclude': [],
                }
                call_command('dumpdata', model, **opts)
                models.append({
                    'name': model,
                    'fixture': output,
                })

                # we need to remove m2m fields from models in order to avoid
                # double insertions errors
                if model in self.m2m_fields:
                    fixture_f = open(output, 'r')
                    fixture = json.load(fixture_f)
                    fixture_f.close()

                    with open(output, 'w') as fixture_f:
                        for instance in fixture:
                            for field in self.m2m_fields[model]:
                                instance['fields'][field] = []
                        json.dump(fixture, fixture_f)

            manage_path = os.path.join(os.path.abspath(settings.BASE_DIR), 'manage.py')
            template = Template(DENIS_SH_TEMPLATE)
            ctx = Context({'models': models, 'managepy': manage_path, 'database': database})
            f.write(template.render(ctx))
<commit_msg>Rework object collection<commit_after># -*- coding: utf-8 -*-
from collections import defaultdict

from django.contrib.admin.utils import NestedObjects
from django.core.management import call_command
from django.template import Context, Template
from django.conf import settings

import json
import os

DENIS_SH_TEMPLATE = """#!/bin/sh

if [ -z $MANAGE ]; then
    MANAGE={{ managepy }}
fi

{% for model in models %}
echo PROCESSING {{ model.name }}
$MANAGE loaddata --database {{ database }} {{ model.fixture }}
{% endfor %}
"""

class Denis(object):

    def __init__(self, qs, outdir, using, **kwargs):
        self.qs = qs
        self.output_dir = outdir
        self.using = using

        self.models_blacklist = kwargs.get('blacklist')
        if not self.models_blacklist:
            self.models_blacklist = []

        # dumpdata options
        self.indent = kwargs.get('indent')
        if not self.indent:
            self.indent = 2
        self.format = kwargs.get('format')
        if not self.format:
            self.format = 'json'

    def collect(self):
        def collect_objects(obj):
            try:
                iter(obj)
                for o in obj:
                    collect_objects(o)
            except TypeError:
                flattened.append(obj)

        collector = NestedObjects(using=self.using)
        collector.collect(self.qs)
        objs = collector.nested()
        flattened = []

        for obj in objs:
            collect_objects(obj)

        return flattened

    def _parse_collected_objects(self, objects):
        def model_name(obj):
            return "%s.%s" % (obj._meta.app_label, obj._meta.model_name)

        def get_m2m_fields(model):
            return [
                f.name for f in model._meta.get_fields()
                    if (f.many_to_many and not f.auto_created)
            ]

        models = []
        objects_ids = defaultdict(list)
        m2m_fields = {}

        for obj in objects:
            name = model_name(obj)
            if name not in models and name not in self.models_blacklist:
                models.append(name)

                m2ms = get_m2m_fields(obj)
                m2m_fields[name] = m2ms
            objects_ids[name].append(str(obj.pk))

        return models, objects_ids, m2m_fields

    def dump(self, database=None):
        outdir = os.path.abspath(self.output_dir)
        try:
            os.mkdir(outdir)
        except OSError:
            pass
        except:
            raise

        if not database:
            database = 'default'

        def model_fixture(outdir, model, format):
            return os.path.join(outdir, '%s.%s' % (model, self.format))

        script = os.path.join(outdir, 'denis.sh')
        with open(script, 'w+') as f:
            flat_models = self.collect()
            models, objects_ids, m2m_fields = self._parse_collected_objects(flat_models)

            models_to_dump = []
            for model in models:
                pks = ','.join(objects_ids[model])
                output = model_fixture(outdir, model, self.format)
                if os.path.exists(output):
                    os.unlink(output)

                opts = {
                    'primary_keys': pks,
                    'output': output,
                    'format': self.format,
                    'database': self.using,
                    'indent': self.indent,
                    'exclude': [],
                }
                call_command('dumpdata', model, **opts)
                models_to_dump.append({
                    'name': model,
                    'fixture': output,
                })

                # we need to remove m2m fields from models in order to avoid
                # double insertions errors
                if model in m2m_fields:
                    fixture_f = open(output, 'r')
                    fixture = json.load(fixture_f)
                    fixture_f.close()

                    with open(output, 'w') as fixture_f:
                        for instance in fixture:
                            for field in m2m_fields[model]:
                                instance['fields'][field] = []
                        json.dump(fixture, fixture_f)

            manage_path = os.path.join(os.path.abspath(settings.BASE_DIR), 'manage.py')
            template = Template(DENIS_SH_TEMPLATE)
            ctx = Context({'models': models_to_dump, 'managepy': manage_path, 'database': database})
            f.write(template.render(ctx))
<|endoftext|>


<commit_before>import time
import socket
import threading

from celery.messaging import EventPublisher, EventConsumer

def create_event(type, fields):
    std = {"type": type,
           "timestamp": fields.get("timestamp") or time.time()}
    return dict(fields, **std)

def Event(type, **fields):
    """Create an event.

    An event is a dictionary, the only required field is the type.

    """
    return create_event(type, fields)

class EventDispatcher(object):
    """Send events as messages.

    :param connection: Carrot connection.

    :keyword hostname: Hostname to identify ourselves as,
        by default uses the hostname returned by :func:`socket.gethostname`.

    :keyword enabled: Set to ``False`` to not actually publish any events,
        making :meth:`send` a noop operation.

    You need to :meth:`close` this after use.

    """

    def __init__(self, connection, hostname=None, enabled=True):
        self.connection = connection
        self.hostname = hostname or socket.gethostname()
        self.enabled = enabled
        self._lock = threading.Lock()
        self.publisher = None

        if self.enabled:
            self.enable()

    def enable(self):
        self.enabled = True
        self.publisher = EventPublisher(self.connection)

    def disable(self):
        self.enabled = False
        if self.publisher is not None:
            self.publisher.close()
            self.publisher = None

    def send(self, type, **fields):
        """Send event.

        :param type: Kind of event.
        :keyword \*\*fields: Event arguments.

        """
        if not self.enabled:
            return

        self._lock.acquire()
        try:
            self.publisher.send(Event(type, hostname=self.hostname, **fields))
        finally:
            self._lock.release()

    def close(self):
        """Close the event dispatcher."""
        self._lock.locked() and self._lock.release()
        self.publisher and self.publisher.close()

class EventReceiver(object):
    """Capture events.

    :param connection: Carrot connection.
    :keyword handlers: Event handlers.

    :attr:`handlers`` is a dict of event types and their handlers,
    the special handler ``"*`"`` captures all events that doesn't have a
    handler.

    """
    handlers = {}

    def __init__(self, connection, handlers=None):
        self.connection = connection
        if handlers is not None:
            self.handlers = handlers

    def process(self, type, event):
        """Process the received event by dispatching it to the appropriate
        handler."""
        handler = self.handlers.get(type) or self.handlers.get("*")
        handler and handler(event)

    def consumer(self):
        consumer = EventConsumer(self.connection)
        consumer.register_callback(self._receive)
        return consumer

    def capture(self, limit=None):
        """Open up a consumer capturing events.

        This has to run in the main process, and it will never
        stop unless forced via :exc:`KeyboardInterrupt` or :exc:`SystemExit`.

        """
        consumer = self.consumer()
        it = consumer.iterconsume(limit=limit)
        while True:
            it.next()

    def _receive(self, message_data, message):
        type = message_data.pop("type").lower()
        self.process(type, create_event(type, message_data))
<commit_msg>events.EventReceiver.capture now uses connection.drain_events, and supports the timeout argument.<commit_after>import time
import socket
import threading

from itertools import count

from celery.messaging import EventPublisher, EventConsumer

def create_event(type, fields):
    std = {"type": type,
           "timestamp": fields.get("timestamp") or time.time()}
    return dict(fields, **std)

def Event(type, **fields):
    """Create an event.

    An event is a dictionary, the only required field is the type.

    """
    return create_event(type, fields)

class EventDispatcher(object):
    """Send events as messages.

    :param connection: Carrot connection.

    :keyword hostname: Hostname to identify ourselves as,
        by default uses the hostname returned by :func:`socket.gethostname`.

    :keyword enabled: Set to ``False`` to not actually publish any events,
        making :meth:`send` a noop operation.

    You need to :meth:`close` this after use.

    """

    def __init__(self, connection, hostname=None, enabled=True):
        self.connection = connection
        self.hostname = hostname or socket.gethostname()
        self.enabled = enabled
        self._lock = threading.Lock()
        self.publisher = None

        if self.enabled:
            self.enable()

    def enable(self):
        self.enabled = True
        self.publisher = EventPublisher(self.connection)

    def disable(self):
        self.enabled = False
        if self.publisher is not None:
            self.publisher.close()
            self.publisher = None

    def send(self, type, **fields):
        """Send event.

        :param type: Kind of event.
        :keyword \*\*fields: Event arguments.

        """
        if not self.enabled:
            return

        self._lock.acquire()
        try:
            self.publisher.send(Event(type, hostname=self.hostname, **fields))
        finally:
            self._lock.release()

    def close(self):
        """Close the event dispatcher."""
        self._lock.locked() and self._lock.release()
        self.publisher and self.publisher.close()

class EventReceiver(object):
    """Capture events.

    :param connection: Carrot connection.
    :keyword handlers: Event handlers.

    :attr:`handlers`` is a dict of event types and their handlers,
    the special handler ``"*`"`` captures all events that doesn't have a
    handler.

    """
    handlers = {}

    def __init__(self, connection, handlers=None):
        self.connection = connection
        if handlers is not None:
            self.handlers = handlers

    def process(self, type, event):
        """Process the received event by dispatching it to the appropriate
        handler."""
        handler = self.handlers.get(type) or self.handlers.get("*")
        handler and handler(event)

    def consumer(self):
        consumer = EventConsumer(self.connection)
        consumer.register_callback(self._receive)
        return consumer

    def capture(self, limit=None, timeout=None):
        """Open up a consumer capturing events.

        This has to run in the main process, and it will never
        stop unless forced via :exc:`KeyboardInterrupt` or :exc:`SystemExit`.

        """
        consumer = self.consumer()
        consumer.consume()
        try:
            for iteration in count(0):
                if limit and iteration > limit:
                    break
                consumer.connection.drain_events(timeout=timeout)
        finally:
            consumer.close()

    def _receive(self, message_data, message):
        type = message_data.pop("type").lower()
        self.process(type, create_event(type, message_data))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import sys

import alagitpull

# Get the project root dir, which is the parent dir of this
cwd = os.getcwd()
project_root = os.path.dirname(cwd)

sys.path.insert(0, project_root)
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext")))

# package data
about = {}
with open("../django_docutils/__about__.py") as fp:
    exec(fp.read(), about)

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.napoleon',
    'alagitpull',
    'sphinx_issues',
]

issues_github_path = about['__github__']

templates_path = ['_templates']

source_suffix = '.rst'

master_doc = 'index'

project = about['__title__']
copyright = about['__copyright__']

version = '%s' % ('.'.join(about['__version__'].split('.'))[:2])
release = '%s' % (about['__version__'])

exclude_patterns = ['_build']

pygments_style = 'sphinx'

html_theme_path = [alagitpull.get_path()]
html_static_path = ['_static']
html_extra_path = ['manifest.json']
html_theme = 'alagitpull'
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',
        'more.html',
        'searchbox.html',
    ]
}

html_theme_options = {
    'logo': "img/icons/logo.svg",
    'github_user': 'tony',
    'github_repo': 'django-docutils',
    'github_type': 'star',
    'github_banner': True,
    'projects': alagitpull.projects,
    'project_name': about['__title__'],
    'project_title': about['__title__'],
    'project_description': about['__description__'],
    'project_url': about['__docs__'],
    'show_meta_manifest_tag': True,
    'show_meta_og_tags': True,
    'show_meta_app_icon_tags': True,
}

alagitpull_internal_hosts = [
    'django-docutils.git-pull.com',
    '0.0.0.0',
]
alagitpull_external_hosts_new_window = True

htmlhelp_basename = '%sdoc' % about['__title__']

latex_documents = [
    (
        'index',
        '{0}.tex'.format(about['__package_name__']),
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        'manual',
    ),
]

man_pages = [
    (
        'index',
        about['__package_name__'],
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        1,
    ),
]

texinfo_documents = [
    (
        'index',
        '{0}'.format(about['__package_name__']),
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        about['__package_name__'],
        about['__description__'],
        'Miscellaneous',
    ),
]

intersphinx_mapping = {
    'python': ('http://docs.python.org/', None),
}
<commit_msg>docs: Fix gh link path<commit_after># -*- coding: utf-8 -*-
import os
import sys

import alagitpull

# Get the project root dir, which is the parent dir of this
cwd = os.getcwd()
project_root = os.path.dirname(cwd)

sys.path.insert(0, project_root)
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext")))

# package data
about = {}
with open("../django_docutils/__about__.py") as fp:
    exec(fp.read(), about)

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    'sphinx.ext.napoleon',
    'alagitpull',
    'sphinx_issues',
]

issues_github_path = about['__github__'].replace('https://github.com/', '')

templates_path = ['_templates']

source_suffix = '.rst'

master_doc = 'index'

project = about['__title__']
copyright = about['__copyright__']

version = '%s' % ('.'.join(about['__version__'].split('.'))[:2])
release = '%s' % (about['__version__'])

exclude_patterns = ['_build']

pygments_style = 'sphinx'

html_theme_path = [alagitpull.get_path()]
html_static_path = ['_static']
html_extra_path = ['manifest.json']
html_theme = 'alagitpull'
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',
        'more.html',
        'searchbox.html',
    ]
}

html_theme_options = {
    'logo': "img/icons/logo.svg",
    'github_user': 'tony',
    'github_repo': 'django-docutils',
    'github_type': 'star',
    'github_banner': True,
    'projects': alagitpull.projects,
    'project_name': about['__title__'],
    'project_title': about['__title__'],
    'project_description': about['__description__'],
    'project_url': about['__docs__'],
    'show_meta_manifest_tag': True,
    'show_meta_og_tags': True,
    'show_meta_app_icon_tags': True,
}

alagitpull_internal_hosts = [
    'django-docutils.git-pull.com',
    '0.0.0.0',
]
alagitpull_external_hosts_new_window = True

htmlhelp_basename = '%sdoc' % about['__title__']

latex_documents = [
    (
        'index',
        '{0}.tex'.format(about['__package_name__']),
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        'manual',
    ),
]

man_pages = [
    (
        'index',
        about['__package_name__'],
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        1,
    ),
]

texinfo_documents = [
    (
        'index',
        '{0}'.format(about['__package_name__']),
        '{0} Documentation'.format(about['__title__']),
        about['__author__'],
        about['__package_name__'],
        about['__description__'],
        'Miscellaneous',
    ),
]

intersphinx_mapping = {
    'python': ('http://docs.python.org/', None),
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import copy
import datetime
import hashlib
import os
import re

import click
import requests
import taskcluster

from cli_common.log import get_logger

logger = get_logger(__name__)

TASKCLUSTER_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'

with open(taskcluster._client_importer.__file__) as f:
    TASKCLUSTER_SERVICES = [
        line.split(' ')[1][1:]
        for line in f.read().split('\n')
        if line
    ]

def read_hosts():
    '''
    Read /etc/hosts to get hostnames
    on a Nix env (used for taskclusterProxy)
    Only reads ipv4 entries to avoid duplicates
    '''
    out = {}
    regex = re.compile(r'([\w:\-\.]+)')
    for line in open('/etc/hosts').readlines():
        if ':' in line:  # only ipv4
            continue
        x = regex.findall(line)
        if not x:
            continue
        ip, names = x[0], x[1:]
        out.update(dict(zip(names, [ip] * len(names))))

    return out

def get_options(client_id=None, access_token=None):
    '''
    Build Taskcluster credentials options
    '''

    if client_id is not None and access_token is not None:
        # Use provided credentials
        tc_options = {
            'credentials': {
                'clientId': client_id,
                'accessToken': access_token,
            },
            'rootUrl': 'https://taskcluster.net',
        }

    else:
        # Get taskcluster proxy host
        # as /etc/hosts is not used in the Nix image (?)
        hosts = read_hosts()
        if 'taskcluster' not in hosts:
            raise Exception('Missing taskcluster in /etc/hosts')

        # Load secrets from TC task context
        # with taskclusterProxy
        root_url = 'http://{}'.format(hosts['taskcluster'])
        logger.info('Taskcluster Proxy enabled', url=root_url)
        tc_options = {
            'rootUrl': root_url
        }

    tc_options['maxRetries'] = 12

    return tc_options

def get_service(service_name, client_id=None, access_token=None):
    '''
    Build a Taskcluster service instance from the environment
    Supports:
     * directly provided credentials
     * credentials from click
     * credentials from environment variables
     * taskclusterProxy
    '''
    if service_name not in TASKCLUSTER_SERVICES:
        raise Exception('Service `{}` does not exists.'.format(service_name))

    # Credentials preference: Use click variables
    if client_id is None and access_token is None:
        try:
            ctx = click.get_current_context()
            client_id = ctx.params.get('taskcluster_client_id')
            access_token = ctx.params.get('taskcluster_access_token')
        except RuntimeError:
            pass  # no active context

    # Credentials preference: Use env. variables
    if client_id is None and access_token is None:
        client_id = os.environ.get('TASKCLUSTER_CLIENT_ID')
        access_token = os.environ.get('TASKCLUSTER_ACCESS_TOKEN')

    # Instanciate service
    options = get_options(client_id, access_token)
    return getattr(taskcluster, service_name.capitalize())(options)

def get_secrets(name,
                project_name,
                required=[],
                existing=dict(),
                taskcluster_client_id=None,
                taskcluster_access_token=None,
                ):
    '''
    Fetch a specific set of secrets by name and verify that the required
    secrets exist.

    Merge secrets in the following order (the latter overrides the former):
        - `existing` argument
        - common secrets, specified under the `common` key in the secrets
          object
        - project specific secrets, specified under the `project_name` key in
          the secrets object
    '''

    secrets = dict()
    if existing:
        secrets = copy.deepcopy(existing)

    all_secrets = dict()
    if name:
        secrets_service = get_service('secrets',
                                      taskcluster_client_id,
                                      taskcluster_access_token,
                                      )
        all_secrets = secrets_service.get(name).get('secret', dict())

    secrets_common = all_secrets.get('common', dict())
    secrets.update(secrets_common)

    secrets_app = all_secrets.get(project_name, dict())
    secrets.update(secrets_app)

    for required_secret in required:
        if required_secret not in secrets:
            raise Exception('Missing value {} in secrets.'.format(required_secret))

    return secrets

def get_hook_artifact(hook_group_id, hook_id, artifact_name, client_id=None,
                      access_token=None):
    '''
    Load an artifact from the last execution of an hook
    '''

    # Get last run from hook
    hooks = get_service('hooks', client_id, access_token)
    hook_status = hooks.getHookStatus(hook_group_id, hook_id)
    last_fire = hook_status.get('lastFire')
    if last_fire is None:
        raise Exception('Hook did not fire')
    task_id = last_fire['taskId']

    # Get successful run for this task
    queue = get_service('queue', client_id, access_token)
    task_status = queue.status(task_id)
    if task_status['status']['state'] != 'completed':
        raise Exception('Task {} is not completed'.format(task_id))
    run_id = None
    for run in task_status['status']['runs']:
        if run['state'] == 'completed':
            run_id = run['runId']
            break
    if run_id is None:
        raise Exception('No completed run found')

    # Load artifact from task run
    return queue.getArtifact(task_id, run_id, artifact_name)

def create_blob_artifact(queue_service, task_id, run_id, path, content, content_type, ttl):
    '''
    Manually create and upload a blob artifact to use a specific content type
    '''
    assert isinstance(content, str)
    assert isinstance(ttl, datetime.timedelta)

    # Create artifact on Taskcluster
    sha256 = hashlib.sha256(content.encode('utf-8')).hexdigest()
    resp = queue_service.createArtifact(
        task_id,
        run_id,
        path,
        {
            'storageType': 'blob',
            'expires': (datetime.datetime.utcnow() + ttl).strftime(TASKCLUSTER_DATE_FORMAT),
            'contentType': content_type,
            'contentSha256': sha256,
            'contentLength': len(content),
        }
    )
    assert resp['storageType'] == 'blob', 'Not a blob storage'
    assert len(resp['requests']) == 1, 'Should only get one request'
    request = resp['requests'][0]
    assert request['method'] == 'PUT', 'Should get a PUT request'

    # Push the artifact on storage service
    push = requests.put(
        url=request['url'],
        headers=request['headers'],
        data=content,
    )
    push.raise_for_status()

    # Build the absolute url
    return f'https://queue.taskcluster.net/v1/task/{task_id}/runs/{run_id}/artifacts/{path}'
<commit_msg>cli_common: mark artifact as completed when uploaded manually, fixes #1755, refs #1758<commit_after># -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import copy
import datetime
import hashlib
import os
import re

import click
import requests
import taskcluster

from cli_common.log import get_logger

logger = get_logger(__name__)

TASKCLUSTER_DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'

with open(taskcluster._client_importer.__file__) as f:
    TASKCLUSTER_SERVICES = [
        line.split(' ')[1][1:]
        for line in f.read().split('\n')
        if line
    ]

def read_hosts():
    '''
    Read /etc/hosts to get hostnames
    on a Nix env (used for taskclusterProxy)
    Only reads ipv4 entries to avoid duplicates
    '''
    out = {}
    regex = re.compile(r'([\w:\-\.]+)')
    for line in open('/etc/hosts').readlines():
        if ':' in line:  # only ipv4
            continue
        x = regex.findall(line)
        if not x:
            continue
        ip, names = x[0], x[1:]
        out.update(dict(zip(names, [ip] * len(names))))

    return out

def get_options(client_id=None, access_token=None):
    '''
    Build Taskcluster credentials options
    '''

    if client_id is not None and access_token is not None:
        # Use provided credentials
        tc_options = {
            'credentials': {
                'clientId': client_id,
                'accessToken': access_token,
            },
            'rootUrl': 'https://taskcluster.net',
        }

    else:
        # Get taskcluster proxy host
        # as /etc/hosts is not used in the Nix image (?)
        hosts = read_hosts()
        if 'taskcluster' not in hosts:
            raise Exception('Missing taskcluster in /etc/hosts')

        # Load secrets from TC task context
        # with taskclusterProxy
        root_url = 'http://{}'.format(hosts['taskcluster'])
        logger.info('Taskcluster Proxy enabled', url=root_url)
        tc_options = {
            'rootUrl': root_url
        }

    tc_options['maxRetries'] = 12

    return tc_options

def get_service(service_name, client_id=None, access_token=None):
    '''
    Build a Taskcluster service instance from the environment
    Supports:
     * directly provided credentials
     * credentials from click
     * credentials from environment variables
     * taskclusterProxy
    '''
    if service_name not in TASKCLUSTER_SERVICES:
        raise Exception('Service `{}` does not exists.'.format(service_name))

    # Credentials preference: Use click variables
    if client_id is None and access_token is None:
        try:
            ctx = click.get_current_context()
            client_id = ctx.params.get('taskcluster_client_id')
            access_token = ctx.params.get('taskcluster_access_token')
        except RuntimeError:
            pass  # no active context

    # Credentials preference: Use env. variables
    if client_id is None and access_token is None:
        client_id = os.environ.get('TASKCLUSTER_CLIENT_ID')
        access_token = os.environ.get('TASKCLUSTER_ACCESS_TOKEN')

    # Instanciate service
    options = get_options(client_id, access_token)
    return getattr(taskcluster, service_name.capitalize())(options)

def get_secrets(name,
                project_name,
                required=[],
                existing=dict(),
                taskcluster_client_id=None,
                taskcluster_access_token=None,
                ):
    '''
    Fetch a specific set of secrets by name and verify that the required
    secrets exist.

    Merge secrets in the following order (the latter overrides the former):
        - `existing` argument
        - common secrets, specified under the `common` key in the secrets
          object
        - project specific secrets, specified under the `project_name` key in
          the secrets object
    '''

    secrets = dict()
    if existing:
        secrets = copy.deepcopy(existing)

    all_secrets = dict()
    if name:
        secrets_service = get_service('secrets',
                                      taskcluster_client_id,
                                      taskcluster_access_token,
                                      )
        all_secrets = secrets_service.get(name).get('secret', dict())

    secrets_common = all_secrets.get('common', dict())
    secrets.update(secrets_common)

    secrets_app = all_secrets.get(project_name, dict())
    secrets.update(secrets_app)

    for required_secret in required:
        if required_secret not in secrets:
            raise Exception('Missing value {} in secrets.'.format(required_secret))

    return secrets

def get_hook_artifact(hook_group_id, hook_id, artifact_name, client_id=None,
                      access_token=None):
    '''
    Load an artifact from the last execution of an hook
    '''

    # Get last run from hook
    hooks = get_service('hooks', client_id, access_token)
    hook_status = hooks.getHookStatus(hook_group_id, hook_id)
    last_fire = hook_status.get('lastFire')
    if last_fire is None:
        raise Exception('Hook did not fire')
    task_id = last_fire['taskId']

    # Get successful run for this task
    queue = get_service('queue', client_id, access_token)
    task_status = queue.status(task_id)
    if task_status['status']['state'] != 'completed':
        raise Exception('Task {} is not completed'.format(task_id))
    run_id = None
    for run in task_status['status']['runs']:
        if run['state'] == 'completed':
            run_id = run['runId']
            break
    if run_id is None:
        raise Exception('No completed run found')

    # Load artifact from task run
    return queue.getArtifact(task_id, run_id, artifact_name)

def create_blob_artifact(queue_service, task_id, run_id, path, content, content_type, ttl):
    '''
    Manually create and upload a blob artifact to use a specific content type
    '''
    assert isinstance(content, str)
    assert isinstance(ttl, datetime.timedelta)

    # Create artifact on Taskcluster
    sha256 = hashlib.sha256(content.encode('utf-8')).hexdigest()
    resp = queue_service.createArtifact(
        task_id,
        run_id,
        path,
        {
            'storageType': 'blob',
            'expires': (datetime.datetime.utcnow() + ttl).strftime(TASKCLUSTER_DATE_FORMAT),
            'contentType': content_type,
            'contentSha256': sha256,
            'contentLength': len(content),
        }
    )
    assert resp['storageType'] == 'blob', 'Not a blob storage'
    assert len(resp['requests']) == 1, 'Should only get one request'
    request = resp['requests'][0]
    assert request['method'] == 'PUT', 'Should get a PUT request'

    # Push the artifact on storage service
    push = requests.put(
        url=request['url'],
        headers=request['headers'],
        data=content,
    )
    push.raise_for_status()

    # Mark artifact as completed
    queue_service.completeArtifact(
        task_id,
        run_id,
        path,
        {
            'etags': [
                push.headers['ETag'],
            ],
        }
    )

    # Build the absolute url
    return f'https://queue.taskcluster.net/v1/task/{task_id}/runs/{run_id}/artifacts/{path}'
<|endoftext|>