<commit_before>"""Config-driven font project builder.

This utility wraps fontmake and a number of post-processing fixes to
build variable, static OTF, static TTF and webfonts from Glyphs,
Designspace/UFO or UFO sources.

It should be instantiated with a configuration file, typically ``config.yaml``,
which looks like this::

    sources:
      - Texturina.glyphs
      - Texturina-Italic.glyphs
    axisOrder:
      - opsz
      - wght
    outputDir: "../fonts"
    familyName: Texturina
    stat:
      - name: Width
        tag: wdth
        values:
        - name: UltraCondensed
          value: 50
          ...
      - name: Weight
        tag: wght
        values:
        - name: Hairline
          rangeMinValue: 1
          nominalValue: 1
          rangeMaxValue: 50
        ...

To build a font family from the command line, use:

    gftools builder path/to/config.yaml

The config file may contain the following keys. The ``sources`` key is
required, all others have sensible defaults:

* ``sources``: Required. An array of Glyphs, UFO or designspace source files.
* ``logLevel``: Debugging log level. Defaults to ``INFO``.
* ``stylespaceFile``: A statmake ``.stylespace`` file.
* ``stat``: A STAT table configuration. This may be either a list of axes and
    values as demonstrated above, or a dictionary mapping each source file to a
    per-source list. If neither ``stylespaceFile`` or ``stat`` are provided, a
    STAT table is generated automatically using ``gftools.stat``.
* ``buildVariable``: Build variable fonts. Defaults to true.
* ``buildStatic``: Build static fonts. Defaults to true.
* ``buildWebfont``: Build WOFF2 fonts. Defaults to ``$buildStatic``.
* ``outputDir``: Where to put the fonts. Defaults to ``../fonts/``
* ``vfDir``: Where to put variable fonts. Defaults to ``$outputDir/variable``.
* ``ttDir``: Where to put TrueType static fonts. Defaults to ``$outputDir/ttf``.
* ``otDir``: Where to put CFF static fonts. Defaults to ``$outputDir/otf``.
* ``woffDir``: Where to put WOFF2 static fonts. Defaults to ``$outputDir/webfonts``.
* ``cleanUp`: Whether or not to remove temporary files. Defaults to ``true``.
* ``autohintTTF`: Whether or not to autohint TTF files. Defaults to ``true``.
* ``axisOrder``: STAT table axis order. Defaults to fvar order.
* ``familyName``: Family name for variable fonts. Defaults to family name of first source file.

"""

from fontTools import designspaceLib
from fontTools.ttLib import TTFont
from fontmake.font_project import FontProject
from ufo2ft import CFFOptimization
from gftools.fix import fix_font
from gftools.stat import gen_stat_tables, gen_stat_tables_from_config
from gftools.utils import font_is_italic
from fontTools.otlLib.builder import buildStatTable
import statmake.classes
import statmake.lib
from babelfont import Babelfont
import sys
import os
import shutil
import glyphsLib
import tempfile
from fontTools.ttLib.woff2 import main as woff2_main
import logging
import yaml

class GFBuilder:
    def __init__(self, configfile=None, config=None):
        if configfile:
            self.config = yaml.load(open(configfile), Loader=yaml.SafeLoader)
            if os.path.dirname(configfile):
                os.chdir(os.path.dirname(configfile))
        else:
            self.config = config
        self.masters = {}
        self.logger = logging.getLogger("GFBuilder")
        self.fill_config_defaults()

    def build(self):
        loglevel = getattr(logging, self.config["logLevel"].upper())
        self.logger.setLevel(loglevel)
        # Shut up irrelevant loggers
        if loglevel != logging.DEBUG:
            for irrelevant in [
                "glyphsLib.classes",
                "glyphsLib.builder.components",
                "ufo2ft",
                "ufo2ft.filters",
                "ufo2ft.postProcessor",
                "fontTools.varLib",
                "cu2qu.ufo",
                "glyphsLib.builder.builders.UFOBuilder",
            ]:
                logging.getLogger(irrelevant).setLevel(logging.ERROR)
        if self.config["buildVariable"]:
            self.build_variable()
        if self.config["buildStatic"]:
            self.build_static()
        # All done
        self.logger.info(
            "Building %s completed. All done!" % (
                ", ".join([
                    b.lstrip("build").lower()
                    for b in ["buildVariable", "buildStatic", "buildWebfont"]
                    if self.config[b]
                ])
            )
        )

    def load_masters(self):
        if self.masters:
            return
        for src in self.config["sources"]:
            if src.endswith("glyphs"):
                gsfont = glyphsLib.GSFont(src)
                self.masters[src] = []
                for master in gsfont.masters:
                    self.masters[src].append(Babelfont.open(src, master=master.name))
            elif src.endswith("designspace"):
                continue
            else:
                self.masters[src] = [Babelfont.open(src)]

    def fill_config_defaults(self):
        if "familyName" not in self.config:
            self.logger.info("Deriving family name (this takes a while)")
            if  self.config["sources"][0].endswith("designspace"):
                designspace = designspaceLib.DesignSpaceDocument.fromfile(self.config["sources"][0])
                self.config["familyName"] = designspace.sources[0].familyName
            else:
                self.load_masters()
                familynames = set([x.info.familyName for x in self.masters.values()])
                if len(familynames) != 1:
                    raise ValueError(
                        "Inconsistent family names in sources (%s). Set familyName in config instead"
                        % familynames
                    )
                self.config["familyName"] = list(familynames)[0]
        if "outputDir" not in self.config:
            self.config["outputDir"] = "../fonts"
        if "vfDir" not in self.config:
            self.config["vfDir"] = self.config["outputDir"] + "/variable"
        if "ttDir" not in self.config:
            self.config["ttDir"] = self.config["outputDir"] + "/ttf"
        if "otDir" not in self.config:
            self.config["otDir"] = self.config["outputDir"] + "/otf"
        if "woffDir" not in self.config:
            self.config["woffDir"] = self.config["outputDir"] + "/webfonts"

        if "buildVariable" not in self.config:
            self.config["buildVariable"] = True
        if "buildStatic" not in self.config:
            self.config["buildStatic"] = True
        if "buildWebfont" not in self.config:
            self.config["buildWebfont"] = self.config["buildStatic"]
        if "autohintTTF" not in self.config:
            self.config["autohintTTF"] = True
        if "logLevel" not in self.config:
            self.config["logLevel"] = "INFO"
        if "cleanUp" not in self.config:
            self.config["cleanUp"] = True
        if "includeSourceFixes" not in self.config:
            self.config["includeSourceFixes"] = False

    def build_variable(self):
        self.mkdir(self.config["vfDir"], clean=True)
        args = {"output": ["variable"], "family_name": self.config["familyName"]}
        ttFonts = []
        for source in self.config["sources"]:
            if not source.endswith(".designspace") and not source.endswith("glyphs"):
                continue
            self.logger.info("Creating variable fonts from %s" % source)
            sourcebase = os.path.splitext(os.path.basename(source))[0]
            args["output_path"] = os.path.join(
                self.config["vfDir"], sourcebase + "-VF.ttf",
            )
            output_files = self.run_fontmake(source, args)
            newname = self.rename_variable(output_files[0])
            self.post_process(newname)
            ttFont = TTFont(newname)
            ttFonts.append(ttFont)
        self.gen_stat(ttFonts)

    def run_fontmake(self, source, args):
        if "output_dir" in args:
            original_output_dir = args["output_dir"]
            tmpdir = tempfile.TemporaryDirectory()
            args["output_dir"] = tmpdir.name

        if source.endswith(".glyphs"):
            FontProject().run_from_glyphs(source, **args)
        elif source.endswith(".designspace"):
            FontProject().run_from_designspace(source, **args)
        elif source.endswith(".ufo"):
            FontProject().run_from_ufos([source], **args)
        else:
            raise ValueError("Can't build from unknown source file: %s" % source)
        if "output_path" in args:
            return [args["output_path"]]
        else:
            # Move it to where it should be...
            file_names = os.listdir(args["output_dir"])
            for file_name in file_names:
                shutil.move(
                    os.path.join(args["output_dir"], file_name), original_output_dir
                )
            tmpdir.cleanup()
            args["output_dir"] = original_output_dir
            return [os.path.join(original_output_dir, x) for x in file_names]

    def rename_variable(self, fontfile):
        if "axisOrder" not in self.config:
            font = TTFont(fontfile)
            self.config["axisOrder"] = [ax.axisTag for ax in font["fvar"].axes]
        axes = ",".join(self.config["axisOrder"])
        newname = fontfile.replace("-VF.ttf", "[%s].ttf" % axes)
        os.rename(fontfile, newname)
        return newname

    def gen_stat(self, varfonts):
        if len(varfonts) > 1 and "ital" not in self.config["axisOrder"]:
            # *Are* any italic? Presumably, but test
            if any(font_is_italic(f) for f in varfonts):
                self.config["axisOrder"].append("ital")

        if "stylespaceFile" in self.config and self.config["stylespaceFile"]:
            self.gen_stat_stylespace(self.config["stylespaceFile"], varfonts)
        elif "stat" in self.config:
            gen_stat_tables_from_config(self.config["stat"], varfonts)
        else:
            gen_stat_tables(varfonts, self.config["axisOrder"])

        for ttFont in varfonts:
            ttFont.save(ttFont.reader.file.name)

    def gen_stat_stylespace(self, stylespaceFile, varfonts):
        import warnings
        warnings.warn(".stylespace files are supported for compatibility but"
            "you are encouraged to specify your STAT table axes in the config file")
        stylespace = statmake.classes.Stylespace.from_file(stylespaceFile)
        for ttFont in varfonts:
            if "ital" in self.config["axisOrder"]:
                if font_is_italic(ttFont):
                    additional_locations = {"Italic": 1}
                else:
                    additional_locations = {"Italic": 0}
            else:
                additional_locations = {}
            statmake.lib.apply_stylespace_to_variable_font(
                stylespace, ttFont, additional_locations
            )

    def build_static(self):
        self.build_a_static_format("otf", self.config["otDir"], self.post_process)
        if self.config["buildWebfont"]:
            self.mkdir(self.config["woffDir"], clean=True)
        self.build_a_static_format("ttf", self.config["ttDir"], self.post_process_ttf)
        if self.config["cleanUp"]:
            self.rmdir("instance_ufos")

    def build_a_static_format(self, format, directory, postprocessor):
        self.mkdir(directory, clean=True)
        args = {
            "output": [format],
            "output_dir": directory,
            "optimize_cff": CFFOptimization.SUBROUTINIZE,
        }
        for source in self.config["sources"]:
            if source.endswith("ufo"):
                if "interpolate" in args:
                    del args["interpolate"]
            else:
                args["interpolate"] = True
            self.logger.info("Creating static fonts from %s" % source)
            for fontfile in self.run_fontmake(source, args):
                self.logger.info("Created static font %s" % fontfile)
                postprocessor(fontfile)

    def rmdir(self, directory):
        shutil.rmtree(directory, ignore_errors=True)

    def mkdir(self, directory, clean=False):
        if clean:
            self.rmdir(directory)
        os.makedirs(directory)

    def post_process(self, filename):
        self.logger.info("Postprocessing font %s" % filename)
        font = TTFont(filename)
        fix_font(font, include_source_fixes=self.config["includeSourceFixes"])
        font.save(filename)

    def post_process_ttf(self, filename):
        if self.config["autohintTTF"]:
            self.logger.debug("Autohinting")
            self.autohint(filename)
        self.post_process(filename)
        if self.config["buildWebfont"]:
            self.logger.debug("Building webfont")
            woff2_main(["compress", filename])
            self.move_webfont(filename)

    def autohint(self, filename):
        from ttfautohint.options import parse_args as ttfautohint_parse_args
        from ttfautohint import ttfautohint

        ttfautohint(**ttfautohint_parse_args([filename, filename]))

    def move_webfont(self, filename):
        wf_filename = filename.replace(".ttf", ".woff2")
        os.rename(
            wf_filename,
            wf_filename.replace(self.config["ttDir"], self.config["woffDir"]),
        )

if __name__ == "__main__":
    GFBuilder(sys.argv[1]).build()
<commit_msg>builder: Fix getting familyName if config does not specify one<commit_after>"""Config-driven font project builder.

This utility wraps fontmake and a number of post-processing fixes to
build variable, static OTF, static TTF and webfonts from Glyphs,
Designspace/UFO or UFO sources.

It should be instantiated with a configuration file, typically ``config.yaml``,
which looks like this::

    sources:
      - Texturina.glyphs
      - Texturina-Italic.glyphs
    axisOrder:
      - opsz
      - wght
    outputDir: "../fonts"
    familyName: Texturina
    stat:
      - name: Width
        tag: wdth
        values:
        - name: UltraCondensed
          value: 50
          ...
      - name: Weight
        tag: wght
        values:
        - name: Hairline
          rangeMinValue: 1
          nominalValue: 1
          rangeMaxValue: 50
        ...

To build a font family from the command line, use:

    gftools builder path/to/config.yaml

The config file may contain the following keys. The ``sources`` key is
required, all others have sensible defaults:

* ``sources``: Required. An array of Glyphs, UFO or designspace source files.
* ``logLevel``: Debugging log level. Defaults to ``INFO``.
* ``stylespaceFile``: A statmake ``.stylespace`` file.
* ``stat``: A STAT table configuration. This may be either a list of axes and
    values as demonstrated above, or a dictionary mapping each source file to a
    per-source list. If neither ``stylespaceFile`` or ``stat`` are provided, a
    STAT table is generated automatically using ``gftools.stat``.
* ``buildVariable``: Build variable fonts. Defaults to true.
* ``buildStatic``: Build static fonts. Defaults to true.
* ``buildWebfont``: Build WOFF2 fonts. Defaults to ``$buildStatic``.
* ``outputDir``: Where to put the fonts. Defaults to ``../fonts/``
* ``vfDir``: Where to put variable fonts. Defaults to ``$outputDir/variable``.
* ``ttDir``: Where to put TrueType static fonts. Defaults to ``$outputDir/ttf``.
* ``otDir``: Where to put CFF static fonts. Defaults to ``$outputDir/otf``.
* ``woffDir``: Where to put WOFF2 static fonts. Defaults to ``$outputDir/webfonts``.
* ``cleanUp`: Whether or not to remove temporary files. Defaults to ``true``.
* ``autohintTTF`: Whether or not to autohint TTF files. Defaults to ``true``.
* ``axisOrder``: STAT table axis order. Defaults to fvar order.
* ``familyName``: Family name for variable fonts. Defaults to family name of first source file.

"""

from fontTools import designspaceLib
from fontTools.ttLib import TTFont
from fontmake.font_project import FontProject
from ufo2ft import CFFOptimization
from gftools.fix import fix_font
from gftools.stat import gen_stat_tables, gen_stat_tables_from_config
from gftools.utils import font_is_italic
from fontTools.otlLib.builder import buildStatTable
import statmake.classes
import statmake.lib
from babelfont import Babelfont
import sys
import os
import shutil
import glyphsLib
import tempfile
from fontTools.ttLib.woff2 import main as woff2_main
import logging
import yaml

class GFBuilder:
    def __init__(self, configfile=None, config=None):
        if configfile:
            self.config = yaml.load(open(configfile), Loader=yaml.SafeLoader)
            if os.path.dirname(configfile):
                os.chdir(os.path.dirname(configfile))
        else:
            self.config = config
        self.masters = {}
        self.logger = logging.getLogger("GFBuilder")
        self.fill_config_defaults()

    def build(self):
        loglevel = getattr(logging, self.config["logLevel"].upper())
        self.logger.setLevel(loglevel)
        # Shut up irrelevant loggers
        if loglevel != logging.DEBUG:
            for irrelevant in [
                "glyphsLib.classes",
                "glyphsLib.builder.components",
                "ufo2ft",
                "ufo2ft.filters",
                "ufo2ft.postProcessor",
                "fontTools.varLib",
                "cu2qu.ufo",
                "glyphsLib.builder.builders.UFOBuilder",
            ]:
                logging.getLogger(irrelevant).setLevel(logging.ERROR)
        if self.config["buildVariable"]:
            self.build_variable()
        if self.config["buildStatic"]:
            self.build_static()
        # All done
        self.logger.info(
            "Building %s completed. All done!" % (
                ", ".join([
                    b.lstrip("build").lower()
                    for b in ["buildVariable", "buildStatic", "buildWebfont"]
                    if self.config[b]
                ])
            )
        )

    def load_masters(self):
        if self.masters:
            return
        for src in self.config["sources"]:
            if src.endswith("glyphs"):
                gsfont = glyphsLib.GSFont(src)
                self.masters[src] = []
                for master in gsfont.masters:
                    self.masters[src].append(Babelfont.open(src, master=master.name))
            elif src.endswith("designspace"):
                continue
            else:
                self.masters[src] = [Babelfont.open(src)]

    def fill_config_defaults(self):
        if "familyName" not in self.config:
            self.logger.info("Deriving family name (this takes a while)")
            if  self.config["sources"][0].endswith("designspace"):
                designspace = designspaceLib.DesignSpaceDocument.fromfile(self.config["sources"][0])
                self.config["familyName"] = designspace.sources[0].familyName
            else:
                self.load_masters()

                familynames = set()
                for masters in self.masters.values():
                    for master in masters:
                        familynames.add(master.info.familyName)

                if len(familynames) != 1:
                    raise ValueError(
                        "Inconsistent family names in sources (%s). Set familyName in config instead"
                        % familynames
                    )
                self.config["familyName"] = list(familynames)[0]
        if "outputDir" not in self.config:
            self.config["outputDir"] = "../fonts"
        if "vfDir" not in self.config:
            self.config["vfDir"] = self.config["outputDir"] + "/variable"
        if "ttDir" not in self.config:
            self.config["ttDir"] = self.config["outputDir"] + "/ttf"
        if "otDir" not in self.config:
            self.config["otDir"] = self.config["outputDir"] + "/otf"
        if "woffDir" not in self.config:
            self.config["woffDir"] = self.config["outputDir"] + "/webfonts"

        if "buildVariable" not in self.config:
            self.config["buildVariable"] = True
        if "buildStatic" not in self.config:
            self.config["buildStatic"] = True
        if "buildWebfont" not in self.config:
            self.config["buildWebfont"] = self.config["buildStatic"]
        if "autohintTTF" not in self.config:
            self.config["autohintTTF"] = True
        if "logLevel" not in self.config:
            self.config["logLevel"] = "INFO"
        if "cleanUp" not in self.config:
            self.config["cleanUp"] = True
        if "includeSourceFixes" not in self.config:
            self.config["includeSourceFixes"] = False

    def build_variable(self):
        self.mkdir(self.config["vfDir"], clean=True)
        args = {"output": ["variable"], "family_name": self.config["familyName"]}
        ttFonts = []
        for source in self.config["sources"]:
            if not source.endswith(".designspace") and not source.endswith("glyphs"):
                continue
            self.logger.info("Creating variable fonts from %s" % source)
            sourcebase = os.path.splitext(os.path.basename(source))[0]
            args["output_path"] = os.path.join(
                self.config["vfDir"], sourcebase + "-VF.ttf",
            )
            output_files = self.run_fontmake(source, args)
            newname = self.rename_variable(output_files[0])
            self.post_process(newname)
            ttFont = TTFont(newname)
            ttFonts.append(ttFont)
        self.gen_stat(ttFonts)

    def run_fontmake(self, source, args):
        if "output_dir" in args:
            original_output_dir = args["output_dir"]
            tmpdir = tempfile.TemporaryDirectory()
            args["output_dir"] = tmpdir.name

        if source.endswith(".glyphs"):
            FontProject().run_from_glyphs(source, **args)
        elif source.endswith(".designspace"):
            FontProject().run_from_designspace(source, **args)
        elif source.endswith(".ufo"):
            FontProject().run_from_ufos([source], **args)
        else:
            raise ValueError("Can't build from unknown source file: %s" % source)
        if "output_path" in args:
            return [args["output_path"]]
        else:
            # Move it to where it should be...
            file_names = os.listdir(args["output_dir"])
            for file_name in file_names:
                shutil.move(
                    os.path.join(args["output_dir"], file_name), original_output_dir
                )
            tmpdir.cleanup()
            args["output_dir"] = original_output_dir
            return [os.path.join(original_output_dir, x) for x in file_names]

    def rename_variable(self, fontfile):
        if "axisOrder" not in self.config:
            font = TTFont(fontfile)
            self.config["axisOrder"] = [ax.axisTag for ax in font["fvar"].axes]
        axes = ",".join(self.config["axisOrder"])
        newname = fontfile.replace("-VF.ttf", "[%s].ttf" % axes)
        os.rename(fontfile, newname)
        return newname

    def gen_stat(self, varfonts):
        if len(varfonts) > 1 and "ital" not in self.config["axisOrder"]:
            # *Are* any italic? Presumably, but test
            if any(font_is_italic(f) for f in varfonts):
                self.config["axisOrder"].append("ital")

        if "stylespaceFile" in self.config and self.config["stylespaceFile"]:
            self.gen_stat_stylespace(self.config["stylespaceFile"], varfonts)
        elif "stat" in self.config:
            gen_stat_tables_from_config(self.config["stat"], varfonts)
        else:
            gen_stat_tables(varfonts, self.config["axisOrder"])

        for ttFont in varfonts:
            ttFont.save(ttFont.reader.file.name)

    def gen_stat_stylespace(self, stylespaceFile, varfonts):
        import warnings
        warnings.warn(".stylespace files are supported for compatibility but"
            "you are encouraged to specify your STAT table axes in the config file")
        stylespace = statmake.classes.Stylespace.from_file(stylespaceFile)
        for ttFont in varfonts:
            if "ital" in self.config["axisOrder"]:
                if font_is_italic(ttFont):
                    additional_locations = {"Italic": 1}
                else:
                    additional_locations = {"Italic": 0}
            else:
                additional_locations = {}
            statmake.lib.apply_stylespace_to_variable_font(
                stylespace, ttFont, additional_locations
            )

    def build_static(self):
        self.build_a_static_format("otf", self.config["otDir"], self.post_process)
        if self.config["buildWebfont"]:
            self.mkdir(self.config["woffDir"], clean=True)
        self.build_a_static_format("ttf", self.config["ttDir"], self.post_process_ttf)
        if self.config["cleanUp"]:
            self.rmdir("instance_ufos")

    def build_a_static_format(self, format, directory, postprocessor):
        self.mkdir(directory, clean=True)
        args = {
            "output": [format],
            "output_dir": directory,
            "optimize_cff": CFFOptimization.SUBROUTINIZE,
        }
        for source in self.config["sources"]:
            if source.endswith("ufo"):
                if "interpolate" in args:
                    del args["interpolate"]
            else:
                args["interpolate"] = True
            self.logger.info("Creating static fonts from %s" % source)
            for fontfile in self.run_fontmake(source, args):
                self.logger.info("Created static font %s" % fontfile)
                postprocessor(fontfile)

    def rmdir(self, directory):
        shutil.rmtree(directory, ignore_errors=True)

    def mkdir(self, directory, clean=False):
        if clean:
            self.rmdir(directory)
        os.makedirs(directory)

    def post_process(self, filename):
        self.logger.info("Postprocessing font %s" % filename)
        font = TTFont(filename)
        fix_font(font, include_source_fixes=self.config["includeSourceFixes"])
        font.save(filename)

    def post_process_ttf(self, filename):
        if self.config["autohintTTF"]:
            self.logger.debug("Autohinting")
            self.autohint(filename)
        self.post_process(filename)
        if self.config["buildWebfont"]:
            self.logger.debug("Building webfont")
            woff2_main(["compress", filename])
            self.move_webfont(filename)

    def autohint(self, filename):
        from ttfautohint.options import parse_args as ttfautohint_parse_args
        from ttfautohint import ttfautohint

        ttfautohint(**ttfautohint_parse_args([filename, filename]))

    def move_webfont(self, filename):
        wf_filename = filename.replace(".ttf", ".woff2")
        os.rename(
            wf_filename,
            wf_filename.replace(self.config["ttDir"], self.config["woffDir"]),
        )

if __name__ == "__main__":
    GFBuilder(sys.argv[1]).build()
<|endoftext|>


<commit_before>import pymongo

from eduid_signup.compat import urlparse

DEFAULT_MONGODB_HOST = 'localhost'
DEFAULT_MONGODB_PORT = 27017
DEFAULT_MONGODB_NAME = 'eduid'
DEFAULT_MONGODB_URI = 'mongodb://%s:%d/%s' % (DEFAULT_MONGODB_HOST,
                                              DEFAULT_MONGODB_PORT,
                                              DEFAULT_MONGODB_NAME)

class MongoDB(object):
    """Simple wrapper to get pymongo real objects from the settings uri"""

    def __init__(self, db_uri=DEFAULT_MONGODB_URI,
                 connection_factory=pymongo.Connection):
        self.db_uri = urlparse.urlparse(db_uri)
        self.connection = connection_factory(
            host=self.db_uri.hostname or DEFAULT_MONGODB_HOST,
            port=self.db_uri.port or DEFAULT_MONGODB_PORT,
            tz_aware=True)

        if self.db_uri.path:
            self.database_name = self.db_uri.path[1:]
        else:
            self.database_name = DEFAULT_MONGODB_NAME

    def get_connection(self):
        return self.connection

    def get_database(self):
        database = self.connection[self.database_name]
        if self.db_uri.username and self.db_uri.password:
            database.authenticate(self.db_uri.username, self.db_uri.password)

        return database

def get_db(request):
    return request.registry.settings['mongodb'].get_database()
<commit_msg>Allow Mongo connections to Mongo Replicaset Cluster<commit_after>import pymongo

DEFAULT_MONGODB_HOST = 'localhost'
DEFAULT_MONGODB_PORT = 27017
DEFAULT_MONGODB_NAME = 'eduid'
DEFAULT_MONGODB_URI = 'mongodb://%s:%d/%s' % (DEFAULT_MONGODB_HOST,
                                              DEFAULT_MONGODB_PORT,
                                              DEFAULT_MONGODB_NAME)

class MongoDB(object):
    """Simple wrapper to get pymongo real objects from the settings uri"""

    def __init__(self, db_uri=DEFAULT_MONGODB_URI,
                 connection_factory=pymongo.MongoClient):
        self.db_uri = db_uri
        self.connection = connection_factory(
            host=self.db_uri,
            tz_aware=True)

        if self.db_uri.path:
            self.database_name = self.db_uri.path[1:]
        else:
            self.database_name = DEFAULT_MONGODB_NAME

    def get_connection(self):
        return self.connection

    def get_database(self):
        database = self.connection[self.database_name]
        if self.db_uri.username and self.db_uri.password:
            database.authenticate(self.db_uri.username, self.db_uri.password)

        return database

def get_db(request):
    return request.registry.settings['mongodb'].get_database()
<|endoftext|>


<commit_before>"""Benchmark some basic import use-cases.

The assumption is made that this benchmark is run in a fresh interpreter and
thus has no external changes made to import-related attributes in sys.

"""
from . import util
from .source import util as source_util
import decimal
import imp
import importlib
import json
import os
import py_compile
import sys
import timeit

def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3):
    """Bench the given statement as many times as necessary until total
    executions take one second."""
    stmt = "__import__({!r})".format(name)
    timer = timeit.Timer(stmt)
    for x in range(repeat):
        total_time = 0
        count = 0
        while total_time < seconds:
            try:
                total_time += timer.timeit(1)
            finally:
                cleanup()
            count += 1
        else:
            # One execution too far
            if total_time > seconds:
                count -= 1
        yield count // seconds

def from_cache(seconds, repeat):
    """sys.modules"""
    name = '<benchmark import>'
    module = imp.new_module(name)
    module.__file__ = '<test>'
    module.__package__ = ''
    with util.uncache(name):
        sys.modules[name] = module
        for result in bench(name, repeat=repeat, seconds=seconds):
            yield result

def builtin_mod(seconds, repeat):
    """Built-in module"""
    name = 'errno'
    if name in sys.modules:
        del sys.modules[name]
    # Relying on built-in importer being implicit.
    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                        seconds=seconds):
        yield result

def source_wo_bytecode(seconds, repeat):
    """Source w/o bytecode: simple"""
    sys.dont_write_bytecode = True
    try:
        name = '__importlib_test_benchmark__'
        # Clears out sys.modules and puts an entry at the front of sys.path.
        with source_util.create_modules(name) as mapping:
            assert not os.path.exists(imp.cache_from_source(mapping[name]))
            for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                                seconds=seconds):
                yield result
    finally:
        sys.dont_write_bytecode = False

def decimal_wo_bytecode(seconds, repeat):
    """Source w/o bytecode: decimal"""
    name = 'decimal'
    decimal_bytecode = imp.cache_from_source(decimal.__file__)
    if os.path.exists(decimal_bytecode):
        os.unlink(decimal_bytecode)
    sys.dont_write_bytecode = True
    try:
        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                            seconds=seconds):
            yield result
    finally:
        sys.dont_write_bytecode = False

def source_writing_bytecode(seconds, repeat):
    """Source writing bytecode: simple"""
    assert not sys.dont_write_bytecode
    name = '__importlib_test_benchmark__'
    with source_util.create_modules(name) as mapping:
        def cleanup():
            sys.modules.pop(name)
            os.unlink(imp.cache_from_source(mapping[name]))
        for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
            assert not os.path.exists(imp.cache_from_source(mapping[name]))
            yield result

def decimal_writing_bytecode(seconds, repeat):
    """Source writing bytecode: decimal"""
    assert not sys.dont_write_bytecode
    name = 'decimal'
    def cleanup():
        sys.modules.pop(name)
        os.unlink(imp.cache_from_source(decimal.__file__))
    for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
        yield result

def source_using_bytecode(seconds, repeat):
    """Bytecode w/ source: simple"""
    name = '__importlib_test_benchmark__'
    with source_util.create_modules(name) as mapping:
        py_compile.compile(mapping[name])
        assert os.path.exists(imp.cache_from_source(mapping[name]))
        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                            seconds=seconds):
            yield result

def decimal_using_bytecode(seconds, repeat):
    """Bytecode w/ source: decimal"""
    name = 'decimal'
    py_compile.compile(decimal.__file__)
    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                        seconds=seconds):
        yield result

def main(import_, filename=None, benchmark=None):
    if filename and os.path.exists(filename):
        with open(filename, 'r') as file:
            prev_results = json.load(file)
    else:
        prev_results = {}
    __builtins__.__import__ = import_
    benchmarks = (from_cache, builtin_mod,
                  source_using_bytecode, source_wo_bytecode,
                  source_writing_bytecode,
                  decimal_using_bytecode, decimal_writing_bytecode,
                  decimal_wo_bytecode,)
    if benchmark:
        for b in benchmarks:
            if b.__doc__ == benchmark:
                benchmarks = [b]
                break
        else:
            print('Unknown benchmark: {!r}'.format(benchmark, file=sys.stderr))
            sys.exit(1)
    seconds = 1
    seconds_plural = 's' if seconds > 1 else ''
    repeat = 3
    header = ('Measuring imports/second over {} second{}, best out of {}\n'
              'Entire benchmark run should take about {} seconds\n')
    print(header.format(seconds, seconds_plural, repeat,
                        len(benchmarks) * seconds * repeat))
    new_results = {}
    for benchmark in benchmarks:
        print(benchmark.__doc__, "[", end=' ')
        sys.stdout.flush()
        results = []
        for result in benchmark(seconds=seconds, repeat=repeat):
            results.append(result)
            print(result, end=' ')
            sys.stdout.flush()
        assert not sys.dont_write_bytecode
        print("]", "best is", format(max(results), ',d'))
        new_results[benchmark.__doc__] = results
    prev_results[import_.__module__] = new_results
    if 'importlib._bootstrap' in prev_results and 'builtins' in prev_results:
        print('\n\nComparing importlib vs. __import__\n')
        importlib_results = prev_results['importlib._bootstrap']
        builtins_results = prev_results['builtins']
        for benchmark in benchmarks:
            benchmark_name = benchmark.__doc__
            importlib_result = max(importlib_results[benchmark_name])
            builtins_result = max(builtins_results[benchmark_name])
            result = '{:,d} vs. {:,d} ({:%})'.format(importlib_result,
                                                     builtins_result,
                                              importlib_result/builtins_result)
            print(benchmark_name, ':', result)
    if filename:
        with open(filename, 'w') as file:
            json.dump(prev_results, file, indent=2)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-b', '--builtin', dest='builtin', action='store_true',
                        default=False, help="use the built-in __import__")
    parser.add_argument('-f', '--file', dest='filename', default=None,
                        help='file to read/write results from/to'
                             '(incompatible w/ --benchmark)')
    parser.add_argument('--benchmark', dest='benchmark',
                        help='specific benchmark to run '
                             '(incompatible w/ --file')
    options = parser.parse_args()
    if options.filename and options.benchmark:
        print('Cannot specify a benchmark *and* read/write results')
        sys.exit(1)
    import_ = __import__
    if not options.builtin:
        import_ = importlib.__import__

    main(import_, options.filename, options.benchmark)
<commit_msg>Fix a minor output typo as found by Terry Reedy.<commit_after>"""Benchmark some basic import use-cases.

The assumption is made that this benchmark is run in a fresh interpreter and
thus has no external changes made to import-related attributes in sys.

"""
from . import util
from .source import util as source_util
import decimal
import imp
import importlib
import json
import os
import py_compile
import sys
import timeit

def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3):
    """Bench the given statement as many times as necessary until total
    executions take one second."""
    stmt = "__import__({!r})".format(name)
    timer = timeit.Timer(stmt)
    for x in range(repeat):
        total_time = 0
        count = 0
        while total_time < seconds:
            try:
                total_time += timer.timeit(1)
            finally:
                cleanup()
            count += 1
        else:
            # One execution too far
            if total_time > seconds:
                count -= 1
        yield count // seconds

def from_cache(seconds, repeat):
    """sys.modules"""
    name = '<benchmark import>'
    module = imp.new_module(name)
    module.__file__ = '<test>'
    module.__package__ = ''
    with util.uncache(name):
        sys.modules[name] = module
        for result in bench(name, repeat=repeat, seconds=seconds):
            yield result

def builtin_mod(seconds, repeat):
    """Built-in module"""
    name = 'errno'
    if name in sys.modules:
        del sys.modules[name]
    # Relying on built-in importer being implicit.
    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                        seconds=seconds):
        yield result

def source_wo_bytecode(seconds, repeat):
    """Source w/o bytecode: simple"""
    sys.dont_write_bytecode = True
    try:
        name = '__importlib_test_benchmark__'
        # Clears out sys.modules and puts an entry at the front of sys.path.
        with source_util.create_modules(name) as mapping:
            assert not os.path.exists(imp.cache_from_source(mapping[name]))
            for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                                seconds=seconds):
                yield result
    finally:
        sys.dont_write_bytecode = False

def decimal_wo_bytecode(seconds, repeat):
    """Source w/o bytecode: decimal"""
    name = 'decimal'
    decimal_bytecode = imp.cache_from_source(decimal.__file__)
    if os.path.exists(decimal_bytecode):
        os.unlink(decimal_bytecode)
    sys.dont_write_bytecode = True
    try:
        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                            seconds=seconds):
            yield result
    finally:
        sys.dont_write_bytecode = False

def source_writing_bytecode(seconds, repeat):
    """Source writing bytecode: simple"""
    assert not sys.dont_write_bytecode
    name = '__importlib_test_benchmark__'
    with source_util.create_modules(name) as mapping:
        def cleanup():
            sys.modules.pop(name)
            os.unlink(imp.cache_from_source(mapping[name]))
        for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
            assert not os.path.exists(imp.cache_from_source(mapping[name]))
            yield result

def decimal_writing_bytecode(seconds, repeat):
    """Source writing bytecode: decimal"""
    assert not sys.dont_write_bytecode
    name = 'decimal'
    def cleanup():
        sys.modules.pop(name)
        os.unlink(imp.cache_from_source(decimal.__file__))
    for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
        yield result

def source_using_bytecode(seconds, repeat):
    """Bytecode w/ source: simple"""
    name = '__importlib_test_benchmark__'
    with source_util.create_modules(name) as mapping:
        py_compile.compile(mapping[name])
        assert os.path.exists(imp.cache_from_source(mapping[name]))
        for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                            seconds=seconds):
            yield result

def decimal_using_bytecode(seconds, repeat):
    """Bytecode w/ source: decimal"""
    name = 'decimal'
    py_compile.compile(decimal.__file__)
    for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
                        seconds=seconds):
        yield result

def main(import_, filename=None, benchmark=None):
    if filename and os.path.exists(filename):
        with open(filename, 'r') as file:
            prev_results = json.load(file)
    else:
        prev_results = {}
    __builtins__.__import__ = import_
    benchmarks = (from_cache, builtin_mod,
                  source_using_bytecode, source_wo_bytecode,
                  source_writing_bytecode,
                  decimal_using_bytecode, decimal_writing_bytecode,
                  decimal_wo_bytecode,)
    if benchmark:
        for b in benchmarks:
            if b.__doc__ == benchmark:
                benchmarks = [b]
                break
        else:
            print('Unknown benchmark: {!r}'.format(benchmark, file=sys.stderr))
            sys.exit(1)
    seconds = 1
    seconds_plural = 's' if seconds > 1 else ''
    repeat = 3
    header = ('Measuring imports/second over {} second{}, best out of {}\n'
              'Entire benchmark run should take about {} seconds\n')
    print(header.format(seconds, seconds_plural, repeat,
                        len(benchmarks) * seconds * repeat))
    new_results = {}
    for benchmark in benchmarks:
        print(benchmark.__doc__, "[", end=' ')
        sys.stdout.flush()
        results = []
        for result in benchmark(seconds=seconds, repeat=repeat):
            results.append(result)
            print(result, end=' ')
            sys.stdout.flush()
        assert not sys.dont_write_bytecode
        print("]", "best is", format(max(results), ',d'))
        new_results[benchmark.__doc__] = results
    prev_results[import_.__module__] = new_results
    if 'importlib._bootstrap' in prev_results and 'builtins' in prev_results:
        print('\n\nComparing importlib vs. __import__\n')
        importlib_results = prev_results['importlib._bootstrap']
        builtins_results = prev_results['builtins']
        for benchmark in benchmarks:
            benchmark_name = benchmark.__doc__
            importlib_result = max(importlib_results[benchmark_name])
            builtins_result = max(builtins_results[benchmark_name])
            result = '{:,d} vs. {:,d} ({:%})'.format(importlib_result,
                                                     builtins_result,
                                              importlib_result/builtins_result)
            print(benchmark_name, ':', result)
    if filename:
        with open(filename, 'w') as file:
            json.dump(prev_results, file, indent=2)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-b', '--builtin', dest='builtin', action='store_true',
                        default=False, help="use the built-in __import__")
    parser.add_argument('-f', '--file', dest='filename', default=None,
                        help='file to read/write results from/to'
                             '(incompatible w/ --benchmark)')
    parser.add_argument('--benchmark', dest='benchmark',
                        help='specific benchmark to run '
                             '(incompatible w/ --file)')
    options = parser.parse_args()
    if options.filename and options.benchmark:
        print('Cannot specify a benchmark *and* read/write results')
        sys.exit(1)
    import_ = __import__
    if not options.builtin:
        import_ = importlib.__import__

    main(import_, options.filename, options.benchmark)
<|endoftext|>


<commit_before>import logging
import multiprocessing
import time

from radosgw_agent import worker
from radosgw_agent import client
from radosgw_agent.exceptions import NotFound, HttpError

log = logging.getLogger(__name__)

# the replica log api only supports one entry, and updating it
# requires sending a daemon id that matches the existing one. This
# doesn't make a whole lot of sense with the current structure of
# radosgw-agent, so just use a constant value for the daemon id.
DAEMON_ID = 'radosgw-agent'

def prepare_sync(syncer, error_delay):
    """Attempt to prepare a syncer for running a sync.

    :param error_delay: seconds to wait before retrying

    This will retry forever so the sync agent continues if radosgws
    are unavailable temporarily.
    """
    while True:
        try:
            syncer.prepare()
            break
        except Exception:
            log.warn('error preparing for sync, will retry. Traceback:',
                     exc_info=True)
            time.sleep(error_delay)

def incremental_sync(meta_syncer, data_syncer, num_workers, lock_timeout,
                     incremental_sync_delay, metadata_only, error_delay):
    """Run a continuous incremental sync.

    This will run forever, pausing between syncs by a
    incremental_sync_delay seconds.
    """
    while True:
        try:
            meta_syncer.sync(num_workers, lock_timeout)
            if not metadata_only:
                data_syncer.sync(num_workers, lock_timeout)
        except Exception:
            log.warn('error doing incremental sync, will try again. Traceback:',
                     exc_info=True)

        # prepare data before sleeping due to rgw_log_bucket_window
        if not metadata_only:
            prepare_sync(data_syncer, error_delay)
        log.info('waiting %d seconds until next sync',
                 incremental_sync_delay)
        time.sleep(incremental_sync_delay)
        prepare_sync(meta_syncer, error_delay)

class Syncer(object):
    def __init__(self, src, dest, max_entries, *args, **kwargs):
        self.src = src
        self.dest = dest
        self.src_conn = client.connection(src)
        self.dest_conn = client.connection(dest)
        self.daemon_id = DAEMON_ID
        self.worker_cls = None # filled in by subclass constructor
        self.num_shards = None
        self.max_entries = max_entries
        self.object_sync_timeout = kwargs.get('object_sync_timeout')

    def init_num_shards(self):
        if self.num_shards is not None:
            return
        try:
            self.num_shards = client.num_log_shards(self.src_conn, self.type)
            log.debug('%d shards to check', self.num_shards)
        except Exception:
            log.error('finding number of shards failed')
            raise

    def shard_num_for_key(self, key):
        key = key.encode('utf8')
        hash_val = 0
        for char in key:
            c = ord(char)
            hash_val = (hash_val + (c << 4) + (c >> 4)) * 11
        return hash_val % self.num_shards

    def prepare(self):
        """Setup any state required before syncing starts.

        This must be called before sync().
        """
        pass

    def generate_work(self):
        """Generate items to be place in a queue or processing"""
        pass

    def wait_until_ready(self):
        pass

    def complete_item(self, shard_num, retries):
        """Called when syncing a single item completes successfully"""
        marker = self.shard_info.get(shard_num)
        if not marker:
            return
        try:
            data = [dict(name=retry, time=worker.DEFAULT_TIME)
                    for retry in retries]
            client.set_worker_bound(self.dest_conn,
                                    self.type,
                                    marker,
                                    worker.DEFAULT_TIME,
                                    self.daemon_id,
                                    shard_num,
                                    data)
        except Exception:
            log.warn('could not set worker bounds, may repeat some work.'
                     'Traceback:', exc_info=True)

    def sync(self, num_workers, log_lock_time):
        workQueue = multiprocessing.Queue()
        resultQueue = multiprocessing.Queue()

        processes = [self.worker_cls(workQueue,
                                     resultQueue,
                                     log_lock_time,
                                     self.src,
                                     self.dest,
                                     daemon_id=self.daemon_id,
                                     max_entries=self.max_entries,
                                     object_sync_timeout=self.object_sync_timeout,
                                     )
                     for i in xrange(num_workers)]
        for process in processes:
            process.daemon = True
            process.start()

        self.wait_until_ready()

        log.info('Starting sync')
        # enqueue the shards to be synced
        num_items = 0
        for item in self.generate_work():
            num_items += 1
            workQueue.put(item)

        # add a poison pill for each worker
        for i in xrange(num_workers):
            workQueue.put(None)

        # pull the results out as they are produced
        retries = {}
        for i in xrange(num_items):
            result, item = resultQueue.get()
            shard_num, retries = item
            if result == worker.RESULT_SUCCESS:
                log.debug('synced item %r successfully', item)
                self.complete_item(shard_num, retries)
            else:
                log.error('error syncing shard %d', shard_num)
                retries.append(shard_num)

            log.info('%d/%d items processed', i + 1, num_items)
        if retries:
            log.error('Encountered errors syncing these %d shards: %r',
                      len(retries), retries)

class IncrementalSyncer(Syncer):

    def get_worker_bound(self, shard_num):
        bound = client.get_worker_bound(
            self.dest_conn,
            self.type,
            shard_num)

        marker = bound['marker']
        retries = bound['retries']

        log.debug('oldest marker and time for shard %d are: %r %r',
                  shard_num, marker, bound['oldest_time'])
        log.debug('%d items to retrie are: %r', len(retries), retries)

        return marker, retries

    def get_log_entries(self, shard_num, marker):
        try:
            result = client.get_log(self.src_conn, self.type,
                                    marker, self.max_entries,
                                    shard_num)
            last_marker = result['marker']
            log_entries = result['entries']
            if len(log_entries) == self.max_entries:
                log.warn('shard %d log has fallen behind - log length >= %d',
                         shard_num, self.max_entries)
        except NotFound:
            # no entries past this marker yet, but we my have retries
            last_marker = ' '
            log_entries = []
        return last_marker, log_entries

    def prepare(self):
        self.init_num_shards()

        self.shard_info = {}
        self.shard_work = {}
        for shard_num in xrange(self.num_shards):
            marker, retries = self.get_worker_bound(shard_num)
            last_marker, log_entries = self.get_log_entries(shard_num, marker)
            self.shard_work[shard_num] = log_entries, retries
            self.shard_info[shard_num] = last_marker

        self.prepared_at = time.time()

    def generate_work(self):
        return self.shard_work.iteritems()

class MetaSyncerInc(IncrementalSyncer):

    def __init__(self, *args, **kwargs):
        super(MetaSyncerInc, self).__init__(*args, **kwargs)
        self.worker_cls = worker.MetadataWorkerIncremental
        self.type = 'metadata'

class DataSyncerInc(IncrementalSyncer):

    def __init__(self, *args, **kwargs):
        super(DataSyncerInc, self).__init__(*args, **kwargs)
        self.worker_cls = worker.DataWorkerIncremental
        self.type = 'data'
        self.rgw_data_log_window = kwargs.get('rgw_data_log_window', 30)

    def wait_until_ready(self):
        log.info('waiting to make sure bucket log is consistent')
        while time.time() < self.prepared_at + self.rgw_data_log_window:
            time.sleep(1)

class DataSyncerFull(Syncer):

    def __init__(self, *args, **kwargs):
        super(DataSyncerFull, self).__init__(*args, **kwargs)
        self.worker_cls = worker.DataWorkerFull
        self.type = 'data'
        self.rgw_data_log_window = kwargs.get('rgw_data_log_window', 30)

    def prepare(self):
        self.init_num_shards()

        # save data log markers for each shard
        self.shard_info = {}
        for shard in xrange(self.num_shards):
            info = client.get_log_info(self.src_conn, 'data', shard)
            # setting an empty marker returns an error
            if info['marker']:
                self.shard_info[shard] = info['marker']
            else:
                self.shard_info[shard] = ' '

        # get list of buckets after getting any markers to avoid skipping
        # entries added before we got the marker info
        buckets = client.get_bucket_list(self.src_conn)

        self.prepared_at = time.time()

        self.buckets_by_shard = {}
        for bucket in buckets:
            shard = self.shard_num_for_key(bucket)
            self.buckets_by_shard.setdefault(shard, [])
            self.buckets_by_shard[shard].append(bucket)

    def generate_work(self):
        return self.buckets_by_shard.iteritems()

    def wait_until_ready(self):
        log.info('waiting to make sure bucket log is consistent')
        while time.time() < self.prepared_at + self.rgw_data_log_window:
            time.sleep(1)

class MetaSyncerFull(Syncer):
    def __init__(self, *args, **kwargs):
        super(MetaSyncerFull, self).__init__(*args, **kwargs)
        self.worker_cls = worker.MetadataWorkerFull
        self.type = 'metadata'

    def prepare(self):
        try:
            self.sections = client.get_metadata_sections(self.src_conn)
        except HttpError as e:
            log.error('Error listing metadata sections: %s', e)
            raise

        # grab the lastest shard markers and timestamps before we sync
        self.shard_info = {}
        self.init_num_shards()
        for shard_num in xrange(self.num_shards):
            info = client.get_log_info(self.src_conn, 'metadata', shard_num)
            # setting an empty marker returns an error
            if info['marker']:
                self.shard_info[shard_num] = info['marker']
            else:
                self.shard_info[shard_num] = ' '

        self.metadata_by_shard = {}
        for section in self.sections:
            try:
                for key in client.list_metadata_keys(self.src_conn, section):
                    shard = self.shard_num_for_key(section + ':' + key)
                    self.metadata_by_shard.setdefault(shard, [])
                    self.metadata_by_shard[shard].append((section, key))
            except NotFound:
                # no keys of this type exist
                continue
            except HttpError as e:
                log.error('Error listing metadata for section %s: %s',
                          section, e)
                raise

    def generate_work(self):
        return self.metadata_by_shard.iteritems()
<commit_msg>inform the user about data sync operations<commit_after>import logging
import multiprocessing
import time

from radosgw_agent import worker
from radosgw_agent import client
from radosgw_agent.exceptions import NotFound, HttpError

log = logging.getLogger(__name__)

# the replica log api only supports one entry, and updating it
# requires sending a daemon id that matches the existing one. This
# doesn't make a whole lot of sense with the current structure of
# radosgw-agent, so just use a constant value for the daemon id.
DAEMON_ID = 'radosgw-agent'

def prepare_sync(syncer, error_delay):
    """Attempt to prepare a syncer for running a sync.

    :param error_delay: seconds to wait before retrying

    This will retry forever so the sync agent continues if radosgws
    are unavailable temporarily.
    """
    while True:
        try:
            syncer.prepare()
            break
        except Exception:
            log.warn('error preparing for sync, will retry. Traceback:',
                     exc_info=True)
            time.sleep(error_delay)

def incremental_sync(meta_syncer, data_syncer, num_workers, lock_timeout,
                     incremental_sync_delay, metadata_only, error_delay):
    """Run a continuous incremental sync.

    This will run forever, pausing between syncs by a
    incremental_sync_delay seconds.
    """
    while True:
        try:
            meta_syncer.sync(num_workers, lock_timeout)
            if not metadata_only:
                data_syncer.sync(num_workers, lock_timeout)
        except Exception:
            log.warn('error doing incremental sync, will try again. Traceback:',
                     exc_info=True)

        # prepare data before sleeping due to rgw_log_bucket_window
        if not metadata_only:
            prepare_sync(data_syncer, error_delay)
        log.info('waiting %d seconds until next sync',
                 incremental_sync_delay)
        time.sleep(incremental_sync_delay)
        prepare_sync(meta_syncer, error_delay)

class Syncer(object):
    def __init__(self, src, dest, max_entries, *args, **kwargs):
        self.src = src
        self.dest = dest
        self.src_conn = client.connection(src)
        self.dest_conn = client.connection(dest)
        self.daemon_id = DAEMON_ID
        self.worker_cls = None # filled in by subclass constructor
        self.num_shards = None
        self.max_entries = max_entries
        self.object_sync_timeout = kwargs.get('object_sync_timeout')

    def init_num_shards(self):
        if self.num_shards is not None:
            return
        try:
            self.num_shards = client.num_log_shards(self.src_conn, self.type)
            log.debug('%d shards to check', self.num_shards)
        except Exception:
            log.error('finding number of shards failed')
            raise

    def shard_num_for_key(self, key):
        key = key.encode('utf8')
        hash_val = 0
        for char in key:
            c = ord(char)
            hash_val = (hash_val + (c << 4) + (c >> 4)) * 11
        return hash_val % self.num_shards

    def prepare(self):
        """Setup any state required before syncing starts.

        This must be called before sync().
        """
        pass

    def generate_work(self):
        """Generate items to be place in a queue or processing"""
        pass

    def wait_until_ready(self):
        pass

    def complete_item(self, shard_num, retries):
        """Called when syncing a single item completes successfully"""
        marker = self.shard_info.get(shard_num)
        if not marker:
            return
        try:
            data = [dict(name=retry, time=worker.DEFAULT_TIME)
                    for retry in retries]
            client.set_worker_bound(self.dest_conn,
                                    self.type,
                                    marker,
                                    worker.DEFAULT_TIME,
                                    self.daemon_id,
                                    shard_num,
                                    data)
        except Exception:
            log.warn('could not set worker bounds, may repeat some work.'
                     'Traceback:', exc_info=True)

    def sync(self, num_workers, log_lock_time):
        workQueue = multiprocessing.Queue()
        resultQueue = multiprocessing.Queue()

        processes = [self.worker_cls(workQueue,
                                     resultQueue,
                                     log_lock_time,
                                     self.src,
                                     self.dest,
                                     daemon_id=self.daemon_id,
                                     max_entries=self.max_entries,
                                     object_sync_timeout=self.object_sync_timeout,
                                     )
                     for i in xrange(num_workers)]
        for process in processes:
            process.daemon = True
            process.start()

        self.wait_until_ready()

        log.info('Starting sync')
        # enqueue the shards to be synced
        num_items = 0
        for item in self.generate_work():
            num_items += 1
            workQueue.put(item)

        # add a poison pill for each worker
        for i in xrange(num_workers):
            workQueue.put(None)

        # pull the results out as they are produced
        retries = {}
        for i in xrange(num_items):
            result, item = resultQueue.get()
            shard_num, retries = item
            if result == worker.RESULT_SUCCESS:
                log.debug('synced item %r successfully', item)
                self.complete_item(shard_num, retries)
            else:
                log.error('error syncing shard %d', shard_num)
                retries.append(shard_num)

            log.info('%d/%d items processed', i + 1, num_items)
        if retries:
            log.error('Encountered errors syncing these %d shards: %r',
                      len(retries), retries)

class IncrementalSyncer(Syncer):

    def get_worker_bound(self, shard_num):
        bound = client.get_worker_bound(
            self.dest_conn,
            self.type,
            shard_num)

        marker = bound['marker']
        retries = bound['retries']

        log.debug('oldest marker and time for shard %d are: %r %r',
                  shard_num, marker, bound['oldest_time'])
        log.debug('%d items to retrie are: %r', len(retries), retries)

        return marker, retries

    def get_log_entries(self, shard_num, marker):
        try:
            result = client.get_log(self.src_conn, self.type,
                                    marker, self.max_entries,
                                    shard_num)
            last_marker = result['marker']
            log_entries = result['entries']
            if len(log_entries) == self.max_entries:
                log.warn('shard %d log has fallen behind - log length >= %d',
                         shard_num, self.max_entries)
        except NotFound:
            # no entries past this marker yet, but we my have retries
            last_marker = ' '
            log_entries = []
        return last_marker, log_entries

    def prepare(self):
        self.init_num_shards()

        self.shard_info = {}
        self.shard_work = {}
        for shard_num in xrange(self.num_shards):
            marker, retries = self.get_worker_bound(shard_num)
            last_marker, log_entries = self.get_log_entries(shard_num, marker)
            self.shard_work[shard_num] = log_entries, retries
            self.shard_info[shard_num] = last_marker

        self.prepared_at = time.time()

    def generate_work(self):
        return self.shard_work.iteritems()

class MetaSyncerInc(IncrementalSyncer):

    def __init__(self, *args, **kwargs):
        super(MetaSyncerInc, self).__init__(*args, **kwargs)
        self.worker_cls = worker.MetadataWorkerIncremental
        self.type = 'metadata'

class DataSyncerInc(IncrementalSyncer):

    def __init__(self, *args, **kwargs):
        super(DataSyncerInc, self).__init__(*args, **kwargs)
        self.worker_cls = worker.DataWorkerIncremental
        self.type = 'data'
        self.rgw_data_log_window = kwargs.get('rgw_data_log_window', 30)

    def wait_until_ready(self):
        log.info('waiting to make sure bucket log is consistent')
        while time.time() < self.prepared_at + self.rgw_data_log_window:
            time.sleep(1)

class DataSyncerFull(Syncer):

    def __init__(self, *args, **kwargs):
        super(DataSyncerFull, self).__init__(*args, **kwargs)
        self.worker_cls = worker.DataWorkerFull
        self.type = 'data'
        self.rgw_data_log_window = kwargs.get('rgw_data_log_window', 30)

    def prepare(self):
        log.info('preparing to do a full data sync')
        self.init_num_shards()

        # save data log markers for each shard
        self.shard_info = {}
        for shard in xrange(self.num_shards):
            info = client.get_log_info(self.src_conn, 'data', shard)
            # setting an empty marker returns an error
            if info['marker']:
                self.shard_info[shard] = info['marker']
            else:
                self.shard_info[shard] = ' '

        # get list of buckets after getting any markers to avoid skipping
        # entries added before we got the marker info
        log.debug('getting bucket list')
        buckets = client.get_bucket_list(self.src_conn)

        self.prepared_at = time.time()

        self.buckets_by_shard = {}
        for bucket in buckets:
            shard = self.shard_num_for_key(bucket)
            self.buckets_by_shard.setdefault(shard, [])
            self.buckets_by_shard[shard].append(bucket)

    def generate_work(self):
        return self.buckets_by_shard.iteritems()

    def wait_until_ready(self):
        log.info('waiting to make sure bucket log is consistent')
        while time.time() < self.prepared_at + self.rgw_data_log_window:
            time.sleep(1)

class MetaSyncerFull(Syncer):
    def __init__(self, *args, **kwargs):
        super(MetaSyncerFull, self).__init__(*args, **kwargs)
        self.worker_cls = worker.MetadataWorkerFull
        self.type = 'metadata'

    def prepare(self):
        try:
            self.sections = client.get_metadata_sections(self.src_conn)
        except HttpError as e:
            log.error('Error listing metadata sections: %s', e)
            raise

        # grab the lastest shard markers and timestamps before we sync
        self.shard_info = {}
        self.init_num_shards()
        for shard_num in xrange(self.num_shards):
            info = client.get_log_info(self.src_conn, 'metadata', shard_num)
            # setting an empty marker returns an error
            if info['marker']:
                self.shard_info[shard_num] = info['marker']
            else:
                self.shard_info[shard_num] = ' '

        self.metadata_by_shard = {}
        for section in self.sections:
            try:
                for key in client.list_metadata_keys(self.src_conn, section):
                    shard = self.shard_num_for_key(section + ':' + key)
                    self.metadata_by_shard.setdefault(shard, [])
                    self.metadata_by_shard[shard].append((section, key))
            except NotFound:
                # no keys of this type exist
                continue
            except HttpError as e:
                log.error('Error listing metadata for section %s: %s',
                          section, e)
                raise

    def generate_work(self):
        return self.metadata_by_shard.iteritems()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import importlib
import re
import string
import sys; sys.path.append(".."); sys.path.append("../fireplace/cards/data")
from fireplace import cards
from fireplace.enums import CardType, CardSet

import buffs

GREEN = "\033[92m"
RED = "\033[91m"
ENDC = "\033[0m"
PREFIXES = {
	GREEN: "Implemented",
	RED: "Not implemented",
}

SOLVED_KEYWORDS = [
	"Windfury", "Charge", "Divine Shield", "Taunt", "Stealth",
	"Can't be targeted by spells or Hero Powers",
	"Destroy any minion damaged by this minion.",
	"50% chance to attack the wrong enemy",
	"Can't attack",
	"Always wins Brawls.",
	r"Your Hero Power deals \d+ extra damage.",
	r"Spell Damage \+\d+",
	r"Overload: \(\d+\)",
]

CARD_SETS = {
	"debug": None,
	"game": None,
	"tutorial": None,
	"classic": None,
	"naxxramas": None,
	"gvg": None,
	"blackrock": None,
	"tgt": None,
}

DUMMY_CARDS = (
	"CS1_113e",  # Mind Control
	"CS2_022e",  # Polymorph
	"EX1_246e",  # Hexxed
	"Mekka4e",  # Transformed
	"NEW1_025e",  # Bolstered (Unused)
	"XXX_058e",  # Weapon Nerf Enchant

	# Dynamic buffs set by their parent
	"EX1_304e",  # Consume (Void Terror)
	"NEW1_018e",  # Treasure Crazed (Bloodsail Raider)
)

for cardset in CARD_SETS:
	CARD_SETS[cardset] = importlib.import_module("fireplace.cards.%s" % (cardset))

def cleanup_description(description):
	ret = description
	ret = re.sub(".+", "", ret)
	ret = re.sub("(|)", "", ret)
	ret = re.sub("(" + "|".join(SOLVED_KEYWORDS) + ")", "", ret)
	ret = re.sub("<[^>]*>", "", ret)
	exclude_chars = string.punctuation + string.whitespace
	ret = "".join([ch for ch in ret if ch not in exclude_chars])
	return ret

def main():
	impl = 0
	unimpl = 0

	p = argparse.ArgumentParser()
	p.add_argument(
		"--implemented",
		action="store_true",
		dest="implemented",
		help="Show only implemented cards"
	)
	p.add_argument(
		"--unimplemented",
		action="store_true",
		dest="unimplemented",
		help="Show only unimplemented cards"
	)
	args = p.parse_args(sys.argv[1:])

	if not args.implemented and not args.unimplemented:
		args.implemented = True
		args.unimplemented = True

	for id in sorted(cards.db):
		card = cards.db[id]
		description = cleanup_description(card.description)
		implemented = False

		if not description:
			# Minions without card text or with basic abilities are implemented
			implemented = True
		elif card.type == CardType.ENCHANTMENT:
			if id in buffs.__dict__:
				implemented = True
			else:
				implemented = False
		elif card.card_set == CardSet.CREDITS:
			implemented = True

		if id in DUMMY_CARDS:
			implemented = True

		for set in CARD_SETS.values():
			if hasattr(set, id):
				implemented = True
				break
		else:
			if "Enrage" in card.description or card.choose_cards:
				implemented = True

		color = GREEN if implemented else RED
		name = color + "%s: %s" % (PREFIXES[color], card.name) + ENDC

		if implemented:
			impl += 1
			if args.implemented:
				print("%s (%s)" % (name, id))
		else:
			unimpl += 1
			if args.unimplemented:
				print("%s (%s)" % (name, id))

	total = impl + unimpl

	print("%i / %i cards implemented (%i%%)" % (impl, total, (impl / total) * 100))

if __name__ == "__main__":
	main()
<commit_msg>implemented.py: Ignore Empty Enchant<commit_after>#!/usr/bin/env python
import argparse
import importlib
import re
import string
import sys; sys.path.append(".."); sys.path.append("../fireplace/cards/data")
from fireplace import cards
from fireplace.enums import CardType, CardSet

import buffs

GREEN = "\033[92m"
RED = "\033[91m"
ENDC = "\033[0m"
PREFIXES = {
	GREEN: "Implemented",
	RED: "Not implemented",
}

SOLVED_KEYWORDS = [
	"Windfury", "Charge", "Divine Shield", "Taunt", "Stealth",
	"Can't be targeted by spells or Hero Powers",
	"Destroy any minion damaged by this minion.",
	"50% chance to attack the wrong enemy",
	"Can't attack",
	"Always wins Brawls.",
	r"Your Hero Power deals \d+ extra damage.",
	r"Spell Damage \+\d+",
	r"Overload: \(\d+\)",
]

CARD_SETS = {
	"debug": None,
	"game": None,
	"tutorial": None,
	"classic": None,
	"naxxramas": None,
	"gvg": None,
	"blackrock": None,
	"tgt": None,
}

DUMMY_CARDS = (
	"CS1_113e",  # Mind Control
	"CS2_022e",  # Polymorph
	"EX1_246e",  # Hexxed
	"Mekka4e",  # Transformed
	"NEW1_025e",  # Bolstered (Unused)
	"XXX_009e",  # Empty Enchant
	"XXX_058e",  # Weapon Nerf Enchant

	# Dynamic buffs set by their parent
	"EX1_304e",  # Consume (Void Terror)
	"NEW1_018e",  # Treasure Crazed (Bloodsail Raider)
)

for cardset in CARD_SETS:
	CARD_SETS[cardset] = importlib.import_module("fireplace.cards.%s" % (cardset))

def cleanup_description(description):
	ret = description
	ret = re.sub(".+", "", ret)
	ret = re.sub("(|)", "", ret)
	ret = re.sub("(" + "|".join(SOLVED_KEYWORDS) + ")", "", ret)
	ret = re.sub("<[^>]*>", "", ret)
	exclude_chars = string.punctuation + string.whitespace
	ret = "".join([ch for ch in ret if ch not in exclude_chars])
	return ret

def main():
	impl = 0
	unimpl = 0

	p = argparse.ArgumentParser()
	p.add_argument(
		"--implemented",
		action="store_true",
		dest="implemented",
		help="Show only implemented cards"
	)
	p.add_argument(
		"--unimplemented",
		action="store_true",
		dest="unimplemented",
		help="Show only unimplemented cards"
	)
	args = p.parse_args(sys.argv[1:])

	if not args.implemented and not args.unimplemented:
		args.implemented = True
		args.unimplemented = True

	for id in sorted(cards.db):
		card = cards.db[id]
		description = cleanup_description(card.description)
		implemented = False

		if not description:
			# Minions without card text or with basic abilities are implemented
			implemented = True
		elif card.type == CardType.ENCHANTMENT:
			if id in buffs.__dict__:
				implemented = True
			else:
				implemented = False
		elif card.card_set == CardSet.CREDITS:
			implemented = True

		if id in DUMMY_CARDS:
			implemented = True

		for set in CARD_SETS.values():
			if hasattr(set, id):
				implemented = True
				break
		else:
			if "Enrage" in card.description or card.choose_cards:
				implemented = True

		color = GREEN if implemented else RED
		name = color + "%s: %s" % (PREFIXES[color], card.name) + ENDC

		if implemented:
			impl += 1
			if args.implemented:
				print("%s (%s)" % (name, id))
		else:
			unimpl += 1
			if args.unimplemented:
				print("%s (%s)" % (name, id))

	total = impl + unimpl

	print("%i / %i cards implemented (%i%%)" % (impl, total, (impl / total) * 100))

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before><commit_msg>Fixed multiproc version of the correlation_fitter class version of corrfitx<commit_after><|endoftext|>


<commit_before>"""SCGI XMLRPC Transport.

XMLRPC in Python only supports HTTP(S). This module extends the transport to also support SCGI.

SCGI is required by rTorrent if you want to communicate directly with an instance.

Example:
  Small usage example

  literal blocks::
    from xmlrpc.client import ServerProxy

    proxy = ServerProxy('http://127.0.0.1:8000/', transport=SCGITransport())
    proxy.system.listMethods()

License:
    Public Domain (no attribution needed).
    The license only applies to THIS file.
"""

import socket
import urllib

from io import BytesIO
from xmlrpclib import Transport

def encode_netstring(input):
    return str(len(input)) + ':' + input + ','

def encode_header(key, value):
    return key + '\x00' + value + '\x00'

class SCGITransport(Transport):
    def single_request(self, host, handler, request_body, verbose=False):
        self.verbose = verbose
        host, port = urllib.splitport(host)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((host, int(port)))

        request = encode_header('CONTENT_LENGTH', str(len(request_body)))
        request += encode_header('SCGI', '1')
        request += encode_header('REQUEST_METHOD', 'POST')
        request += encode_header('REQUEST_URI', handler)

        request = encode_netstring(request)
        request += request_body

        s.send(request)
        print host, handler, request_body

        response = ''
        while True:
            r = s.recv(1024)
            if not r:
                break
            response += r

        response_body = BytesIO('\r\n\r\n'.join(response.split('\r\n\r\n')[1:]))

        return self.parse_response(response_body)

if not hasattr(Transport, 'single_request'):
    SCGITransport.request = SCGITransport.single_request<commit_msg>removed debug<commit_after>"""SCGI XMLRPC Transport.

XMLRPC in Python only supports HTTP(S). This module extends the transport to also support SCGI.

SCGI is required by rTorrent if you want to communicate directly with an instance.

Example:
  Small usage example

  literal blocks::
    from xmlrpc.client import ServerProxy

    proxy = ServerProxy('http://127.0.0.1:8000/', transport=SCGITransport())
    proxy.system.listMethods()

License:
    Public Domain (no attribution needed).
    The license only applies to THIS file.
"""

import socket
import urllib

from io import BytesIO
from xmlrpclib import Transport

def encode_netstring(input):
    return str(len(input)) + ':' + input + ','

def encode_header(key, value):
    return key + '\x00' + value + '\x00'

class SCGITransport(Transport):
    def single_request(self, host, handler, request_body, verbose=False):
        self.verbose = verbose
        host, port = urllib.splitport(host)

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((host, int(port)))

        request = encode_header('CONTENT_LENGTH', str(len(request_body)))
        request += encode_header('SCGI', '1')
        request += encode_header('REQUEST_METHOD', 'POST')
        request += encode_header('REQUEST_URI', handler)

        request = encode_netstring(request)
        request += request_body

        s.send(request)

        response = ''
        while True:
            r = s.recv(1024)
            if not r:
                break
            response += r

        response_body = BytesIO('\r\n\r\n'.join(response.split('\r\n\r\n')[1:]))

        return self.parse_response(response_body)

if not hasattr(Transport, 'single_request'):
    SCGITransport.request = SCGITransport.single_request
<|endoftext|>