<commit_before>"""setuptools.command.egg_info

Create a distribution's .egg-info directory and contents"""

# This module should be kept compatible with Python 2.3
import os, re, sys
from setuptools import Command
from distutils.errors import *
from distutils import log
from setuptools.command.sdist import sdist
from distutils.util import convert_path
from distutils.filelist import FileList as _FileList
from pkg_resources import parse_requirements, safe_name, parse_version, \
    safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename
from sdist import walk_revctrl

class egg_info(Command):
    description = "create a distribution's .egg-info directory"

    user_options = [
        ('egg-base=', 'e', "directory containing .egg-info directories"
                           " (default: top of the source tree)"),
        ('tag-svn-revision', 'r',
            "Add subversion revision ID to version number"),
        ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"),
        ('tag-build=', 'b', "Specify explicit tag to add to version number"),
        ('no-svn-revision', 'R',
            "Don't add subversion revision ID [default]"),
        ('no-date', 'D', "Don't include date stamp [default]"),
    ]

    boolean_options = ['tag-date', 'tag-svn-revision']
    negative_opt = {'no-svn-revision': 'tag-svn-revision',
                    'no-date': 'tag-date'}

    def initialize_options(self):
        self.egg_name = None
        self.egg_version = None
        self.egg_base = None
        self.egg_info = None
        self.tag_build = None
        self.tag_svn_revision = 0
        self.tag_date = 0
        self.broken_egg_info = False
        self.vtags = None

    def save_version_info(self, filename):
        from setopt import edit_config
        edit_config(
            filename,
            {'egg_info':
                {'tag_svn_revision':0, 'tag_date': 0, 'tag_build': self.tags()}
            }
        )

    def finalize_options (self):
        self.egg_name = safe_name(self.distribution.get_name())
        self.vtags = self.tags()
        self.egg_version = self.tagged_version()

        try:
            list(
                parse_requirements('%s==%s' % (self.egg_name,self.egg_version))
            )
        except ValueError:
            raise DistutilsOptionError(
                "Invalid distribution name or version syntax: %s-%s" %
                (self.egg_name,self.egg_version)
            )

        if self.egg_base is None:
            dirs = self.distribution.package_dir
            self.egg_base = (dirs or {}).get('',os.curdir)

        self.ensure_dirname('egg_base')
        self.egg_info = to_filename(self.egg_name)+'.egg-info'
        if self.egg_base != os.curdir:
            self.egg_info = os.path.join(self.egg_base, self.egg_info)
        if '-' in self.egg_name: self.check_broken_egg_info()

        # Set package version for the benefit of dumber commands
        # (e.g. sdist, bdist_wininst, etc.)
        #
        self.distribution.metadata.version = self.egg_version

        # If we bootstrapped around the lack of a PKG-INFO, as might be the
        # case in a fresh checkout, make sure that any special tags get added
        # to the version info
        #
        pd = self.distribution._patched_dist
        if pd is not None and pd.key==self.egg_name.lower():
            pd._version = self.egg_version
            pd._parsed_version = parse_version(self.egg_version)
            self.distribution._patched_dist = None

    def write_or_delete_file(self, what, filename, data, force=False):
        """Write `data` to `filename` or delete if empty

        If `data` is non-empty, this routine is the same as ``write_file()``.
        If `data` is empty but not ``None``, this is the same as calling
        ``delete_file(filename)`.  If `data` is ``None``, then this is a no-op
        unless `filename` exists, in which case a warning is issued about the
        orphaned file (if `force` is false), or deleted (if `force` is true).
        """
        if data:
            self.write_file(what, filename, data)
        elif os.path.exists(filename):
            if data is None and not force:
                log.warn(
                    "%s not set in setup(), but %s exists", what, filename
                )
                return
            else:
                self.delete_file(filename)

    def write_file(self, what, filename, data):
        """Write `data` to `filename` (if not a dry run) after announcing it

        `what` is used in a log message to identify what is being written
        to the file.
        """
        log.info("writing %s to %s", what, filename)
        if sys.version_info >= (3,):
            data = data.encode("utf-8")
        if not self.dry_run:
            f = open(filename, 'wb')
            f.write(data)
            f.close()

    def delete_file(self, filename):
        """Delete `filename` (if not a dry run) after announcing it"""
        log.info("deleting %s", filename)
        if not self.dry_run:
            os.unlink(filename)

    def tagged_version(self):
        version = self.distribution.get_version()
        # egg_info may be called more than once for a distribution,
        # in which case the version string already contains all tags.
        if self.vtags and version.endswith(self.vtags):
            return safe_version(version)
        return safe_version(version + self.vtags)

    def run(self):
        self.mkpath(self.egg_info)
        installer = self.distribution.fetch_build_egg
        for ep in iter_entry_points('egg_info.writers'):
            writer = ep.load(installer=installer)
            writer(self, ep.name, os.path.join(self.egg_info,ep.name))

        # Get rid of native_libs.txt if it was put there by older bdist_egg
        nl = os.path.join(self.egg_info, "native_libs.txt")
        if os.path.exists(nl):
            self.delete_file(nl)

        self.find_sources()

    def tags(self):
        version = ''
        if self.tag_build:
            version+=self.tag_build
        if self.tag_svn_revision and (
            os.path.exists('.svn') or os.path.exists('PKG-INFO')
        ):  version += '-r%s' % self.get_svn_revision()
        if self.tag_date:
            import time; version += time.strftime("-%Y%m%d")
        return version

    def get_svn_revision(self):
        revision = 0
        urlre = re.compile('url="([^"]+)"')
        revre = re.compile('committed-rev="(\d+)"')

        for base,dirs,files in os.walk(os.curdir):
            if '.svn' not in dirs:
                dirs[:] = []
                continue    # no sense walking uncontrolled subdirs
            dirs.remove('.svn')
            f = open(os.path.join(base,'.svn','entries'))
            data = f.read()
            f.close()

            if data.startswith('10') or data.startswith('9') or data.startswith('8'):
                data = map(str.splitlines,data.split('\n\x0c\n'))
                del data[0][0]  # get rid of the '8' or '9' or '10'
                dirurl = data[0][3]
                localrev = max([int(d[9]) for d in data if len(d)>9 and d[9]]+[0])
            elif data.startswith('<?xml'):
                dirurl = urlre.search(data).group(1)    # get repository URL
                localrev = max([int(m.group(1)) for m in revre.finditer(data)]+[0])
            else:
                log.warn("unrecognized .svn/entries format; skipping %s", base)
                dirs[:] = []
                continue
            if base==os.curdir:
                base_url = dirurl+'/'   # save the root url
            elif not dirurl.startswith(base_url):
                dirs[:] = []
                continue    # not part of the same svn tree, skip it
            revision = max(revision, localrev)

        return str(revision or get_pkg_info_revision())

    def find_sources(self):
        """Generate SOURCES.txt manifest file"""
        manifest_filename = os.path.join(self.egg_info,"SOURCES.txt")
        mm = manifest_maker(self.distribution)
        mm.manifest = manifest_filename
        mm.run()
        self.filelist = mm.filelist

    def check_broken_egg_info(self):
        bei = self.egg_name+'.egg-info'
        if self.egg_base != os.curdir:
            bei = os.path.join(self.egg_base, bei)
        if os.path.exists(bei):
            log.warn(
                "-"*78+'\n'
                "Note: Your current .egg-info directory has a '-' in its name;"
                '\nthis will not work correctly with "setup.py develop".\n\n'
                'Please rename %s to %s to correct this problem.\n'+'-'*78,
                bei, self.egg_info
            )
            self.broken_egg_info = self.egg_info
            self.egg_info = bei     # make it work for now

class FileList(_FileList):
    """File list that accepts only existing, platform-independent paths"""

    def append(self, item):
        if item.endswith('\r'):     # Fix older sdists built on Windows
            item = item[:-1]
        path = convert_path(item)
        # Filter unused template files all of which have ASCII names
        try:
            if sys.version_info >= (3,):
                path.encode('ascii')
        except UnicodeEncodeError:
            self.files.append(path)
        else:
            if os.path.exists(path):
                self.files.append(path)

class manifest_maker(sdist):

    template = "MANIFEST.in"

    def initialize_options (self):
        self.use_defaults = 1
        self.prune = 1
        self.manifest_only = 1
        self.force_manifest = 1

    def finalize_options(self):
        pass

    def run(self):
        self.filelist = FileList()
        if not os.path.exists(self.manifest):
            self.write_manifest()   # it must exist so it'll get in the list
        self.filelist.findall()
        self.add_defaults()
        if os.path.exists(self.template):
            self.read_template()
        self.prune_file_list()
        self.filelist.sort()
        self.filelist.remove_duplicates()
        self.write_manifest()

    def write_manifest (self):
        """Write the file list in 'self.filelist' (presumably as filled in
        by 'add_defaults()' and 'read_template()') to the manifest file
        named by 'self.manifest'.
        """
        # The manifest must be UTF-8 encodable. See #303.
        if sys.version_info >= (3,):
            files = []
            for file in self.filelist.files:
                try:
                    file.encode("utf-8")
                except UnicodeEncodeError:
                    log.warn("'%s' not UTF-8 encodable -- skipping" % file)
                else:
                    files.append(file)
            self.filelist.files = files

        files = self.filelist.files
        if os.sep!='/':
            files = [f.replace(os.sep,'/') for f in files]
        self.execute(write_file, (self.manifest, files),
                     "writing manifest file '%s'" % self.manifest)

    def warn(self, msg):    # suppress missing-file warnings from sdist
        if not msg.startswith("standard file not found:"):
            sdist.warn(self, msg)

    def add_defaults(self):
        sdist.add_defaults(self)
        self.filelist.append(self.template)
        self.filelist.append(self.manifest)
        rcfiles = list(walk_revctrl())
        if rcfiles:
            self.filelist.extend(rcfiles)
        elif os.path.exists(self.manifest):
            self.read_manifest()
        ei_cmd = self.get_finalized_command('egg_info')
        self.filelist.include_pattern("*", prefix=ei_cmd.egg_info)

    def prune_file_list (self):
        build = self.get_finalized_command('build')
        base_dir = self.distribution.get_fullname()
        self.filelist.exclude_pattern(None, prefix=build.build_base)
        self.filelist.exclude_pattern(None, prefix=base_dir)
        sep = re.escape(os.sep)
        self.filelist.exclude_pattern(sep+r'(RCS|CVS|\.svn)'+sep, is_regex=1)

def write_file (filename, contents):
    """Create a file with the specified name and write 'contents' (a
    sequence of strings without line terminators) to it.
    """
    contents = "\n".join(contents)
    if sys.version_info >= (3,):
        contents = contents.encode("utf-8")
    f = open(filename, "wb")        # always write POSIX-style manifest
    f.write(contents)
    f.close()

def write_pkg_info(cmd, basename, filename):
    log.info("writing %s", filename)
    if not cmd.dry_run:
        metadata = cmd.distribution.metadata
        metadata.version, oldver = cmd.egg_version, metadata.version
        metadata.name, oldname   = cmd.egg_name, metadata.name
        try:
            # write unescaped data to PKG-INFO, so older pkg_resources
            # can still parse it
            metadata.write_pkg_info(cmd.egg_info)
        finally:
            metadata.name, metadata.version = oldname, oldver

        safe = getattr(cmd.distribution,'zip_safe',None)
        import bdist_egg; bdist_egg.write_safety_flag(cmd.egg_info, safe)

def warn_depends_obsolete(cmd, basename, filename):
    if os.path.exists(filename):
        log.warn(
            "WARNING: 'depends.txt' is not used by setuptools 0.6!\n"
            "Use the install_requires/extras_require setup() args instead."
        )

def write_requirements(cmd, basename, filename):
    dist = cmd.distribution
    data = ['\n'.join(yield_lines(dist.install_requires or ()))]
    for extra,reqs in (dist.extras_require or {}).items():
        data.append('\n\n[%s]\n%s' % (extra, '\n'.join(yield_lines(reqs))))
    cmd.write_or_delete_file("requirements", filename, ''.join(data))

def write_toplevel_names(cmd, basename, filename):
    pkgs = dict.fromkeys(
        [k.split('.',1)[0]
            for k in cmd.distribution.iter_distribution_names()
        ]
    )
    cmd.write_file("top-level names", filename, '\n'.join(pkgs)+'\n')

def overwrite_arg(cmd, basename, filename):
    write_arg(cmd, basename, filename, True)

def write_arg(cmd, basename, filename, force=False):
    argname = os.path.splitext(basename)[0]
    value = getattr(cmd.distribution, argname, None)
    if value is not None:
        value = '\n'.join(value)+'\n'
    cmd.write_or_delete_file(argname, filename, value, force)

def write_entries(cmd, basename, filename):
    ep = cmd.distribution.entry_points

    if isinstance(ep,basestring) or ep is None:
        data = ep
    elif ep is not None:
        data = []
        for section, contents in ep.items():
            if not isinstance(contents,basestring):
                contents = EntryPoint.parse_group(section, contents)
                contents = '\n'.join(map(str,contents.values()))
            data.append('[%s]\n%s\n\n' % (section,contents))
        data = ''.join(data)

    cmd.write_or_delete_file('entry points', filename, data, True)

def get_pkg_info_revision():
    # See if we can get a -r### off of PKG-INFO, in case this is an sdist of
    # a subversion revision
    #
    if os.path.exists('PKG-INFO'):
        f = open('PKG-INFO','rU')
        for line in f:
            match = re.match(r"Version:.*-r(\d+)\s*$", line)
            if match:
                return int(match.group(1))
        f.close()
    return 0

#
<commit_msg>Revert yesterday's misguided attempt at being smart.<commit_after>"""setuptools.command.egg_info

Create a distribution's .egg-info directory and contents"""

# This module should be kept compatible with Python 2.3
import os, re, sys
from setuptools import Command
from distutils.errors import *
from distutils import log
from setuptools.command.sdist import sdist
from distutils.util import convert_path
from distutils.filelist import FileList as _FileList
from pkg_resources import parse_requirements, safe_name, parse_version, \
    safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename
from sdist import walk_revctrl

class egg_info(Command):
    description = "create a distribution's .egg-info directory"

    user_options = [
        ('egg-base=', 'e', "directory containing .egg-info directories"
                           " (default: top of the source tree)"),
        ('tag-svn-revision', 'r',
            "Add subversion revision ID to version number"),
        ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"),
        ('tag-build=', 'b', "Specify explicit tag to add to version number"),
        ('no-svn-revision', 'R',
            "Don't add subversion revision ID [default]"),
        ('no-date', 'D', "Don't include date stamp [default]"),
    ]

    boolean_options = ['tag-date', 'tag-svn-revision']
    negative_opt = {'no-svn-revision': 'tag-svn-revision',
                    'no-date': 'tag-date'}

    def initialize_options(self):
        self.egg_name = None
        self.egg_version = None
        self.egg_base = None
        self.egg_info = None
        self.tag_build = None
        self.tag_svn_revision = 0
        self.tag_date = 0
        self.broken_egg_info = False
        self.vtags = None

    def save_version_info(self, filename):
        from setopt import edit_config
        edit_config(
            filename,
            {'egg_info':
                {'tag_svn_revision':0, 'tag_date': 0, 'tag_build': self.tags()}
            }
        )

    def finalize_options (self):
        self.egg_name = safe_name(self.distribution.get_name())
        self.vtags = self.tags()
        self.egg_version = self.tagged_version()

        try:
            list(
                parse_requirements('%s==%s' % (self.egg_name,self.egg_version))
            )
        except ValueError:
            raise DistutilsOptionError(
                "Invalid distribution name or version syntax: %s-%s" %
                (self.egg_name,self.egg_version)
            )

        if self.egg_base is None:
            dirs = self.distribution.package_dir
            self.egg_base = (dirs or {}).get('',os.curdir)

        self.ensure_dirname('egg_base')
        self.egg_info = to_filename(self.egg_name)+'.egg-info'
        if self.egg_base != os.curdir:
            self.egg_info = os.path.join(self.egg_base, self.egg_info)
        if '-' in self.egg_name: self.check_broken_egg_info()

        # Set package version for the benefit of dumber commands
        # (e.g. sdist, bdist_wininst, etc.)
        #
        self.distribution.metadata.version = self.egg_version

        # If we bootstrapped around the lack of a PKG-INFO, as might be the
        # case in a fresh checkout, make sure that any special tags get added
        # to the version info
        #
        pd = self.distribution._patched_dist
        if pd is not None and pd.key==self.egg_name.lower():
            pd._version = self.egg_version
            pd._parsed_version = parse_version(self.egg_version)
            self.distribution._patched_dist = None

    def write_or_delete_file(self, what, filename, data, force=False):
        """Write `data` to `filename` or delete if empty

        If `data` is non-empty, this routine is the same as ``write_file()``.
        If `data` is empty but not ``None``, this is the same as calling
        ``delete_file(filename)`.  If `data` is ``None``, then this is a no-op
        unless `filename` exists, in which case a warning is issued about the
        orphaned file (if `force` is false), or deleted (if `force` is true).
        """
        if data:
            self.write_file(what, filename, data)
        elif os.path.exists(filename):
            if data is None and not force:
                log.warn(
                    "%s not set in setup(), but %s exists", what, filename
                )
                return
            else:
                self.delete_file(filename)

    def write_file(self, what, filename, data):
        """Write `data` to `filename` (if not a dry run) after announcing it

        `what` is used in a log message to identify what is being written
        to the file.
        """
        log.info("writing %s to %s", what, filename)
        if sys.version_info >= (3,):
            data = data.encode("utf-8")
        if not self.dry_run:
            f = open(filename, 'wb')
            f.write(data)
            f.close()

    def delete_file(self, filename):
        """Delete `filename` (if not a dry run) after announcing it"""
        log.info("deleting %s", filename)
        if not self.dry_run:
            os.unlink(filename)

    def tagged_version(self):
        version = self.distribution.get_version()
        # egg_info may be called more than once for a distribution,
        # in which case the version string already contains all tags.
        if self.vtags and version.endswith(self.vtags):
            return safe_version(version)
        return safe_version(version + self.vtags)

    def run(self):
        self.mkpath(self.egg_info)
        installer = self.distribution.fetch_build_egg
        for ep in iter_entry_points('egg_info.writers'):
            writer = ep.load(installer=installer)
            writer(self, ep.name, os.path.join(self.egg_info,ep.name))

        # Get rid of native_libs.txt if it was put there by older bdist_egg
        nl = os.path.join(self.egg_info, "native_libs.txt")
        if os.path.exists(nl):
            self.delete_file(nl)

        self.find_sources()

    def tags(self):
        version = ''
        if self.tag_build:
            version+=self.tag_build
        if self.tag_svn_revision and (
            os.path.exists('.svn') or os.path.exists('PKG-INFO')
        ):  version += '-r%s' % self.get_svn_revision()
        if self.tag_date:
            import time; version += time.strftime("-%Y%m%d")
        return version

    def get_svn_revision(self):
        revision = 0
        urlre = re.compile('url="([^"]+)"')
        revre = re.compile('committed-rev="(\d+)"')

        for base,dirs,files in os.walk(os.curdir):
            if '.svn' not in dirs:
                dirs[:] = []
                continue    # no sense walking uncontrolled subdirs
            dirs.remove('.svn')
            f = open(os.path.join(base,'.svn','entries'))
            data = f.read()
            f.close()

            if data.startswith('10') or data.startswith('9') or data.startswith('8'):
                data = map(str.splitlines,data.split('\n\x0c\n'))
                del data[0][0]  # get rid of the '8' or '9' or '10'
                dirurl = data[0][3]
                localrev = max([int(d[9]) for d in data if len(d)>9 and d[9]]+[0])
            elif data.startswith('<?xml'):
                dirurl = urlre.search(data).group(1)    # get repository URL
                localrev = max([int(m.group(1)) for m in revre.finditer(data)]+[0])
            else:
                log.warn("unrecognized .svn/entries format; skipping %s", base)
                dirs[:] = []
                continue
            if base==os.curdir:
                base_url = dirurl+'/'   # save the root url
            elif not dirurl.startswith(base_url):
                dirs[:] = []
                continue    # not part of the same svn tree, skip it
            revision = max(revision, localrev)

        return str(revision or get_pkg_info_revision())

    def find_sources(self):
        """Generate SOURCES.txt manifest file"""
        manifest_filename = os.path.join(self.egg_info,"SOURCES.txt")
        mm = manifest_maker(self.distribution)
        mm.manifest = manifest_filename
        mm.run()
        self.filelist = mm.filelist

    def check_broken_egg_info(self):
        bei = self.egg_name+'.egg-info'
        if self.egg_base != os.curdir:
            bei = os.path.join(self.egg_base, bei)
        if os.path.exists(bei):
            log.warn(
                "-"*78+'\n'
                "Note: Your current .egg-info directory has a '-' in its name;"
                '\nthis will not work correctly with "setup.py develop".\n\n'
                'Please rename %s to %s to correct this problem.\n'+'-'*78,
                bei, self.egg_info
            )
            self.broken_egg_info = self.egg_info
            self.egg_info = bei     # make it work for now

class FileList(_FileList):
    """File list that accepts only existing, platform-independent paths"""

    def append(self, item):
        if item.endswith('\r'):     # Fix older sdists built on Windows
            item = item[:-1]
        path = convert_path(item)
        try:
            if os.path.exists(path):
                self.files.append(path)
            elif path != manifest_maker.template:
                log.warn("%r not found -- skipping", path)
        except UnicodeEncodeError:
            log.warn("%r not %s encodable -- skipping", path,
                sys.getfilesystemencoding())

class manifest_maker(sdist):

    template = "MANIFEST.in"

    def initialize_options (self):
        self.use_defaults = 1
        self.prune = 1
        self.manifest_only = 1
        self.force_manifest = 1

    def finalize_options(self):
        pass

    def run(self):
        self.filelist = FileList()
        if not os.path.exists(self.manifest):
            self.write_manifest()   # it must exist so it'll get in the list
        self.filelist.findall()
        self.add_defaults()
        if os.path.exists(self.template):
            self.read_template()
        self.prune_file_list()
        self.filelist.sort()
        self.filelist.remove_duplicates()
        self.write_manifest()

    def write_manifest (self):
        """Write the file list in 'self.filelist' (presumably as filled in
        by 'add_defaults()' and 'read_template()') to the manifest file
        named by 'self.manifest'.
        """
        # The manifest must be UTF-8 encodable. See #303.
        if sys.version_info >= (3,):
            files = []
            for file in self.filelist.files:
                try:
                    file.encode("utf-8")
                except UnicodeEncodeError:
                    log.warn("'%s' not UTF-8 encodable -- skipping" % file)
                else:
                    files.append(file)
            self.filelist.files = files

        files = self.filelist.files
        if os.sep!='/':
            files = [f.replace(os.sep,'/') for f in files]
        self.execute(write_file, (self.manifest, files),
                     "writing manifest file '%s'" % self.manifest)

    def warn(self, msg):    # suppress missing-file warnings from sdist
        if not msg.startswith("standard file not found:"):
            sdist.warn(self, msg)

    def add_defaults(self):
        sdist.add_defaults(self)
        self.filelist.append(self.template)
        self.filelist.append(self.manifest)
        rcfiles = list(walk_revctrl())
        if rcfiles:
            self.filelist.extend(rcfiles)
        elif os.path.exists(self.manifest):
            self.read_manifest()
        ei_cmd = self.get_finalized_command('egg_info')
        self.filelist.include_pattern("*", prefix=ei_cmd.egg_info)

    def prune_file_list (self):
        build = self.get_finalized_command('build')
        base_dir = self.distribution.get_fullname()
        self.filelist.exclude_pattern(None, prefix=build.build_base)
        self.filelist.exclude_pattern(None, prefix=base_dir)
        sep = re.escape(os.sep)
        self.filelist.exclude_pattern(sep+r'(RCS|CVS|\.svn)'+sep, is_regex=1)

def write_file (filename, contents):
    """Create a file with the specified name and write 'contents' (a
    sequence of strings without line terminators) to it.
    """
    contents = "\n".join(contents)
    if sys.version_info >= (3,):
        contents = contents.encode("utf-8")
    f = open(filename, "wb")        # always write POSIX-style manifest
    f.write(contents)
    f.close()

def write_pkg_info(cmd, basename, filename):
    log.info("writing %s", filename)
    if not cmd.dry_run:
        metadata = cmd.distribution.metadata
        metadata.version, oldver = cmd.egg_version, metadata.version
        metadata.name, oldname   = cmd.egg_name, metadata.name
        try:
            # write unescaped data to PKG-INFO, so older pkg_resources
            # can still parse it
            metadata.write_pkg_info(cmd.egg_info)
        finally:
            metadata.name, metadata.version = oldname, oldver

        safe = getattr(cmd.distribution,'zip_safe',None)
        import bdist_egg; bdist_egg.write_safety_flag(cmd.egg_info, safe)

def warn_depends_obsolete(cmd, basename, filename):
    if os.path.exists(filename):
        log.warn(
            "WARNING: 'depends.txt' is not used by setuptools 0.6!\n"
            "Use the install_requires/extras_require setup() args instead."
        )

def write_requirements(cmd, basename, filename):
    dist = cmd.distribution
    data = ['\n'.join(yield_lines(dist.install_requires or ()))]
    for extra,reqs in (dist.extras_require or {}).items():
        data.append('\n\n[%s]\n%s' % (extra, '\n'.join(yield_lines(reqs))))
    cmd.write_or_delete_file("requirements", filename, ''.join(data))

def write_toplevel_names(cmd, basename, filename):
    pkgs = dict.fromkeys(
        [k.split('.',1)[0]
            for k in cmd.distribution.iter_distribution_names()
        ]
    )
    cmd.write_file("top-level names", filename, '\n'.join(pkgs)+'\n')

def overwrite_arg(cmd, basename, filename):
    write_arg(cmd, basename, filename, True)

def write_arg(cmd, basename, filename, force=False):
    argname = os.path.splitext(basename)[0]
    value = getattr(cmd.distribution, argname, None)
    if value is not None:
        value = '\n'.join(value)+'\n'
    cmd.write_or_delete_file(argname, filename, value, force)

def write_entries(cmd, basename, filename):
    ep = cmd.distribution.entry_points

    if isinstance(ep,basestring) or ep is None:
        data = ep
    elif ep is not None:
        data = []
        for section, contents in ep.items():
            if not isinstance(contents,basestring):
                contents = EntryPoint.parse_group(section, contents)
                contents = '\n'.join(map(str,contents.values()))
            data.append('[%s]\n%s\n\n' % (section,contents))
        data = ''.join(data)

    cmd.write_or_delete_file('entry points', filename, data, True)

def get_pkg_info_revision():
    # See if we can get a -r### off of PKG-INFO, in case this is an sdist of
    # a subversion revision
    #
    if os.path.exists('PKG-INFO'):
        f = open('PKG-INFO','rU')
        for line in f:
            match = re.match(r"Version:.*-r(\d+)\s*$", line)
            if match:
                return int(match.group(1))
        f.close()
    return 0

#
<|endoftext|>


<commit_before>"""
  Cloud connected autonomous RC car.

  Copyright 2016 Visible Energy Inc. All Rights Reserved.
"""
__license__ = """
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import time
import subprocess as sp
import numpy as np
import cv2

# filename to fetch telemetry from -- updated atomically by the controller loop at 30 Hz
TELEMFNAME = '/tmpfs/meta.txt'

# filename where to store the last frame -- used by the application loop and as parameter for the CNN prediction
FRAMEFNAME = '/tmpfs/frame.yuv'

log=None
config=None
camera=None
car=None

def init(conf, logger, carobj):
  global config, log, camera

  config=conf
  log=logger
  car=carobj

  videodevs = ["/dev/" + x for x in os.listdir("/dev/") if x.startswith("video") ]
  if len(videodevs) == 0:
    log("Fatal error. Cannot proceed without cameras connected.")
    return None

  print videodevs
  # create an empty telemetry file
  s = 'echo > ' + TELEMFNAME
  sp.check_call(s, shell=True)

  # use the first camera
  camera=videodevs[0]

  # set resolution and encoding (Logitech C920)
  s = 'v4l2-ctl --device=' + camera + ' --set-fmt-video=width=320,height=240,pixelformat=1'
  log("Setting camera: %s" % s)
  # execute and wait for completion
  sp.check_call(s, shell=True)

  return camera

def video_stop():
  """ Stop running video capture streamer """
  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot stop the video streamer. Streamer not defined in config.json")
    return None

  s = 'killall ' + pname
  FNULL = open(os.devnull, 'w')
  try:
    # execute and wait for completion
    sp.check_call(s, shell=True, stderr=FNULL)
  except Exception, e:
    # fails when no ffmpeg is running
    if config['app_params']['verbose']:
      log("Error stopping streamer. %s" % e)
    else:
      pass
  return

def __video_start(telem):
  """ Start a video streamer """
  global config, log, camera

  # insure no streamer is running
  video_stop()
  time.sleep(1)

  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot start the video streamer. Streamer not defined in config.json")
    return None

  # set video codec depending on platform
  vcodec = 'h264_omx' if 'raspberrypi' in os.uname() else 'h264'
  # to suppress output when running the streamer
  FNULL = open(os.devnull, 'w')
  if telem:
    # streaming video with embedded telemetry
    params = [pname, '-r','30', '-use_wallclock_as_timestamps', '1', '-thread_queue_size', '512', '-f', 'v4l2', '-i', camera, '-c:v ', vcodec, '-maxrate', '768k', '-bufsize', '960k']
    format = 'format=yuv444p,drawbox=y=ih-h:color=black@0.9:width=40:height=12:t=max,drawtext=fontfile=OpenSans-Regular.ttf:textfile=' + TELEMFNAME + ':reload=1:fontsize=10:fontcolor=white:x=0:y=(h-th-2),format=yuv420p'
    url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']
    params = params + ['-vf', format, '-threads', '4', '-r', '30', '-g', '60', '-f', 'flv', url]
    # spawn a process and do not wait
    pid = sp.Popen(params, stderr=FNULL)
  else:
    # streaming video and saving the last frame for CNN prediction
    params = [pname, '-r','30', '-use_wallclock_as_timestamps', '1', '-thread_queue_size', '512', '-f', 'v4l2', '-i', camera, '-c:v ', vcodec, '-maxrate', '768k', '-bufsize', '960k']
    url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']
    params = params + ['-threads', '4', '-r', '30', '-g', '60', '-f', 'flv', url]
    params = params + ['-vcodec', 'rawvideo', '-an', '-updatefirst', '1', '-y', '-f', 'image2', FRAMEFNAME]
    # to transcode use format YUYV422:
    # $ ffmpeg -vcodec rawvideo -s 320x240 -r 1 -pix_fmt  yuyv422  -i frame.yuv rawframe.jpg
    # spawn a process and do not wait
    pid = sp.Popen(params, stderr=FNULL)
  return pid

def video_start(telem):

  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot start the video streamer. Streamer not defined in config.json")
    return None

  i_command = [ pname,
            '-r', '30',
            '-use_wallclock_as_timestamps', '1',
            '-f', 'v4l2',
            '-i', '/dev/video0',
            '-vb','1000k',
            '-f', 'image2pipe',
            '-pix_fmt', 'yuyv422',
            '-vcodec', 'rawvideo', '-']
  i_pipe = sp.Popen(i_command, stdout = sp.PIPE, bufsize=10**5)

  url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']

  o_command = [ pname,
        '-f', 'rawvideo',
        '-vcodec','rawvideo',
        '-s', '320x240', # size of one frame
#         '-pix_fmt', 'rgb24',
        '-pix_fmt', 'rgb24', #'yuyv422',
        '-r', '30', # frames per second
        '-i', 'pipe:0', # The imput comes from a pipe
        '-an', # Tells FFMPEG not to expect any audio
        '-c:v','libx264',
        '-profile:v','main',
        '-preset','ultrafast',
        '-pix_fmt', 'yuv420p',
        '-b:v','1000k',
        '-bufsize','2000k',
        '-g','30',
        '-f','flv',
        url ]
  o_pipe = sp.Popen(o_command, stdin=sp.PIPE, stderr=sp.PIPE)

  width = 320 # 640
  height = 240 #480

  rows = height
  cols = width

  image_size = rows * cols *3 #* 2

  while True:
    raw_image = i_pipe.stdout.read(image_size)

    if telem:
      msg = car.telemetry()
      car.com.send_data(json.dumps(msg))

    f = np.fromstring(raw_image, dtype=np.uint8)
    i_pipe.stdout.flush()

    img = f.reshape(rows, cols, 3) # 2)

    # convert to RGB
    #rgb_img =  cv2.cvtColor(img, cv2.COLOR_YUV2RGB_YUY2)  # working w camera format yuyv422

    rgb_img = img
  #  rgb_img =  cv2.cvtColor(img, cv2.COLOR_YUV420p2RGB)

    # draw a center rectangle
    cv2.rectangle(rgb_img,(130,100),(190,140),(255,0,0),2)
  #  M = cv2.getRotationMatrix2D((width/2,height/2),180,1)

    # rotate the image 90 degrees twice and bring back to normal
  #  dst = cv2.warpAffine(rgb_img,M,(width,height))

    # output the image
    o_pipe.stdin.write(dst.tostring())<commit_msg>properly use streamer in config<commit_after>"""
  Cloud connected autonomous RC car.

  Copyright 2016 Visible Energy Inc. All Rights Reserved.
"""
__license__ = """
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import time
import subprocess as sp
import numpy as np
import cv2

# filename to fetch telemetry from -- updated atomically by the controller loop at 30 Hz
TELEMFNAME = '/tmpfs/meta.txt'

# filename where to store the last frame -- used by the application loop and as parameter for the CNN prediction
FRAMEFNAME = '/tmpfs/frame.yuv'

log=None
config=None
camera=None
car=None

def init(conf, logger, carobj):
  global config, log, camera

  config=conf
  log=logger
  car=carobj

  videodevs = ["/dev/" + x for x in os.listdir("/dev/") if x.startswith("video") ]
  if len(videodevs) == 0:
    log("Fatal error. Cannot proceed without cameras connected.")
    return None

  print videodevs
  # create an empty telemetry file
  s = 'echo > ' + TELEMFNAME
  sp.check_call(s, shell=True)

  # use the first camera
  camera=videodevs[0]

  # set resolution and encoding (Logitech C920)
  s = 'v4l2-ctl --device=' + camera + ' --set-fmt-video=width=320,height=240,pixelformat=1'
  log("Setting camera: %s" % s)
  # execute and wait for completion
  sp.check_call(s, shell=True)

  return camera

def video_stop():
  """ Stop running video capture streamer """
  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot stop the video streamer. Streamer not defined in config.json")
    return None

  s = 'killall ' + pname
  FNULL = open(os.devnull, 'w')
  try:
    # execute and wait for completion
    sp.check_call(s, shell=True, stderr=FNULL)
  except Exception, e:
    # fails when no ffmpeg is running
    if config['app_params']['verbose']:
      log("Error stopping streamer. %s" % e)
    else:
      pass
  return

def __video_start(telem):
  """ Start a video streamer """
  global config, log, camera

  # insure no streamer is running
  video_stop()
  time.sleep(1)

  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot start the video streamer. Streamer not defined in config.json")
    return None

  # set video codec depending on platform
  vcodec = 'h264_omx' if 'raspberrypi' in os.uname() else 'h264'
  # to suppress output when running the streamer
  FNULL = open(os.devnull, 'w')
  if telem:
    # streaming video with embedded telemetry
    params = [pname, '-r','30', '-use_wallclock_as_timestamps', '1', '-thread_queue_size', '512', '-f', 'v4l2', '-i', camera, '-c:v ', vcodec, '-maxrate', '768k', '-bufsize', '960k']
    format = 'format=yuv444p,drawbox=y=ih-h:color=black@0.9:width=40:height=12:t=max,drawtext=fontfile=OpenSans-Regular.ttf:textfile=' + TELEMFNAME + ':reload=1:fontsize=10:fontcolor=white:x=0:y=(h-th-2),format=yuv420p'
    url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']
    params = params + ['-vf', format, '-threads', '4', '-r', '30', '-g', '60', '-f', 'flv', url]
    # spawn a process and do not wait
    pid = sp.Popen(params, stderr=FNULL)
  else:
    # streaming video and saving the last frame for CNN prediction
    params = [pname, '-r','30', '-use_wallclock_as_timestamps', '1', '-thread_queue_size', '512', '-f', 'v4l2', '-i', camera, '-c:v ', vcodec, '-maxrate', '768k', '-bufsize', '960k']
    url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']
    params = params + ['-threads', '4', '-r', '30', '-g', '60', '-f', 'flv', url]
    params = params + ['-vcodec', 'rawvideo', '-an', '-updatefirst', '1', '-y', '-f', 'image2', FRAMEFNAME]
    # to transcode use format YUYV422:
    # $ ffmpeg -vcodec rawvideo -s 320x240 -r 1 -pix_fmt  yuyv422  -i frame.yuv rawframe.jpg
    # spawn a process and do not wait
    pid = sp.Popen(params, stderr=FNULL)
  return pid

def video_start(telem):

  try:
    pname = config['video']['streamer']
  except:
    log("Error cannot start the video streamer. Streamer not defined in config.json")
    return None

  i_command = [ pname,
            '-r', '30',
            '-use_wallclock_as_timestamps', '1',
            '-f', 'v4l2',
            '-i', '/dev/video0',
            '-vb','1000k',
            '-f', 'image2pipe',
            '-pix_fmt', 'yuyv422',
            '-vcodec', 'rawvideo', '-']
  i_pipe = sp.Popen(i_command, stdout = sp.PIPE, bufsize=10**5)

  url = 'rtmp://' + config['video']['server'] + ':' + config['video']['port'] + '/src/' + config['video']['key']

  o_command = [ pname,
        '-f', 'rawvideo',
        '-vcodec','rawvideo',
        '-s', '320x240', # size of one frame
#         '-pix_fmt', 'rgb24',
        '-pix_fmt', 'rgb24', #'yuyv422',
        '-r', '30', # frames per second
        '-i', 'pipe:0', # The imput comes from a pipe
        '-an', # Tells FFMPEG not to expect any audio
        '-c:v','libx264',
        '-profile:v','main',
        '-preset','ultrafast',
        '-pix_fmt', 'yuv420p',
        '-b:v','1000k',
        '-bufsize','2000k',
        '-g','30',
        '-f','flv',
        url ]
  o_pipe = sp.Popen(o_command, stdin=sp.PIPE, stderr=sp.PIPE)

  width = 320 # 640
  height = 240 #480

  rows = height
  cols = width

  image_size = rows * cols *3 #* 2

  while True:
    raw_image = i_pipe.stdout.read(image_size)

    if telem:
      msg = car.telemetry()
      car.com.send_data(json.dumps(msg))

    f = np.fromstring(raw_image, dtype=np.uint8)
    i_pipe.stdout.flush()

    img = f.reshape(rows, cols, 3) # 2)

    # convert to RGB
    #rgb_img =  cv2.cvtColor(img, cv2.COLOR_YUV2RGB_YUY2)  # working w camera format yuyv422

    rgb_img = img
  #  rgb_img =  cv2.cvtColor(img, cv2.COLOR_YUV420p2RGB)

    # draw a center rectangle
    cv2.rectangle(rgb_img,(130,100),(190,140),(255,0,0),2)
  #  M = cv2.getRotationMatrix2D((width/2,height/2),180,1)

    # rotate the image 90 degrees twice and bring back to normal
  #  dst = cv2.warpAffine(rgb_img,M,(width,height))

    # output the image
    o_pipe.stdin.write(rgb_img.tostring())<|endoftext|>


<commit_before># coding=utf-8

__version__ = '0.1.3'
__author__ = 'ricter'
__email__ = 'ricterzheng@gmail.com'
<commit_msg>0.2.0<commit_after># coding=utf-8

__version__ = '0.2.0'
__author__ = 'ricter'
__email__ = 'ricterzheng@gmail.com'
<|endoftext|>


<commit_before>#http://www-rohan.sdsu.edu/~gawron/python_for_ss/course_core/book_draft/web/simple_scraping.html
from lxml.html import parse
from urllib2 import urlopen
from pandas.io.parsers import TextParser

def _unpack (row,kind='td'):
    elts = row.findall('.//%s' % kind)
    return [val.text for val in elts]

def parse_options_data (table):
    rows = table.findall('.//tr')
    header = _unpack(rows[0],kind='th')
    data = [_unpack(r) for r in rows[1:]]
    return TextParser(data,names=header).get_chunk()

if __name__ == '__main__':

    #parsed = parse('http://finance.yahoo.com/q/op?s=AAPL+Options')
    #parsed = parse('http://www-rohan.sdsu.edu/~gawron')

    #parsed = parse('http://www.lajollasurf.org/cgi-bin/plottide.pl')
    url = 'http://www.ezfshn.com/tides/usa/california/san%20diego'
    parsed = parse(url)
    #id="ctl00_ctl00_Content_MCC_RadDatePicker_calendar_Top"

    doc = parsed.getroot()

    links = doc.findall('.//a')

    links_sub_list = links[15:20]
    lnk = links_sub_list[0]

    sample_url = lnk.get('href')

    sample_display_text = lnk.text_content()

    tables = doc.findall('.//table')
    ## Look at tables,  find a table of interest
    #puts = tables[9]
    ## Ditto
    #calls = tables[13]
    dt = tables[0]

    rows = dt.findall('.//tr')

    headers = _unpack(rows[0],kind='th')

    row_vals =  _unpack(rows[1],kind='td')

    #call_data = parse_options_data(calls)
    tide_data = parse_options_data(dt)

    print tide_data[:10]<commit_msg>Delete test03.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

import os
import sys
import argparse
import readline

import lexer      as l
import parser     as p
import evaluator  as e
import context    as c
import obj        as o

def main():
    parser = argparse.ArgumentParser(description="The interpreter for Pluto")

    parser.add_argument("-f", "--file", action="store", dest="file", type=str, help="the file to execute")
    parser.add_argument("-p", "--parse", action="store_true", default=False, help="just parse the file - don't execute it")
    parser.add_argument("-t", "--tree", action="store_true", default=False, help="print the parse tree")
    parser.add_argument("-i", "--interactive", action="store_true", default=False, help="enter interactive mode after the file has been run")
    parser.add_argument("-v", "--version", action="version", version="Pluto, early beta version")

    args = parser.parse_args()

    if args.file == None:
        ctx = c.Context()
        import_prelude(ctx)
        repl(ctx)
    else:
        try:
            text = open(args.file).read()

            if args.parse or args.tree:
                tokens = l.lex(text)
                parse = p.Parser(tokens)
                program = parse.parse_program()

                if len(parse.errors) > 0:
                    parse.print_errors()
                elif args.tree:
                    print(program)

                return

            ctx = c.Context()
            import_prelude(ctx)
            execute(text, False, ctx)

            if args.interactive:
                print()
                repl(ctx)

        except FileNotFoundError:
            print("File not found: %s" % args.file)
            return

def execute(text, print_result, ctx):
    tokens = l.lex(text)
    parser = p.Parser(tokens)
    program = parser.parse_program()

    if len(parser.errors) > 0:
        parser.print_errors()
    else:
        result = e.evaluate(program, ctx)

        if (print_result and type(result) != o.Null) or type(result) == o.Error:
            print(result)

def repl(ctx):
    print("Pluto REPL - https://pluto.zacgarby.co.uk")
    print("Copyright (c) Zac Garby <me@zacgarby.co.uk>")

    while True:
        try:
            string = input(">> ")

            if string[:-1] == "exit":
                break

            execute(string, True, ctx)
        except (KeyboardInterrupt, EOFError):
            print("Goodbye!")
            sys.exit()

def import_prelude(ctx):
    print("Importing prelude")
    src_path = os.path.dirname(os.path.realpath(__file__))
    prelude_path = os.path.join(src_path, "lib/prelude.pluto")

    with open(prelude_path, "r") as prelude:
        text = prelude.read()
        execute(text, False, ctx)

def run_file():
    with open(sys.argv[1], "r") as f:
        content = f.read()
        ctx = c.Context()
        import_prelude(ctx)
        execute(content, False, ctx)

if __name__ == "__main__":
    main()

<commit_msg>Remove debug printing<commit_after>#!/usr/bin/env python3

import os
import sys
import argparse
import readline

import lexer      as l
import parser     as p
import evaluator  as e
import context    as c
import obj        as o

def main():
    parser = argparse.ArgumentParser(description="The interpreter for Pluto")

    parser.add_argument("-f", "--file", action="store", dest="file", type=str, help="the file to execute")
    parser.add_argument("-p", "--parse", action="store_true", default=False, help="just parse the file - don't execute it")
    parser.add_argument("-t", "--tree", action="store_true", default=False, help="print the parse tree")
    parser.add_argument("-i", "--interactive", action="store_true", default=False, help="enter interactive mode after the file has been run")
    parser.add_argument("-v", "--version", action="version", version="Pluto, early beta version")

    args = parser.parse_args()

    if args.file == None:
        ctx = c.Context()
        import_prelude(ctx)
        repl(ctx)
    else:
        try:
            text = open(args.file).read()

            if args.parse or args.tree:
                tokens = l.lex(text)
                parse = p.Parser(tokens)
                program = parse.parse_program()

                if len(parse.errors) > 0:
                    parse.print_errors()
                elif args.tree:
                    print(program)

                return

            ctx = c.Context()
            import_prelude(ctx)
            execute(text, False, ctx)

            if args.interactive:
                print()
                repl(ctx)

        except FileNotFoundError:
            print("File not found: %s" % args.file)
            return

def execute(text, print_result, ctx):
    tokens = l.lex(text)
    parser = p.Parser(tokens)
    program = parser.parse_program()

    if len(parser.errors) > 0:
        parser.print_errors()
    else:
        result = e.evaluate(program, ctx)

        if (print_result and type(result) != o.Null) or type(result) == o.Error:
            print(result)

def repl(ctx):
    print("Pluto REPL - https://pluto.zacgarby.co.uk")
    print("Copyright (c) Zac Garby <me@zacgarby.co.uk>")

    while True:
        try:
            string = input(">> ")

            if string[:-1] == "exit":
                break

            execute(string, True, ctx)
        except (KeyboardInterrupt, EOFError):
            print("Goodbye!")
            sys.exit()

def import_prelude(ctx):
    src_path = os.path.dirname(os.path.realpath(__file__))
    prelude_path = os.path.join(src_path, "lib/prelude.pluto")

    with open(prelude_path, "r") as prelude:
        text = prelude.read()
        execute(text, False, ctx)

def run_file():
    with open(sys.argv[1], "r") as f:
        content = f.read()
        ctx = c.Context()
        import_prelude(ctx)
        execute(content, False, ctx)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>import unittest

from datetime import date

from DateObjects import Date

class TestDateOperations(unittest.TestCase):

    def test_eq(self):
        self.assertTrue(Date(1991, 10, 25) == Date(1991, 10, 25))
        self.assertFalse(Date(1991, 10, 25) == Date(1992, 10, 25))
        self.assertFalse(Date(1991, 10, 25) == Date(1991, 10, 26))
        self.assertFalse(Date(1991, 10, 25) == Date(1991, 11, 25))

    def text_lt(self):
        self.assertFalse(Date(1991, 10, 25) < Date(1991, 10, 25))
        self.assertTrue(Date(1991, 10, 25) < Date(1992, 10, 25))
        self.assertTrue(Date(1991, 10, 25) < Date(1991, 10, 26))
        self.assertTrue(Date(1991, 10, 25) < Date(1991, 11, 25))

    def text_le(self):
        self.assertTrue(Date(1991, 10, 25) <= Date(1991, 10, 25))
        self.assertTrue(Date(1991, 10, 25) <= Date(1992, 10, 25))
        self.assertTrue(Date(1991, 10, 25) <= Date(1991, 10, 26))
        self.assertTrue(Date(1991, 10, 25) <= Date(1991, 11, 25))

    def text_ne(self):
        self.assertFalse(Date(1991, 10, 25) != Date(1991, 10, 25))
        self.assertTrue(Date(1991, 10, 25) != Date(1992, 10, 25))
        self.assertTrue(Date(1991, 10, 25) != Date(1991, 10, 26))
        self.assertTrue(Date(1991, 10, 25) != Date(1991, 11, 25))

    def text_gt(self):
        self.assertFalse(Date(1991, 10, 25) > Date(1991, 10, 25))
        self.assertTrue(Date(1991, 11, 25) > Date(1991, 10, 25))
        self.assertTrue(Date(1991, 10, 26) > Date(1991, 10, 25))
        self.assertTrue(Date(1992, 10, 25) > Date(1991, 10, 25))

    def text_ge(self):
        self.assertTrue(Date(1991, 10, 25) >= Date(1991, 10, 25))
        self.assertTrue(Date(1991, 11, 25) >= Date(1991, 10, 25))
        self.assertTrue(Date(1991, 10, 26) >= Date(1991, 10, 25))
        self.assertTrue(Date(1992, 10, 25) >= Date(1991, 10, 25))

    def test_to_datetime(self):
        self.assertEqual(Date(1991, 10, 25).to_datetime(), date(1991, 10, 25))

    def test_get_last_day_of_month(self):
        self.assertEqual(Date(1991, 1, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2020, 2, 1).get_last_day_of_month(), 29)
        self.assertEqual(Date(2021, 2, 1).get_last_day_of_month(), 28)
        self.assertEqual(Date(2021, 3, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 4, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 5, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 6, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 7, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 8, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 9, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 10, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 11, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 12, 1).get_last_day_of_month(), 31)

    def is_last_day(self):
        self.assertTrue(Date(1991, 2, 28).is_last_day())
        self.assertTrue(Date(2020, 2, 29).is_last_day())
        self.assertFalse(Date(1991, 2, 1).is_last_day())
        self.assertFalse(Date(2020, 2, 28).is_last_day())

    def is_last_month(self):
        self.assertFalse(Date(1991, 10, 25).is_last_day())
        self.assertTrue(Date(1991, 12, 25).is_last_day())

    def is_first_day(self):
        self.assertFalse(Date(1991, 10, 2).is_last_day())
        self.assertTrue(Date(1991, 10, 1).is_last_day())

    def is_first_month(self):
        self.assertFalse(Date(1991, 12, 2).is_last_day())
        self.assertTrue(Date(1991, 1991, 2).is_last_day())

    def is_leap(self):
        self.assertFalse(Date(1201, 1, 1).is_last_day())
        self.assertFalse(Date(1330, 1, 1).is_last_day())
        self.assertFalse(Date(1646, 1, 1).is_last_day())
        self.assertTrue(Date(1288, 1, 1).is_last_day())
        self.assertTrue(Date(1460, 1, 1).is_last_day())
        self.assertTrue(Date(1772, 1, 1).is_last_day())

    def test_invalid_date(self):
        self.assertRaises(ValueError, lambda: Date(1991, 0, 1))
        self.assertRaises(ValueError, lambda: Date(1991, 13, 1))
        self.assertRaises(ValueError, lambda: Date(1991, 1, 0))
        self.assertRaises(ValueError, lambda: Date(1991, 1, 32))

if __name__ == '__main__':
    unittest.main()
<commit_msg>more tests<commit_after>import unittest

from datetime import date

from DateObjects import Date, Day, Month, Year

class TestDayOperations(unittest.TestCase):
    def test_eq(self):
        self.assertEqual(Day(1), Day(1))

    def test_ne(self):
        self.assertNotEqual(Day(1), Day(2))

    def test_lt(self):
        self.assertLess(Day(1), Day(2))

    def test_le(self):
        self.assertLessEqual(Day(1), Day(2))
        self.assertLessEqual(Day(2), Day(2))

    def test_gt(self):
        self.assertGreater(Day(2), Day(1))

    def test_ge(self):
        self.assertGreaterEqual(Day(2), Day(1))
        self.assertGreaterEqual(Day(2), Day(2))

    def test_is_first(self):
        self.assertTrue(Day(1).is_first())
        self.assertFalse(Day(2).is_first())

    def test_is_last(self):
        self.assertFalse(Day(1).is_last())
        self.assertTrue(Day(31).is_last())
        self.assertTrue(Day(28).is_last(month=Month(2)))
        self.assertTrue(Day(29).is_last(month=Month(2), year=Year(2020)))

    def test_invalid_day(self):
        self.assertRaises(ValueError, lambda: Day(0))
        self.assertRaises(ValueError, lambda: Day(32))

class TestMonthOperations(unittest.TestCase):
    def test_eq(self):
        self.assertEqual(Month(1), Month(1))

    def test_ne(self):
        self.assertNotEqual(Month(1), Month(2))

    def test_lt(self):
        self.assertLess(Month(1), Month(2))

    def test_le(self):
        self.assertLessEqual(Month(1), Month(2))
        self.assertLessEqual(Month(2), Month(2))

    def test_gt(self):
        self.assertGreater(Month(2), Month(1))

    def test_ge(self):
        self.assertGreaterEqual(Month(2), Month(1))
        self.assertGreaterEqual(Month(2), Month(2))

    def test_is_first(self):
        self.assertTrue(Month(1).is_first())
        self.assertFalse(Month(2).is_first())

    def test_is_last(self):
        self.assertFalse(Month(1).is_last())
        self.assertTrue(Month(12).is_last())

    def test_get_last_day(self):
        self.assertEqual(Month(1).get_last_day(), 31)
        self.assertEqual(Month(2).get_last_day(), 28)
        self.assertEqual(Month(2).get_last_day(year=Year(1991)), 28)
        self.assertEqual(Month(2).get_last_day(year=Year(1992)), 29)
        self.assertEqual(Month(3).get_last_day(), 31)
        self.assertEqual(Month(4).get_last_day(), 30)
        self.assertEqual(Month(5).get_last_day(), 31)
        self.assertEqual(Month(6).get_last_day(), 30)
        self.assertEqual(Month(7).get_last_day(), 31)
        self.assertEqual(Month(8).get_last_day(), 31)
        self.assertEqual(Month(9).get_last_day(), 30)
        self.assertEqual(Month(10).get_last_day(), 31)
        self.assertEqual(Month(11).get_last_day(), 30)
        self.assertEqual(Month(12).get_last_day(), 31)

    def test_invalid_month(self):
        self.assertRaises(ValueError, lambda: Month(0))
        self.assertRaises(ValueError, lambda: Month(13))

class TestYearOperations(unittest.TestCase):
    def test_eq(self):
        self.assertEqual(Year(1), Year(1))

    def test_ne(self):
        self.assertNotEqual(Year(1), Year(2))

    def test_lt(self):
        self.assertLess(Year(1), Year(2))

    def test_le(self):
        self.assertLessEqual(Year(1), Year(2))
        self.assertLessEqual(Year(2), Year(2))

    def test_gt(self):
        self.assertGreater(Year(2), Year(1))

    def test_ge(self):
        self.assertGreaterEqual(Year(2), Year(1))
        self.assertGreaterEqual(Year(2), Year(2))

    def test_is_leap(self):
        self.assertTrue(Year(2020).is_leap())
        self.assertTrue(Year(1992).is_leap())
        self.assertFalse(Year(1991).is_leap())

class TestDateOperations(unittest.TestCase):

    def test_eq(self):
        self.assertEqual(Date(1991, 10, 25), Date(1991, 10, 25))

    def test_ne(self):
        self.assertNotEqual(Date(1991, 10, 25), Date(1992, 10, 25))
        self.assertNotEqual(Date(1991, 10, 25), Date(1991, 10, 26))
        self.assertNotEqual(Date(1991, 10, 25), Date(1991, 11, 25))

    def test_lt(self):
        self.assertFalse(Date(1991, 10, 25) < Date(1991, 10, 25))
        self.assertLess(Date(1991, 10, 25), Date(1992, 10, 25))
        self.assertLess(Date(1991, 10, 25), Date(1991, 10, 26))
        self.assertLess(Date(1991, 10, 25), Date(1991, 11, 25))

    def test_le(self):
        self.assertLessEqual(Date(1991, 10, 25), Date(1991, 10, 25))
        self.assertLessEqual(Date(1991, 10, 25), Date(1992, 10, 25))
        self.assertLessEqual(Date(1991, 10, 25), Date(1991, 10, 26))
        self.assertLessEqual(Date(1991, 10, 25), Date(1991, 11, 25))

    def test_gt(self):
        self.assertFalse(Date(1991, 10, 25) > Date(1991, 10, 25))
        self.assertGreater(Date(1991, 11, 25), Date(1991, 10, 25))
        self.assertGreater(Date(1991, 10, 26), Date(1991, 10, 25))
        self.assertGreater(Date(1992, 10, 25), Date(1991, 10, 25))

    def test_ge(self):
        self.assertGreaterEqual(Date(1991, 10, 25), Date(1991, 10, 25))
        self.assertGreaterEqual(Date(1991, 11, 25), Date(1991, 10, 25))
        self.assertGreaterEqual(Date(1991, 10, 26), Date(1991, 10, 25))
        self.assertGreaterEqual(Date(1992, 10, 25), Date(1991, 10, 25))

    def test_to_datetime(self):
        self.assertEqual(Date(1991, 10, 25).to_datetime(), date(1991, 10, 25))

    def test_get_last_day_of_month(self):
        self.assertEqual(Date(1991, 1, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2020, 2, 1).get_last_day_of_month(), 29)
        self.assertEqual(Date(2021, 2, 1).get_last_day_of_month(), 28)
        self.assertEqual(Date(2021, 3, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 4, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 5, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 6, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 7, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 8, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 9, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 10, 1).get_last_day_of_month(), 31)
        self.assertEqual(Date(2021, 11, 1).get_last_day_of_month(), 30)
        self.assertEqual(Date(2021, 12, 1).get_last_day_of_month(), 31)

    def test_is_last_day(self):
        self.assertTrue(Date(1991, 2, 28).is_last_day())
        self.assertTrue(Date(2020, 2, 29).is_last_day())
        self.assertFalse(Date(1991, 2, 1).is_last_day())
        self.assertFalse(Date(2020, 2, 28).is_last_day())

    def test_is_last_month(self):
        self.assertFalse(Date(1991, 10, 25).is_last_day())
        self.assertTrue(Date(1991, 12, 31).is_last_day())

    def test_is_first_day(self):
        self.assertFalse(Date(1991, 10, 2).is_last_day())
        self.assertTrue(Date(1991, 10, 31).is_last_day())
        self.assertTrue(Date(1991, 2, 28).is_last_day())
        self.assertTrue(Date(1992, 2, 29).is_last_day())

    def test_is_first_month(self):
        self.assertFalse(Date(1991, 1, 2).is_last_month())
        self.assertTrue(Date(1991, 12, 28).is_last_month())

    def test_is_leap(self):
        self.assertFalse(Date(1201, 1, 1).is_leap())
        self.assertFalse(Date(1330, 1, 1).is_leap())
        self.assertFalse(Date(1646, 1, 1).is_leap())
        self.assertTrue(Date(1288, 1, 1).is_leap())
        self.assertTrue(Date(1460, 1, 1).is_leap())
        self.assertTrue(Date(1772, 1, 1).is_leap())

    def test_invalid_date(self):
        self.assertRaises(ValueError, lambda: Date(1991, 0, 1))
        self.assertRaises(ValueError, lambda: Date(1991, 13, 1))
        self.assertRaises(ValueError, lambda: Date(1991, 1, 0))
        self.assertRaises(ValueError, lambda: Date(1991, 1, 32))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>{
  'targets': [
    {
      'target_name': 'tests',
      'type': 'executable',
      'dependencies': [
        'project.gyp:runtime',
        'project.gyp:compiler'
      ],
      'include_dirs': [
        'src',
        'spec',
        'externals/bandit',
        'externals/utf8proc',
      ],
      'sources': [
        'spec/specs.cc',
        '<!@(find spec/compiler -name "*.cc")',
        '<!@(find spec/runtime -name "*.cc")',
        '<!@(find spec/integration -name "*.cc")',
        '<!@(find spec/helpers -name "*.cc")',
      ],
      'default_configuration': 'Debug',
      'configurations': {'Debug': {}, 'Release': {}},
      'cflags': [
        '-g',
        '-O0',
        '-Wall',
        '-Wextra',
        '-Wno-unused-parameter',
        '-Wno-unknown-pragmas',
      ],
      'cflags_c': [
        '-std=c99',
      ],
      'cflags_cc': [
        '-std=c++0x',
      ],
      'ldflags': [
        '-g',
      ],
      'variables': {
        'USE_LIBPROFILER%': 'true',
      },
      'conditions': [
        ['USE_LIBPROFILER != "false"', {
          'libraries': ['-lprofiler'],
        }]
      ],
      'xcode_settings': {
        'CLANG_CXX_LANGUAGE_STANDARD': 'c++11',
        'OTHER_LDFLAGS': ['-g'],
        'GCC_OPTIMIZATION_LEVEL': '0',
        'ALWAYS_SEARCH_USER_PATHS': 'NO',
        'WARNING_CFLAGS': [
          '-Wall',
          '-Wextra',
          '-Wno-unused-parameter'
        ],
      },
    }
  ]
}
<commit_msg>Link to libld in specs<commit_after>{
  'targets': [
    {
      'target_name': 'tests',
      'type': 'executable',
      'dependencies': [
        'project.gyp:runtime',
        'project.gyp:compiler'
      ],
      'include_dirs': [
        'src',
        'spec',
        'externals/bandit',
        'externals/utf8proc',
      ],
      'sources': [
        'spec/specs.cc',
        '<!@(find spec/compiler -name "*.cc")',
        '<!@(find spec/runtime -name "*.cc")',
        '<!@(find spec/integration -name "*.cc")',
        '<!@(find spec/helpers -name "*.cc")',
      ],
      'libraries': [
        '-ldl'
      ],
      'default_configuration': 'Debug',
      'configurations': {'Debug': {}, 'Release': {}},
      'cflags': [
        '-g',
        '-O0',
        '-Wall',
        '-Wextra',
        '-Wno-unused-parameter',
        '-Wno-unknown-pragmas',
      ],
      'cflags_c': [
        '-std=c99',
      ],
      'cflags_cc': [
        '-std=c++0x',
      ],
      'ldflags': [
        '-g',
      ],
      'variables': {
        'USE_LIBPROFILER%': 'true',
      },
      'conditions': [
        ['USE_LIBPROFILER != "false"', {
          'libraries': ['-lprofiler'],
        }]
      ],
      'xcode_settings': {
        'CLANG_CXX_LANGUAGE_STANDARD': 'c++11',
        'OTHER_LDFLAGS': ['-g'],
        'GCC_OPTIMIZATION_LEVEL': '0',
        'ALWAYS_SEARCH_USER_PATHS': 'NO',
        'WARNING_CFLAGS': [
          '-Wall',
          '-Wextra',
          '-Wno-unused-parameter'
        ],
      },
    }
  ]
}
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import division

import copy
import math
from shadow.net_spec import Shadow

def get_arg(shadow_op, arg_name, arg_type, default_value):
    for arg in shadow_op.arg:
        if arg.name == arg_name:
            if arg_type == 's_i':
                return arg.s_i
            elif arg_type == 's_f':
                return arg.s_f
            elif arg_type == 's_s':
                return arg.s_s
            elif arg_type == 'v_i':
                return arg.v_i
            elif arg_type == 'v_f':
                return arg.v_f
            elif arg_type == 'v_s':
                return arg.v_s
    return default_value

def convert_batch_norm(conv_op, bn_op, scale_op, merged_net, copy_params):
    merged_op = merged_net.add_op()
    merged_op.name = conv_op.name
    merged_op.type = conv_op.type
    merged_op.top.extend(scale_op.top)
    merged_op.bottom.extend(conv_op.bottom)

    for arg in conv_op.arg:
        if arg.name != 'bias_term':
            merged_op.arg.add().CopyFrom(arg)

    if not copy_params:
        return

    has_bias = get_arg(conv_op, 'bias_term', 's_i', 1)
    weight = copy.deepcopy(conv_op.blobs[0].data_f)
    out_c = conv_op.blobs[0].shape[0]
    bias = [0 for i in range(0, out_c)]
    if has_bias:
        bias = copy.deepcopy(conv_op.blobs[1].data_f)

    bn_mean = copy.deepcopy(bn_op.blobs[0].data_f)
    bn_var = copy.deepcopy(bn_op.blobs[1].data_f)
    bn_scale = bn_op.blobs[2].data_f[0]
    if bn_scale != 0:
        bn_scale = 1 / bn_scale
    for i in range(0, len(bn_mean)):
        bn_mean[i] *= bn_scale
        bn_var[i] = 1 / math.sqrt(abs(bn_var[i]) * bn_scale + 0.000001)

    scale_scale = copy.deepcopy(scale_op.blobs[0].data_f)
    scale_bias = copy.deepcopy(scale_op.blobs[1].data_f)
    spatial = int(len(weight) / out_c)
    for i in range(0, len(weight)):
        c = int(i / spatial)
        weight[i] *= bn_var[c] * scale_scale[c]
    for i in range(0, len(bias)):
        bias[i] = (bias[i] - bn_mean[i]) * bn_var[i] * scale_scale[i] + scale_bias[i]

    merge_weight_blob = merged_op.blobs.add()
    merge_bias_blob = merged_op.blobs.add()
    merge_weight_blob.shape.extend(conv_op.blobs[0].shape)
    merge_weight_blob.data_f.extend(weight)
    merge_bias_blob.shape.append(out_c)
    merge_bias_blob.data_f.extend(bias)

def convert_activate(conv_op, ac_op, merged_net):
    merged_op = merged_net.add_op()
    merged_op.name = conv_op.name
    merged_op.type = conv_op.type
    merged_op.top.extend(ac_op.top)
    merged_op.bottom.extend(conv_op.bottom)

    for blob in conv_op.blobs:
        merged_op.blobs.add().CopyFrom(blob)

    for arg in conv_op.arg:
        merged_op.arg.add().CopyFrom(arg)

    bias_arg = merged_op.arg.add()
    bias_arg.name = 'type'
    bias_arg.s_i = 1

def MergeBatchNorm(shadow_net, copy_params):
    merged_net = Shadow()
    for n, ori_net in enumerate(shadow_net.get_nets()):
        merged_net.set_net(n)
        merged_net.set_net_name(ori_net.name)
        merged_net.set_net_num_class(ori_net.num_class)
        merged_net.set_net_out_blob(ori_net.out_blob)
        merged_net.copy_net_arg(ori_net.arg)
        op_size = len(ori_net.op)
        o = 0
        while o < op_size:
            op = ori_net.op[o]
            if 'Conv' in op.type:
                bn_index, scale_index = o + 1, o + 2
                has_bn, has_scale = False, False
                if bn_index < op_size:
                    has_bn = ori_net.op[bn_index].type == 'BatchNorm'
                if scale_index < op_size:
                    has_scale = ori_net.op[scale_index].type == 'Scale'
                if has_bn and has_scale:
                    convert_batch_norm(op, ori_net.op[bn_index], ori_net.op[scale_index], merged_net, copy_params)
                    o += 3
                    continue
            merged_net.add_op().CopyFrom(op)
            o += 1
    return merged_net

def MergeActivate(shadow_net):
    merged_net = Shadow()
    for n, ori_net in enumerate(shadow_net.get_nets()):
        merged_net.set_net(n)
        merged_net.set_net_name(ori_net.name)
        merged_net.set_net_num_class(ori_net.num_class)
        merged_net.set_net_out_blob(ori_net.out_blob)
        merged_net.copy_net_arg(ori_net.arg)
        op_size = len(ori_net.op)
        o = 0
        while o < op_size:
            op = ori_net.op[o]
            if 'Conv' in op.type:
                ac_index = o + 1
                has_ac = False
                if ac_index < op_size:
                    ac_op = ori_net.op[ac_index]
                    has_ac = ac_op.type == 'Activate' and get_arg(ac_op, 'type', 's_i', -1) == 1
                if has_ac:
                    convert_activate(op, ori_net.op[ac_index], merged_net)
                    o += 2
                    continue
            merged_net.add_op().CopyFrom(op)
            o += 1
    return merged_net

def Merge(shadow_net, copy_params, merge_activate=True):
    merged_bn_net = MergeBatchNorm(shadow_net, copy_params)
    if merge_activate:
        return MergeActivate(merged_bn_net)
    else:
        return merged_bn_net
<commit_msg>fix bug: fix bn eps in merger<commit_after>from __future__ import print_function
from __future__ import division

import copy
import math
from shadow.net_spec import Shadow

def get_arg(shadow_op, arg_name, arg_type, default_value):
    for arg in shadow_op.arg:
        if arg.name == arg_name:
            if arg_type == 's_i':
                return arg.s_i
            elif arg_type == 's_f':
                return arg.s_f
            elif arg_type == 's_s':
                return arg.s_s
            elif arg_type == 'v_i':
                return arg.v_i
            elif arg_type == 'v_f':
                return arg.v_f
            elif arg_type == 'v_s':
                return arg.v_s
    return default_value

def convert_batch_norm(conv_op, bn_op, scale_op, merged_net, copy_params):
    merged_op = merged_net.add_op()
    merged_op.name = conv_op.name
    merged_op.type = conv_op.type
    merged_op.top.extend(scale_op.top)
    merged_op.bottom.extend(conv_op.bottom)

    merge_weight_blob = merged_op.blobs.add()
    merge_bias_blob = merged_op.blobs.add()

    merge_weight_blob.shape.extend(conv_op.blobs[0].shape)
    merge_bias_blob.shape.append(conv_op.blobs[0].shape[0])

    for arg in conv_op.arg:
        if arg.name != 'bias_term':
            merged_op.arg.add().CopyFrom(arg)

    if not copy_params:
        return

    has_bias = get_arg(conv_op, 'bias_term', 's_i', 1)
    weight = copy.deepcopy(conv_op.blobs[0].data_f)
    out_c = conv_op.blobs[0].shape[0]
    bias = [0] * out_c
    if has_bias:
        bias = copy.deepcopy(conv_op.blobs[1].data_f)

    eps = get_arg(bn_op, 'eps', 's_f', 1e-5)
    bn_mean = copy.deepcopy(bn_op.blobs[0].data_f)
    bn_var = copy.deepcopy(bn_op.blobs[1].data_f)
    bn_scale = bn_op.blobs[2].data_f[0]
    if bn_scale != 0:
        bn_scale = 1 / bn_scale
    for i in range(0, len(bn_mean)):
        bn_mean[i] *= bn_scale
        bn_var[i] = 1 / math.sqrt(abs(bn_var[i]) * bn_scale + eps)

    scale_scale = copy.deepcopy(scale_op.blobs[0].data_f)
    scale_bias = copy.deepcopy(scale_op.blobs[1].data_f)
    spatial = int(len(weight) / out_c)
    for i in range(0, len(weight)):
        c = int(i / spatial)
        weight[i] *= bn_var[c] * scale_scale[c]
    for i in range(0, len(bias)):
        bias[i] = (bias[i] - bn_mean[i]) * bn_var[i] * scale_scale[i] + scale_bias[i]

    merge_weight_blob.data_f.extend(weight)
    merge_bias_blob.data_f.extend(bias)

def convert_activate(conv_op, ac_op, merged_net):
    merged_op = merged_net.add_op()
    merged_op.name = conv_op.name
    merged_op.type = conv_op.type
    merged_op.top.extend(ac_op.top)
    merged_op.bottom.extend(conv_op.bottom)

    for blob in conv_op.blobs:
        merged_op.blobs.add().CopyFrom(blob)

    for arg in conv_op.arg:
        merged_op.arg.add().CopyFrom(arg)

    bias_arg = merged_op.arg.add()
    bias_arg.name = 'type'
    bias_arg.s_i = 1

def MergeBatchNorm(shadow_net, copy_params):
    merged_net = Shadow()
    for n, ori_net in enumerate(shadow_net.get_nets()):
        merged_net.set_net(n)
        merged_net.set_net_name(ori_net.name)
        merged_net.set_net_num_class(ori_net.num_class)
        merged_net.set_net_out_blob(ori_net.out_blob)
        merged_net.copy_net_arg(ori_net.arg)
        op_size = len(ori_net.op)
        o = 0
        while o < op_size:
            op = ori_net.op[o]
            if 'Conv' in op.type:
                bn_index, scale_index = o + 1, o + 2
                has_bn, has_scale = False, False
                if bn_index < op_size:
                    has_bn = ori_net.op[bn_index].type == 'BatchNorm'
                if scale_index < op_size:
                    has_scale = ori_net.op[scale_index].type == 'Scale'
                if has_bn and has_scale:
                    convert_batch_norm(op, ori_net.op[bn_index], ori_net.op[scale_index], merged_net, copy_params)
                    o += 3
                    continue
            merged_net.add_op().CopyFrom(op)
            o += 1
    return merged_net

def MergeActivate(shadow_net):
    merged_net = Shadow()
    for n, ori_net in enumerate(shadow_net.get_nets()):
        merged_net.set_net(n)
        merged_net.set_net_name(ori_net.name)
        merged_net.set_net_num_class(ori_net.num_class)
        merged_net.set_net_out_blob(ori_net.out_blob)
        merged_net.copy_net_arg(ori_net.arg)
        op_size = len(ori_net.op)
        o = 0
        while o < op_size:
            op = ori_net.op[o]
            if 'Conv' in op.type:
                ac_index = o + 1
                has_ac = False
                if ac_index < op_size:
                    ac_op = ori_net.op[ac_index]
                    has_ac = ac_op.type == 'Activate' and get_arg(ac_op, 'type', 's_i', -1) == 1
                if has_ac:
                    convert_activate(op, ori_net.op[ac_index], merged_net)
                    o += 2
                    continue
            merged_net.add_op().CopyFrom(op)
            o += 1
    return merged_net

def Merge(shadow_net, copy_params, merge_activate=True):
    merged_bn_net = MergeBatchNorm(shadow_net, copy_params)
    if merge_activate:
        return MergeActivate(merged_bn_net)
    else:
        return merged_bn_net
<|endoftext|>