<commit_before>import os
import sys
import bz2
import zipfile
from cStringIO import StringIO
from collections import defaultdict
from os.path import basename, getsize, isfile, isdir, join

from egginst.utils import pprint_fn_action, rm_rf
from enstaller.utils import comparable_version, md5_file, write_data_from_url
import metadata
import dist_naming
from requirement import Req, add_Reqs_to_spec, filter_name, dist_as_req

class Chain(object):

    def __init__(self, repos=[], verbose=False):
        self.verbose = verbose

        # maps distributions to specs
        self.index = {}

        # maps cnames to the list of distributions (in repository order)
        self.groups = defaultdict(list)

        # Chain of repositories, either local or remote
        self.repos = []
        for repo in repos:
            # These are file:// (optionally indexed) or http:// (indexed)
            self.add_repo(repo)

        if self.verbose:
            self.print_repos()

    def print_repos(self):
        print 'Repositories:'
        for r in self.repos:
            print '  %r' % r

    def add_repo(self, repo, index_fn='index-depend.bz2'):
        """
        Add a repo to the chain, i.e. read the index file of the url,
        parse it and update the index.
        """
        if self.verbose:
            print "Adding repository:"
            print "   URL:", repo
        repo = dist_naming.cleanup_reponame(repo)

        self.repos.append(repo)

        index_url = repo + index_fn

        if index_url.startswith('file://'):
            if isfile(index_url[7:]):
                # A local url with index file
                if self.verbose:
                    print "    found index", index_url
            else:
                # A local url without index file
                self.index_all_files(repo)
                return

        if self.verbose:
            print " index:", index_fn

        faux = StringIO()
        write_data_from_url(faux, index_url)
        index_data = faux.getvalue()
        faux.close()

        if self.verbose:
            import hashlib
            print "   md5:", hashlib.md5(index_data).hexdigest()
            print

        if index_fn.endswith('.bz2'):
            index_data = bz2.decompress(index_data)

        new_index = metadata.parse_depend_index(index_data)
        for spec in new_index.itervalues():
            add_Reqs_to_spec(spec)

        for distname, spec in new_index.iteritems():
            dist = repo + distname
            self.index[dist] = spec
            self.groups[spec['cname']].append(dist)

    def get_version_build(self, dist):
        """
        Returns a tuple(version, build) for a distribution, version is a
        RationalVersion object (see verlib).  This method is used below
        for determining the distribution with the largest version and build
        number.
        """
        return dist_naming.comparable_spec(self.index[dist])

    def iter_dists(self, req):
        """
        iterate over all distributions matching the requirement (in
        repository order)
        """
        assert req.strictness >= 1
        for dist in self.groups[req.name]:
            if req.matches(self.index[dist]):
                yield dist

    def get_repo(self, req):
        """
        return the first repository in which the requirement matches at least
        one distribution
        """
        for dist in self.iter_dists(req):
            return dist_naming.repo_dist(dist)
        return None

    def get_dist(self, req):
        """
        return the distributions with the largest version and build number
        from the first repository which contains any matches
        """
        repo = self.get_repo(req)
        if repo is None:
            return None

        matches = []
        for dist in self.iter_dists(req):
            if dist_naming.repo_dist(dist) == repo:
                matches.append(dist)
        return max(matches, key=self.get_version_build)

    def reqs_dist(self, dist):
        """
        return the set of requirement objects listed by the given
        distribution
        """
        return self.index[dist]['Reqs']

    def cname_dist(self, dist):
        return self.index[dist]['cname']

    def determine_install_order(self, dists):
        """
        given the distributions 'dists' (which are already complete, i.e.
        the for each distribution all dependencies are also included in
        the 'dists'), return a list of the same distribution in the correct
        install order
        """
        dists = list(dists)
        # make sure each project name is listed only once
        assert len(dists) == len(set(self.cname_dist(d) for d in dists)), \
                                                                      dists

        # the distributions corresponding to the requirements must be sorted
        # because the output of this function is otherwise not deterministic
        dists.sort(key=self.cname_dist)

        # maps dist -> set of required (project) names
        rns = {}
        for dist in dists:
            rns[dist] = set(r.name for r in self.reqs_dist(dist))

        # as long as we have things missing, simply look for things which
        # can be added, i.e. all the requirements have been added already
        result = []
        names_inst = set()
        while len(result) < len(dists):
            n = len(result)
            for dist in dists:
                if dist in result:
                    continue
                # see if all required packages were added already
                if all(bool(name in names_inst) for name in rns[dist]):
                    result.append(dist)
                    names_inst.add(self.index[dist]['cname'])
                    assert len(names_inst) == len(result)

            if len(result) == n:
                # nothing was added
                raise Exception(
                    "Loop in dependency graph or incomplete "
                    "distributions:\n%r" %
                    [dist_naming.filename_dist(d) for d in dists])

        return result

    def handle_multiple_dists(self):
        cnames = set(self.cname_dist(d) for d in self._dists)
        if len(self._dists) == len(cnames):
            return

        for cname in cnames:
            ds = [d for d in self._dists if self.cname_dist(d) == cname]
            assert len(ds) != 0
            if len(ds) == 1:
                continue

            if self.verbose:
                print 'mutiple: %s' % cname
                for d in ds:
                    print '    %s' % d

            r = max(self._reqs_deep[cname], key=lambda r: r.strictness)
            assert r.name == cname
            self._dists = [d for d in self._dists
                           if self.cname_dist(d) != cname]
            self._dists.append(self.get_dist(r))

    def add_dependents(self, dist):
        for r in self.reqs_dist(dist):
            self._reqs_deep[r.name].add(r)
            if (r.name in self._reqs_shallow  and
                    r.strictness < self._reqs_shallow[r.name].strictness):
                continue
            d = self.get_dist(r)
            self._dists.add(d)
            self.add_dependents(d)

    def order(self, req, mode='recur'):
        """
        Return the list of distributions which need to be installed.
        The returned list is given in dependency order.
        The 'mode' may be:

        'single':  only the distribution for the requirement itself is
                   contained in the result (but not any dependencies)

        'flat':    dependencies are handled only one level deep

        'recur':   dependencies are handled recursively (default)
        """
        if self.verbose:
            print "Determining install order for %r" % req
        root = self.get_dist(req)
        if root is None:
            return None

        if mode =='single':
            return [root]

        if mode == 'flat':
            dists = [self.get_dist(r) for r in self.reqs_dist(root)]
            dists.append(root)
            return self.determine_install_order(dists)

        if mode == 'recur':
            self._reqs_shallow = {}
            for r in self.reqs_dist(root):
                self._reqs_shallow[r.name] = r
            self._reqs_deep = defaultdict(set)
            self._dists = set([root])
            self.add_dependents(root)
            self.handle_multiple_dists()
            return self.determine_install_order(self._dists)

        raise Exception('did not expect mode: %r' % mode)

    # ---------------------------------------------------------------- old

    def select_new_reqs(self, reqs, dist):
        """
        Selects new requirements, which are listed as dependencies in the
        distribution 'dist', and are not already in the requirements 'reqs',
        unless the distribution requires something more strict.
        """
        result = set()
        for r in self.reqs_dist(dist):
            # from all the reqs (we already have collected) filter the
            # ones with the same project name
            rs2 = filter_name(reqs, r.name)
            if rs2:
                # if there are requirements for an existing project name,
                # only add if it is more strict
                for r2 in rs2:
                    if r2.strictness > r.strictness:
                        result.add(r2)
            else:
                # otherwise, just add it, there is no requirement for this
                # project yet
                result.add(r)
        return result

    def add_reqs(self, reqs, req, level=1):
        """
        Finds requirements of 'req', recursively and adds them to 'reqs',
        which is a dictionary mapping requirements to a
        tuple(recursion level, distribution which requires the requirement)
        """
        for dist in self.iter_dists(req):
            for r in self.select_new_reqs(reqs, dist):
                if r in reqs:
                    continue
                reqs[r] = (level, dist)
                self.add_reqs(reqs, r, level + 1)

    def get_reqs(self, req):
        """
        Returns a dictionary mapping all requirements found recursively
        to the distribution which requires it.
        """
        # the root requirement (in the argument) itself maps to recursion
        # level 0 and a non-existent distribution (because the required by
        # the argument of this function and not any other distribution)
        assert req.strictness == 3, req
        reqs1 = {req: (0, 'ROOT')}

        # add all requirements for the root requirement
        self.add_reqs(reqs1, req)

        if self.verbose:
            print "Requirements: (-level, strictness)"
            for r in sorted(reqs1):
                print '\t%-33r %3i %3i' % (r, -reqs1[r][0], r.strictness)

        reqs2 = {}
        for name in set(r.name for r in reqs1):
            # get all requirements for the name
            rs = []
            for r in filter_name(reqs1, name):
                # append a tuple with:
                #   * tuple(negative recursion level, strictness)
                #   * requirement itself
                #   * distribution requiring it
                rs.append(((-reqs1[r][0], r.strictness), r, reqs1[r][1]))

            rs.sort()
            r, d = rs[-1][1:]
            reqs2[r] = d

        return reqs2

    def install_order(self, req, recur=True):
        """
        Return the list of distributions which need to be installed.
        The returned list is given in dependency order, i.e. the
        distributions can be installed in this order without any package
        being installed before its dependencies got installed.
        """
        if self.verbose:
            print "Determining install order for %r" % req
        dist_required = self.get_dist(req)
        if dist_required is None:
            return None

        if not recur:
            return [dist_required]

        req = dist_as_req(dist_required)
        if self.verbose:
            print dist_required
            print "Requirement: %r" % req

        dists = []
        for r, d in self.get_reqs(req).iteritems():
            dist = self.get_dist(r)
            if dist:
                dists.append(dist)
                continue
            print 'ERROR: No distribution found for: %r' % r
            if d != 'ROOT':
                print '       required by: %s' % d
            sys.exit(1)

        return self.determine_install_order(dists)

    def list_versions(self, name):
        """
        given the name of a package, retruns a sorted list of versions for
        package `name` found in any repo.
        """
        versions = set()

        req = Req(name)
        for dist in self.groups[req.name]:
            spec = self.index[dist]
            if req.matches(spec):
                versions.add(spec['version'])

        return sorted(versions, key=comparable_version)

    def fetch_dist(self, dist, fetch_dir, force=False, check_md5=False,
                   dry_run=False):
        """
        Get a distribution, i.e. copy or download the distribution into
        fetch_dir.

        force:
            force download or copy

        check_md5:
            when determining if a file needs to be downloaded or copied,
            check it's MD5.  This is, of course, slower but more reliable
            then just checking the file-size (which is always done first).
            Note:
              * This option has nothing to do with checking the MD5 of the
                download.  The md5 is always checked when files are
                downloaded (regardless of this option).
              * If force=True, this option is has no effect, because the file
                is forcefully downloaded, ignoring any existing file (as well
                as the MD5).
        """
        md5 = self.index[dist].get('md5')
        size = self.index[dist].get('size')

        fn = dist_naming.filename_dist(dist)
        dst = join(fetch_dir, fn)
        # if force is not used, see if (i) the file exists (ii) its size is
        # the expected (iii) optionally, make sure the md5 is the expected.
        if (not force and isfile(dst) and getsize(dst) == size and
                   (not check_md5 or md5_file(dst) == md5)):
            if self.verbose:
                print "Not forcing refetch, %r already exists" % dst
            return

        pprint_fn_action(fn, ('copying', 'downloading')
                             [dist.startswith(('http://', 'https://'))])
        if dry_run:
            return

        if self.verbose:
            print "Copying: %r" % dist
            print "     to: %r" % dst

        fo = open(dst + '.part', 'wb')
        write_data_from_url(fo, dist, md5, size)
        fo.close()
        rm_rf(dst)
        os.rename(dst + '.part', dst)

    def dirname_repo(self, repo):
        if repo.startswith('file://'):
            return repo[7:].rstrip(r'\/')
        return None

    def index_file(self, filename, repo):
        """
        Add an unindexed distribution, which must already exist in a local
        repository to the index (in memory).  Note that the index file on
        disk remains unchanged.
        """
        assert filename == basename(filename), filename
        dist = repo + filename
        if self.verbose:
            print "Adding %r to index" % dist

        arcname = 'EGG-INFO/spec/depend'
        z = zipfile.ZipFile(join(self.dirname_repo(repo), filename))
        if arcname not in z.namelist():
            z.close()
            raise Exception("zipfile %r has no arcname=%r" %
                            (filename, arcname))

        spec = metadata.parse_data(z.read(arcname))
        z.close()
        add_Reqs_to_spec(spec)
        self.index[dist] = spec

    def index_all_files(self, repo):
        """
        Add all distributions to the index, see index_file() above.
        Note that no index file is written to disk.
        """
        dir_path = self.dirname_repo(repo)
        assert isdir(dir_path), dir_path
        for fn in os.listdir(dir_path):
            if not fn.endswith('.egg'):
                continue
            if not dist_naming.is_valid_eggname(fn):
                print "WARNING: ignoring invalid egg name:", join(dir_path, fn)
                continue
            self.index_file(fn, repo)
<commit_msg>move function<commit_after>import os
import sys
import bz2
import zipfile
from cStringIO import StringIO
from collections import defaultdict
from os.path import basename, getsize, isfile, isdir, join

from egginst.utils import pprint_fn_action, rm_rf
from enstaller.utils import comparable_version, md5_file, write_data_from_url
import metadata
import dist_naming
from requirement import Req, add_Reqs_to_spec, filter_name, dist_as_req

class Chain(object):

    def __init__(self, repos=[], verbose=False):
        self.verbose = verbose

        # maps distributions to specs
        self.index = {}

        # maps cnames to the list of distributions (in repository order)
        self.groups = defaultdict(list)

        # Chain of repositories, either local or remote
        self.repos = []
        for repo in repos:
            # These are file:// (optionally indexed) or http:// (indexed)
            self.add_repo(repo)

        if self.verbose:
            self.print_repos()

    def print_repos(self):
        print 'Repositories:'
        for r in self.repos:
            print '  %r' % r

    def add_repo(self, repo, index_fn='index-depend.bz2'):
        """
        Add a repo to the chain, i.e. read the index file of the url,
        parse it and update the index.
        """
        if self.verbose:
            print "Adding repository:"
            print "   URL:", repo
        repo = dist_naming.cleanup_reponame(repo)

        self.repos.append(repo)

        index_url = repo + index_fn

        if index_url.startswith('file://'):
            if isfile(index_url[7:]):
                # A local url with index file
                if self.verbose:
                    print "    found index", index_url
            else:
                # A local url without index file
                self.index_all_files(repo)
                return

        if self.verbose:
            print " index:", index_fn

        faux = StringIO()
        write_data_from_url(faux, index_url)
        index_data = faux.getvalue()
        faux.close()

        if self.verbose:
            import hashlib
            print "   md5:", hashlib.md5(index_data).hexdigest()
            print

        if index_fn.endswith('.bz2'):
            index_data = bz2.decompress(index_data)

        new_index = metadata.parse_depend_index(index_data)
        for spec in new_index.itervalues():
            add_Reqs_to_spec(spec)

        for distname, spec in new_index.iteritems():
            dist = repo + distname
            self.index[dist] = spec
            self.groups[spec['cname']].append(dist)

    def get_version_build(self, dist):
        """
        Returns a tuple(version, build) for a distribution, version is a
        RationalVersion object (see verlib).  This method is used below
        for determining the distribution with the largest version and build
        number.
        """
        return dist_naming.comparable_spec(self.index[dist])

    def iter_dists(self, req):
        """
        iterate over all distributions matching the requirement (in
        repository order)
        """
        assert req.strictness >= 1
        for dist in self.groups[req.name]:
            if req.matches(self.index[dist]):
                yield dist

    def get_repo(self, req):
        """
        return the first repository in which the requirement matches at least
        one distribution
        """
        for dist in self.iter_dists(req):
            return dist_naming.repo_dist(dist)
        return None

    def get_dist(self, req):
        """
        return the distributions with the largest version and build number
        from the first repository which contains any matches
        """
        repo = self.get_repo(req)
        if repo is None:
            return None

        matches = []
        for dist in self.iter_dists(req):
            if dist_naming.repo_dist(dist) == repo:
                matches.append(dist)
        return max(matches, key=self.get_version_build)

    def reqs_dist(self, dist):
        """
        return the set of requirement objects listed by the given
        distribution
        """
        return self.index[dist]['Reqs']

    def cname_dist(self, dist):
        return self.index[dist]['cname']

    def determine_install_order(self, dists):
        """
        given the distributions 'dists' (which are already complete, i.e.
        the for each distribution all dependencies are also included in
        the 'dists'), return a list of the same distribution in the correct
        install order
        """
        dists = list(dists)
        # make sure each project name is listed only once
        assert len(dists) == len(set(self.cname_dist(d) for d in dists)), \
                                                                      dists

        # the distributions corresponding to the requirements must be sorted
        # because the output of this function is otherwise not deterministic
        dists.sort(key=self.cname_dist)

        # maps dist -> set of required (project) names
        rns = {}
        for dist in dists:
            rns[dist] = set(r.name for r in self.reqs_dist(dist))

        # as long as we have things missing, simply look for things which
        # can be added, i.e. all the requirements have been added already
        result = []
        names_inst = set()
        while len(result) < len(dists):
            n = len(result)
            for dist in dists:
                if dist in result:
                    continue
                # see if all required packages were added already
                if all(bool(name in names_inst) for name in rns[dist]):
                    result.append(dist)
                    names_inst.add(self.index[dist]['cname'])
                    assert len(names_inst) == len(result)

            if len(result) == n:
                # nothing was added
                raise Exception(
                    "Loop in dependency graph or incomplete "
                    "distributions:\n%r" %
                    [dist_naming.filename_dist(d) for d in dists])

        return result

    def handle_multiple_dists(self):
        cnames = set(self.cname_dist(d) for d in self._dists)
        if len(self._dists) == len(cnames):
            return

        for cname in cnames:
            ds = [d for d in self._dists if self.cname_dist(d) == cname]
            assert len(ds) != 0
            if len(ds) == 1:
                continue

            if self.verbose:
                print 'mutiple: %s' % cname
                for d in ds:
                    print '    %s' % d

            r = max(self._reqs_deep[cname], key=lambda r: r.strictness)
            assert r.name == cname
            self._dists = [d for d in self._dists
                           if self.cname_dist(d) != cname]
            self._dists.append(self.get_dist(r))

    def order(self, req, mode='recur'):
        """
        Return the list of distributions which need to be installed.
        The returned list is given in dependency order.
        The 'mode' may be:

        'single':  only the distribution for the requirement itself is
                   contained in the result (but not any dependencies)

        'flat':    dependencies are handled only one level deep

        'recur':   dependencies are handled recursively (default)
        """
        if self.verbose:
            print "Determining install order for %r" % req
        root = self.get_dist(req)
        if root is None:
            return None

        if mode =='single':
            return [root]

        if mode == 'flat':
            dists = [self.get_dist(r) for r in self.reqs_dist(root)]
            dists.append(root)
            return self.determine_install_order(dists)

        if mode == 'recur':
            def add_dependents(dist):
                for r in self.reqs_dist(dist):
                    self._reqs_deep[r.name].add(r)
                    if (r.name in reqs_shallow  and
                            r.strictness < reqs_shallow[r.name].strictness):
                        continue
                    d = self.get_dist(r)
                    self._dists.add(d)
                    add_dependents(d)

            reqs_shallow = {}
            for r in self.reqs_dist(root):
                reqs_shallow[r.name] = r
            self._reqs_deep = defaultdict(set)
            self._dists = set([root])
            add_dependents(root)
            self.handle_multiple_dists()
            return self.determine_install_order(self._dists)

        raise Exception('did not expect mode: %r' % mode)

    # ---------------------------------------------------------------- old

    def select_new_reqs(self, reqs, dist):
        """
        Selects new requirements, which are listed as dependencies in the
        distribution 'dist', and are not already in the requirements 'reqs',
        unless the distribution requires something more strict.
        """
        result = set()
        for r in self.reqs_dist(dist):
            # from all the reqs (we already have collected) filter the
            # ones with the same project name
            rs2 = filter_name(reqs, r.name)
            if rs2:
                # if there are requirements for an existing project name,
                # only add if it is more strict
                for r2 in rs2:
                    if r2.strictness > r.strictness:
                        result.add(r2)
            else:
                # otherwise, just add it, there is no requirement for this
                # project yet
                result.add(r)
        return result

    def add_reqs(self, reqs, req, level=1):
        """
        Finds requirements of 'req', recursively and adds them to 'reqs',
        which is a dictionary mapping requirements to a
        tuple(recursion level, distribution which requires the requirement)
        """
        for dist in self.iter_dists(req):
            for r in self.select_new_reqs(reqs, dist):
                if r in reqs:
                    continue
                reqs[r] = (level, dist)
                self.add_reqs(reqs, r, level + 1)

    def get_reqs(self, req):
        """
        Returns a dictionary mapping all requirements found recursively
        to the distribution which requires it.
        """
        # the root requirement (in the argument) itself maps to recursion
        # level 0 and a non-existent distribution (because the required by
        # the argument of this function and not any other distribution)
        assert req.strictness == 3, req
        reqs1 = {req: (0, 'ROOT')}

        # add all requirements for the root requirement
        self.add_reqs(reqs1, req)

        if self.verbose:
            print "Requirements: (-level, strictness)"
            for r in sorted(reqs1):
                print '\t%-33r %3i %3i' % (r, -reqs1[r][0], r.strictness)

        reqs2 = {}
        for name in set(r.name for r in reqs1):
            # get all requirements for the name
            rs = []
            for r in filter_name(reqs1, name):
                # append a tuple with:
                #   * tuple(negative recursion level, strictness)
                #   * requirement itself
                #   * distribution requiring it
                rs.append(((-reqs1[r][0], r.strictness), r, reqs1[r][1]))

            rs.sort()
            r, d = rs[-1][1:]
            reqs2[r] = d

        return reqs2

    def install_order(self, req, recur=True):
        """
        Return the list of distributions which need to be installed.
        The returned list is given in dependency order, i.e. the
        distributions can be installed in this order without any package
        being installed before its dependencies got installed.
        """
        if self.verbose:
            print "Determining install order for %r" % req
        dist_required = self.get_dist(req)
        if dist_required is None:
            return None

        if not recur:
            return [dist_required]

        req = dist_as_req(dist_required)
        if self.verbose:
            print dist_required
            print "Requirement: %r" % req

        dists = []
        for r, d in self.get_reqs(req).iteritems():
            dist = self.get_dist(r)
            if dist:
                dists.append(dist)
                continue
            print 'ERROR: No distribution found for: %r' % r
            if d != 'ROOT':
                print '       required by: %s' % d
            sys.exit(1)

        return self.determine_install_order(dists)

    def list_versions(self, name):
        """
        given the name of a package, retruns a sorted list of versions for
        package `name` found in any repo.
        """
        versions = set()

        req = Req(name)
        for dist in self.groups[req.name]:
            spec = self.index[dist]
            if req.matches(spec):
                versions.add(spec['version'])

        return sorted(versions, key=comparable_version)

    def fetch_dist(self, dist, fetch_dir, force=False, check_md5=False,
                   dry_run=False):
        """
        Get a distribution, i.e. copy or download the distribution into
        fetch_dir.

        force:
            force download or copy

        check_md5:
            when determining if a file needs to be downloaded or copied,
            check it's MD5.  This is, of course, slower but more reliable
            then just checking the file-size (which is always done first).
            Note:
              * This option has nothing to do with checking the MD5 of the
                download.  The md5 is always checked when files are
                downloaded (regardless of this option).
              * If force=True, this option is has no effect, because the file
                is forcefully downloaded, ignoring any existing file (as well
                as the MD5).
        """
        md5 = self.index[dist].get('md5')
        size = self.index[dist].get('size')

        fn = dist_naming.filename_dist(dist)
        dst = join(fetch_dir, fn)
        # if force is not used, see if (i) the file exists (ii) its size is
        # the expected (iii) optionally, make sure the md5 is the expected.
        if (not force and isfile(dst) and getsize(dst) == size and
                   (not check_md5 or md5_file(dst) == md5)):
            if self.verbose:
                print "Not forcing refetch, %r already exists" % dst
            return

        pprint_fn_action(fn, ('copying', 'downloading')
                             [dist.startswith(('http://', 'https://'))])
        if dry_run:
            return

        if self.verbose:
            print "Copying: %r" % dist
            print "     to: %r" % dst

        fo = open(dst + '.part', 'wb')
        write_data_from_url(fo, dist, md5, size)
        fo.close()
        rm_rf(dst)
        os.rename(dst + '.part', dst)

    def dirname_repo(self, repo):
        if repo.startswith('file://'):
            return repo[7:].rstrip(r'\/')
        return None

    def index_file(self, filename, repo):
        """
        Add an unindexed distribution, which must already exist in a local
        repository to the index (in memory).  Note that the index file on
        disk remains unchanged.
        """
        assert filename == basename(filename), filename
        dist = repo + filename
        if self.verbose:
            print "Adding %r to index" % dist

        arcname = 'EGG-INFO/spec/depend'
        z = zipfile.ZipFile(join(self.dirname_repo(repo), filename))
        if arcname not in z.namelist():
            z.close()
            raise Exception("zipfile %r has no arcname=%r" %
                            (filename, arcname))

        spec = metadata.parse_data(z.read(arcname))
        z.close()
        add_Reqs_to_spec(spec)
        self.index[dist] = spec

    def index_all_files(self, repo):
        """
        Add all distributions to the index, see index_file() above.
        Note that no index file is written to disk.
        """
        dir_path = self.dirname_repo(repo)
        assert isdir(dir_path), dir_path
        for fn in os.listdir(dir_path):
            if not fn.endswith('.egg'):
                continue
            if not dist_naming.is_valid_eggname(fn):
                print "WARNING: ignoring invalid egg name:", join(dir_path, fn)
                continue
            self.index_file(fn, repo)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This scripts generates a tabular report of draftifications over a specified
date range.

The following arguments are required:

-page             The page to output the report to.

The following arguments are supported:

-end              The end date of the range for the report.
                  If -end is not provided, it will be yesterday.

-start            The start date of the range for the report.
                  If -start is not provided, it will be the same as -end.
"""
# Author : JJMC89
# License: MIT
import datetime
from datetime import date, timedelta
import re
from dateutil.parser import parse as parse_date
import pywikibot
from pywikibot import pagegenerators

BOT_START_END = re.compile(
    r'^(.*?<!--\s*bot start\s*-->).*?(<!--\s*bot end\s*-->.*)$',
    flags=re.S | re.I
)

def validate_options(options, site):
    """
    Validate the options and return bool.

    @param options: options to validate
    @type options: dict

    @rtype: bool
    """
    pywikibot.log('Options:')
    required_keys = ['end', 'page', 'start']
    has_keys = list()
    result = True
    if 'start' not in options:
        options['start'] = options['end']
    for key, value in options.items():
        pywikibot.log('-{} = {}'.format(key, value))
        if key in required_keys:
            has_keys.append(key)
        if key in 'end' 'start':
            if not isinstance(value, date):
                try:
                    options[key] = parse_date(value).date()
                except ValueError as e:
                    pywikibot.log('Invalid date: {}'.format(e))
                    result = False
        elif key == 'page':
            if not isinstance(value, str):
                pywikibot.log('Must be a string.')
                result = False
            options[key] = pywikibot.Page(site, value)
        pywikibot.log('\u2192{} = {}'.format(key, options[key]))
    if sorted(has_keys) != sorted(required_keys):
        pywikibot.log('Missing one more required keys.')
        result = False
    if options['end'] < options['start']:
        pywikibot.log('end cannot be before start.')
        result = False
    return result

def get_xfds(pages):
    """
    Return a set of XfDs for the pages.

    @param pages: Pages to get XfDs for
    @type pages: iterable of L{pywikibot.Page}

    @rtype: set
    """
    xfds = set()
    for page in pages:
        if page.namespace() == page.site.namespaces.MAIN:
            prefix = 'Articles for deletion/'
        else:
            prefix = 'Miscellany for deletion/'
        prefix += page.title()
        gen = pagegenerators.PrefixingPageGenerator(
            prefix, namespace=page.site.namespaces.PROJECT, site=page.site)
        xfds = xfds.union([xfd_page.title(asLink=True) for xfd_page in gen])
    return xfds

def iterable_to_wikitext(items):
    """
    Convert iterable to wikitext.

    @param items: Items to iterate
    @type items: iterable

    @rtype: str
    """
    if len(items) == 1:
        return '{}'.format(next(iter(items)))
    text = ''
    for item in items:
        text += '\n* {}'.format(item)
    return text

def save_bot_start_end(save_text, page, summary):
    """
    Writes the text to the given page.

    @param save_text: Text to save
    @type save_text: str
    @param page: Page to save to
    @type page: L{pywikibot.Page}
    @param summary: Edit summary
    @type summary: str
    """
    #print(save_text)
    save_text = save_text.strip()
    if page.exists():
        if BOT_START_END.match(page.text):
            page.text = BOT_START_END.sub(r'\1\n{}\2'.format(save_text),
                                          page.text)
        else:
            page.text = save_text
        page.save(summary=summary, minor=False, botflag=False)
        #print(page.text)
    else:
        pywikibot.error('{} does not exist. Skipping.'.format(page.title()))

def output_move_log(page=None, start=None, end=None):
    """
    Writes move logevents to a page.

    @param page: The page to output to
    @type page: L{pywikibot.Page}
    """
    text = ''
    for logevent in page.site.logevents(logtype='move',
                                        namespace=page.site.namespaces.MAIN.id,
                                        start=start, end=end, reverse=True):
        if (logevent.target_ns not in (page.site.namespaces.DRAFT,
                                       page.site.namespaces.USER)
                or logevent.target_title.startswith('Draft:Move/')):
            # Only want moves to Draft or User.
            # Skip page swaps.
            continue
        current_page = None
        creator = creation = last_edit = num_editors = '(Unknown)'
        if logevent.target_page.exists():
            current_page = logevent.target_page
            if (current_page.isRedirectPage()
                    and current_page.getRedirectTarget().exists()
                    and (current_page.getRedirectTarget().namespace()
                         in (page.site.namespaces.DRAFT,
                             page.site.namespaces.USER,
                             page.site.namespaces.MAIN))
               ):
                current_page = current_page.getRedirectTarget()
        elif logevent.page().exists():
            current_page = logevent.page()
        if current_page:
            if current_page.oldest_revision.user:
                creator = '[[User:{}]]'.format(
                    current_page.oldest_revision.user)
            creation = ('[[Special:PermaLink/{rev.revid}|{rev.timestamp}]]'
                        .format(rev=current_page.oldest_revision))
            last_edit = '[[Special:Diff/{rev.revid}|{rev.timestamp}]]'.format(
                rev=current_page.latest_revision)
            editors = set()
            for rev in current_page.revisions():
                if rev.user:
                    editors.add(rev.user)
            num_editors = len(editors)
        text += (
            '\n|-\n| {page} || {target} || [[User:{log[user]}]] || '
            '{log[timestamp]} || <nowiki>{log[comment]}</nowiki> || '
            '{creator} || {creation} || {editors} || {last_edit} || {notes}'
            .format(
                page=logevent.page().title(asLink=True, textlink=True),
                target=logevent.target_page.title(asLink=True, textlink=True),
                log=logevent.data,
                creator=creator,
                creation=creation,
                editors=num_editors,
                last_edit=last_edit,
                notes=iterable_to_wikitext(
                    get_xfds([logevent.page(), logevent.target_page])
                )
            )
        )
    if text:
        if start.date() == end.date():
            caption = 'Report for {}'.format(start.date().isoformat())
        else:
            caption = 'Report for {} to {}'.format(start.date().isoformat(),
                                                   end.date().isoformat())
        caption += '; Last updated: ~~~~~'
        text = (
            '\n{{| class="wikitable sortable plainlinks"\n|+ {caption}'
            '\n! Page !! Target !! Mover !! Move date/time !! Move summary !! '
            'Creator !! Creation !! Editors !! Last edit !! Notes{body}\n|}}'
            .format(caption=caption, body=text)
        )
    else:
        text = 'None'
    save_bot_start_end(text, page, 'Updating draftification report')

def main(*args):
    """
    Process command line arguments and invoke bot.

    @param args: command line arguments
    @type args: list of unicode
    """
    options = {
        'end': date.today() - timedelta(days=1)
    }
    # Process global arguments
    local_args = pywikibot.handle_args(args)
    site = pywikibot.Site()
    site.login()
    # Parse command line arguments
    for arg in local_args:
        arg, _, value = arg.partition(':')
        arg = arg[1:]
        if arg in 'end' 'page' 'start':
            if not value:
                value = pywikibot.input(
                    'Please enter a value for {}'.format(arg),
                    default=None
                )
            options[arg] = value
        else:
            options[arg] = True
    if not validate_options(options, site):
        pywikibot.error('Invalid options.')
        return False

    # Output logs
    output_move_log(
        page=options['page'],
        start=datetime.datetime.combine(options['start'], datetime.time.min),
        end=datetime.datetime.combine(options['end'], datetime.time.max)
    )

if __name__ == "__main__":
    main()
<commit_msg>Handle circular redirects<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This scripts generates a tabular report of draftifications over a specified
date range.

The following arguments are required:

-page             The page to output the report to.

The following arguments are supported:

-end              The end date of the range for the report.
                  If -end is not provided, it will be yesterday.

-start            The start date of the range for the report.
                  If -start is not provided, it will be the same as -end.
"""
# Author : JJMC89
# License: MIT
import datetime
from datetime import date, timedelta
import re
from dateutil.parser import parse as parse_date
import pywikibot
from pywikibot import pagegenerators

BOT_START_END = re.compile(
    r'^(.*?<!--\s*bot start\s*-->).*?(<!--\s*bot end\s*-->.*)$',
    flags=re.S | re.I
)

def validate_options(options, site):
    """
    Validate the options and return bool.

    @param options: options to validate
    @type options: dict

    @rtype: bool
    """
    pywikibot.log('Options:')
    required_keys = ['end', 'page', 'start']
    has_keys = list()
    result = True
    if 'start' not in options:
        options['start'] = options['end']
    for key, value in options.items():
        pywikibot.log('-{} = {}'.format(key, value))
        if key in required_keys:
            has_keys.append(key)
        if key in 'end' 'start':
            if not isinstance(value, date):
                try:
                    options[key] = parse_date(value).date()
                except ValueError as e:
                    pywikibot.log('Invalid date: {}'.format(e))
                    result = False
        elif key == 'page':
            if not isinstance(value, str):
                pywikibot.log('Must be a string.')
                result = False
            options[key] = pywikibot.Page(site, value)
        pywikibot.log('\u2192{} = {}'.format(key, options[key]))
    if sorted(has_keys) != sorted(required_keys):
        pywikibot.log('Missing one more required keys.')
        result = False
    if options['end'] < options['start']:
        pywikibot.log('end cannot be before start.')
        result = False
    return result

def get_xfds(pages):
    """
    Return a set of XfDs for the pages.

    @param pages: Pages to get XfDs for
    @type pages: iterable of L{pywikibot.Page}

    @rtype: set
    """
    xfds = set()
    for page in pages:
        if page.namespace() == page.site.namespaces.MAIN:
            prefix = 'Articles for deletion/'
        else:
            prefix = 'Miscellany for deletion/'
        prefix += page.title()
        gen = pagegenerators.PrefixingPageGenerator(
            prefix, namespace=page.site.namespaces.PROJECT, site=page.site)
        xfds = xfds.union([xfd_page.title(asLink=True) for xfd_page in gen])
    return xfds

def iterable_to_wikitext(items):
    """
    Convert iterable to wikitext.

    @param items: Items to iterate
    @type items: iterable

    @rtype: str
    """
    if len(items) == 1:
        return '{}'.format(next(iter(items)))
    text = ''
    for item in items:
        text += '\n* {}'.format(item)
    return text

def save_bot_start_end(save_text, page, summary):
    """
    Writes the text to the given page.

    @param save_text: Text to save
    @type save_text: str
    @param page: Page to save to
    @type page: L{pywikibot.Page}
    @param summary: Edit summary
    @type summary: str
    """
    save_text = save_text.strip()
    if page.exists():
        if BOT_START_END.match(page.text):
            page.text = BOT_START_END.sub(r'\1\n{}\2'.format(save_text),
                                          page.text)
        else:
            page.text = save_text
        page.save(summary=summary, minor=False, botflag=False)
    else:
        pywikibot.error('{} does not exist. Skipping.'.format(page.title()))

def output_move_log(page=None, start=None, end=None):
    """
    Writes move logevents to a page.

    @param page: The page to output to
    @type page: L{pywikibot.Page}
    """
    text = ''
    for logevent in page.site.logevents(logtype='move',
                                        namespace=page.site.namespaces.MAIN.id,
                                        start=start, end=end, reverse=True):
        if (logevent.target_ns not in (page.site.namespaces.DRAFT,
                                       page.site.namespaces.USER)
                or logevent.target_title.startswith('Draft:Move/')):
            # Only want moves to Draft or User.
            # Skip page swaps.
            continue
        current_page = None
        creator = creation = last_edit = num_editors = '(Unknown)'
        if logevent.target_page.exists():
            current_page = logevent.target_page
            if current_page.isRedirectPage():
                try:
                    redirect_target = current_page.getRedirectTarget()
                except pywikibot.CircularRedirect:
                    pywikibot.log('{} is a circular redirect.'.format(
                        current_page.title(asLink=True)))
                else:
                    if (redirect_target.exists()
                            and (redirect_target.namespace()
                                 in (page.site.namespaces.DRAFT,
                                     page.site.namespaces.USER,
                                     page.site.namespaces.MAIN))
                       ):
                        current_page = redirect_target
        elif logevent.page().exists():
            current_page = logevent.page()
        if current_page:
            if current_page.oldest_revision.user:
                creator = '[[User:{}]]'.format(
                    current_page.oldest_revision.user)
            creation = ('[[Special:PermaLink/{rev.revid}|{rev.timestamp}]]'
                        .format(rev=current_page.oldest_revision))
            last_edit = '[[Special:Diff/{rev.revid}|{rev.timestamp}]]'.format(
                rev=current_page.latest_revision)
            editors = set()
            for rev in current_page.revisions():
                if rev.user:
                    editors.add(rev.user)
            num_editors = len(editors)
        text += (
            '\n|-\n| {page} || {target} || [[User:{log[user]}]] || '
            '{log[timestamp]} || <nowiki>{log[comment]}</nowiki> || '
            '{creator} || {creation} || {editors} || {last_edit} || {notes}'
            .format(
                page=logevent.page().title(asLink=True, textlink=True),
                target=logevent.target_page.title(asLink=True, textlink=True),
                log=logevent.data,
                creator=creator,
                creation=creation,
                editors=num_editors,
                last_edit=last_edit,
                notes=iterable_to_wikitext(
                    get_xfds([logevent.page(), logevent.target_page])
                )
            )
        )
    if text:
        if start.date() == end.date():
            caption = 'Report for {}'.format(start.date().isoformat())
        else:
            caption = 'Report for {} to {}'.format(start.date().isoformat(),
                                                   end.date().isoformat())
        caption += '; Last updated: ~~~~~'
        text = (
            '\n{{| class="wikitable sortable plainlinks"\n|+ {caption}'
            '\n! Page !! Target !! Mover !! Move date/time !! Move summary !! '
            'Creator !! Creation !! Editors !! Last edit !! Notes{body}\n|}}'
            .format(caption=caption, body=text)
        )
    else:
        text = 'None'
    save_bot_start_end(text, page, 'Updating draftification report')

def main(*args):
    """
    Process command line arguments and invoke bot.

    @param args: command line arguments
    @type args: list of unicode
    """
    options = {
        'end': date.today() - timedelta(days=1)
    }
    # Process global arguments
    local_args = pywikibot.handle_args(args)
    site = pywikibot.Site()
    site.login()
    # Parse command line arguments
    for arg in local_args:
        arg, _, value = arg.partition(':')
        arg = arg[1:]
        if arg in 'end' 'page' 'start':
            if not value:
                value = pywikibot.input(
                    'Please enter a value for {}'.format(arg),
                    default=None
                )
            options[arg] = value
        else:
            options[arg] = True
    if not validate_options(options, site):
        pywikibot.error('Invalid options.')
        return False

    # Output logs
    output_move_log(
        page=options['page'],
        start=datetime.datetime.combine(options['start'], datetime.time.min),
        end=datetime.datetime.combine(options['end'], datetime.time.max)
    )

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright (c) 2013, Universal Resource Trading Limited and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
from __future__ import print_function
import __builtin__ as builtins

import frappe
from frappe import msgprint
from frappe.utils import cstr
from datetime import date

from ebay_active_listings import generate_active_ebay_data, sync_ebay_ids
import ugssettings
from revise_items import revise_ebay_price

def better_print(*args, **kwargs):
    with open("price_sync_log.log", "a") as f:
        builtins.print (file=f, *args, **kwargs)

print = better_print

@frappe.whitelist()
def price_sync():
    """
    Initiates automated price reduction
    and sync to eBay
    Note: el.price is price on eBay (ie. inc vat)
    """

    print("Price sync to eBay run on ", date.today())

    generate_active_ebay_data()
    sync_ebay_ids()
    sync_prices_to_ebay()
    frappe.msgprint("Price revision completed")

    return 1

def sync_prices_to_ebay():
    """
    Mass update eBay prices to match the system price
    # TODO alternative is to set up an event driven solution similar to the custom script

    NOTE: el.price is actual eBay price (ie. inc VAT)
    """

    # First get the mis-matched prices
    records = get_mismatched_prices()

    # Call the revise price function
    for r in records:
        # revise_ebay_price takes exc vat pricing
        result = revise_ebay_price(r.item_code, r.standard_rate, False)
        print(result)

def get_mismatched_prices():
    """
    return items where ebay price <> system price
    """

    sql = """
    select it.item_code, it.ebay_id,
    round(ifnull(ip.price_list_rate, 0.0),2) as price_list_rate,
    round(ifnull(it.standard_rate,0.0),2),
    round(ifnull(it.vat_inclusive_price,0.0),2),
    round(ifnull(el.price,0.0),2) as ebay_inc_vat

    from `tabItem Price` ip

    left join `tabItem` it
    on it.item_code = ip.item_code

    left join `zEbayListings` el
    on el.sku = it.item_code

    where it.ebay_id REGEXP '[0-9]'
    and round(ifnull(it.vat_inclusive_price, 0.0),2) <> round(ifnull(el.price, 0.0),2)
    and it.standard_rate > 0.0
    and ifnull(el.price,0.0) > 0
    """
    records = frappe.db.sql(sql, as_dict=1)

    return records

<commit_msg>Update ebay_price_sync.py<commit_after># Copyright (c) 2013, Universal Resource Trading Limited and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
from __future__ import print_function
import __builtin__ as builtins

import frappe
from frappe import msgprint
from frappe.utils import cstr
from datetime import date

from ebay_active_listings import generate_active_ebay_data, sync_ebay_ids
import ugssettings
from revise_items import revise_ebay_price

def better_print(*args, **kwargs):
    with open("price_sync_log.log", "a") as f:
        builtins.print (file=f, *args, **kwargs)

print = better_print

@frappe.whitelist()
def price_sync():
    """
    Initiates automated price reduction
    and sync to eBay
    Note: el.price is price on eBay (ie. inc vat)
    """

    print("Price sync to eBay run on ", date.today())

    generate_active_ebay_data()
    sync_ebay_ids()
    sync_prices_to_ebay()
    frappe.msgprint("Price revision completed")

    return 1

def sync_prices_to_ebay():
    """
    Mass update eBay prices to match the system price
    # TODO alternative is to set up an event driven solution similar to the custom script

    NOTE: el.price is actual eBay price (ie. inc VAT)
    """

    # First get the mis-matched prices
    records = get_mismatched_prices()

    # Call the revise price function
    for r in records:
        # revise_ebay_price takes exc vat pricing
        result = revise_ebay_price(r.item_code, r.price_list_rate, False)
        print(result)

def get_mismatched_prices():
    """
    return items where ebay price <> system price
    """

    sql = """
    select distinct(it.item_code), it.ebay_id,
    round(ifnull(ip.price_list_rate, 0.0),2) as price_list_rate,
    ip.price_list,
    round(ifnull(it.standard_rate,0.0),2) as standard_rate,
    round(ifnull(it.vat_inclusive_price,0.0),2) as vat_inclusive_price,
    round(ifnull(el.price,0.0),2) as ebay_inc_vat

    from `tabItem Price` ip

    left join `tabItem` it
    on it.item_code = ip.item_code

    left join `zEbayListings` el
    on el.sku = it.item_code

    where it.ebay_id REGEXP '[0-9]'
    and round(ifnull(it.vat_inclusive_price, 0.0),2) <> round(ifnull(el.price, 0.0),2)
    and it.standard_rate > 0.0
    and ifnull(el.price,0.0) > 0
    and ip.price_list = 'eBay Selling'
    """
    records = frappe.db.sql(sql, as_dict=1)

    return records

<|endoftext|>


<commit_before># Copyright (c) 2017, Frappe and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe

def execute():
	sms_sender_name = frappe.db.get_single_value("SMS Settings", "sms_sender_name")
	if sms_sender_name:
		frappe.reload_doctype("SMS Settings")
		sms_settings = frappe.get_doc("SMS Settings")
		sms_settings.append("parameters", {
			"parameter": "sender_name",
			"value": sms_sender_name
		})
		sms_settings.save()<commit_msg>[hotfix] ingore_mandatory fields while saving the sms settings in add_sms_sender_name_as_paramenters patch (#4342)<commit_after># Copyright (c) 2017, Frappe and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals
import frappe

def execute():
	sms_sender_name = frappe.db.get_single_value("SMS Settings", "sms_sender_name")
	if sms_sender_name:
		frappe.reload_doctype("SMS Settings")
		sms_settings = frappe.get_doc("SMS Settings")
		sms_settings.append("parameters", {
			"parameter": "sender_name",
			"value": sms_sender_name
		})
		sms_settings.flags.ignore_mandatory = True
		sms_settings.flags.ignore_permissions = True
		sms_settings.save()<|endoftext|>


<commit_before># This is your project's main settings file that can be committed to your
# repo. If you need to override a setting locally, use settings_local.py

from funfactory.settings_base import *

# Name of the top-level module where you put all your apps.
# If you did not install Playdoh with the funfactory installer script
# you may need to edit this value. See the docs about installing from a
# clone.
PROJECT_MODULE = 'mozindia'

# Defines the views served for root URLs.
ROOT_URLCONF = '%s.urls' % PROJECT_MODULE

INSTALLED_APPS = list(INSTALLED_APPS) + [
    # Application base, containing global templates.
    '%s.base' % PROJECT_MODULE,
    # Example code. Can (and should) be removed for actual projects.
    '%s.examples' % PROJECT_MODULE,
]

# Note! If you intend to add `south` to INSTALLED_APPS,
# make sure it comes BEFORE `django_nose`.
#INSTALLED_APPS.remove('django_nose')
#INSTALLED_APPS.append('django_nose')

LOCALE_PATHS = (
    os.path.join(ROOT, PROJECT_MODULE, 'locale'),
)

# Because Jinja2 is the default template loader, add any non-Jinja templated
# apps here:
JINGO_EXCLUDE_APPS = [
    'admin',
    'registration',
]

# BrowserID configuration
AUTHENTICATION_BACKENDS = [
    'django_browserid.auth.BrowserIDBackend',
    'django.contrib.auth.backends.ModelBackend',
]

SITE_URL = 'http://127.0.0.1:8000'
LOGIN_URL = '/'
LOGIN_REDIRECT_URL = 'examples.home'
LOGIN_REDIRECT_URL_FAILURE = 'examples.home'

TEMPLATE_CONTEXT_PROCESSORS = list(TEMPLATE_CONTEXT_PROCESSORS) + [
    'django_browserid.context_processors.browserid_form',
]

# Should robots.txt deny everything or disallow a calculated list of URLs we
# don't want to be crawled?  Default is false, disallow everything.
# Also see http://www.google.com/support/webmasters/bin/answer.py?answer=93710
ENGAGE_ROBOTS = False

# Always generate a CSRF token for anonymous users.
ANON_ALWAYS = True

# Tells the extract script what files to look for L10n in and what function
# handles the extraction. The Tower library expects this.
DOMAIN_METHODS['messages'] = [
    ('%s/**.py' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_python'),
    ('%s/**/templates/**.html' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_template'),
    ('templates/**.html',
        'tower.management.commands.extract.extract_tower_template'),
]

# # Use this if you have localizable HTML files:
# DOMAIN_METHODS['lhtml'] = [
#    ('**/templates/**.lhtml',
#        'tower.management.commands.extract.extract_tower_template'),
# ]

# # Use this if you have localizable JS files:
# DOMAIN_METHODS['javascript'] = [
#    # Make sure that this won't pull in strings from external libraries you
#    # may use.
#    ('media/js/**.js', 'javascript'),
# ]

LOGGING = dict(loggers=dict(playdoh = {'level': logging.DEBUG}))
<commit_msg>Added mainsite to INSTALLED_APPS in the base settings.<commit_after># This is your project's main settings file that can be committed to your
# repo. If you need to override a setting locally, use settings_local.py

from funfactory.settings_base import *

# Name of the top-level module where you put all your apps.
# If you did not install Playdoh with the funfactory installer script
# you may need to edit this value. See the docs about installing from a
# clone.
PROJECT_MODULE = 'mozindia'

# Defines the views served for root URLs.
ROOT_URLCONF = '%s.urls' % PROJECT_MODULE

INSTALLED_APPS = list(INSTALLED_APPS) + [
    # Application base, containing global templates.
    '%s.base' % PROJECT_MODULE,
    '{}.mainsite'.format(PROJECT_MODULE),
]

# Note! If you intend to add `south` to INSTALLED_APPS,
# make sure it comes BEFORE `django_nose`.
#INSTALLED_APPS.remove('django_nose')
#INSTALLED_APPS.append('django_nose')

LOCALE_PATHS = (
    os.path.join(ROOT, PROJECT_MODULE, 'locale'),
)

# Because Jinja2 is the default template loader, add any non-Jinja templated
# apps here:
JINGO_EXCLUDE_APPS = [
    'admin',
    'registration',
]

# BrowserID configuration
AUTHENTICATION_BACKENDS = [
    'django_browserid.auth.BrowserIDBackend',
    'django.contrib.auth.backends.ModelBackend',
]

SITE_URL = 'http://127.0.0.1:8000'
LOGIN_URL = '/'
LOGIN_REDIRECT_URL = 'examples.home'
LOGIN_REDIRECT_URL_FAILURE = 'examples.home'

TEMPLATE_CONTEXT_PROCESSORS = list(TEMPLATE_CONTEXT_PROCESSORS) + [
    'django_browserid.context_processors.browserid_form',
]

# Should robots.txt deny everything or disallow a calculated list of URLs we
# don't want to be crawled?  Default is false, disallow everything.
# Also see http://www.google.com/support/webmasters/bin/answer.py?answer=93710
ENGAGE_ROBOTS = False

# Always generate a CSRF token for anonymous users.
ANON_ALWAYS = True

# Tells the extract script what files to look for L10n in and what function
# handles the extraction. The Tower library expects this.
DOMAIN_METHODS['messages'] = [
    ('%s/**.py' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_python'),
    ('%s/**/templates/**.html' % PROJECT_MODULE,
        'tower.management.commands.extract.extract_tower_template'),
    ('templates/**.html',
        'tower.management.commands.extract.extract_tower_template'),
]

# # Use this if you have localizable HTML files:
# DOMAIN_METHODS['lhtml'] = [
#    ('**/templates/**.lhtml',
#        'tower.management.commands.extract.extract_tower_template'),
# ]

# # Use this if you have localizable JS files:
# DOMAIN_METHODS['javascript'] = [
#    # Make sure that this won't pull in strings from external libraries you
#    # may use.
#    ('media/js/**.js', 'javascript'),
# ]

LOGGING = dict(loggers=dict(playdoh = {'level': logging.DEBUG}))
<|endoftext|>


<commit_before>from __future__ import division, unicode_literals, print_function

"""
Defines various firetasks
"""

import sys
import re
import socket
import copy
import logging

from pymatgen.io.vaspio.vasp_input import Incar, Poscar
from pymatgen.io.vaspio.vasp_input import Potcar, Kpoints
from pymatgen.apps.borg.queen import BorgQueen

from fireworks.core.firework import FireTaskBase, Firework, FWAction
from fireworks.core.launchpad import LaunchPad
from fireworks.utilities.fw_serializers import FWSerializable
from fireworks.utilities.fw_utilities import explicit_serialize
from fireworks.user_objects.queue_adapters.common_adapter import CommonAdapter

from mpinterfaces.calibrate import CalibrateMolecule, CalibrateSlab
from mpinterfaces.calibrate import CalibrateBulk
from mpinterfaces.measurement import Measurement
from mpinterfaces.database import MPINTVaspToDbTaskDrone

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
sh = logging.StreamHandler(stream=sys.stdout)
sh.setFormatter(formatter)
logger.addHandler(sh)

def load_class(mod, name):
    """
    load class named name from module mod
    this function is adapted from materialsproject
    """
    mod = __import__(mod, globals(), locals(), [name], 0)
    return getattr(mod, name)

def get_qadapter(nnodes=1, nprocs=16, walltime='24:00:00',
                 job_bin='/home/km468/Software/VASP/vasp.5.3.5/vasp'):
    d = {}
    #hipergator
    if 'ufhpc' in socket.gethostname():
        job_bin='/home/km468/Software/VASP/vasp.5.3.5/vasp'
        d = {'type':'PBS',
             'params':
                 {
                'nnodes': str(nnodes),
                'ppnode': str(int(nprocs/nnodes)),
                'walltime': walltime,
                'job_name': 'vasp_job',
                'pre_rocket': '#PBS -l pmem=1000mb',
                'rocket_launch': 'mpirun '+job_bin
                }
             }
    #stampede
    elif 'stampede' in socket.gethostname():
        job_bin='/home1/01682/km468/Software/VASP/vasp.5.3.5/vasp'
        d = {'type':'SLURM',
             'params':
                 {
                'nodes': str(nnodes),
                'ntasks': str(nprocs),
                'walltime': walltime,
                'queue':'normal',
                'account':'TG-DMR050028N',
                'job_name': 'vasp_job',
                'rocket_launch': 'ibrun '+job_bin
                }
             }
    if d:
        return CommonAdapter(d['type'], **d['params'])
    else:
        return None

def get_cal_obj(d):
    """
    construct a calibration object from the input dictionary, d

    returns a calibration object
    """
    incar = Incar.from_dict(d["incar"])
    poscar = Poscar.from_dict(d["poscar"])
    symbols = poscar.site_symbols
    potcar = Potcar(symbols)
    kpoints = Kpoints.from_dict(d["kpoints"])
    turn_knobs = d["turn_knobs"]
    qadapter = None
    job_cmd = None
    if d['que_params']:
        qadapter = get_qadapter(**d['que_params'])
    #henniggroup machines
    if qadapter is None:
        if socket.gethostname() in ['hydrogen',
                                    'helium',
                                    'lithium',
                                    'beryllium',
                                    'carbon']:
            job_cmd = ['nohup',
                       '/opt/openmpi_intel/bin/mpirun',
                       '-n', '24',
                       '/home/km468/Software/VASP/vasp.5.3.5/vasp']
            if d.get("other_params") is not None:
                d.get("other_params").update({'wait':False})
            else:
                d["other_params"] = {'wait':False}
        else:
            job_cmd=['ls', '-lt']
    if d.get('job_cmd'):
        job_cmd = d.get('job_cmd')
    cal =  load_class("mpinterfaces.calibrate",
                      d["calibrate"])(incar, poscar, potcar, kpoints,
                                      system = d.get("system", None),
                                      qadapter=qadapter, job_cmd=job_cmd,
                                      turn_knobs=turn_knobs,
                                      **d.get("other_params", {}))
    if d.get('job_dir_list'):
        cal.job_dir_list = d.get('job_dir_list')
    return cal

@explicit_serialize
class MPINTCalibrateTask(FireTaskBase, FWSerializable):
    """
    Calibration Task
    """
    required_params = ["incar", "poscar", "kpoints", "calibrate",
                       "que_params", "turn_knobs"]
    optional_params = ["job_cmd", "system", "other_params"]

    def run_task(self, fw_spec):
        """
        launch jobs to the queue
        """
        cal = get_cal_obj(self)
        cal.setup()
        cal.run()
        d = cal.as_dict()
        d.update({'que_params':self['que_params']})
        return FWAction(mod_spec=[{'_push': {'cal_objs':d}}])

@explicit_serialize
class MPINTMeasurementTask(FireTaskBase, FWSerializable):
    """
    Measurement Task
    """
    required_params = ["measurement", "que_params"]
    optional_params = ["job_cmd", "other_params", "fw_id"]

    def run_task(self, fw_spec):
        """
        setup up a measurement task using the prior calibration jobs
        and run
        """
        cal_objs = []
        logger.info('The measurement task will be constructed from {} calibration objects'
                    .format(len(fw_spec['cal_objs'])) )
        for calparams in fw_spec['cal_objs']:
            calparams.update({'que_params':self['que_params']})
            cal = get_cal_obj(calparams)
            cal_objs.append(cal)
        done = load_class("mpinterfaces.calibrate", "Calibrate").check_calcs(cal_objs)
        if not done:
            logger.info('Calibration not done yet. Try again later')
            if self.get("fw_id"):
                fw_id = int(self.get("fw_id")) + 1
                self["fw_id"] = fw_id
                new_fw = Firework(MPINTMeasurementTask(self),
                                  spec={'cal_objs':fw_spec['cal_objs']},
                                  name = 'new_fw', fw_id = fw_id)
            else:
                new_fw = Firework(MPINTMeasurementTask(self),
                                  spec={'cal_objs':fw_spec['cal_objs']},
                                  name = 'new_fw')

            return FWAction(additions=new_fw)
        else:
            measure = load_class("mpinterfaces.measurement",self['measurement'])(cal_objs,
                                                                                **self.get("other_params", {}))
            job_cmd = None
            if self.get("job_cmd", None) is not None:
                job_cmd = self.get("job_cmd")
            measure.setup()
            measure.run(job_cmd = job_cmd)
            d = {}
            for cal in measure.cal_objs:
                d = cal.as_dict()
                d.update({'que_params':self['que_params']})
            return FWAction(update_spec={'cal_objs':d})

@explicit_serialize
class MPINTDatabaseTask(FireTaskBase, FWSerializable):
    """
    submit data to the database firetask
    """
    required_params = ["measure_dir"]
    optional_params = ["dbase_params"]

    def run_task(self, fw_spec):
        """
        go through the measurement job dirs and
        put the measurement jobs in the database
        """
        drone = MPINTVaspToDbTaskDrone(**d.get("dbase_params", {}))
        queen = BorgQueen(drone)#, number_of_drones=ncpus)
        queen.serial_assimilate(self["measure_dir"])
        return FWAction()
<commit_msg>updated firetasks<commit_after>from __future__ import division, unicode_literals, print_function

"""
Defines various firetasks
"""

import sys
import re
import socket
import copy
import logging

from pymatgen.io.vaspio.vasp_input import Incar, Poscar
from pymatgen.io.vaspio.vasp_input import Potcar, Kpoints
from pymatgen.apps.borg.queen import BorgQueen

from fireworks.core.firework import FireTaskBase, Firework, FWAction
from fireworks.core.launchpad import LaunchPad
from fireworks.utilities.fw_serializers import FWSerializable
from fireworks.utilities.fw_utilities import explicit_serialize
from fireworks.user_objects.queue_adapters.common_adapter import CommonAdapter

from mpinterfaces.calibrate import CalibrateMolecule, CalibrateSlab
from mpinterfaces.calibrate import CalibrateBulk
from mpinterfaces.measurement import Measurement
from mpinterfaces.database import MPINTVaspToDbTaskDrone

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
sh = logging.StreamHandler(stream=sys.stdout)
sh.setFormatter(formatter)
logger.addHandler(sh)

def load_class(mod, name):
    """
    load class named name from module mod
    this function is adapted from materialsproject
    """
    mod = __import__(mod, globals(), locals(), [name], 0)
    return getattr(mod, name)

def get_qadapter(nnodes=1, nprocs=16, walltime='24:00:00',
                 job_bin='/home/km468/Software/VASP/vasp.5.3.5/vasp'):
    d = {}
    #hipergator
    if 'ufhpc' in socket.gethostname():
        job_bin='/home/km468/Software/VASP/vasp.5.3.5/vasp'
        d = {'type':'PBS',
             'params':
                 {
                'nnodes': str(nnodes),
                'ppnode': str(int(nprocs/nnodes)),
                'walltime': walltime,
                'job_name': 'vasp_job',
                'pre_rocket': '#PBS -l pmem=1000mb',
                'rocket_launch': 'mpirun '+job_bin
                }
             }
    #stampede
    elif 'stampede' in socket.gethostname():
        job_bin='/home1/01682/km468/Software/VASP/vasp.5.3.5/vasp'
        d = {'type':'SLURM',
             'params':
                 {
                'nodes': str(nnodes),
                'ntasks': str(nprocs),
                'walltime': walltime,
                'queue':'normal',
                'account':'TG-DMR050028N',
                'job_name': 'vasp_job',
                'rocket_launch': 'ibrun '+job_bin
                }
             }
    if d:
        return CommonAdapter(d['type'], **d['params'])
    else:
        return None

def get_cal_obj(d):
    """
    construct a calibration object from the input dictionary, d

    returns a calibration object
    """
    incar = Incar.from_dict(d["incar"])
    poscar = Poscar.from_dict(d["poscar"])
    symbols = poscar.site_symbols
    potcar = Potcar(symbols)
    kpoints = Kpoints.from_dict(d["kpoints"])
    turn_knobs = d["turn_knobs"]
    qadapter = None
    job_cmd = None
    if d['que_params']:
        qadapter = get_qadapter(**d['que_params'])
    #henniggroup machines
    if qadapter is None:
        if socket.gethostname() in ['hydrogen',
                                    'helium',
                                    'lithium',
                                    'beryllium',
                                    'carbon']:
            job_cmd = ['nohup',
                       '/opt/openmpi_intel/bin/mpirun',
                       '-n', '24',
                       '/home/km468/Software/VASP/vasp.5.3.5/vasp']
            if d.get("other_params") is not None:
                d.get("other_params").update({'wait':False})
            else:
                d["other_params"] = {'wait':False}
        else:
            job_cmd=['ls', '-lt']
    if d.get('job_cmd'):
        job_cmd = d.get('job_cmd')
    cal =  load_class("mpinterfaces.calibrate",
                      d["calibrate"])(incar, poscar, potcar, kpoints,
                                      system = d.get("system", None),
                                      qadapter=qadapter, job_cmd=job_cmd,
                                      turn_knobs=turn_knobs,
                                      **d.get("other_params", {}))
    if d.get('job_dir_list'):
        cal.job_dir_list = d.get('job_dir_list')
    return cal

@explicit_serialize
class MPINTCalibrateTask(FireTaskBase, FWSerializable):
    """
    Calibration Task
    """
    required_params = ["incar", "poscar", "kpoints", "calibrate",
                       "que_params", "turn_knobs"]
    optional_params = ["job_cmd", "system", "other_params"]

    def run_task(self, fw_spec):
        """
        launch jobs to the queue
        """
        cal = get_cal_obj(self)
        cal.setup()
        cal.run()
        d = cal.as_dict()
        d.update({'que_params':self['que_params']})
        return FWAction(mod_spec=[{'_push': {'cal_objs':d}}])

@explicit_serialize
class MPINTMeasurementTask(FireTaskBase, FWSerializable):
    """
    Measurement Task
    """
    required_params = ["measurement", "que_params"]
    optional_params = ["job_cmd", "other_params", "fw_id"]

    def run_task(self, fw_spec):
        """
        setup up a measurement task using the prior calibration jobs
        and run
        """
        cal_objs = []
        logger.info('The measurement task will be constructed from {} calibration objects'
                    .format(len(fw_spec['cal_objs'])) )
        for calparams in fw_spec['cal_objs']:
            calparams.update({'que_params':self['que_params']})
            cal = get_cal_obj(calparams)
            cal_objs.append(cal)
        done = load_class("mpinterfaces.calibrate", "Calibrate").check_calcs(cal_objs)
        if not done:
            logger.info('Calibration not done yet. Try again later')
            if self.get("fw_id"):
                fw_id = int(self.get("fw_id")) + 1
                self["fw_id"] = fw_id
                new_fw = Firework(MPINTMeasurementTask(self),
                                  spec={'cal_objs':fw_spec['cal_objs']},
                                  name = 'new_fw', fw_id = -fw_id)
            else:
                new_fw = Firework(MPINTMeasurementTask(self),
                                  spec={'cal_objs':fw_spec['cal_objs']},
                                  name = 'new_fw')

            return FWAction(additions=new_fw)
        else:
            measure = load_class("mpinterfaces.measurement",self['measurement'])(cal_objs,
                                                                                **self.get("other_params", {}))
            job_cmd = None
            if self.get("job_cmd", None) is not None:
                job_cmd = self.get("job_cmd")
            measure.setup()
            measure.run(job_cmd = job_cmd)
            d = {}
            for cal in measure.cal_objs:
                d = cal.as_dict()
                d.update({'que_params':self['que_params']})
            return FWAction(update_spec={'cal_objs':d})

@explicit_serialize
class MPINTDatabaseTask(FireTaskBase, FWSerializable):
    """
    submit data to the database firetask
    """
    required_params = ["measure_dir"]
    optional_params = ["dbase_params"]

    def run_task(self, fw_spec):
        """
        go through the measurement job dirs and
        put the measurement jobs in the database
        """
        drone = MPINTVaspToDbTaskDrone(**d.get("dbase_params", {}))
        queen = BorgQueen(drone)#, number_of_drones=ncpus)
        queen.serial_assimilate(self["measure_dir"])
        return FWAction()
<|endoftext|>