<commit_before>"""
tipfy.recipe.appengine
======================
This is a ZC Buildout recipe for setting up a
`Google App Engine <http://code.google.com/appengine/>`_ development
environment for the `tipfy <http://www.tipfy.org/>`_ framework.

Based on `rod.recipe.appengine <http://pypi.python.org/pypi/rod.recipe.appengine>`_

This fixes installation issues on Windows and adds an optional configuration to
install dependencies in a subdirectory of the app dir (e.g., '/app/lib').

It will eventually include specific features for tipfy.

Links
-----
* `Tipfy's website <http://www.tipfy.org/>`_
* `Documentation <http://www.tipfy.org//docs/>`_
"""
from setuptools import setup, find_packages

setup(
    name='tipfy.recipe.appengine',
    version='0.1.5',
    author='Rodrigo Moraes',
    author_email='rodrigo.moraes@gmail.com',
    description='ZC Buildout recipe for tipfy.',
    long_description=__doc__,
    license='LGPL 3',
    keywords='appengine gae zc.buildout recipe zope tipfy',
    url='http://pypi.python.org/pypi/tipfy.recipe.appengine',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Framework :: Buildout',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
        'Topic :: Software Development :: Build Tools',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages=find_packages('src'),
    package_dir={'': 'src'},
    install_requires=[
        'setuptools',
        'zc.buildout',
        'zc.recipe.egg',
    ],
    entry_points={'zc.buildout': ['default = tipfy.recipe.appengine:Recipe']},
    zip_safe=False,
)
<commit_msg>Changing tipfy.recipe.appengine version yo 0.1.6.<commit_after>"""
tipfy.recipe.appengine
======================
This is a ZC Buildout recipe for setting up a
`Google App Engine <http://code.google.com/appengine/>`_ development
environment for the `tipfy <http://www.tipfy.org/>`_ framework.

Based on `rod.recipe.appengine <http://pypi.python.org/pypi/rod.recipe.appengine>`_

This fixes installation issues on Windows and adds an optional configuration to
install dependencies in a subdirectory of the app dir (e.g., '/app/lib').

It will eventually include specific features for tipfy.

Links
-----
* `Tipfy's website <http://www.tipfy.org/>`_
* `Documentation <http://www.tipfy.org//docs/>`_
"""
from setuptools import setup, find_packages

setup(
    name='tipfy.recipe.appengine',
    version='0.1.6',
    author='Rodrigo Moraes',
    author_email='rodrigo.moraes@gmail.com',
    description='ZC Buildout recipe for tipfy.',
    long_description=__doc__,
    license='LGPL 3',
    keywords='appengine gae zc.buildout recipe zope tipfy',
    url='http://pypi.python.org/pypi/tipfy.recipe.appengine',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Framework :: Buildout',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
        'Topic :: Software Development :: Build Tools',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
    packages=find_packages('src'),
    package_dir={'': 'src'},
    install_requires=[
        'setuptools',
        'zc.buildout',
        'zc.recipe.egg',
    ],
    entry_points={'zc.buildout': ['default = tipfy.recipe.appengine:Recipe']},
    zip_safe=False,
)
<|endoftext|>


<commit_before>#    Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com>
#    This file is part of GlusterFS.

#    GlusterFS is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published
#    by the Free Software Foundation; either version 3 of the License,
#    or (at your option) any later version.

#    GlusterFS is distributed in the hope that it will be useful, but
#    WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see
#    <http://www.gnu.org/licenses/>.

import os, sys, string
import subprocess

num_replica = 2
num_stripe = 4
#Cachesize calculator
cache_size = "`echo $[ $(grep 'MemTotal' /proc/meminfo | sed 's/[^0-9]//g') / 5120 ]`"

class CreateVolfile:

    def __init__ (self, server_dict, server, transport,
                  transports, options, server_array):

        self.host_dict = server_dict
        self.host = server
        self.volume_name = options.volume_name
        self.transport = transport
        self.transports = transports
        self.gfs_port = options.port
        self.gfs_ib_port = options.port + 1
        self.auth_parameters = options.auth_param
        self.raid_type = options.raid_type
        self.ib_devport = options.ib_dev
        self.num_servers = len (self.host_dict.keys())
        self.conf_dir = options.conf_dir
        self.host_array = server_array
        self.unused = options.unused
        self.debug = options.debug
        self.volume_size_server = options.size_server
        self.volume_size_client = options.size_client

    def create_mount_volfile (self):

        raid_type = self.raid_type

        if self.conf_dir:
            mount_fd = file ("%s/%s-%s.vol" % (self.conf_dir,
                                               str(self.volume_name),
                                               str(self.transport)), "w")
        else:
            mount_fd = file ("%s-%s.vol" % (str(self.volume_name),
                                            str(self.transport)), "w")

        print "Generating client volfiles.. for transport '%s'" % (self.transport)

        cmdline = string.join (sys.argv, ' ')

        mount_fd.write ("## file auto generated by %s (mount.vol)\n" %
                        sys.argv[0])
        mount_fd.write ("# Cmd line:\n")
        mount_fd.write ("# $ %s\n\n" % cmdline)

        if raid_type is not None:
            # Used for later usage
            mount_fd.write ("# RAID %d\n" % raid_type)

        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
        subvolumes = []
        for host in self.host_dict.keys():
            i = 1
            for exports in self.host_dict[host]:
                mount_fd.write ("volume %s-%s\n" % (host,i))
                mount_fd.write ("    type protocol/client\n")
                mount_fd.write ("    option transport-type %s\n" %
                                self.transport)

                mount_fd.write ("    option remote-host %s\n" % host)
                if self.transport == 'ib-verbs':
                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
                                    self.ib_devport)
                    mount_fd.write ("    option transport.remote-port %d\n" %
                                    self.gfs_ib_port)
                if self.transport == 'tcp':
                    mount_fd.write ("    option transport.socket.nodelay on\n")
                    mount_fd.write ("    option transport.remote-port %d\n" %
                                    self.gfs_port)

                mount_fd.write ("    option remote-subvolume brick%s\n" %
                                i)
                mount_fd.write ("end-volume\n\n")
                i += 1

        exportlist = {}
        for entry in self.host_array:
            node = entry.split(':')[0]
            if not exportlist.has_key(node):
                exportlist[node] = 1
            else:
                exportlist[node] += 1
            subvolumes.append(str(node) + '-' + str(exportlist[node]))

        # Stripe section.. if given
        if raid_type is 0:
            max_stripe_idx = len (subvolumes) / num_stripe
            stripe_idx = 0
            index = 0
            while index < max_stripe_idx:
                mount_fd.write ("volume stripe-%d\n" % index)
                mount_fd.write ("    type cluster/stripe\n")
                if self.unused:
                    mount_fd.write ("#    option block-size 128k\n")
                    mount_fd.write ("#    option use-xattr no\n")

                mount_fd.write ("    subvolumes %s %s %s %s\n" %
                                (subvolumes[stripe_idx],
                                 subvolumes[stripe_idx+1],
                                 subvolumes[stripe_idx+2],
                                 subvolumes[stripe_idx+3]))
                mount_fd.write ("end-volume\n\n")
                stripe_idx += 4
                index +=1

        # Replicate section
        if raid_type is 1:
            max_mirror_idx = len (subvolumes) / num_replica
            mirror_idx = 0
            index = 0
            while index < max_mirror_idx:
                mount_fd.write ("volume mirror-%d\n" % index)
                mount_fd.write ("    type cluster/replicate\n")
                mount_fd.write ("    subvolumes %s %s\n" %
                                (subvolumes[mirror_idx],
                                 subvolumes[mirror_idx+1]))
                mount_fd.write ("end-volume\n\n")
                mirror_idx += 2
                index += 1

        # Distribute section
        if raid_type is 0:
            subvolumes = []
            flag = 0
            while flag < index:
                subvolumes.append ("stripe-%d" % flag)
                flag += 1
        if raid_type is 1:
            subvolumes = []
            flag = 0
            while flag < index:
                subvolumes.append ("mirror-%d" % flag)
                flag += 1

        if len (subvolumes) > 1:
            mount_fd.write ("volume distribute\n")
            mount_fd.write ("    type cluster/distribute\n")
            if self.unused:
                mount_fd.write("# option unhashed-sticky-bit yes # Used for migrating data while adding new nodes\n")
                mount_fd.write("# option min-free-disk 5% # Minimum free disk available on the volume\n")

            mount_fd.write ("    subvolumes %s\n" %
                                 string.join (subvolumes,' '))
            mount_fd.write ("end-volume\n\n")
            subvolumes[0] = "distribute"

        if self.volume_size_client:
            mount_fd.write ("volume quota\n")
            mount_fd.write ("    type features/quota\n")
            mount_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_client)
            if self.unused:
                mount_fd.write ("#  option minimum-free-disk-limit 10GB "
                                "# minimum free disk value (default) 0\n")
                mount_fd.write ("#  option refresh-interval 10\n")
            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
            mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume writebehind\n")
        mount_fd.write ("    type performance/write-behind\n")
        mount_fd.write ("    option cache-size 4MB\n")
        if self.unused:
            mount_fd.write ("#   option enable-trickling-writes yes # Flush final write calls when network is free\n")
            mount_fd.write ("#   option enable-O_SYNC yes # Enable O_SYNC for write-behind\n")
            mount_fd.write ("#   option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n")
        if self.volume_size_client:
            mount_fd.write ("    subvolumes quota\n")
        else:
            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])

        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume readahead\n")
        mount_fd.write ("    type performance/read-ahead\n")
        mount_fd.write ("    option page-count 4\n")

        if self.unused:
            mount_fd.write ("#   option force-atime-update yes # force updating atimes, default off\n")
        mount_fd.write ("    subvolumes writebehind\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume iocache\n")
        mount_fd.write ("    type performance/io-cache\n")
        mount_fd.write ("    option cache-size %sMB\n" % cache_size)
        mount_fd.write ("    option cache-timeout 1\n")
        if self.unused:
            mount_fd.write ("#   option priority *.html:1,abc*:2 # Priority list for iocaching files\n")
        mount_fd.write ("    subvolumes readahead\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume quickread\n")
        mount_fd.write ("    type performance/quick-read\n")
        mount_fd.write ("    option cache-timeout 1\n")
        mount_fd.write ("    option max-file-size 64kB\n")
        mount_fd.write ("    subvolumes iocache\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume statprefetch\n")
        mount_fd.write ("    type performance/stat-prefetch\n")
        mount_fd.write ("    subvolumes quickread\n")
        mount_fd.write ("end-volume\n\n")

        return

    def create_export_volfile (self):

        cmdline = string.join (sys.argv, ' ')

        if self.conf_dir:
            exp_fd = file ("%s/%s-export.vol" %
                           (self.conf_dir,
                            str(self.host + '-' + self.volume_name)),"w")
        else:
            exp_fd = file ("%s-export.vol" %
                           (str(self.host + '-' + self.volume_name)),"w")

        print "Generating server volfiles.. for server '%s'" % (self.host)

        exp_fd.write ("## file auto generated by %s (export.vol)\n" %
                      sys.argv[0])
        exp_fd.write ("# Cmd line:\n")
        exp_fd.write ("# $ %s\n\n" % cmdline)
        total_bricks = []
        i=1
        for export in self.host_dict[self.host]:
            exp_fd.write ("volume posix%d\n" % i)
            exp_fd.write ("  type storage/posix\n")
            if self.unused:
                exp_fd.write("# option o-direct enable # (default: disable) boolean type only\n")
                exp_fd.write("# option export-statfs-size no # (default: yes) boolean type only\n")
                exp_fd.write("# option mandate-attribute off # (default: on) boolean type only\n")
                exp_fd.write("# option span-devices 8 # (default: 0) integer value\n")
                exp_fd.write("# option background-unlink yes # (default: no) boolean type\n")

            exp_fd.write ("  option directory %s\n" % export)
            exp_fd.write ("end-volume\n\n")

            if self.volume_size_server:
                exp_fd.write ("volume quota%d\n" % i)
                exp_fd.write ("  type features/quota\n")
                exp_fd.write ("  option disk-usage-limit %s\n" % self.volume_size_server)
                if self.unused:
                    exp_fd.write ("#  option minimum-free-disk-limit 10GB "
                                  "# minimum free disk value (default) 0\n")
                    exp_fd.write ("#  option refresh-interval 10\n")
                exp_fd.write ("  subvolumes posix%d\n" % i)
                exp_fd.write ("end-volume\n\n")

            exp_fd.write ("volume locks%d\n" % i)
            exp_fd.write ("    type features/locks\n")
            if self.unused:
                exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
            if self.volume_size_server:
                exp_fd.write ("    subvolumes quota%d\n" % i)
            else:
                exp_fd.write ("    subvolumes posix%d\n" % i)
            exp_fd.write ("end-volume\n\n")

            exp_fd.write ("volume brick%d\n" % i)
            exp_fd.write ("    type performance/io-threads\n")
            exp_fd.write ("    option thread-count 8\n")
            if self.unused:
                exp_fd.write ("#    option autoscaling yes # Heuristic for autoscaling threads on demand\n")
                exp_fd.write ("#    option min-threads 2 # min count for thread pool\n")
                exp_fd.write ("#    option max-threads 64 # max count for thread pool\n")

            exp_fd.write ("    subvolumes locks%d\n" % i)
            exp_fd.write ("end-volume\n\n")

            total_bricks.append("brick%s" % i)
            i += 1

        for transport in self.transports:
            exp_fd.write ("volume server-%s\n" % transport)
            exp_fd.write ("    type protocol/server\n")
            exp_fd.write ("    option transport-type %s\n" % transport)
            for brick in total_bricks:
                exp_fd.write ("    option auth.addr.%s.allow %s\n" %
                              (brick, self.auth_parameters))

            if transport == 'ib-verbs':
                exp_fd.write ("    option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port)
                exp_fd.write ("    option transport.ib-verbs.port %d\n" %
                              self.ib_devport)
            if transport == 'tcp':
                exp_fd.write ("    option transport.socket.listen-port %d\n" % self.gfs_port)
                exp_fd.write ("    option transport.socket.nodelay on\n")

            exp_fd.write ("    subvolumes %s\n" %
                          string.join(total_bricks, ' '))
            exp_fd.write ("end-volume\n\n")

        return
<commit_msg>Fix backtick evaluation bug in cache-size<commit_after>#    Copyright (c) 2009-2010 Gluster, Inc. <http://www.gluster.com>
#    This file is part of GlusterFS.

#    GlusterFS is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published
#    by the Free Software Foundation; either version 3 of the License,
#    or (at your option) any later version.

#    GlusterFS is distributed in the hope that it will be useful, but
#    WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see
#    <http://www.gnu.org/licenses/>.

import os, sys, string
import subprocess

num_replica = 2
num_stripe = 4
#Cachesize calculator
cache_size = "`echo $(( $(grep 'MemTotal' /proc/meminfo | sed 's/[^0-9]//g') / 5120 ))`"

class CreateVolfile:

    def __init__ (self, server_dict, server, transport,
                  transports, options, server_array):

        self.host_dict = server_dict
        self.host = server
        self.volume_name = options.volume_name
        self.transport = transport
        self.transports = transports
        self.gfs_port = options.port
        self.gfs_ib_port = options.port + 1
        self.auth_parameters = options.auth_param
        self.raid_type = options.raid_type
        self.ib_devport = options.ib_dev
        self.num_servers = len (self.host_dict.keys())
        self.conf_dir = options.conf_dir
        self.host_array = server_array
        self.unused = options.unused
        self.debug = options.debug
        self.volume_size_server = options.size_server
        self.volume_size_client = options.size_client

    def create_mount_volfile (self):

        raid_type = self.raid_type

        if self.conf_dir:
            mount_fd = file ("%s/%s-%s.vol" % (self.conf_dir,
                                               str(self.volume_name),
                                               str(self.transport)), "w")
        else:
            mount_fd = file ("%s-%s.vol" % (str(self.volume_name),
                                            str(self.transport)), "w")

        print "Generating client volfiles.. for transport '%s'" % (self.transport)

        cmdline = string.join (sys.argv, ' ')

        mount_fd.write ("## file auto generated by %s (mount.vol)\n" %
                        sys.argv[0])
        mount_fd.write ("# Cmd line:\n")
        mount_fd.write ("# $ %s\n\n" % cmdline)

        if raid_type is not None:
            # Used for later usage
            mount_fd.write ("# RAID %d\n" % raid_type)

        mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport)
        subvolumes = []
        for host in self.host_dict.keys():
            i = 1
            for exports in self.host_dict[host]:
                mount_fd.write ("volume %s-%s\n" % (host,i))
                mount_fd.write ("    type protocol/client\n")
                mount_fd.write ("    option transport-type %s\n" %
                                self.transport)

                mount_fd.write ("    option remote-host %s\n" % host)
                if self.transport == 'ib-verbs':
                    mount_fd.write ("    option transport.ib-verbs.port %d\n" %
                                    self.ib_devport)
                    mount_fd.write ("    option transport.remote-port %d\n" %
                                    self.gfs_ib_port)
                if self.transport == 'tcp':
                    mount_fd.write ("    option transport.socket.nodelay on\n")
                    mount_fd.write ("    option transport.remote-port %d\n" %
                                    self.gfs_port)

                mount_fd.write ("    option remote-subvolume brick%s\n" %
                                i)
                mount_fd.write ("end-volume\n\n")
                i += 1

        exportlist = {}
        for entry in self.host_array:
            node = entry.split(':')[0]
            if not exportlist.has_key(node):
                exportlist[node] = 1
            else:
                exportlist[node] += 1
            subvolumes.append(str(node) + '-' + str(exportlist[node]))

        # Stripe section.. if given
        if raid_type is 0:
            max_stripe_idx = len (subvolumes) / num_stripe
            stripe_idx = 0
            index = 0
            while index < max_stripe_idx:
                mount_fd.write ("volume stripe-%d\n" % index)
                mount_fd.write ("    type cluster/stripe\n")
                if self.unused:
                    mount_fd.write ("#    option block-size 128k\n")
                    mount_fd.write ("#    option use-xattr no\n")

                mount_fd.write ("    subvolumes %s %s %s %s\n" %
                                (subvolumes[stripe_idx],
                                 subvolumes[stripe_idx+1],
                                 subvolumes[stripe_idx+2],
                                 subvolumes[stripe_idx+3]))
                mount_fd.write ("end-volume\n\n")
                stripe_idx += 4
                index +=1

        # Replicate section
        if raid_type is 1:
            max_mirror_idx = len (subvolumes) / num_replica
            mirror_idx = 0
            index = 0
            while index < max_mirror_idx:
                mount_fd.write ("volume mirror-%d\n" % index)
                mount_fd.write ("    type cluster/replicate\n")
                mount_fd.write ("    subvolumes %s %s\n" %
                                (subvolumes[mirror_idx],
                                 subvolumes[mirror_idx+1]))
                mount_fd.write ("end-volume\n\n")
                mirror_idx += 2
                index += 1

        # Distribute section
        if raid_type is 0:
            subvolumes = []
            flag = 0
            while flag < index:
                subvolumes.append ("stripe-%d" % flag)
                flag += 1
        if raid_type is 1:
            subvolumes = []
            flag = 0
            while flag < index:
                subvolumes.append ("mirror-%d" % flag)
                flag += 1

        if len (subvolumes) > 1:
            mount_fd.write ("volume distribute\n")
            mount_fd.write ("    type cluster/distribute\n")
            if self.unused:
                mount_fd.write("# option unhashed-sticky-bit yes # Used for migrating data while adding new nodes\n")
                mount_fd.write("# option min-free-disk 5% # Minimum free disk available on the volume\n")

            mount_fd.write ("    subvolumes %s\n" %
                                 string.join (subvolumes,' '))
            mount_fd.write ("end-volume\n\n")
            subvolumes[0] = "distribute"

        if self.volume_size_client:
            mount_fd.write ("volume quota\n")
            mount_fd.write ("    type features/quota\n")
            mount_fd.write ("    option disk-usage-limit %s\n" % self.volume_size_client)
            if self.unused:
                mount_fd.write ("#  option minimum-free-disk-limit 10GB "
                                "# minimum free disk value (default) 0\n")
                mount_fd.write ("#  option refresh-interval 10\n")
            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])
            mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume writebehind\n")
        mount_fd.write ("    type performance/write-behind\n")
        mount_fd.write ("    option cache-size 4MB\n")
        if self.unused:
            mount_fd.write ("#   option enable-trickling-writes yes # Flush final write calls when network is free\n")
            mount_fd.write ("#   option enable-O_SYNC yes # Enable O_SYNC for write-behind\n")
            mount_fd.write ("#   option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n")
        if self.volume_size_client:
            mount_fd.write ("    subvolumes quota\n")
        else:
            mount_fd.write ("    subvolumes %s\n" % subvolumes[0])

        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume readahead\n")
        mount_fd.write ("    type performance/read-ahead\n")
        mount_fd.write ("    option page-count 4\n")

        if self.unused:
            mount_fd.write ("#   option force-atime-update yes # force updating atimes, default off\n")
        mount_fd.write ("    subvolumes writebehind\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume iocache\n")
        mount_fd.write ("    type performance/io-cache\n")
        mount_fd.write ("    option cache-size %sMB\n" % cache_size)
        mount_fd.write ("    option cache-timeout 1\n")
        if self.unused:
            mount_fd.write ("#   option priority *.html:1,abc*:2 # Priority list for iocaching files\n")
        mount_fd.write ("    subvolumes readahead\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume quickread\n")
        mount_fd.write ("    type performance/quick-read\n")
        mount_fd.write ("    option cache-timeout 1\n")
        mount_fd.write ("    option max-file-size 64kB\n")
        mount_fd.write ("    subvolumes iocache\n")
        mount_fd.write ("end-volume\n\n")

        mount_fd.write ("volume statprefetch\n")
        mount_fd.write ("    type performance/stat-prefetch\n")
        mount_fd.write ("    subvolumes quickread\n")
        mount_fd.write ("end-volume\n\n")

        return

    def create_export_volfile (self):

        cmdline = string.join (sys.argv, ' ')

        if self.conf_dir:
            exp_fd = file ("%s/%s-export.vol" %
                           (self.conf_dir,
                            str(self.host + '-' + self.volume_name)),"w")
        else:
            exp_fd = file ("%s-export.vol" %
                           (str(self.host + '-' + self.volume_name)),"w")

        print "Generating server volfiles.. for server '%s'" % (self.host)

        exp_fd.write ("## file auto generated by %s (export.vol)\n" %
                      sys.argv[0])
        exp_fd.write ("# Cmd line:\n")
        exp_fd.write ("# $ %s\n\n" % cmdline)
        total_bricks = []
        i=1
        for export in self.host_dict[self.host]:
            exp_fd.write ("volume posix%d\n" % i)
            exp_fd.write ("  type storage/posix\n")
            if self.unused:
                exp_fd.write("# option o-direct enable # (default: disable) boolean type only\n")
                exp_fd.write("# option export-statfs-size no # (default: yes) boolean type only\n")
                exp_fd.write("# option mandate-attribute off # (default: on) boolean type only\n")
                exp_fd.write("# option span-devices 8 # (default: 0) integer value\n")
                exp_fd.write("# option background-unlink yes # (default: no) boolean type\n")

            exp_fd.write ("  option directory %s\n" % export)
            exp_fd.write ("end-volume\n\n")

            if self.volume_size_server:
                exp_fd.write ("volume quota%d\n" % i)
                exp_fd.write ("  type features/quota\n")
                exp_fd.write ("  option disk-usage-limit %s\n" % self.volume_size_server)
                if self.unused:
                    exp_fd.write ("#  option minimum-free-disk-limit 10GB "
                                  "# minimum free disk value (default) 0\n")
                    exp_fd.write ("#  option refresh-interval 10\n")
                exp_fd.write ("  subvolumes posix%d\n" % i)
                exp_fd.write ("end-volume\n\n")

            exp_fd.write ("volume locks%d\n" % i)
            exp_fd.write ("    type features/locks\n")
            if self.unused:
                exp_fd.write ("#   option mandatory on # Default off, used in specific applications\n")
            if self.volume_size_server:
                exp_fd.write ("    subvolumes quota%d\n" % i)
            else:
                exp_fd.write ("    subvolumes posix%d\n" % i)
            exp_fd.write ("end-volume\n\n")

            exp_fd.write ("volume brick%d\n" % i)
            exp_fd.write ("    type performance/io-threads\n")
            exp_fd.write ("    option thread-count 8\n")
            if self.unused:
                exp_fd.write ("#    option autoscaling yes # Heuristic for autoscaling threads on demand\n")
                exp_fd.write ("#    option min-threads 2 # min count for thread pool\n")
                exp_fd.write ("#    option max-threads 64 # max count for thread pool\n")

            exp_fd.write ("    subvolumes locks%d\n" % i)
            exp_fd.write ("end-volume\n\n")

            total_bricks.append("brick%s" % i)
            i += 1

        for transport in self.transports:
            exp_fd.write ("volume server-%s\n" % transport)
            exp_fd.write ("    type protocol/server\n")
            exp_fd.write ("    option transport-type %s\n" % transport)
            for brick in total_bricks:
                exp_fd.write ("    option auth.addr.%s.allow %s\n" %
                              (brick, self.auth_parameters))

            if transport == 'ib-verbs':
                exp_fd.write ("    option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port)
                exp_fd.write ("    option transport.ib-verbs.port %d\n" %
                              self.ib_devport)
            if transport == 'tcp':
                exp_fd.write ("    option transport.socket.listen-port %d\n" % self.gfs_port)
                exp_fd.write ("    option transport.socket.nodelay on\n")

            exp_fd.write ("    subvolumes %s\n" %
                          string.join(total_bricks, ' '))
            exp_fd.write ("end-volume\n\n")

        return
<|endoftext|>


<commit_before>from time import sleep
from fabric.api import runs_once, settings, task
from fabric.tasks import execute
from .. import hosts as cluster_hosts, util as cluster_util
from ... import db, hilary, puppet

__all__ = ["upgrade", "upgrade_host", "delete_data"]

@runs_once
@task
def upgrade():
    """Runs through a general upgrade procedure for a set of Cassandra nodes.

        This will:

            1.  Ask for a password with which to sudo. All servers must have
                the same sudo passowrd
            2.  Perform a git pull on the puppet node to get the latest
                configuration
            3.  Run puppet on each db node
            4.  On each db node sequentially:
                    a. Run nodetool drain
                    b. Restart dse
                    c. Run nodetool upgradesstables
    """
    cluster_util.ensure_sudo_pass()

    # Stop puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.stop, force=True)

    # Pull the updated puppet configuration
    with settings(hosts=[cluster_hosts.puppet()], parallel=True):
        execute(puppet.git_update)

    # Run puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.run, force=False)

    # Upgrade each db node sequentially
    with settings(hosts=cluster_hosts.db(), parallel=False):
        execute(upgrade_host_internal)

    # Start puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.start)

@runs_once
@task
def upgrade_host():
    """Run through the general upgrade procedure for a Cassandra node, assuming
    puppet has already been updated.

        This will:

            1.  Ask for a password with which to sudo
            2.  Forcefully stop any current puppet runs
            3.  Run puppet on the node
            4.  Run nodetool drain
            5.  Restart dse
            6.  Run nodetool upgradesstables
    """
    cluster_util.ensure_sudo_pass()

    execute(puppet.stop, force=True)
    execute(puppet.run, force=False)
    execute(upgrade_host_internal)
    execute(puppet.start)

@runs_once
@task
def delete_data():
    """Delete all the data from the Cassandra cluster"""
    cluster_util.ensure_sudo_pass()

    # Gather convenience lists of hosts
    db_hosts = cluster_hosts.db()
    activity_hosts = cluster_hosts.activity()
    app_hosts = cluster_hosts.app()
    pp_hosts = cluster_hosts.pp()
    all_hilary_stop_hosts = activity_hosts + app_hosts + pp_hosts
    all_puppet_stop_hosts = db_hosts + activity_hosts + app_hosts + pp_hosts

    # Stop puppet on the db and hilary nodes
    with settings(hosts=all_puppet_stop_hosts, parallel=True):
        execute(puppet.stop, force=True)

    # Shut down all hilary nodes
    with settings(hosts=all_hilary_stop_hosts, parallel=True):
        execute(hilary.stop)

    # Delete data on each of the nodes
    with settings(hosts=cluster_hosts.db(), parallel=False):
        execute(delete_data_internal)

    # Run puppet on the db nodes to recreate data dirs and start them up
    with settings(hosts=cluster_hosts.db(), parallel=False):
        execute(puppet.run, force=False)

    # Wait until all Cassandra nodes are running to continue
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(db.wait_until_ready)
        sleep(5)

    # Start one of the hilary nodes so it can create the keyspace
    with settings(hosts=[all_hilary_stop_hosts[0]]):
        execute(hilary.start)

        # Give sufficient time for Hilary to create the keyspace. If a request is made to Cassandra
        # before the keyspace is created, then it can thwart the app startup process
        sleep(15)

        execute(hilary.wait_until_ready)

    # Start the remainder of the hilary nodes
    with settings(hosts=all_hilary_stop_hosts[1:], parallel=True):
        execute(hilary.start)
        execute(hilary.wait_until_ready)

    # Start puppet on all the nodes
    with settings(hosts=all_puppet_stop_hosts, parallel=True):
        execute(puppet.start)

def delete_data_internal():
    db.drain()
    db.stop()
    db.kill()
    sleep(15)
    db.delete_data()

def upgrade_host_internal():
    db.drain()
    db.stop()
    db.start()
    db.wait_until_ready()
    db.upgradesstables()
<commit_msg>Fiddle with a few things to ensure dbs are all down before delete<commit_after>from time import sleep
from fabric.api import runs_once, settings, task
from fabric.tasks import execute
from .. import hosts as cluster_hosts, util as cluster_util
from ... import db, hilary, puppet

__all__ = ["upgrade", "upgrade_host", "delete_data"]

@runs_once
@task
def upgrade():
    """Runs through a general upgrade procedure for a set of Cassandra nodes.

        This will:

            1.  Ask for a password with which to sudo. All servers must have
                the same sudo passowrd
            2.  Perform a git pull on the puppet node to get the latest
                configuration
            3.  Run puppet on each db node
            4.  On each db node sequentially:
                    a. Run nodetool drain
                    b. Restart dse
                    c. Run nodetool upgradesstables
    """
    cluster_util.ensure_sudo_pass()

    # Stop puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.stop, force=True)

    # Pull the updated puppet configuration
    with settings(hosts=[cluster_hosts.puppet()], parallel=True):
        execute(puppet.git_update)

    # Run puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.run, force=False)

    # Upgrade each db node sequentially
    with settings(hosts=cluster_hosts.db(), parallel=False):
        execute(upgrade_host_internal)

    # Start puppet on the db nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(puppet.start)

@runs_once
@task
def upgrade_host():
    """Run through the general upgrade procedure for a Cassandra node, assuming
    puppet has already been updated.

        This will:

            1.  Ask for a password with which to sudo
            2.  Forcefully stop any current puppet runs
            3.  Run puppet on the node
            4.  Run nodetool drain
            5.  Restart dse
            6.  Run nodetool upgradesstables
    """
    cluster_util.ensure_sudo_pass()

    execute(puppet.stop, force=True)
    execute(puppet.run, force=False)
    execute(upgrade_host_internal)
    execute(puppet.start)

@runs_once
@task
def delete_data():
    """Delete all the data from the Cassandra cluster"""
    cluster_util.ensure_sudo_pass()

    # Gather convenience lists of hosts
    db_hosts = cluster_hosts.db()
    activity_hosts = cluster_hosts.activity()
    app_hosts = cluster_hosts.app()
    pp_hosts = cluster_hosts.pp()
    all_hilary_stop_hosts = activity_hosts + app_hosts + pp_hosts
    all_puppet_stop_hosts = db_hosts + activity_hosts + app_hosts + pp_hosts

    # Stop puppet on the db and hilary nodes
    with settings(hosts=all_puppet_stop_hosts, parallel=True):
        execute(puppet.stop, force=True)

    # Shut down all hilary nodes
    with settings(hosts=all_hilary_stop_hosts, parallel=True):
        execute(hilary.stop)

    # Delete data on each of the nodes
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(delete_data_internal)

    # Run puppet on the db nodes to recreate data dirs and start them up
    with settings(hosts=cluster_hosts.db(), parallel=False):
        execute(puppet.run, force=False)

    # Wait until all Cassandra nodes are running to continue
    with settings(hosts=cluster_hosts.db(), parallel=True):
        execute(db.wait_until_ready)
        sleep(5)

    # Start one of the hilary nodes so it can create the keyspace
    with settings(hosts=[all_hilary_stop_hosts[0]]):
        execute(hilary.start)

        # Give sufficient time for Hilary to create the keyspace. If a request is made to Cassandra
        # before the keyspace is created, then it can thwart the app startup process
        sleep(15)

        execute(hilary.wait_until_ready)

    # Start the remainder of the hilary nodes
    with settings(hosts=all_hilary_stop_hosts[1:], parallel=True):
        execute(hilary.start)
        execute(hilary.wait_until_ready)

    # Start puppet on all the nodes
    with settings(hosts=all_puppet_stop_hosts, parallel=True):
        execute(puppet.start)

def delete_data_internal():
    db.drain()
    db.stop()
    db.kill()
    sleep(5)
    db.delete_data()
    sleep(5)

def upgrade_host_internal():
    db.drain()
    db.stop()
    db.start()
    db.wait_until_ready()
    db.upgradesstables()
<|endoftext|>


<commit_before># Generated by Django 2.2.5 on 2020-01-22 19:46

from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('comprehension', '0004_auto_20200115_0024'),
    ]

    operations = [
        migrations.CreateModel(
            name='RuleSet',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.TextField()),
                ('feedback', models.TextField()),
                ('priority', models.IntegerField(default=1)),
                ('pass_order', models.TextField(choices=[('first', 'first'), ('second', 'second')])),
                ('test_for_contains', models.BooleanField(default=True)),
                ('prompt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, default=None, null=True, related_name='rule_sets', to='comprehension.Prompt')),
            ],
            options={
                'unique_together': {('prompt', 'priority', 'pass_order')},
            },
        ),
        migrations.CreateModel(
            name='Rule',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('regex_text', models.TextField()),
                ('case_sensitive', models.BooleanField(default=False)),
                ('rule_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rules', to='comprehension.RuleSet')),
            ],
            options={
                'abstract': False,
            },
        ),
    ]
<commit_msg>take out old migration edit<commit_after># Generated by Django 2.2.5 on 2020-01-22 19:46

from django.db import migrations, models
import django.db.models.deletion

class Migration(migrations.Migration):

    dependencies = [
        ('comprehension', '0004_auto_20200115_0024'),
    ]

    operations = [
        migrations.CreateModel(
            name='RuleSet',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('name', models.TextField()),
                ('feedback', models.TextField()),
                ('priority', models.IntegerField(default=1)),
                ('pass_order', models.TextField(choices=[('first', 'first'), ('second', 'second')])),
                ('test_for_contains', models.BooleanField(default=True)),
                ('prompt', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rule_sets', to='comprehension.Prompt')),
            ],
            options={
                'unique_together': {('prompt', 'priority', 'pass_order')},
            },
        ),
        migrations.CreateModel(
            name='Rule',
            fields=[
                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                ('regex_text', models.TextField()),
                ('case_sensitive', models.BooleanField(default=False)),
                ('rule_set', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rules', to='comprehension.RuleSet')),
            ],
            options={
                'abstract': False,
            },
        ),
    ]
<|endoftext|>


<commit_before>"""Easy install Tests
"""
import sys
import os
import shutil
import tempfile
import unittest
import site
import contextlib
import textwrap

from setuptools.command.easy_install import easy_install, get_script_args, main
from setuptools.command.easy_install import  PthDistributions
from setuptools.command import easy_install as easy_install_pkg
from setuptools.dist import Distribution
from pkg_resources import Distribution as PRDistribution
import setuptools.tests.server

try:
    # import multiprocessing solely for the purpose of testing its existence
    __import__('multiprocessing')
    import logging
    _LOG = logging.getLogger('test_easy_install')
    logging.basicConfig(level=logging.INFO, stream=sys.stderr)
    _MULTIPROC = True
except ImportError:
    _MULTIPROC = False
    _LOG = None

class FakeDist(object):
    def get_entry_map(self, group):
        if group != 'console_scripts':
            return {}
        return {'name': 'ep'}

    def as_requirement(self):
        return 'spec'

WANTED = """\
#!%s
# EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name'
__requires__ = 'spec'
import sys
from pkg_resources import load_entry_point

if __name__ == '__main__':
    sys.exit(
        load_entry_point('spec', 'console_scripts', 'name')()
    )
""" % sys.executable

SETUP_PY = """\
from setuptools import setup

setup(name='foo')
"""

class TestEasyInstallTest(unittest.TestCase):

    def test_install_site_py(self):
        dist = Distribution()
        cmd = easy_install(dist)
        cmd.sitepy_installed = False
        cmd.install_dir = tempfile.mkdtemp()
        try:
            cmd.install_site_py()
            sitepy = os.path.join(cmd.install_dir, 'site.py')
            self.assert_(os.path.exists(sitepy))
        finally:
            shutil.rmtree(cmd.install_dir)

    def test_get_script_args(self):
        dist = FakeDist()

        old_platform = sys.platform
        try:
            name, script = [i for i in get_script_args(dist).next()][0:2]
        finally:
            sys.platform = old_platform

        self.assertEquals(script, WANTED)

    def test_no_setup_cfg(self):
        # makes sure easy_install as a command (main)
        # doesn't use a setup.cfg file that is located
        # in the current working directory
        dir = tempfile.mkdtemp()
        setup_cfg = open(os.path.join(dir, 'setup.cfg'), 'w')
        setup_cfg.write('[easy_install]\nfind_links = http://example.com')
        setup_cfg.close()
        setup_py = open(os.path.join(dir, 'setup.py'), 'w')
        setup_py.write(SETUP_PY)
        setup_py.close()

        from setuptools.dist import Distribution

        def _parse_command_line(self):
            msg = 'Error: a local setup.cfg was used'
            opts = self.command_options
            if 'easy_install' in opts:
                assert 'find_links' not in opts['easy_install'], msg
            return self._old_parse_command_line

        Distribution._old_parse_command_line = Distribution.parse_command_line
        Distribution.parse_command_line = _parse_command_line

        old_wd = os.getcwd()
        try:
            os.chdir(dir)
            main([])
        finally:
            os.chdir(old_wd)
            shutil.rmtree(dir)

    def test_no_find_links(self):
        # new option '--no-find-links', that blocks find-links added at
        # the project level
        dist = Distribution()
        cmd = easy_install(dist)
        cmd.check_pth_processing = lambda: True
        cmd.no_find_links = True
        cmd.find_links = ['link1', 'link2']
        cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok')
        cmd.args = ['ok']
        cmd.ensure_finalized()
        self.assertEquals(cmd.package_index.scanned_urls, {})

        # let's try without it (default behavior)
        cmd = easy_install(dist)
        cmd.check_pth_processing = lambda: True
        cmd.find_links = ['link1', 'link2']
        cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok')
        cmd.args = ['ok']
        cmd.ensure_finalized()
        keys = cmd.package_index.scanned_urls.keys()
        keys.sort()
        self.assertEquals(keys, ['link1', 'link2'])

class TestPTHFileWriter(unittest.TestCase):
    def test_add_from_cwd_site_sets_dirty(self):
        '''a pth file manager should set dirty
        if a distribution is in site but also the cwd
        '''
        pth = PthDistributions('does-not_exist', [os.getcwd()])
        self.assert_(not pth.dirty)
        pth.add(PRDistribution(os.getcwd()))
        self.assert_(pth.dirty)

    def test_add_from_site_is_ignored(self):
        if os.name != 'nt':
            location = '/test/location/does-not-have-to-exist'
        else:
            location = 'c:\\does_not_exist'
        pth = PthDistributions('does-not_exist', [location, ])
        self.assert_(not pth.dirty)
        pth.add(PRDistribution(location))
        self.assert_(not pth.dirty)

class TestUserInstallTest(unittest.TestCase):

    def setUp(self):
        self.dir = tempfile.mkdtemp()
        setup = os.path.join(self.dir, 'setup.py')
        f = open(setup, 'w')
        f.write(SETUP_PY)
        f.close()
        self.old_cwd = os.getcwd()
        os.chdir(self.dir)
        if sys.version >= "2.6":
            self.old_has_site = easy_install_pkg.HAS_USER_SITE
            self.old_file = easy_install_pkg.__file__
            self.old_base = site.USER_BASE
            site.USER_BASE = tempfile.mkdtemp()
            self.old_site = site.USER_SITE
            site.USER_SITE = tempfile.mkdtemp()
            easy_install_pkg.__file__ = site.USER_SITE

    def tearDown(self):
        os.chdir(self.old_cwd)
        shutil.rmtree(self.dir)
        if sys.version >= "2.6":
            shutil.rmtree(site.USER_BASE)
            shutil.rmtree(site.USER_SITE)
            site.USER_BASE = self.old_base
            site.USER_SITE = self.old_site
            easy_install_pkg.HAS_USER_SITE = self.old_has_site
            easy_install_pkg.__file__ = self.old_file

    def test_user_install_implied(self):
        easy_install_pkg.HAS_USER_SITE = True # disabled sometimes
        #XXX: replace with something meaningfull
        if sys.version < "2.6":
            return #SKIP
        dist = Distribution()
        dist.script_name = 'setup.py'
        cmd = easy_install(dist)
        cmd.args = ['py']
        cmd.ensure_finalized()
        self.assertTrue(cmd.user, 'user should be implied')

    def test_multiproc_atexit(self):
        if not _MULTIPROC:
            return
        _LOG.info('this should not break')

    def test_user_install_not_implied_without_usersite_enabled(self):
        easy_install_pkg.HAS_USER_SITE = False # usually enabled
        #XXX: replace with something meaningfull
        if sys.version < "2.6":
            return #SKIP
        dist = Distribution()
        dist.script_name = 'setup.py'
        cmd = easy_install(dist)
        cmd.args = ['py']
        cmd.initialize_options()
        self.assertFalse(cmd.user, 'NOT user should be implied')

    def test_local_index(self):
        # make sure the local index is used
        # when easy_install looks for installed
        # packages
        new_location = tempfile.mkdtemp()
        target = tempfile.mkdtemp()
        egg_file = os.path.join(new_location, 'foo-1.0.egg-info')
        f = open(egg_file, 'w')
        try:
            f.write('Name: foo\n')
        except:
            f.close()

        sys.path.append(target)
        old_ppath = os.environ.get('PYTHONPATH')
        os.environ['PYTHONPATH'] = os.path.pathsep.join(sys.path)
        try:
            dist = Distribution()
            dist.script_name = 'setup.py'
            cmd = easy_install(dist)
            cmd.install_dir = target
            cmd.args = ['foo']
            cmd.ensure_finalized()
            cmd.local_index.scan([new_location])
            res = cmd.easy_install('foo')
            self.assertEquals(res.location, new_location)
        finally:
            sys.path.remove(target)
            for basedir in [new_location, target, ]:
                if not os.path.exists(basedir) or not os.path.isdir(basedir):
                    continue
                try:
                    shutil.rmtree(basedir)
                except:
                    pass
            if old_ppath is not None:
                os.environ['PYTHONPATH'] = old_ppath
            else:
                del os.environ['PYTHONPATH']

class TestSetupRequires(unittest.TestCase):

    def test_setup_requires_honors_fetch_params(self):
        """
        When easy_install installs a source distribution which specifies
        setup_requires, it should honor the fetch parameters (such as
        allow-hosts, index-url, and find-links).
        """
        # set up a server which will simulate an alternate package index.
        p_index = setuptools.tests.server.MockServer()
        p_index.handle_request_in_thread()
        # create an sdist that has a build-time dependency.
        with TestSetupRequires.create_sdist() as dist_file:
            with tempdir_context() as temp_install_dir:
                with environment_context(PYTHONPATH=temp_install_dir):
                    ei_params = ['--index-url', p_index.url,
                        '--allow-hosts', 'localhost',
                        '--exclude-scripts', '--install-dir', temp_install_dir,
                        dist_file]
                    # attempt to install the dist. It should fail because
                    #  our fake server can't actually supply the dependency
                    self.assertRaises(SystemExit,
                        easy_install_pkg.main, ei_params)
                #self.assertTrue(os.listdir(temp_install_dir))
        self.assertEqual(len(p_index.requests), 1)
        self.assertEqual(p_index.requests[0].path, 'x')

    @staticmethod
    @contextlib.contextmanager
    def create_sdist():
        """
        Return an sdist generated by self.generate_dist()

        We don't generate it dynamically, because we don't want the test suite
        to have to connect to the network for the setup_requires declaration
        just to build the sdist.
        """
        with tempdir_context() as d:
            dist_path = os.path.join(d, 'distribute-test-fetcher-1.0.tar.gz')
            with open(dist_path, 'wb') as dist:
                dist.write("""
                    H4sICLBagE8C/2Rpc3RcZGlzdHJpYnV0ZS10ZXN0LWZldGNoZXItMS4wLnRhcgDtmNtvmzAUh/Ns
                    Kf+DlZduUkmBYJAi5WHaXd2SqlG7h6pCNJwQa9xmm2j57+eQaqGpktKt0F3O9wI+XCJy/JmfCLlU
                    gt8UCgwFUhlzULMFCMPqmyedJ8LUeJ5XbjW723LfsjzHNBmzXV23HJuxDmWdFiikCgSlT/KQ1Yf7
                    SwgP9H97zF8f82+P9SGKDJ7Os5Om+m/brutg///4/oeQQxpCOlv5MU+/yr76rvb8Na7rHui/te0/
                    0/PEdvWgQ03sf+OQDvI/81v+n52+Nz6O301qqHHI/4HFdvwfePp09L8FPoMKwkAFxiUIybN0SHXn
                    u2QcJDCkeyZHl9w9eVokSSBWQ3oxPh1Pvoy75EOWgJEHEVRqrwq1yMS9ggFJwONK+ROfQSqrV74B
                    ORM8V+Uv/qyexYGaZyKplFDndv2fTi7OX7+d7nnt1/ffdHb8dxgboP9tIEEVeT9fkdqLPXnMtCC/
                    lCEfvkpluR/DEuKH5h7SoP81u/D4/M8cC/3H/I88q/81430tNWrn//L7HxswC/3H/I/5/zn932TD
                    2Txq2H/r3vd/13QZ+t8GVzrM+eswd90lKoj8m4LHIR3RzUivDKAH5mYkl6kvYMnX6m+qqNy/73++
                    avr9b3ne3fyv/Tdt9L8NeJJnQtGy1SrLYkm2u/1y9wWhmlQHglFvz2zpHZfnLDepYNTTk+e2VN5B
                    LxrfCi5A6kXj6mgRlTc/uj4mL3H5QBAEQRAEaZsfEynDsQAoAAA=
                    """.decode('base64'))
            yield dist_path

    @classmethod
    def generate_sdist(cls):
        """
        generate the sdist suitable for create_sdist
        """
        with tempdir_context(cd=os.chdir):
            with open('setup.py', 'wb') as setup_script:
                setup_script.write(textwrap.dedent("""
                    import setuptools
                    setuptools.setup(
                        name="distribute-test-fetcher",
                        version="1.0",
                        setup_requires = ['hgtools'],
                    )
                    """).lstrip())
            with argv_context(['setup.py', 'sdist', '-q', '--formats', 'gztar']):
                setuptools.setup(
                    name="distribute-test-fetcher",
                    version = "1.0",
                    setup_requires = ['hgtools'],
                )
            filename = 'distribute-test-fetcher-1.0.tar.gz'
            dist = os.path.join('dist', filename)
            assert os.path.isfile(dist)
            with open(dist, 'rb') as dist_f:
                print("=====================")
                print(dist_f.read().encode('base64'))

@contextlib.contextmanager
def argv_context(repl):
    old_argv = sys.argv[:]
    sys.argv[:] = repl
    yield
    sys.argv[:] = old_argv

@contextlib.contextmanager
def tempdir_context(cd=lambda dir:None):
    temp_dir = tempfile.mkdtemp()
    orig_dir = os.getcwd()
    try:
        cd(temp_dir)
        yield temp_dir
    finally:
        cd(orig_dir)
        shutil.rmtree(temp_dir)

@contextlib.contextmanager
def environment_context(**updates):
    old_env = os.environ.copy()
    os.environ.update(updates)
    try:
        yield
    finally:
        for key in updates:
            del os.environ[key]
        os.environ.update(old_env)
<commit_msg>Create the sdist using tarfile and the code is much simpler<commit_after>"""Easy install Tests
"""
import sys
import os
import shutil
import tempfile
import unittest
import site
import contextlib
import textwrap
import tarfile

from setuptools.command.easy_install import easy_install, get_script_args, main
from setuptools.command.easy_install import  PthDistributions
from setuptools.command import easy_install as easy_install_pkg
from setuptools.dist import Distribution
from pkg_resources import Distribution as PRDistribution
import setuptools.tests.server

try:
    # import multiprocessing solely for the purpose of testing its existence
    __import__('multiprocessing')
    import logging
    _LOG = logging.getLogger('test_easy_install')
    logging.basicConfig(level=logging.INFO, stream=sys.stderr)
    _MULTIPROC = True
except ImportError:
    _MULTIPROC = False
    _LOG = None

class FakeDist(object):
    def get_entry_map(self, group):
        if group != 'console_scripts':
            return {}
        return {'name': 'ep'}

    def as_requirement(self):
        return 'spec'

WANTED = """\
#!%s
# EASY-INSTALL-ENTRY-SCRIPT: 'spec','console_scripts','name'
__requires__ = 'spec'
import sys
from pkg_resources import load_entry_point

if __name__ == '__main__':
    sys.exit(
        load_entry_point('spec', 'console_scripts', 'name')()
    )
""" % sys.executable

SETUP_PY = """\
from setuptools import setup

setup(name='foo')
"""

class TestEasyInstallTest(unittest.TestCase):

    def test_install_site_py(self):
        dist = Distribution()
        cmd = easy_install(dist)
        cmd.sitepy_installed = False
        cmd.install_dir = tempfile.mkdtemp()
        try:
            cmd.install_site_py()
            sitepy = os.path.join(cmd.install_dir, 'site.py')
            self.assert_(os.path.exists(sitepy))
        finally:
            shutil.rmtree(cmd.install_dir)

    def test_get_script_args(self):
        dist = FakeDist()

        old_platform = sys.platform
        try:
            name, script = [i for i in get_script_args(dist).next()][0:2]
        finally:
            sys.platform = old_platform

        self.assertEquals(script, WANTED)

    def test_no_setup_cfg(self):
        # makes sure easy_install as a command (main)
        # doesn't use a setup.cfg file that is located
        # in the current working directory
        dir = tempfile.mkdtemp()
        setup_cfg = open(os.path.join(dir, 'setup.cfg'), 'w')
        setup_cfg.write('[easy_install]\nfind_links = http://example.com')
        setup_cfg.close()
        setup_py = open(os.path.join(dir, 'setup.py'), 'w')
        setup_py.write(SETUP_PY)
        setup_py.close()

        from setuptools.dist import Distribution

        def _parse_command_line(self):
            msg = 'Error: a local setup.cfg was used'
            opts = self.command_options
            if 'easy_install' in opts:
                assert 'find_links' not in opts['easy_install'], msg
            return self._old_parse_command_line

        Distribution._old_parse_command_line = Distribution.parse_command_line
        Distribution.parse_command_line = _parse_command_line

        old_wd = os.getcwd()
        try:
            os.chdir(dir)
            main([])
        finally:
            os.chdir(old_wd)
            shutil.rmtree(dir)

    def test_no_find_links(self):
        # new option '--no-find-links', that blocks find-links added at
        # the project level
        dist = Distribution()
        cmd = easy_install(dist)
        cmd.check_pth_processing = lambda: True
        cmd.no_find_links = True
        cmd.find_links = ['link1', 'link2']
        cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok')
        cmd.args = ['ok']
        cmd.ensure_finalized()
        self.assertEquals(cmd.package_index.scanned_urls, {})

        # let's try without it (default behavior)
        cmd = easy_install(dist)
        cmd.check_pth_processing = lambda: True
        cmd.find_links = ['link1', 'link2']
        cmd.install_dir = os.path.join(tempfile.mkdtemp(), 'ok')
        cmd.args = ['ok']
        cmd.ensure_finalized()
        keys = cmd.package_index.scanned_urls.keys()
        keys.sort()
        self.assertEquals(keys, ['link1', 'link2'])

class TestPTHFileWriter(unittest.TestCase):
    def test_add_from_cwd_site_sets_dirty(self):
        '''a pth file manager should set dirty
        if a distribution is in site but also the cwd
        '''
        pth = PthDistributions('does-not_exist', [os.getcwd()])
        self.assert_(not pth.dirty)
        pth.add(PRDistribution(os.getcwd()))
        self.assert_(pth.dirty)

    def test_add_from_site_is_ignored(self):
        if os.name != 'nt':
            location = '/test/location/does-not-have-to-exist'
        else:
            location = 'c:\\does_not_exist'
        pth = PthDistributions('does-not_exist', [location, ])
        self.assert_(not pth.dirty)
        pth.add(PRDistribution(location))
        self.assert_(not pth.dirty)

class TestUserInstallTest(unittest.TestCase):

    def setUp(self):
        self.dir = tempfile.mkdtemp()
        setup = os.path.join(self.dir, 'setup.py')
        f = open(setup, 'w')
        f.write(SETUP_PY)
        f.close()
        self.old_cwd = os.getcwd()
        os.chdir(self.dir)
        if sys.version >= "2.6":
            self.old_has_site = easy_install_pkg.HAS_USER_SITE
            self.old_file = easy_install_pkg.__file__
            self.old_base = site.USER_BASE
            site.USER_BASE = tempfile.mkdtemp()
            self.old_site = site.USER_SITE
            site.USER_SITE = tempfile.mkdtemp()
            easy_install_pkg.__file__ = site.USER_SITE

    def tearDown(self):
        os.chdir(self.old_cwd)
        shutil.rmtree(self.dir)
        if sys.version >= "2.6":
            shutil.rmtree(site.USER_BASE)
            shutil.rmtree(site.USER_SITE)
            site.USER_BASE = self.old_base
            site.USER_SITE = self.old_site
            easy_install_pkg.HAS_USER_SITE = self.old_has_site
            easy_install_pkg.__file__ = self.old_file

    def test_user_install_implied(self):
        easy_install_pkg.HAS_USER_SITE = True # disabled sometimes
        #XXX: replace with something meaningfull
        if sys.version < "2.6":
            return #SKIP
        dist = Distribution()
        dist.script_name = 'setup.py'
        cmd = easy_install(dist)
        cmd.args = ['py']
        cmd.ensure_finalized()
        self.assertTrue(cmd.user, 'user should be implied')

    def test_multiproc_atexit(self):
        if not _MULTIPROC:
            return
        _LOG.info('this should not break')

    def test_user_install_not_implied_without_usersite_enabled(self):
        easy_install_pkg.HAS_USER_SITE = False # usually enabled
        #XXX: replace with something meaningfull
        if sys.version < "2.6":
            return #SKIP
        dist = Distribution()
        dist.script_name = 'setup.py'
        cmd = easy_install(dist)
        cmd.args = ['py']
        cmd.initialize_options()
        self.assertFalse(cmd.user, 'NOT user should be implied')

    def test_local_index(self):
        # make sure the local index is used
        # when easy_install looks for installed
        # packages
        new_location = tempfile.mkdtemp()
        target = tempfile.mkdtemp()
        egg_file = os.path.join(new_location, 'foo-1.0.egg-info')
        f = open(egg_file, 'w')
        try:
            f.write('Name: foo\n')
        except:
            f.close()

        sys.path.append(target)
        old_ppath = os.environ.get('PYTHONPATH')
        os.environ['PYTHONPATH'] = os.path.pathsep.join(sys.path)
        try:
            dist = Distribution()
            dist.script_name = 'setup.py'
            cmd = easy_install(dist)
            cmd.install_dir = target
            cmd.args = ['foo']
            cmd.ensure_finalized()
            cmd.local_index.scan([new_location])
            res = cmd.easy_install('foo')
            self.assertEquals(res.location, new_location)
        finally:
            sys.path.remove(target)
            for basedir in [new_location, target, ]:
                if not os.path.exists(basedir) or not os.path.isdir(basedir):
                    continue
                try:
                    shutil.rmtree(basedir)
                except:
                    pass
            if old_ppath is not None:
                os.environ['PYTHONPATH'] = old_ppath
            else:
                del os.environ['PYTHONPATH']

class TestSetupRequires(unittest.TestCase):

    def test_setup_requires_honors_fetch_params(self):
        """
        When easy_install installs a source distribution which specifies
        setup_requires, it should honor the fetch parameters (such as
        allow-hosts, index-url, and find-links).
        """
        # set up a server which will simulate an alternate package index.
        p_index = setuptools.tests.server.MockServer()
        p_index.handle_request_in_thread()
        # create an sdist that has a build-time dependency.
        with TestSetupRequires.create_sdist() as dist_file:
            with tempdir_context() as temp_install_dir:
                with environment_context(PYTHONPATH=temp_install_dir):
                    ei_params = ['--index-url', p_index.url,
                        '--allow-hosts', 'localhost',
                        '--exclude-scripts', '--install-dir', temp_install_dir,
                        dist_file]
                    # attempt to install the dist. It should fail because
                    #  our fake server can't actually supply the dependency
                    self.assertRaises(SystemExit,
                        easy_install_pkg.main, ei_params)
                #self.assertTrue(os.listdir(temp_install_dir))
        self.assertEqual(len(p_index.requests), 1)
        self.assertEqual(p_index.requests[0].path, 'x')

    @staticmethod
    @contextlib.contextmanager
    def create_sdist():
        """
        Return an sdist with a setup_requires dependency (of something that
        doesn't exist)
        """
        with tempdir_context() as d:
            with open('setup.py', 'wb') as setup_script:
                setup_script.write(textwrap.dedent("""
                    import setuptools
                    setuptools.setup(
                        name="distribute-test-fetcher",
                        version="1.0",
                        setup_requires = ['does-not-exist'],
                    )
                    """).lstrip())
            dist_path = os.path.join(d, 'distribute-test-fetcher-1.0.tar.gz')
            with tarfile.open(dist_path, 'w:gz') as dist:
                dist.add('setup.py')
            yield dist_path

@contextlib.contextmanager
def tempdir_context(cd=lambda dir:None):
    temp_dir = tempfile.mkdtemp()
    orig_dir = os.getcwd()
    try:
        cd(temp_dir)
        yield temp_dir
    finally:
        cd(orig_dir)
        shutil.rmtree(temp_dir)

@contextlib.contextmanager
def environment_context(**updates):
    old_env = os.environ.copy()
    os.environ.update(updates)
    try:
        yield
    finally:
        for key in updates:
            del os.environ[key]
        os.environ.update(old_env)
<|endoftext|>


<commit_before>__author__ = 'mcsquaredjr'

import os

node_file = os.environ["NODES"]
cad_file = os.environ["CAD"]
procs_per_nod = os.environ["PROCS_PER_NODE"]

def my_lines(ip):
    with open(cad_file, "r") as cad:
        lines = []
        for line in cad:
            ip, port = line.split(":")
            if ip == str(ip):
                line.append(line)

def chunk_number(i):
    if i == 0 or i == 1:
        return 0
    else:
        return i -1

def chunk_count(i):
    with open(cad_file) as cad:
        for i, l in enumerate(cad):
            pass
    return i + 1<commit_msg>Add more variables and bug fixes<commit_after>__author__ = 'mcsquaredjr'

import os
import socket

node_file = os.environ["NODES"]
cad_file = os.environ["CAD"]
procs_per_nod = os.environ["PROCS_PER_NODE"]
itemcount = os.environ["ITEMCOUNT"]
ddp = os.environment["DDP"]

def my_lines(i):
    ip = socket.gethostbyname(socket.gethostname())
    with open(cad_file, "r") as cad:
        lines = []
        for line in cad:
            ip_str, port = line.split(":")
            if ip_str == str(ip):
                lines.append(line)

def chunk_number(i):
    if i == 0 or i == 1:
        return 0
    else:
        return i -1

def chunk_count(i):
    with open(cad_file) as cad:
        for i, l in enumerate(cad):
            pass
    return i + 1 - 2<|endoftext|>


<commit_before># Python
from importlib import import_module

# Django
from django.utils import timezone
from django.conf import settings
from django.contrib.auth import logout
from django.http import HttpRequest
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from django.core.exceptions import ObjectDoesNotExist

class Command(BaseCommand):
    """Expire Django auth sessions for a user/all users"""
    help='Expire Django auth sessions. Will expire all auth sessions if --user option is not supplied.'

    def add_arguments(self, parser):
        parser.add_argument('--user', dest='user', type=str)

    def handle(self, *args, **options):
        # Try to see if the user exist
        try:
            user = User.objects.get(username=options['user']) if options['user'] else None
        except ObjectDoesNotExist:
            raise CommandError('The user does not exist.')
        # We use the following hack to filter out sessions that are still active,
        # with consideration for timezones.
        start = timezone.now()
        sessions = Session.objects.filter(expire_date__gte=start).iterator()
        request = HttpRequest()
        for session in sessions:
            user_id = session.get_decoded().get('_auth_user_id')
            if (user is None) or (user_id and user.id == int(user_id)):
                request.session = request.session = import_module(settings.SESSION_ENGINE).SessionStore(session.session_key)
                logout(request)
<commit_msg>fix expire session command<commit_after># Python
from importlib import import_module

# Django
from django.utils import timezone
from django.conf import settings
from django.contrib.auth import logout
from django.http import HttpRequest
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from django.core.exceptions import ObjectDoesNotExist

class Command(BaseCommand):
    """Expire Django auth sessions for a user/all users"""
    help='Expire Django auth sessions. Will expire all auth sessions if --user option is not supplied.'

    def add_arguments(self, parser):
        parser.add_argument('--user', dest='user', type=str)

    def handle(self, *args, **options):
        # Try to see if the user exist
        try:
            user = User.objects.get(username=options['user']) if options['user'] else None
        except ObjectDoesNotExist:
            raise CommandError('The user does not exist.')
        # We use the following hack to filter out sessions that are still active,
        # with consideration for timezones.
        start = timezone.now()
        sessions = Session.objects.filter(expire_date__gte=start).iterator()
        request = HttpRequest()
        for session in sessions:
            user_id = session.get_decoded().get('_auth_user_id')
            if (user is None) or (user_id and user.id == int(user_id)):
                request.session = import_module(settings.SESSION_ENGINE).SessionStore(session.session_key)
                logout(request)
<|endoftext|>


<commit_before>from .. import mapper, IniConfigFile

@mapper("autofs_conf")
class AutoFSConf(IniConfigFile):
    """
        /etc/autofs.conf is a standard INI style config file.
    """
    pass
<commit_msg>Using correct 'autofs.conf' spec - thank @bfahr<commit_after>from .. import mapper, IniConfigFile

@mapper("autofs.conf")
class AutoFSConf(IniConfigFile):
    """
        /etc/autofs.conf is a standard INI style config file.
    """
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python3.2

'''
Description:

Accepts a tab-delimited input file that has at least molecule ID,
start and stop coordinates.  You must specify the column numbers
in which each of these are found.  You can optionally pass a column
that contains identifiers which will be used in the header of each
sequence region extracted.

Input:

- A (multi-)FASTA file.  The IDs in the header (up to the first
whitespace) must match the identifiers in the tab file.
- A tab file of sequence ranges to extract.  The order can be  user
specified, but these columns must be present:
    - molecule ID
    - start position
    - end position

Optionally, another column can be used to provide a label for the
extracted range, will which appear in the FASTA header.  If not
specified one will be fabricated from the molecule + coordinates.

Column numbers provided are numbered starting at 1

The coordinates in the file are assumed be of the human-readable sort
and are internally transformed into 0-based, interbase.  A future
feature will be to add direct support for this coordinate system.

Output:

Output will be multi-FASTA data printed to STDOUT or a file if you
pass the -o option.

Examples:

'''

import argparse
import os
import sys
import biocodeutils

def main():
    parser = argparse.ArgumentParser( description='Put a description of your script here')

    ## output file to be written
    parser.add_argument('-f', '--fasta_file', type=str, required=True, help='Path to an input FASTA file' )
    parser.add_argument('-c', '--coords_file', type=str, required=True, help='Path to a tab-delimited file with coordinates' )
    parser.add_argument('-m', '--mol_col', type=int, required=True, help='Tabdel file column with molecule identifiers' )
    parser.add_argument('-x', '--start_coord_col', type=int, required=True, help='Tabdel file column with coordinate start positions' )
    parser.add_argument('-y', '--stop_coord_col', type=int, required=True, help='Tabdel file column with coordinate stop positions' )
    parser.add_argument('-n', '--name_col', type=int, required=False, default=None, help='Optional tabdel file column with name for exported fragment' )
    parser.add_argument('-o', '--output_file', type=str, required=False, default=None, help='Optional Path to an output file to be created' )
    args = parser.parse_args()

    ## output will either be a file or STDOUT
    fout = sys.stdout
    if args.output_file is not None:
        fout = open(args.output_file, 'wt')

    seqs = biocodeutils.fasta_dict_from_file( args.fasta_file )

    start_col = args.start_coord_col - 1
    stop_col  = args.stop_coord_col - 1
    mol_col   = args.mol_col - 1

    for line in open(args.coords_file):
        line = line.rstrip()
        cols = line.split('\t')

        if len(cols) < 3:
            continue

        (fmin, fmax, strand) = biocodeutils.humancoords_to_0interbase( int(cols[start_col]), int(cols[stop_col]) )
        mol_id = cols[mol_col]

        if mol_id not in seqs:
            raise Exception("ERROR: molecule ID ({0}) not found in FASTA file".format(mol_id))

        seq = seqs[mol_id]['s'][fmin:fmax]

        seq_id = None
        if args.name_col is None:
            seq_id = "{0}___{1}.{2}.{3}".format( mol_id, fmin, fmax, strand  )
        else:
            seq_id = cols[int(args.name_col) - 1]

        if strand == -1:
            seq = biocodeutils.reverse_complement(seq)

        ## write this sequence, 60bp per line
        fout.write(">{0}\n".format(seq_id))
        for i in range(0, len(seq), 60):
            fout.write(seq[i : i + 60] + "\n")

if __name__ == '__main__':
    main()

<commit_msg>minor: changed python3.2 to python3 in extract_fasta_regions.py<commit_after>#!/usr/bin/env python3

'''
Description:

Accepts a tab-delimited input file that has at least molecule ID,
start and stop coordinates.  You must specify the column numbers
in which each of these are found.  You can optionally pass a column
that contains identifiers which will be used in the header of each
sequence region extracted.

Input:

- A (multi-)FASTA file.  The IDs in the header (up to the first
whitespace) must match the identifiers in the tab file.
- A tab file of sequence ranges to extract.  The order can be  user
specified, but these columns must be present:
    - molecule ID
    - start position
    - end position

Optionally, another column can be used to provide a label for the
extracted range, will which appear in the FASTA header.  If not
specified one will be fabricated from the molecule + coordinates.

Column numbers provided are numbered starting at 1

The coordinates in the file are assumed be of the human-readable sort
and are internally transformed into 0-based, interbase.  A future
feature will be to add direct support for this coordinate system.

Output:

Output will be multi-FASTA data printed to STDOUT or a file if you
pass the -o option.

Examples:

'''

import argparse
import os
import sys
import biocodeutils

def main():
    parser = argparse.ArgumentParser( description='Put a description of your script here')

    ## output file to be written
    parser.add_argument('-f', '--fasta_file', type=str, required=True, help='Path to an input FASTA file' )
    parser.add_argument('-c', '--coords_file', type=str, required=True, help='Path to a tab-delimited file with coordinates' )
    parser.add_argument('-m', '--mol_col', type=int, required=True, help='Tabdel file column with molecule identifiers' )
    parser.add_argument('-x', '--start_coord_col', type=int, required=True, help='Tabdel file column with coordinate start positions' )
    parser.add_argument('-y', '--stop_coord_col', type=int, required=True, help='Tabdel file column with coordinate stop positions' )
    parser.add_argument('-n', '--name_col', type=int, required=False, default=None, help='Optional tabdel file column with name for exported fragment' )
    parser.add_argument('-o', '--output_file', type=str, required=False, default=None, help='Optional Path to an output file to be created' )
    args = parser.parse_args()

    ## output will either be a file or STDOUT
    fout = sys.stdout
    if args.output_file is not None:
        fout = open(args.output_file, 'wt')

    seqs = biocodeutils.fasta_dict_from_file( args.fasta_file )

    start_col = args.start_coord_col - 1
    stop_col  = args.stop_coord_col - 1
    mol_col   = args.mol_col - 1

    for line in open(args.coords_file):
        line = line.rstrip()
        cols = line.split('\t')

        if len(cols) < 3:
            continue

        (fmin, fmax, strand) = biocodeutils.humancoords_to_0interbase( int(cols[start_col]), int(cols[stop_col]) )
        mol_id = cols[mol_col]

        if mol_id not in seqs:
            raise Exception("ERROR: molecule ID ({0}) not found in FASTA file".format(mol_id))

        seq = seqs[mol_id]['s'][fmin:fmax]

        seq_id = None
        if args.name_col is None:
            seq_id = "{0}___{1}.{2}.{3}".format( mol_id, fmin, fmax, strand  )
        else:
            seq_id = cols[int(args.name_col) - 1]

        if strand == -1:
            seq = biocodeutils.reverse_complement(seq)

        ## write this sequence, 60bp per line
        fout.write(">{0}\n".format(seq_id))
        for i in range(0, len(seq), 60):
            fout.write(seq[i : i + 60] + "\n")

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from AccessControl import ClassSecurityInfo
from Products.ATContentTypes.content import schemata
from Products.Archetypes import atapi
from Products.Archetypes.ArchetypeTool import registerType
from Products.CMFCore import permissions
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.utils import safe_unicode
from bika.lims.browser import BrowserView
from bika.lims.browser.bika_listing import BikaListingView
from bika.lims.config import PROJECTNAME
from plone.app.layout.globals.interfaces import IViewView
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t
from Products.Archetypes import PloneMessageFactory as _p
from bika.lims.interfaces import IStorageLocations
from bika.lims.content.bikaschema import BikaFolderSchema
from plone.app.content.browser.interfaces import IFolderContentsView
from plone.app.folder.folder import ATFolder, ATFolderSchema
from zope.interface.declarations import implements
from Products.CMFCore.utils import getToolByName
import json
import plone

class StorageLocationsView(BikaListingView):
    implements(IFolderContentsView, IViewView)

    def __init__(self, context, request):
        super(StorageLocationsView, self).__init__(context, request)
        self.catalog = 'bika_setup_catalog'
        self.contentFilter = {'portal_type': 'StorageLocation',
                              'sort_on': 'sortable_title'}
        self.context_actions = {_('Add'):
                            {'url': 'createObject?type_name=StorageLocation',
                             'icon': '++resource++bika.lims.images/add.png'}}
        self.title = self.context.translate(_("Storage Locations"))
        self.icon = self.portal_url + "/++resource++bika.lims.images/storagelocation_big.png"
        self.description = ""
        self.show_sort_column = False
        self.show_select_row = False
        self.show_select_column = True
        self.pagesize = 25

        self.columns = {
            'Title': {'title': _('Storage Location'),
                      'index':'sortable_title'},
            'Room': {'title': _('Room'),
                     'toggle': True},
            'Type': {'title': _('Storage Type'),
                     'toggle': True},
            'Hierarchy': {'title': _('Hierarchy'),
                          'toggle': True},
            'StockItem': {'title': _('Stock Item'),
                          'toggle': True},
            'Sampletemp': {'title': _('Sample'),
                          'toggle': True},
        }

        # ________________________________ #
        #    HOCINE ADD POSITION WORKFLOW  #
        # ________________________________ #
        self.review_states = [
            {'id':'default',
             'title': _('Active'),
             'contentFilter': {'inactive_state': 'active',
                               'sort_on': 'created',
                               'sort_order': 'ascending'},
             'transitions': [{'id':'deactivate'},
                             {'id': 'reserve'},
                             {'id': 'occupy'},],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Sampletemp',
                        ]},
            {'id':'inactive',
             'title': _('Dormant'),
             'contentFilter': {'inactive_state': 'inactive'},
             'transitions': [{'id':'activate'}, ],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Sampletemp',
                        ]},
            {'id': 'position_free',
             'title': _('Free'),
             'contentFilter': {'review_state': 'position_free',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'reserve'},
                             {'id': 'occupy'}],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         ]},
            {'id': 'position_reserved',
             'title': _('Reserved'),
             'contentFilter': {'review_state': 'position_reserved',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'free'},
                             {'id': 'occupy'}],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         ]},
            {'id': 'position_occupied',
             'title': _('Occupied'),
             'contentFilter': {'review_state': 'position_occupied',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'free'},
                             {'id': 'reserve'},],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Sampletemp',
                         ]},
            {'id':'all',
             'title': _('All'),
             'contentFilter':{'sort_on': 'created',
                              'sort_order': 'ascending'},
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Sampletemp',
                        ]},
        ]

    def folderitems(self):
        items = BikaListingView.folderitems(self)
        for x in range(len(items)):
            if not items[x].has_key('obj'): continue
            obj = items[x]['obj']
            items[x]['replace']['Title'] = "%s" % \
                 (items[x]['url'], items[x]['Title'])
            items[x]['Room'] = obj.getRoom()
            items[x]['Type'] = obj.getStorageType()
            items[x]['Hierarchy'] = obj.getHierarchy()
            items[x]['StockItem'] = obj.getProduct() and obj.getProduct().Title() or ''
            items[x]['Sampletemp'] = obj.getSampletemp() and obj.getSampletemp().Title() or ''
            # if obj.aq_parent.portal_type == 'Client':
            #     items[x]['Owner'] = obj.aq_parent.Title()
            # else:
            #     items[x]['Owner'] = self.context.bika_setup.laboratory.Title()
        return items

schema = ATFolderSchema.copy()

class StorageLocations(ATFolder):
    implements(IStorageLocations)
    displayContentsTab = False
    schema = schema

schemata.finalizeATCTSchema(schema, folderish = True, moveDiscussion = False)
atapi.registerType(StorageLocations, PROJECTNAME)

class ajax_StorageLocations(BrowserView):
    """ The autocomplete data source for storage location selection widgets.
        Returns a JSON list of storage location titles.

        Request parameters:

        - term: the string which will be searched against all Storage Location
          titles.

        - _authenticator: The plone.protect authenticator.

    """

    def filter_list(self, items, searchterm):
        if searchterm and len(searchterm) < 3:
            # Items that start with A or AA
            res = [s.getObject()
                     for s in items
                     if s.title.lower().startswith(searchterm)]
            if not res:
                # or, items that contain A or AA
                res = [s.getObject()
                         for s in items
                         if s.title.lower().find(searchterm) > -1]
        else:
            # or, items that contain searchterm.
            res = [s.getObject()
                     for s in items
                     if s.title.lower().find(searchterm) > -1]
        return res

    def __call__(self):
        plone.protect.CheckAuthenticator(self.request)
        bsc = getToolByName(self.context, 'bika_setup_catalog')
        term = safe_unicode(self.request.get('term', '')).lower()
        if not term:
            return json.dumps([])

        client_items = lab_items = []

        # User (client) storage locations
        if self.context.portal_type == 'Client':
            client_path = self.context.getPhysicalPath()
            client_items = list(
                bsc(portal_type = "StorageLocation",
                    path = {"query": "/".join(client_path), "level" : 0 },
                    inactive_state = 'active',
                    sort_on='sortable_title'))

        # Global (lab) storage locations
        lab_path = \
                self.context.bika_setup.bika_storagelocations.getPhysicalPath()
        lab_items = list(
            bsc(portal_type = "StorageLocation",
                path = {"query": "/".join(lab_path), "level" : 0 },
                inactive_state = 'active',
                sort_on='sortable_title'))

        client_items = [callable(s.Title) and s.Title() or s.title
                 for s in self.filter_list(client_items, term)]
        lab_items = [callable(s.Title) and s.Title() or s.title
                 for s in self.filter_list(lab_items, term)]
        lab_items = ["%s: %s" % (_("Lab"), safe_unicode(i))
                     for i in lab_items]

        items = client_items + lab_items

        return json.dumps(items)
<commit_msg>changing name sampletemp to aliquot<commit_after>from AccessControl import ClassSecurityInfo
from Products.ATContentTypes.content import schemata
from Products.Archetypes import atapi
from Products.Archetypes.ArchetypeTool import registerType
from Products.CMFCore import permissions
from Products.CMFCore.utils import getToolByName
from Products.CMFPlone.utils import safe_unicode
from bika.lims.browser import BrowserView
from bika.lims.browser.bika_listing import BikaListingView
from bika.lims.config import PROJECTNAME
from plone.app.layout.globals.interfaces import IViewView
from bika.lims import bikaMessageFactory as _
from bika.lims.utils import t
from Products.Archetypes import PloneMessageFactory as _p
from bika.lims.interfaces import IStorageLocations
from bika.lims.content.bikaschema import BikaFolderSchema
from plone.app.content.browser.interfaces import IFolderContentsView
from plone.app.folder.folder import ATFolder, ATFolderSchema
from zope.interface.declarations import implements
from Products.CMFCore.utils import getToolByName
import json
import plone

class StorageLocationsView(BikaListingView):
    implements(IFolderContentsView, IViewView)

    def __init__(self, context, request):
        super(StorageLocationsView, self).__init__(context, request)
        self.catalog = 'bika_setup_catalog'
        self.contentFilter = {'portal_type': 'StorageLocation',
                              'sort_on': 'sortable_title'}
        self.context_actions = {_('Add'):
                            {'url': 'createObject?type_name=StorageLocation',
                             'icon': '++resource++bika.lims.images/add.png'}}
        self.title = self.context.translate(_("Storage Locations"))
        self.icon = self.portal_url + "/++resource++bika.lims.images/storagelocation_big.png"
        self.description = ""
        self.show_sort_column = False
        self.show_select_row = False
        self.show_select_column = True
        self.pagesize = 25

        self.columns = {
            'Title': {'title': _('Storage Location'),
                      'index':'sortable_title'},
            'Room': {'title': _('Room'),
                     'toggle': True},
            'Type': {'title': _('Storage Type'),
                     'toggle': True},
            'Hierarchy': {'title': _('Hierarchy'),
                          'toggle': True},
            'StockItem': {'title': _('Stock Item'),
                          'toggle': True},
            'Aliquot': {'title': _('Aliquot'),
                          'toggle': True},
        }

        # ________________________________ #
        #    HOCINE ADD POSITION WORKFLOW  #
        # ________________________________ #
        self.review_states = [
            {'id':'default',
             'title': _('Active'),
             'contentFilter': {'inactive_state': 'active',
                               'sort_on': 'created',
                               'sort_order': 'ascending'},
             'transitions': [{'id':'deactivate'},
                             {'id': 'reserve'},
                             {'id': 'occupy'},],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Aliquot',
                        ]},
            {'id':'inactive',
             'title': _('Dormant'),
             'contentFilter': {'inactive_state': 'inactive'},
             'transitions': [{'id':'activate'}, ],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Aliquot',
                        ]},
            {'id': 'position_free',
             'title': _('Free'),
             'contentFilter': {'review_state': 'position_free',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'reserve'},
                             {'id': 'occupy'}],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         ]},
            {'id': 'position_reserved',
             'title': _('Reserved'),
             'contentFilter': {'review_state': 'position_reserved',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'free'},
                             {'id': 'occupy'}],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         ]},
            {'id': 'position_occupied',
             'title': _('Occupied'),
             'contentFilter': {'review_state': 'position_occupied',
                               'sort_on': 'created',
                               'sort_order': 'reverse'},
             'transitions': [{'id': 'deactivate'},
                             {'id': 'free'},
                             {'id': 'reserve'},],
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Aliquot',
                         ]},
            {'id':'all',
             'title': _('All'),
             'contentFilter':{'sort_on': 'created',
                              'sort_order': 'ascending'},
             'columns': ['Title',
                         'Room',
                         'Type',
                         'Hierarchy',
                         'StockItem',
                         'Aliquot',
                        ]},
        ]

    def folderitems(self):
        items = BikaListingView.folderitems(self)
        for x in range(len(items)):
            if not items[x].has_key('obj'): continue
            obj = items[x]['obj']
            items[x]['replace']['Title'] = "%s" % \
                 (items[x]['url'], items[x]['Title'])
            items[x]['Room'] = obj.getRoom()
            items[x]['Type'] = obj.getStorageType()
            items[x]['Hierarchy'] = obj.getHierarchy()
            items[x]['StockItem'] = obj.getProduct() and obj.getProduct().Title() or ''
            items[x]['Aliquot'] = obj.getAliquot() and obj.getAliquot().Title() or ''
            # if obj.aq_parent.portal_type == 'Client':
            #     items[x]['Owner'] = obj.aq_parent.Title()
            # else:
            #     items[x]['Owner'] = self.context.bika_setup.laboratory.Title()
        return items

schema = ATFolderSchema.copy()

class StorageLocations(ATFolder):
    implements(IStorageLocations)
    displayContentsTab = False
    schema = schema

schemata.finalizeATCTSchema(schema, folderish = True, moveDiscussion = False)
atapi.registerType(StorageLocations, PROJECTNAME)

class ajax_StorageLocations(BrowserView):
    """ The autocomplete data source for storage location selection widgets.
        Returns a JSON list of storage location titles.

        Request parameters:

        - term: the string which will be searched against all Storage Location
          titles.

        - _authenticator: The plone.protect authenticator.

    """

    def filter_list(self, items, searchterm):
        if searchterm and len(searchterm) < 3:
            # Items that start with A or AA
            res = [s.getObject()
                     for s in items
                     if s.title.lower().startswith(searchterm)]
            if not res:
                # or, items that contain A or AA
                res = [s.getObject()
                         for s in items
                         if s.title.lower().find(searchterm) > -1]
        else:
            # or, items that contain searchterm.
            res = [s.getObject()
                     for s in items
                     if s.title.lower().find(searchterm) > -1]
        return res

    def __call__(self):
        plone.protect.CheckAuthenticator(self.request)
        bsc = getToolByName(self.context, 'bika_setup_catalog')
        term = safe_unicode(self.request.get('term', '')).lower()
        if not term:
            return json.dumps([])

        client_items = lab_items = []

        # User (client) storage locations
        if self.context.portal_type == 'Client':
            client_path = self.context.getPhysicalPath()
            client_items = list(
                bsc(portal_type = "StorageLocation",
                    path = {"query": "/".join(client_path), "level" : 0 },
                    inactive_state = 'active',
                    sort_on='sortable_title'))

        # Global (lab) storage locations
        lab_path = \
                self.context.bika_setup.bika_storagelocations.getPhysicalPath()
        lab_items = list(
            bsc(portal_type = "StorageLocation",
                path = {"query": "/".join(lab_path), "level" : 0 },
                inactive_state = 'active',
                sort_on='sortable_title'))

        client_items = [callable(s.Title) and s.Title() or s.title
                 for s in self.filter_list(client_items, term)]
        lab_items = [callable(s.Title) and s.Title() or s.title
                 for s in self.filter_list(lab_items, term)]
        lab_items = ["%s: %s" % (_("Lab"), safe_unicode(i))
                     for i in lab_items]

        items = client_items + lab_items

        return json.dumps(items)
<|endoftext|>