<commit_before># -*- coding: utf-8 -*-
import os
import re
import logging
import mimetypes
import hashlib
import json
import time
import calendar

from django.conf import settings
from django.test.client import Client

from django_perseus.exceptions import RendererException
from django_perseus.renderers.base import BaseRenderer
from django_perseus.renderers.default import DefaultRenderer
from django.core.urlresolvers import reverse

from .models import Guide, Content
from .serializers import GuidesSerializer

logger = logging.getLogger('perseus')

def date_handler(obj):
    """
    Simple python handler that conver
    a datetime.date to human date
    """
    return obj.isoformat() if hasattr(obj, 'isoformat') else obj

class GuideRenderer(BaseRenderer):

    """
    This serializer extend from Django perseus - BaseRenderer
    this serializer convert the name of a url to hashlib and
    append the web file extention.
    """

    regex = re.compile("^/guia/(?P<pk>\d+)/$")
    about_regex = re.compile("^/acerca-de/$")

    def render_path(self, path=None, view=None):

        if path:
            # create deploy dir if not exists
            deploy_dir = settings.PERSEUS_SOURCE_DIR
            outpath = os.path.join(deploy_dir, '')
            android_filename = ".nomedia"
            manifest_filename = "manifest.json"
            if not os.path.exists(deploy_dir):
                os.makedirs(deploy_dir)

            # create the no media file for the android app
            nomedia_dir = os.path.join(deploy_dir, android_filename)
            nomedia_file = open(nomedia_dir, "w")
            nomedia_file.close()

            # create the Manifest file for the android app
            manifest_dir = os.path.join(deploy_dir, manifest_filename)
            manifest_file = open(manifest_dir, "w")
            json.dump(self.create_manifest(), manifest_file, indent=4)
            manifest_file.close()

            # create the renders page
            if self.regex and self.regex.findall(path):
                # if the guide is 1 then name it index.html
                response, mime = self.render_page(path)
                outpath = os.path.join(outpath, 'index{0}'.format(mime))
                self.save_page(response, outpath)
                return
            elif self.about_regex.findall(path):
                # if the url in path is /acerca-de/ rename it
                # to about.html
                response, mime = self.render_page(path)
                outpath = os.path.join(outpath, 'about{0}'.format(mime))
                self.save_page(response, outpath)
                return
            else:
                # otherwise call it path with sha1 name
                response, mime = self.render_page(path)
                name = '%s.html' % hashlib.sha1(path).hexdigest()
                outpath = os.path.join(deploy_dir, name)
                self.save_page(response, outpath)
                return

    def render_page(self, path):
        response = self.client.get(path)
        if response.status_code is not 200:
            raise RendererException(
                'Path: {0} returns status code: {1}.'.format(path, response.status_code))
        return response, self.get_mime(response)

    def get_mime(self, response):
        mime = response['Content-Type']
        encoding = mime.split(';', 1)[0]
        return mimetypes.guess_extension(encoding)

    def save_page(self, response, outpath):
        logger.debug(outpath)
        with open(outpath, 'wb') as f:
            f.write(response.content)

    def generate(self):
        self.client = Client()
        for path in self.paths():
            self.render_path(path=path)

class HomeRenderer(GuideRenderer):

    """
    This class make the render from dinamic pages,
    the filter is from the base command
    """

    def __init__(self, guide_number=None):
        self.guide_number = guide_number

        if not guide_number:
            self.regex = re.compile('^/static_index/$')

    def create_manifest(self):
        """
        Simple method that return our manifest
        dict used in the manifest.json
        """
        guides = Guide.objects.all()
        serializer = GuidesSerializer(guides, many=True)
        content = json.dumps(serializer.data, default=date_handler, indent=4)
        manifest_dict = {
            "manifest_version": 1,
            "guide_id": self.guide_number,
            "guide_version": self.guide_number,
            'generation_date': calendar.timegm(time.gmtime()),
            'contents': json.loads(content)
        }
        return manifest_dict

    def paths(self):
        """
        Create a dict with all paths ( urls )
        used when we render all the elements
        """
        paths = set([])
        if self.guide_number:
            guides = Guide.objects.filter(number=self.guide_number)
            contents = Content.objects.filter(section__guide=self.guide_number)
        else:
            guides = Guide.objects.all()
            contents = Content.objects.all()

            # Add index view to our path list
            paths.add(reverse('home_static'))

        for guide in guides:
            paths.add(guide.get_absolute_url())
        for content in contents:
            paths.add(content.get_absolute_url())

        # Add the 'acerca-de' url to the render path
        paths.add(reverse('about'))

        return paths

class AboutUsRenderer(DefaultRenderer):

    def paths(self):
        return [reverse('about')]

renderers = [HomeRenderer, ]
<commit_msg>usa numero de guia, no id<commit_after># -*- coding: utf-8 -*-
import os
import re
import logging
import mimetypes
import hashlib
import json
import time
import calendar

from django.conf import settings
from django.test.client import Client

from django_perseus.exceptions import RendererException
from django_perseus.renderers.base import BaseRenderer
from django_perseus.renderers.default import DefaultRenderer
from django.core.urlresolvers import reverse

from .models import Guide, Content
from .serializers import GuidesSerializer

logger = logging.getLogger('perseus')

def date_handler(obj):
    """
    Simple python handler that conver
    a datetime.date to human date
    """
    return obj.isoformat() if hasattr(obj, 'isoformat') else obj

class GuideRenderer(BaseRenderer):

    """
    This serializer extend from Django perseus - BaseRenderer
    this serializer convert the name of a url to hashlib and
    append the web file extention.
    """

    regex = re.compile("^/guia/(?P<pk>\d+)/$")
    about_regex = re.compile("^/acerca-de/$")

    def render_path(self, path=None, view=None):

        if path:
            # create deploy dir if not exists
            deploy_dir = settings.PERSEUS_SOURCE_DIR
            outpath = os.path.join(deploy_dir, '')
            android_filename = ".nomedia"
            manifest_filename = "manifest.json"
            if not os.path.exists(deploy_dir):
                os.makedirs(deploy_dir)

            # create the no media file for the android app
            nomedia_dir = os.path.join(deploy_dir, android_filename)
            nomedia_file = open(nomedia_dir, "w")
            nomedia_file.close()

            # create the Manifest file for the android app
            manifest_dir = os.path.join(deploy_dir, manifest_filename)
            manifest_file = open(manifest_dir, "w")
            json.dump(self.create_manifest(), manifest_file, indent=4)
            manifest_file.close()

            # create the renders page
            if self.regex and self.regex.findall(path):
                # if the guide is 1 then name it index.html
                response, mime = self.render_page(path)
                outpath = os.path.join(outpath, 'index{0}'.format(mime))
                self.save_page(response, outpath)
                return
            elif self.about_regex.findall(path):
                # if the url in path is /acerca-de/ rename it
                # to about.html
                response, mime = self.render_page(path)
                outpath = os.path.join(outpath, 'about{0}'.format(mime))
                self.save_page(response, outpath)
                return
            else:
                # otherwise call it path with sha1 name
                response, mime = self.render_page(path)
                name = '%s.html' % hashlib.sha1(path).hexdigest()
                outpath = os.path.join(deploy_dir, name)
                self.save_page(response, outpath)
                return

    def render_page(self, path):
        response = self.client.get(path)
        if response.status_code is not 200:
            raise RendererException(
                'Path: {0} returns status code: {1}.'.format(path, response.status_code))
        return response, self.get_mime(response)

    def get_mime(self, response):
        mime = response['Content-Type']
        encoding = mime.split(';', 1)[0]
        return mimetypes.guess_extension(encoding)

    def save_page(self, response, outpath):
        logger.debug(outpath)
        with open(outpath, 'wb') as f:
            f.write(response.content)

    def generate(self):
        self.client = Client()
        for path in self.paths():
            self.render_path(path=path)

class HomeRenderer(GuideRenderer):

    """
    This class make the render from dinamic pages,
    the filter is from the base command
    """

    def __init__(self, guide_number=None):
        self.guide_number = guide_number

        if not guide_number:
            self.regex = re.compile('^/static_index/$')

    def create_manifest(self):
        """
        Simple method that return our manifest
        dict used in the manifest.json
        """
        guides = Guide.objects.all()
        serializer = GuidesSerializer(guides, many=True)
        content = json.dumps(serializer.data, default=date_handler, indent=4)
        manifest_dict = {
            "manifest_version": 1,
            "guide_id": self.guide_number,
            "guide_version": self.guide_number,
            'generation_date': calendar.timegm(time.gmtime()),
            'contents': json.loads(content)
        }
        return manifest_dict

    def paths(self):
        """
        Create a dict with all paths ( urls )
        used when we render all the elements
        """
        paths = set([])
        if self.guide_number:
            guides = Guide.objects.filter(number=self.guide_number)
            contents = Content.objects.filter(section__guide__number=self.guide_number)
        else:
            guides = Guide.objects.all()
            contents = Content.objects.all()

            # Add index view to our path list
            paths.add(reverse('home_static'))

        for guide in guides:
            paths.add(guide.get_absolute_url())
        for content in contents:
            paths.add(content.get_absolute_url())

        # Add the 'acerca-de' url to the render path
        paths.add(reverse('about'))

        return paths

class AboutUsRenderer(DefaultRenderer):

    def paths(self):
        return [reverse('about')]

renderers = [HomeRenderer, ]
<|endoftext|>


<commit_before>## @package tags
# Module caffe2.python.layers.tags
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from caffe2.python import context

@context.define_context(allow_default=True)
class TagContext(object):
    """
    Scope driven way to provide tags to the layers.
    """

    def __init__(self, tags=None):
        # Tags is expected to be list to keep order of adding/removing things
        self.tags = tags or []

    def add_tags(self, tags):
        self.tags.extend(tags)

    def remove_tags(self, tags):
        assert self.tags[-len(tags):] == tags
        self.tags = self.tags[:-len(tags)]

class Tags(object):
    # TODO(amalevich): Tags might need to live in their own contexts, add this
    # split later
    EXCLUDE_FROM_TRAIN = 'exclude_from_train'
    EXCLUDE_FROM_EVAL = 'exclude_from_eval'
    EXCLUDE_FROM_PREDICTION = 'exclude_from_prediction'
    EXCLUDE_FROM_ACCUMULATE_PRED = 'exclude_from_accumulate_pred'
    PREPROCESSING = 'preprocessing'
    HANDLE_AS_SPARSE_LAYER = 'handle_as_sparse_layer'
    GRADIENT_FROM_PS = 'gradient_from_ps'
    PREFER_GPU = 'prefer_gpu'

    # In certain cases we want to have different schema for training and
    # prediction, as an example in prediction we might need to have only
    # subset of ids present in the orignal schema. This tag is one of the ways
    # to mark operators that will be removed from prediction and should
    # override schema for predictors.
    PREDICTION_SCHEMA = 'prediction_schema'

    def __init__(self, tags):
        if not isinstance(tags, list):
            tags = [tags]
        self.tags = tags

    def __enter__(self):
        TagContext.current().add_tags(self.tags)
        return self

    def __exit__(self, type, value, traceback):
        TagContext.current().remove_tags(self.tags)

Tags.TRAIN_ONLY = [Tags.EXCLUDE_FROM_PREDICTION, Tags.EXCLUDE_FROM_EVAL,
                   Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
Tags.EVAL_ONLY = [Tags.EXCLUDE_FROM_PREDICTION, Tags.EXCLUDE_FROM_TRAIN,
                  Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
Tags.PREDICTION_ONLY = [Tags.EXCLUDE_FROM_TRAIN, Tags.EXCLUDE_FROM_EVAL,
                        Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
<commit_msg>offline_all_gpu_experiment<commit_after>## @package tags
# Module caffe2.python.layers.tags
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from caffe2.python import context

@context.define_context(allow_default=True)
class TagContext(object):
    """
    Scope driven way to provide tags to the layers.
    """

    def __init__(self, tags=None):
        # Tags is expected to be list to keep order of adding/removing things
        self.tags = tags or []

    def add_tags(self, tags):
        self.tags.extend(tags)

    def remove_tags(self, tags):
        assert self.tags[-len(tags):] == tags
        self.tags = self.tags[:-len(tags)]

class Tags(object):
    # TODO(amalevich): Tags might need to live in their own contexts, add this
    # split later
    EXCLUDE_FROM_TRAIN = 'exclude_from_train'
    EXCLUDE_FROM_EVAL = 'exclude_from_eval'
    EXCLUDE_FROM_PREDICTION = 'exclude_from_prediction'
    EXCLUDE_FROM_ACCUMULATE_PRED = 'exclude_from_accumulate_pred'
    PREPROCESSING = 'preprocessing'
    HANDLE_AS_SPARSE_LAYER = 'handle_as_sparse_layer'
    GRADIENT_FROM_PS = 'gradient_from_ps'
    PREFER_GPU = 'prefer_gpu'
    CPU_ONLY = 'cpu_only'

    # In certain cases we want to have different schema for training and
    # prediction, as an example in prediction we might need to have only
    # subset of ids present in the orignal schema. This tag is one of the ways
    # to mark operators that will be removed from prediction and should
    # override schema for predictors.
    PREDICTION_SCHEMA = 'prediction_schema'

    def __init__(self, tags):
        if not isinstance(tags, list):
            tags = [tags]
        self.tags = tags

    def __enter__(self):
        TagContext.current().add_tags(self.tags)
        return self

    def __exit__(self, type, value, traceback):
        TagContext.current().remove_tags(self.tags)

Tags.TRAIN_ONLY = [Tags.EXCLUDE_FROM_PREDICTION, Tags.EXCLUDE_FROM_EVAL,
                   Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
Tags.EVAL_ONLY = [Tags.EXCLUDE_FROM_PREDICTION, Tags.EXCLUDE_FROM_TRAIN,
                  Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
Tags.PREDICTION_ONLY = [Tags.EXCLUDE_FROM_TRAIN, Tags.EXCLUDE_FROM_EVAL,
                        Tags.EXCLUDE_FROM_ACCUMULATE_PRED]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download the latest CAL-ACCESS database ZIP.
"""
from __future__ import unicode_literals
import os
import logging
import requests
from datetime import datetime
from hurry.filesize import size
from clint.textui import progress
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime
from django.core.files import File
from django.utils.timezone import utc, now
from calaccess_raw import get_download_directory
from django.template.loader import render_to_string
from django.core.management.base import CommandError
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.models.tracking import RawDataVersion
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Download the latest CAL-ACCESS database ZIP.
    """
    help = "Download the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Download the ZIP archive without asking permission"
        )
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # get the dir where data goes from app settings
        self.data_dir = get_download_directory()
        # if data_dir doesn't exist, create it
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)

        # downloaded zip file will go in data_dir
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])

        self.download_metadata = self.get_download_metadata()
        logger.debug('Server: %s <-- (from HEAD)' % self.download_metadata['server'])
        logger.debug('ETag: %s <-- (from HEAD)' % self.download_metadata['etag'])
        logger.debug(
            'Last-Modified: %s <-- (from HEAD)' % self.download_metadata['last-modified']
        )
        logger.debug(
            'Content-Length %s <-- (from HEAD):' % self.download_metadata['content-length']
        )

        # get or create the RawDataVersion
        self.version, created = RawDataVersion.objects.get_or_create(
            release_datetime=self.download_metadata['last-modified'],
            expected_size=self.download_metadata['content-length'],
        )

        # if not called from command-line, assume called by update command
        if not self._called_from_command_line:
            # get the version that the update command is working on
            last_update_started = RawDataVersion.objects.filter(
                download_start_datetime__isnull=False
            ).latest('update_start_datetime')
            # confirm that update and download commands are working with same version
            if self.version != last_update_started:
                raise CommandError(
                    'Version available to download does not match version of update.'
                )

        # log if a new version was found
        if created:
            logger.info('New CAL-ACCESS version available.')

        # if download is stalled, zip file is there, and user did not invoke restart
        if (
            self.version.download_stalled and
            os.path.exists(self.zip_path) and
            not options['restart']
        ):
            # enable resuming
            self.resume = True
            # set current size to partially downloaded zip
            self.local_file_size = os.path.getsize(self.zip_path)
            # set the datetime of last download to last modified date of zip file
            timestamp = os.path.getmtime(self.zip_path)
            self.local_file_datetime = datetime.fromtimestamp(timestamp, utc)
        else:
            self.resume = False
            self.local_file_size = 0
            self.local_file_datetime = None

        if not options['noinput']:
            # get downloaded versions
            downloaded_versions = RawDataVersion.objects.filter(
                download_finish_datetime__isnull=False
            )
            # if there are any
            if downloaded_versions:
                # get the last version downloaded datetime
                last_download_datetime = downloaded_versions.latest(
                    'download_finish_datetime'
                ).download_finish_datetime
                since_last_download = naturaltime(last_download_datetime)
            else:
                last_download_datetime = None

            # setting up the prompt
            prompt_context = dict(
                latest_version=self.version,
                resume=self.resume,
                local_file_size=size(self.local_file_size),
                local_file_datetime=self.local_file_datetime,
                download_dir=self.data_dir,
                since_previous_download=since_last_download,
            )

            prompt = render_to_string(
                'calaccess_raw/downloadcalaccessrawdata.txt',
                prompt_context,
            )

            # if the user doesn't confirm initially
            if not self.confirm_proceed(prompt):
                # but the user can resume
                if self.resume:
                    # prompt to restart
                    confirm_restart = self.confirm_proceed(
                        'Do you want re-start your update?\n'
                    )
                    # if user confirms restart, do not resume
                    if confirm_restart:
                        self.resume = False
                    else:
                        # if the user doesn't confirm restart, cancel
                        raise CommandError("Download cancelled")
                # if the user doesn't confirm initial and can't resume
                elif not self.resume:
                    # cancel
                    raise CommandError("Download cancelled")

        # if not resuming, store the download start time
        if not self.resume:
            self.version.download_start_datetime = now()
        # either way, reset the finish time
        self.version.download_finish_datetime = None
        # save here in case the command doesn't finish
        self.version.save()

        # check if local zip file is already completely downloaded before trying
        if self.local_file_size < self.version.expected_size:
            self.download()

        logger.debug('Download zip size: %s bytes' % os.path.getsize(self.zip_path))
        # log warning if downloaded zip size is not same as expected size
        if self.version.expected_size != os.path.getsize(self.zip_path):
            raise CommandError(
                'Expected {0} byte zip, but download {1} byte zip.'.format(
                    self.version.expected_size,
                    os.path.getsize(self.zip_path)
                )
            )

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()

        # store download finish time
        self.version.download_finish_datetime = now()
        # and save the RawDataVersion
        self.version.save()

    def download(self):
        """
        Download the ZIP file in pieces.
        """
        if self.verbosity:
            if self.resume:
                self.header("Resuming download of ZIP file")
            else:
                self.header("Downloading ZIP file")

        # Prep
        expected_size = self.version.expected_size
        headers = dict()
        if os.path.exists(self.zip_path):
            if self.resume:
                headers['Range'] = 'bytes=%d-' % self.local_file_size
                expected_size = expected_size - self.local_file_size
            else:
                os.remove(self.zip_path)

        # Stream the download
        chunk_size = 1024
        req = requests.get(self.url, stream=True, headers=headers)

        logger.debug('Server: %s <--  (from GET)' % req.headers['server'])
        logger.debug('ETag: %s <-- (from GET)' % req.headers['etag'])
        logger.debug('Last-Modified: %s <--  (from GET)' % req.headers['last-modified'])
        logger.debug('Content-Length: %s <-- (from GET)' % req.headers['content-length'])

        if self.download_metadata['etag'] != req.headers['etag']:
            raise CommandError("ETags in HEAD and GET requests don't match.")

        # in Python 2, need to convert this to long int
        try:
            divisor = long(chunk_size + 1)
        except NameError:
            # no long() in Python 3, all ints are long ints
            divisor = chunk_size + 1

        n_iters = float(expected_size) / divisor

        with open(self.zip_path, 'ab') as fp:
            for chunk in progress.bar(req.iter_content(chunk_size=chunk_size),
                                      expected_size=n_iters):
                fp.write(chunk)
                fp.flush()

    def archive(self):
        """
        Save a copy of the download zip file and each file inside.
        """
        if self.verbosity:
            self.log(" Archiving {0}".format(os.path.basename(self.zip_path)))
        # Remove previous zip file
        self.version.download_zip_archive.delete()
        # Store the actual download zip file size
        self.version.download_zip_size = os.path.getsize(self.zip_path)
        # Open up the zipped file so we can wrap it in the Django File obj
        zipped_file = open(self.zip_path)
        # Save the zip on the raw data version
        self.version.download_zip_archive.save(
            self.url.split('/')[-1],
            File(zipped_file)
        )
        zipped_file.close()
<commit_msg>Include release datetime in downloading/resuming logs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Download the latest CAL-ACCESS database ZIP.
"""
from __future__ import unicode_literals
import os
import logging
import requests
from datetime import datetime
from hurry.filesize import size
from clint.textui import progress
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime
from django.core.files import File
from django.utils.timezone import utc, now
from calaccess_raw import get_download_directory
from django.template.loader import render_to_string
from django.core.management.base import CommandError
from calaccess_raw.management.commands import CalAccessCommand
from calaccess_raw.models.tracking import RawDataVersion
logger = logging.getLogger(__name__)

class Command(CalAccessCommand):
    """
    Download the latest CAL-ACCESS database ZIP.
    """
    help = "Download the latest CAL-ACCESS database ZIP"

    def add_arguments(self, parser):
        """
        Adds custom arguments specific to this command.
        """
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            "--noinput",
            action="store_true",
            dest="noinput",
            default=False,
            help="Download the ZIP archive without asking permission"
        )
        parser.add_argument(
            "--force-restart",
            "--restart",
            action="store_true",
            dest="restart",
            default=False,
            help="Force re-start (overrides auto-resume)."
        )

    def handle(self, *args, **options):
        """
        Make it happen.
        """
        super(Command, self).handle(*args, **options)

        # get the dir where data goes from app settings
        self.data_dir = get_download_directory()
        # if data_dir doesn't exist, create it
        os.path.exists(self.data_dir) or os.makedirs(self.data_dir)

        # downloaded zip file will go in data_dir
        self.zip_path = os.path.join(self.data_dir, self.url.split('/')[-1])

        self.download_metadata = self.get_download_metadata()
        logger.debug('Server: %s <-- (from HEAD)' % self.download_metadata['server'])
        logger.debug('ETag: %s <-- (from HEAD)' % self.download_metadata['etag'])
        logger.debug(
            'Last-Modified: %s <-- (from HEAD)' % self.download_metadata['last-modified']
        )
        logger.debug(
            'Content-Length %s <-- (from HEAD):' % self.download_metadata['content-length']
        )

        # get or create the RawDataVersion
        self.version, created = RawDataVersion.objects.get_or_create(
            release_datetime=self.download_metadata['last-modified'],
            expected_size=self.download_metadata['content-length'],
        )

        # if not called from command-line, assume called by update command
        if not self._called_from_command_line:
            # get the version that the update command is working on
            last_update_started = RawDataVersion.objects.filter(
                download_start_datetime__isnull=False
            ).latest('update_start_datetime')
            # confirm that update and download commands are working with same version
            if self.version != last_update_started:
                raise CommandError(
                    'Version available to download does not match version of update.'
                )

        # log if a new version was found
        if created:
            logger.info('New CAL-ACCESS version available.')

        # if download is stalled, zip file is there, and user did not invoke restart
        if (
            self.version.download_stalled and
            os.path.exists(self.zip_path) and
            not options['restart']
        ):
            # enable resuming
            self.resume = True
            # set current size to partially downloaded zip
            self.local_file_size = os.path.getsize(self.zip_path)
            # set the datetime of last download to last modified date of zip file
            timestamp = os.path.getmtime(self.zip_path)
            self.local_file_datetime = datetime.fromtimestamp(timestamp, utc)
        else:
            self.resume = False
            self.local_file_size = 0
            self.local_file_datetime = None

        if not options['noinput']:
            # get downloaded versions
            downloaded_versions = RawDataVersion.objects.filter(
                download_finish_datetime__isnull=False
            )
            # if there are any
            if downloaded_versions:
                # get the last version downloaded datetime
                last_download_datetime = downloaded_versions.latest(
                    'download_finish_datetime'
                ).download_finish_datetime
                since_last_download = naturaltime(last_download_datetime)
            else:
                last_download_datetime = None

            # setting up the prompt
            prompt_context = dict(
                latest_version=self.version,
                resume=self.resume,
                local_file_size=size(self.local_file_size),
                local_file_datetime=self.local_file_datetime,
                download_dir=self.data_dir,
                since_previous_download=since_last_download,
            )

            prompt = render_to_string(
                'calaccess_raw/downloadcalaccessrawdata.txt',
                prompt_context,
            )

            # if the user doesn't confirm initially
            if not self.confirm_proceed(prompt):
                # but the user can resume
                if self.resume:
                    # prompt to restart
                    confirm_restart = self.confirm_proceed(
                        'Do you want re-start your update?\n'
                    )
                    # if user confirms restart, do not resume
                    if confirm_restart:
                        self.resume = False
                    else:
                        # if the user doesn't confirm restart, cancel
                        raise CommandError("Download cancelled")
                # if the user doesn't confirm initial and can't resume
                elif not self.resume:
                    # cancel
                    raise CommandError("Download cancelled")

        # if not resuming, store the download start time
        if not self.resume:
            self.version.download_start_datetime = now()
        # either way, reset the finish time
        self.version.download_finish_datetime = None
        # save here in case the command doesn't finish
        self.version.save()

        # check if local zip file is already completely downloaded before trying
        if self.local_file_size < self.version.expected_size:
            self.download()

        logger.debug('Download zip size: %s bytes' % os.path.getsize(self.zip_path))
        # log warning if downloaded zip size is not same as expected size
        if self.version.expected_size != os.path.getsize(self.zip_path):
            raise CommandError(
                'Expected {0} byte zip, but download {1} byte zip.'.format(
                    self.version.expected_size,
                    os.path.getsize(self.zip_path)
                )
            )

        if getattr(settings, 'CALACCESS_STORE_ARCHIVE', False):
            self.archive()

        # store download finish time
        self.version.download_finish_datetime = now()
        # and save the RawDataVersion
        self.version.save()

    def download(self):
        """
        Download the ZIP file in pieces.
        """
        if self.verbosity:
            if self.resume:
                self.header(
                    "Resuming download of {:%m-%d-%Y %H:%M:%S} ZIP".format(
                        self.version.release_datetime
                    )
                )
            else:
                self.header("Downloading {:%m-%d-%Y %H:%M:%S} ZIP".format(
                      self.version.release_datetime
                    )
                )

        # Prep
        expected_size = self.version.expected_size
        headers = dict()
        if os.path.exists(self.zip_path):
            if self.resume:
                headers['Range'] = 'bytes=%d-' % self.local_file_size
                expected_size = expected_size - self.local_file_size
            else:
                os.remove(self.zip_path)

        # Stream the download
        chunk_size = 1024
        req = requests.get(self.url, stream=True, headers=headers)

        logger.debug('Server: %s <--  (from GET)' % req.headers['server'])
        logger.debug('ETag: %s <-- (from GET)' % req.headers['etag'])
        logger.debug('Last-Modified: %s <--  (from GET)' % req.headers['last-modified'])
        logger.debug('Content-Length: %s <-- (from GET)' % req.headers['content-length'])

        if self.download_metadata['etag'] != req.headers['etag']:
            raise CommandError("ETags in HEAD and GET requests don't match.")

        # in Python 2, need to convert this to long int
        try:
            divisor = long(chunk_size + 1)
        except NameError:
            # no long() in Python 3, all ints are long ints
            divisor = chunk_size + 1

        n_iters = float(expected_size) / divisor

        with open(self.zip_path, 'ab') as fp:
            for chunk in progress.bar(req.iter_content(chunk_size=chunk_size),
                                      expected_size=n_iters):
                fp.write(chunk)
                fp.flush()

    def archive(self):
        """
        Save a copy of the download zip file and each file inside.
        """
        if self.verbosity:
            self.log(" Archiving {0}".format(os.path.basename(self.zip_path)))
        # Remove previous zip file
        self.version.download_zip_archive.delete()
        # Store the actual download zip file size
        self.version.download_zip_size = os.path.getsize(self.zip_path)
        # Open up the zipped file so we can wrap it in the Django File obj
        zipped_file = open(self.zip_path)
        # Save the zip on the raw data version
        self.version.download_zip_archive.save(
            self.url.split('/')[-1],
            File(zipped_file)
        )
        zipped_file.close()
<|endoftext|>


<commit_before>import numpy as np
import yt
import ytree
import sys

def select_root_halos(a, criteria, fields):
    """
    Select desired halos given a set of criteria

    Parameters
    -----------
    a        : ytree object
    criteria : sequence of conditional strings that can be
               applied to the tree nodes
    fields   : fields used in the criteria

    Returns
    --------
    roots    : List of root halos in tree node that satisfy the
               criteria
    """
    selection = a.select_halos(criteria, fields = fields)
    roots = [x.find_root() for x in selection]

    return roots

def get_root_halos(a, criteria, fields,
                      unique = True,
                      mutually_exclusive = ''):
    """
    Loop over a list of criteria and return root halos
    that satisfy each criteria.

    Parameters
    -----------
    a        : ytree object
    criteria : sequence of conditional strings that can be
               applied to the tree nodes
    fields   : fields used in the criteria

    unique   : bool, optional
               ensure that the root nodes within each criteria selection
               are unique. Default : True

    mutually_exlcusive : bool, optional
        Method to ensure that halos across criteria are mutually exclusive.
        By default this is '', and this is not used. If set to 'sequential',
        this is meant to be used with criteria that are (in effect) nested
        such that the first is the most restrictive and the last the most
        general, such that it is likely that all roots in each subsequent
        selection are also included in the previous. Default : 'sequential'

    Returns
    ---------
    final_roots : list
        list of the list of root nodes that satisfy the
        criteria
    """

    num_criteria = len(criteria)

    all_roots = []
    for c in criteria:
        all_roots.append(select_root_halos(a, c, fields))

    # now make sure these are unique

    final_roots = [None]*num_criteria
    if mutually_exclusive == 'sequential':

        final_roots[0] = all_roots[0] # always in this mode

        for i in np.arange(1, num_criteria):
            final_roots[i] = [all_roots[i][ii] for ii in np.arange(np.size(all_roots[i])) if (not(all_roots[i][ii] in all_roots[i-1]))]

    elif mutually_exclusive == '':
        final_roots = all_roots
    else:
        print("Mutually exclusive selection type " + mutually_exclusive + " not yet supported")
        raise(ValueError)

    if unique:
        #
        # make sure roots halos are unique within each list
        # np.unique does not work on TreeNode objects (unfortunately)
        # so doing this in a really dumb way
        #
        final_roots_ids = [None]*num_criteria
        for i in np.arange(num_criteria):
            final_roots_ids[i] = [x['id'] for x in final_roots[i]]

        final_roots = [None]*num_criteria
        for i in np.arange(num_criteria):
            final_roots[i] = [ a.query(ii) for ii in np.arange(a.size) if a.query(ii)['id'] in final_roots_ids[i]]

    return final_roots

def print_halo_info(a, treenodes):
    """
    Print some information about the list of treenodes

    Parameters
    -----------
    a         :   ytree object from which the nodes come from
    treenodes : list of tree nodes
    """

    for tn in treenodes:
        print("%4i %5.5E %5.5E %.6f %.6f %.6f"%(tn['id'],tn['mass'],tn['virial_radius'],tn['x']/a.box_size,tn['y']/a.box_size,tn['z']/a.box_size))

    return

if __name__ == "__main__":

    #
    #
    #
    all_criteria = [ '(tree["tree", "redshift"] > 8.0) *'
                     '(tree["tree", "redshift"] < 12.5) *'
                     '(tree["tree", "mass"] > 1.0E9)',

                     '(tree["tree", "redshift"] > 7.0) *'
                     '(tree["tree", "redshift"] < 8.1) *'
                     '(tree["tree", "mass"] > 1.0E9)',

                     '(tree["tree", "redshift"] > 6.0) *'
                     '(tree["tree", "redshift"] < 7.1) *'
                     '(tree["tree", "mass"] > 1.0E9)']

    fields = ['redshift','mass']

    a = ytree.load(str(sys.argv[1]))

    all_halos = get_root_halos(a, all_criteria, fields, unique=True, mutually_exclusive='sequential')

    for i in np.arange(len(all_criteria)):
        print_halo_info(a,all_halos[i])
        print("----------------------------")
<commit_msg>Print function now has option to write to file. Plot halo mass evolution<commit_after>import matplotlib
matplotlib.use('agg')
from galaxy_analysis.plot_styles import *
import matplotlib.pyplot as plt

import numpy as np
import yt
import ytree
import sys

def select_root_halos(a, criteria, fields):
    """
    Select desired halos given a set of criteria

    Parameters
    -----------
    a        : ytree object
    criteria : sequence of conditional strings that can be
               applied to the tree nodes
    fields   : fields used in the criteria

    Returns
    --------
    roots    : List of root halos in tree node that satisfy the
               criteria
    """
    selection = a.select_halos(criteria, fields = fields)
    roots = [x.find_root() for x in selection]

    return roots

def get_root_halos(a, criteria, fields,
                      unique = True,
                      mutually_exclusive = ''):
    """
    Loop over a list of criteria and return root halos
    that satisfy each criteria.

    Parameters
    -----------
    a        : ytree object
    criteria : sequence of conditional strings that can be
               applied to the tree nodes
    fields   : fields used in the criteria

    unique   : bool, optional
               ensure that the root nodes within each criteria selection
               are unique. Default : True

    mutually_exlcusive : bool, optional
        Method to ensure that halos across criteria are mutually exclusive.
        By default this is '', and this is not used. If set to 'sequential',
        this is meant to be used with criteria that are (in effect) nested
        such that the first is the most restrictive and the last the most
        general, such that it is likely that all roots in each subsequent
        selection are also included in the previous. Default : 'sequential'

    Returns
    ---------
    final_roots : list
        list of the list of root nodes that satisfy the
        criteria
    """

    num_criteria = len(criteria)

    all_roots = []
    for c in criteria:
        all_roots.append(select_root_halos(a, c, fields))

    # now make sure these are unique

    final_roots = [None]*num_criteria
    if mutually_exclusive == 'sequential':

        final_roots[0] = all_roots[0] # always in this mode

        for i in np.arange(1, num_criteria):
            final_roots[i] = [all_roots[i][ii] for ii in np.arange(np.size(all_roots[i])) if (not(all_roots[i][ii] in all_roots[i-1]))]

    elif mutually_exclusive == '':
        final_roots = all_roots
    else:
        print("Mutually exclusive selection type " + mutually_exclusive + " not yet supported")
        raise(ValueError)

    if unique:
        #
        # make sure roots halos are unique within each list
        # np.unique does not work on TreeNode objects (unfortunately)
        # so doing this in a really dumb way
        #
        final_roots_ids = [None]*num_criteria
        for i in np.arange(num_criteria):
            final_roots_ids[i] = [x['id'] for x in final_roots[i]]

        final_roots = [None]*num_criteria
        for i in np.arange(num_criteria):
            final_roots[i] = [ a.query(ii) for ii in np.arange(a.size) if a.query(ii)['id'] in final_roots_ids[i]]

    return final_roots

def print_halo_info(a, treenodes, header = None, outfile = None):
    """
    Print some information about the list of treenodes

    Parameters
    -----------
    a         :   ytree object from which the nodes come from
    treenodes : list of tree nodes
    """

    if outfile is None:
        writefunc = lambda x : print(x, end='')
    else:
        f = open(outfile,'w')
        writefunc = f.write

    writefunc(header + '\n')

    for tn in treenodes:
        writefunc("%4i %5.5E %5.5E %.6f %.6f %.6f\n"%(tn['id'],tn['mass'],tn['virial_radius'],tn['x']/a.box_size,tn['y']/a.box_size,tn['z']/a.box_size))

    if not (outfile is None):
        f.close()

    return

def plot_halos(a,treenodes, outfile = None):
    """
    For a list of tree nodes, plot their growth history
    """

    if outfile is None:
        outfile = "halo_growth_history.png"

    fig, ax = plt.subplots()
    fig.set_size_inches(6,6)

    for i,tn in enumerate(treenodes):
        x = tn['tree','redshift']
        y = tn['tree','mass']
        color = "C%01i"%(i)
        ax.scatter(x,y, color = color, s = 40)
        ax.plot(x,y,color=color,lw=3,lable=tn['id'])

    ax.legend(loc='best')
    ax.set_ylim(1.0E8,8.0E9)
    ax.set_ylabel(r"Rockstar Halo Mass (M$_{\odot}$)")
    ax.semilogy()
    ax.set_xlabel("z")
    ax.set_xlim(15.0,5.0)
    plt.minorticks_on()
    plt.tight_layout()
    fig.savefig(outfile, bbox_inches='tight', pad_inches=0.0)

    return

if __name__ == "__main__":

    #
    #
    #
    all_criteria = [ '(tree["tree", "redshift"] > 8.0) *'
                     '(tree["tree", "redshift"] < 12.5) *'
                     '(tree["tree", "mass"] > 1.0E9)',

                     '(tree["tree", "redshift"] > 7.0) *'
                     '(tree["tree", "redshift"] < 8.1) *'
                     '(tree["tree", "mass"] > 1.0E9)',

                     '(tree["tree", "redshift"] > 6.0) *'
                     '(tree["tree", "redshift"] < 7.1) *'
                     '(tree["tree", "mass"] > 1.0E9)']

    fields = ['redshift','mass']

    a = ytree.load(str(sys.argv[1]))

    all_halos = get_root_halos(a, all_criteria, fields, unique=True, mutually_exclusive='sequential')

    outfiles = ['1E9_above_z8.dat','1E9_above_z7.dat', '1E9_above_z6.dat']
    outplots = ['1E9_above_z8.png','1E9_above_z7.png', '1E9_above_z6.png']
    for i in np.arange(len(all_criteria)):
        print_halo_info(a,all_halos[i], header = all_criteria[i], outfile = outfiles[i])
        plot_halos(a,all_halos[i], outfile = outplots[i])
<|endoftext|>


<commit_before>from django.http import HttpResponse
from StringIO import StringIO

def export_data_shared(export_tag, format=None, filename=None,
                       previous_export_id=None, filter=None):
    """
    Shared method for export. If there is data, return an HTTPResponse
    with the appropriate data. If there is not data returns None.
    """
    from couchexport.export import export

    if not filename:
        filename = export_tag

    tmp = StringIO()
    checkpoint = export(export_tag, tmp, format=format,
                        previous_export_id=previous_export_id,
                        filter=filter)
    if checkpoint:
        return export_response(tmp, format, filename, checkpoint)

    else:
        return None

def export_response(file, format, filename, checkpoint=None):
    """
    Get an http response for an export
    """
    from couchexport.export import Format
    if not filename:
        filename = "NAMELESS EXPORT"

    format = Format.from_format(format)
    response = HttpResponse(mimetype=format.mimetype)
    response['Content-Disposition'] = 'attachment; filename=%s.%s' % \
                                    (filename, format.extension)
    if checkpoint:
        response['X-CommCareHQ-Export-Token'] = checkpoint.get_id
    response.write(file.getvalue())
    file.close()
    return response

<commit_msg>support unicode form names in custom exports<commit_after>from django.http import HttpResponse
from StringIO import StringIO
from unidecode import unidecode

def export_data_shared(export_tag, format=None, filename=None,
                       previous_export_id=None, filter=None):
    """
    Shared method for export. If there is data, return an HTTPResponse
    with the appropriate data. If there is not data returns None.
    """
    from couchexport.export import export

    if not filename:
        filename = export_tag

    tmp = StringIO()
    checkpoint = export(export_tag, tmp, format=format,
                        previous_export_id=previous_export_id,
                        filter=filter)
    if checkpoint:
        return export_response(tmp, format, filename, checkpoint)

    else:
        return None

def export_response(file, format, filename, checkpoint=None):
    """
    Get an http response for an export
    """
    from couchexport.export import Format
    if not filename:
        filename = "NAMELESS EXPORT"

    format = Format.from_format(format)
    response = HttpResponse(mimetype=format.mimetype)
    response['Content-Disposition'] = 'attachment; filename=%s.%s' % \
                                    (unidecode(filename), format.extension)
    if checkpoint:
        response['X-CommCareHQ-Export-Token'] = checkpoint.get_id
    response.write(file.getvalue())
    file.close()
    return response

<|endoftext|>


<commit_before>"""
Django settings for councilmatic project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from .settings_local import *

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

ALLOWED_HOSTS = [
    'localhost', '127.0.0.1',
    '.datamade.us',
    '.nyccouncilmatic.org'
]

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'haystack',
    'councilmatic_core',
    'nyc',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
)

ROOT_URLCONF = 'councilmatic.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        #'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'councilmatic_core.views.city_context'
            ],
            'loaders': [
                ('django.template.loaders.cached.Loader', [
                    'django.template.loaders.filesystem.Loader',
                    'django.template.loaders.app_directories.Loader',
                ]),
            ],
        },
    },
]

WSGI_APPLICATION = 'councilmatic.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
<commit_msg>allowed hosts<commit_after>"""
Django settings for councilmatic project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from .settings_local import *

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

ALLOWED_HOSTS = [
    'localhost', '127.0.0.1',
    '.datamade.us',
    '.nyccouncilmatic.org',
    '.councilmatic.org',
]

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'haystack',
    'councilmatic_core',
    'nyc',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
)

ROOT_URLCONF = 'councilmatic.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        #'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'councilmatic_core.views.city_context'
            ],
            'loaders': [
                ('django.template.loaders.cached.Loader', [
                    'django.template.loaders.filesystem.Loader',
                    'django.template.loaders.app_directories.Loader',
                ]),
            ],
        },
    },
]

WSGI_APPLICATION = 'councilmatic.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')

HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'
<|endoftext|>


<commit_before>#!/usr/bin/python

#
# Created by Casey Deccio (cdeccio@verisign.com)
#
# Copyright (c) 2015-2016, VeriSign, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import codecs
import re

import dns.name

def import_names(f, i, p):
    with codecs.open(f, 'r', 'utf-8') as fh:
        n = i
        for line in fh:
            if re.search('BEGIN PRIVATE', line) is not None:
                n = p

            line = line.rstrip()
            line = re.sub(r'//.*', '', line)

            if not line:
                continue
            name = dns.name.from_text(line)

            line = re.sub(r'//.*', '', line)

            n.add(name)

def usage():
    import sys
    sys.stderr.write('Usage: %s <psl_filename> <odup_ns_zone>\n' % (sys.argv[0]))

def main():
    import sys

    args = sys.argv[1:]
    if len(args) != 2:
        usage()
        sys.exit(1)

    public_suffix_file = args[0]
    zone_name = dns.name.from_text(args[1]).to_text()

    if zone_name == '.':
        sys.stderr.write('ODUP NS zone name may not be the root.\n')
        sys.exit(1)

    icann_names = set()
    private_names = set()
    import_names(public_suffix_file, icann_names, private_names)

    print '$ORIGIN _odup.'
    print '$TTL 604800'
    print '@\tSOA\ta.%s root.%s 1 1800 900 604800 86400' % (zone_name, zone_name)
    print '\tNS\ta.%s' % (zone_name)
    print '\tNS\tb.%s' % (zone_name)
    print '\tTXT\t"v=odup1 +bound -all"'

    for name in icann_names:
        if name[0][0].startswith('!'):
            print '%s\tTXT\t"v=odup1 +org"' % str(name).rstrip('.').lstrip('!')
        elif name[0] == '*':
            print '%s\tTXT\t"v=odup1 +bound:%d"' % (str(name).rstrip('.'), len(name) - 2)
        else:
            print '%s\tTXT\t"v=odup1 +bound"' % str(name).rstrip('.')

if __name__ == '__main__':
    main()
<commit_msg>Build _odup sub-zones for each TLD<commit_after>#!/usr/bin/python

#
# Created by Casey Deccio (cdeccio@verisign.com)
#
# Copyright (c) 2015-2016, VeriSign, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import codecs
import os
import re

import dns.name

def import_names(f, i, p):
    with codecs.open(f, 'r', 'utf-8') as fh:
        n = i
        for line in fh:
            if re.search('BEGIN PRIVATE', line) is not None:
                n = p

            line = line.rstrip()
            line = re.sub(r'//.*', '', line)

            if not line:
                continue
            name = dns.name.from_text(line)
            tld = dns.name.from_text(name[-2])

            if tld not in n:
                n[tld] = set()
            if tld != name:
                n[tld].add(name)

def export_zone(tld, names, zonefile_dir, server_names, conffile_fh):

    filename = os.path.join(zonefile_dir, 'db._odup.%s' % (tld.to_text().rstrip('.')))
    conffile_fh.write('zone "_odup.%s" {\n' % tld.to_text())
    conffile_fh.write('\ttype master;\n')
    conffile_fh.write('\tfile "%s";\n' % filename)
    conffile_fh.write('\tallow-transfer { any; };\n')
    conffile_fh.write('};\n')

    with open(filename, 'w+') as fh:
        fh.write('$ORIGIN _odup.%s\n' % (tld.to_text()))
        fh.write('$TTL 604800\n')
        fh.write('@\tSOA\t%s root.nic.%s 1 1800 900 604800 86400\n' % (server_names[0].to_text(), tld.to_text()))
        for server_name in server_names:
            fh.write('\tNS\t%s\n' % (server_name.to_text()))
        fh.write('\tTXT\t"v=odup1 +bound +fetch:axfr:// -all"\n')

        for name in names:
            name = name.relativize(tld)
            if name[0][0].startswith('!'):
                fh.write('%s\tTXT\t"v=odup1 +org"\n' % (name.to_text().lstrip('!')))
            elif name[0] == '*':
                fh.write('%s\tTXT\t"v=odup1 +bound:%d"\n' % (name.to_text(), len(name) - 1))
            else:
                fh.write('%s\tTXT\t"v=odup1 +bound"\n' % (name.to_text()))

def usage():
    import sys
    sys.stderr.write('Usage: %s <psl_filename> <zonefile_dir> <named_conf_inc_filename> [ <server_name> ]\n' % (sys.argv[0]))

def main():
    import sys

    args = sys.argv[1:]
    if len(args) < 3:
        usage()
        sys.exit(1)

    public_suffix_file = args[0]
    zonefile_dir = args[1]
    named_conf_inc_filename = args[2]

    if len(args) > 3:
        server_name = dns.name.from_text(args[3])
    else:
        server_name = None

    icann_names = {}
    private_names = {}
    import_names(public_suffix_file, icann_names, private_names)

    with open(named_conf_inc_filename, 'w+') as fh:
        for tld in icann_names:
            if server_name is not None:
                server_names = (server_name,)
            else:
                server_names = (dns.name.Name(('a', 'odup-servers', tld[-2])),)
            export_zone(tld, icann_names[tld], zonefile_dir, server_names, fh)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt

"""Mixin classes to help make good tests."""

import atexit
import collections
import contextlib
import os
import random
import shutil
import sys
import tempfile
import textwrap

from coverage.backunittest import TestCase
from coverage.backward import StringIO, to_bytes

class Tee(object):
    """A file-like that writes to all the file-likes it has."""

    def __init__(self, *files):
        """Make a Tee that writes to all the files in `files.`"""
        self._files = files
        if hasattr(files[0], "encoding"):
            self.encoding = files[0].encoding

    def write(self, data):
        """Write `data` to all the files."""
        for f in self._files:
            f.write(data)

    def flush(self):
        """Flush the data on all the files."""
        for f in self._files:
            f.flush()

    if 0:
        # Use this if you need to use a debugger, though it makes some tests
        # fail, I'm not sure why...
        def __getattr__(self, name):
            return getattr(self._files[0], name)

@contextlib.contextmanager
def change_dir(new_dir):
    """Change directory, and then change back.

    Use as a context manager, it will give you the new directory, and later
    restore the old one.

    """
    old_dir = os.getcwd()
    os.chdir(new_dir)
    try:
        yield os.getcwd()
    finally:
        os.chdir(old_dir)

@contextlib.contextmanager
def saved_sys_path():
    """Save sys.path, and restore it later."""
    old_syspath = sys.path[:]
    try:
        yield
    finally:
        sys.path = old_syspath

def setup_with_context_manager(testcase, cm):
    """Use a contextmanager to setUp a test case.

    If you have a context manager you like::

        with ctxmgr(a, b, c) as v:
            # do something with v

    and you want to have that effect for a test case, call this function from
    your setUp, and it will start the context manager for your test, and end it
    when the test is done::

        def setUp(self):
            self.v = setup_with_context_manager(self, ctxmgr(a, b, c))

        def test_foo(self):
            # do something with self.v

    """
    val = cm.__enter__()
    testcase.addCleanup(cm.__exit__, None, None, None)
    return val

class ModuleAwareMixin(TestCase):
    """A test case mixin that isolates changes to sys.modules."""

    def setUp(self):
        super(ModuleAwareMixin, self).setUp()

        # Record sys.modules here so we can restore it in cleanup_modules.
        self.old_modules = list(sys.modules)
        self.addCleanup(self.cleanup_modules)

    def cleanup_modules(self):
        """Remove any new modules imported during the test run.

        This lets us import the same source files for more than one test.

        """
        for m in [m for m in sys.modules if m not in self.old_modules]:
            del sys.modules[m]

class SysPathAwareMixin(TestCase):
    """A test case mixin that isolates changes to sys.path."""

    def setUp(self):
        super(SysPathAwareMixin, self).setUp()
        setup_with_context_manager(self, saved_sys_path())

class EnvironmentAwareMixin(TestCase):
    """A test case mixin that isolates changes to the environment."""

    def setUp(self):
        super(EnvironmentAwareMixin, self).setUp()

        # Record environment variables that we changed with set_environ.
        self.environ_undos = {}

        self.addCleanup(self.cleanup_environ)

    def set_environ(self, name, value):
        """Set an environment variable `name` to be `value`.

        The environment variable is set, and record is kept that it was set,
        so that `cleanup_environ` can restore its original value.

        """
        if name not in self.environ_undos:
            self.environ_undos[name] = os.environ.get(name)
        os.environ[name] = value

    def cleanup_environ(self):
        """Undo all the changes made by `set_environ`."""
        for name, value in self.environ_undos.items():
            if value is None:
                del os.environ[name]
            else:
                os.environ[name] = value

class StdStreamCapturingMixin(TestCase):
    """A test case mixin that captures stdout and stderr."""

    def setUp(self):
        super(StdStreamCapturingMixin, self).setUp()

        # Capture stdout and stderr so we can examine them in tests.
        # nose keeps stdout from littering the screen, so we can safely Tee it,
        # but it doesn't capture stderr, so we don't want to Tee stderr to the
        # real stderr, since it will interfere with our nice field of dots.
        old_stdout = sys.stdout
        self.captured_stdout = StringIO()
        sys.stdout = Tee(sys.stdout, self.captured_stdout)

        old_stderr = sys.stderr
        self.captured_stderr = StringIO()
        sys.stderr = self.captured_stderr

        self.addCleanup(self.cleanup_std_streams, old_stdout, old_stderr)

    def cleanup_std_streams(self, old_stdout, old_stderr):
        """Restore stdout and stderr."""
        sys.stdout = old_stdout
        sys.stderr = old_stderr

    def stdout(self):
        """Return the data written to stdout during the test."""
        return self.captured_stdout.getvalue()

    def stderr(self):
        """Return the data written to stderr during the test."""
        return self.captured_stderr.getvalue()

class DelayedAssertionMixin(TestCase):
    """A test case mixin that provides a `delayed_assertions` context manager.

    Use it like this::

        with self.delayed_assertions():
            self.assertEqual(x, y)
            self.assertEqual(z, w)

    All of the assertions will run.  The failures will be displayed at the end
    of the with-statement.

    NOTE: only works with some assert methods, I'm not sure which!

    """
    def __init__(self, *args, **kwargs):
        super(DelayedAssertionMixin, self).__init__(*args, **kwargs)
        # This mixin only works with assert methods that call `self.fail`.  In
        # Python 2.7, `assertEqual` didn't, but we can do what Python 3 does,
        # and use `assertMultiLineEqual` for comparing strings.
        self.addTypeEqualityFunc(str, 'assertMultiLineEqual')
        self._delayed_assertions = None

    @contextlib.contextmanager
    def delayed_assertions(self):
        """The context manager: assert that we didn't collect any assertions."""
        self._delayed_assertions = []
        old_fail = self.fail
        self.fail = self._delayed_fail
        try:
            yield
        finally:
            self.fail = old_fail
        if self._delayed_assertions:
            if len(self._delayed_assertions) == 1:
                self.fail(self._delayed_assertions[0])
            else:
                self.fail(
                    "{} failed assertions:\n{}".format(
                        len(self._delayed_assertions),
                        "\n".join(self._delayed_assertions),
                    )
                )

    def _delayed_fail(self, msg=None):
        """The stand-in for TestCase.fail during delayed_assertions."""
        self._delayed_assertions.append(msg)

class TempDirMixin(SysPathAwareMixin, ModuleAwareMixin, TestCase):
    """A test case mixin that creates a temp directory and files in it.

    Includes SysPathAwareMixin and ModuleAwareMixin, because making and using
    temp directories like this will also need that kind of isolation.

    """

    # Our own setting: most of these tests run in their own temp directory.
    # Set this to False in your subclass if you don't want a temp directory
    # created.
    run_in_temp_dir = True

    # Set this if you aren't creating any files with make_file, but still want
    # the temp directory.  This will stop the test behavior checker from
    # complaining.
    no_files_in_temp_dir = False

    def setUp(self):
        super(TempDirMixin, self).setUp()

        if self.run_in_temp_dir:
            # Create a temporary directory.
            self.temp_dir = self.make_temp_dir("test_cover")
            self.chdir(self.temp_dir)

            # Modules should be importable from this temp directory.  We don't
            # use '' because we make lots of different temp directories and
            # nose's caching importer can get confused.  The full path prevents
            # problems.
            sys.path.insert(0, os.getcwd())

        class_behavior = self.class_behavior()
        class_behavior.tests += 1
        class_behavior.temp_dir = self.run_in_temp_dir
        class_behavior.no_files_ok = self.no_files_in_temp_dir

        self.addCleanup(self.check_behavior)

    def make_temp_dir(self, slug="test_cover"):
        """Make a temp directory that is cleaned up when the test is done."""
        name = "%s_%08d" % (slug, random.randint(0, 99999999))
        temp_dir = os.path.join(tempfile.gettempdir(), name)
        os.makedirs(temp_dir)
        self.addCleanup(shutil.rmtree, temp_dir)
        return temp_dir

    def chdir(self, new_dir):
        """Change directory, and change back when the test is done."""
        old_dir = os.getcwd()
        os.chdir(new_dir)
        self.addCleanup(os.chdir, old_dir)

    def check_behavior(self):
        """Check that we did the right things."""

        class_behavior = self.class_behavior()
        if class_behavior.test_method_made_any_files:
            class_behavior.tests_making_files += 1

    def make_file(self, filename, text="", newline=None):
        """Create a file for testing.

        `filename` is the relative path to the file, including directories if
        desired, which will be created if need be.

        `text` is the content to create in the file, a native string (bytes in
        Python 2, unicode in Python 3).

        If `newline` is provided, it is a string that will be used as the line
        endings in the created file, otherwise the line endings are as provided
        in `text`.

        Returns `filename`.

        """
        # Tests that call `make_file` should be run in a temp environment.
        assert self.run_in_temp_dir
        self.class_behavior().test_method_made_any_files = True

        text = textwrap.dedent(text)
        if newline:
            text = text.replace("\n", newline)

        # Make sure the directories are available.
        dirs, _ = os.path.split(filename)
        if dirs and not os.path.exists(dirs):
            os.makedirs(dirs)

        # Create the file.
        with open(filename, 'wb') as f:
            f.write(to_bytes(text))

        return filename

    # We run some tests in temporary directories, because they may need to make
    # files for the tests. But this is expensive, so we can change per-class
    # whether a temp directory is used or not.  It's easy to forget to set that
    # option properly, so we track information about what the tests did, and
    # then report at the end of the process on test classes that were set
    # wrong.

    class ClassBehavior(object):
        """A value object to store per-class."""
        def __init__(self):
            self.tests = 0
            self.skipped = 0
            self.temp_dir = True
            self.no_files_ok = False
            self.tests_making_files = 0
            self.test_method_made_any_files = False

    # Map from class to info about how it ran.
    class_behaviors = collections.defaultdict(ClassBehavior)

    @classmethod
    def report_on_class_behavior(cls):
        """Called at process exit to report on class behavior."""
        for test_class, behavior in cls.class_behaviors.items():
            bad = ""
            if behavior.tests <= behavior.skipped:
                bad = ""
            elif behavior.temp_dir and behavior.tests_making_files == 0:
                if not behavior.no_files_ok:
                    bad = "Inefficient"
            elif not behavior.temp_dir and behavior.tests_making_files > 0:
                bad = "Unsafe"

            if bad:
                if behavior.temp_dir:
                    where = "in a temp directory"
                else:
                    where = "without a temp directory"
                print(
                    "%s: %s ran %d tests, %d made files %s" % (
                        bad,
                        test_class.__name__,
                        behavior.tests,
                        behavior.tests_making_files,
                        where,
                    )
                )

    def class_behavior(self):
        """Get the ClassBehavior instance for this test."""
        return self.class_behaviors[self.__class__]

# When the process ends, find out about bad classes.
atexit.register(TempDirMixin.report_on_class_behavior)
<commit_msg>Fix 2.6, as usual<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt

"""Mixin classes to help make good tests."""

import atexit
import collections
import contextlib
import os
import random
import shutil
import sys
import tempfile
import textwrap

from coverage.backunittest import TestCase
from coverage.backward import StringIO, to_bytes

class Tee(object):
    """A file-like that writes to all the file-likes it has."""

    def __init__(self, *files):
        """Make a Tee that writes to all the files in `files.`"""
        self._files = files
        if hasattr(files[0], "encoding"):
            self.encoding = files[0].encoding

    def write(self, data):
        """Write `data` to all the files."""
        for f in self._files:
            f.write(data)

    def flush(self):
        """Flush the data on all the files."""
        for f in self._files:
            f.flush()

    if 0:
        # Use this if you need to use a debugger, though it makes some tests
        # fail, I'm not sure why...
        def __getattr__(self, name):
            return getattr(self._files[0], name)

@contextlib.contextmanager
def change_dir(new_dir):
    """Change directory, and then change back.

    Use as a context manager, it will give you the new directory, and later
    restore the old one.

    """
    old_dir = os.getcwd()
    os.chdir(new_dir)
    try:
        yield os.getcwd()
    finally:
        os.chdir(old_dir)

@contextlib.contextmanager
def saved_sys_path():
    """Save sys.path, and restore it later."""
    old_syspath = sys.path[:]
    try:
        yield
    finally:
        sys.path = old_syspath

def setup_with_context_manager(testcase, cm):
    """Use a contextmanager to setUp a test case.

    If you have a context manager you like::

        with ctxmgr(a, b, c) as v:
            # do something with v

    and you want to have that effect for a test case, call this function from
    your setUp, and it will start the context manager for your test, and end it
    when the test is done::

        def setUp(self):
            self.v = setup_with_context_manager(self, ctxmgr(a, b, c))

        def test_foo(self):
            # do something with self.v

    """
    val = cm.__enter__()
    testcase.addCleanup(cm.__exit__, None, None, None)
    return val

class ModuleAwareMixin(TestCase):
    """A test case mixin that isolates changes to sys.modules."""

    def setUp(self):
        super(ModuleAwareMixin, self).setUp()

        # Record sys.modules here so we can restore it in cleanup_modules.
        self.old_modules = list(sys.modules)
        self.addCleanup(self.cleanup_modules)

    def cleanup_modules(self):
        """Remove any new modules imported during the test run.

        This lets us import the same source files for more than one test.

        """
        for m in [m for m in sys.modules if m not in self.old_modules]:
            del sys.modules[m]

class SysPathAwareMixin(TestCase):
    """A test case mixin that isolates changes to sys.path."""

    def setUp(self):
        super(SysPathAwareMixin, self).setUp()
        setup_with_context_manager(self, saved_sys_path())

class EnvironmentAwareMixin(TestCase):
    """A test case mixin that isolates changes to the environment."""

    def setUp(self):
        super(EnvironmentAwareMixin, self).setUp()

        # Record environment variables that we changed with set_environ.
        self.environ_undos = {}

        self.addCleanup(self.cleanup_environ)

    def set_environ(self, name, value):
        """Set an environment variable `name` to be `value`.

        The environment variable is set, and record is kept that it was set,
        so that `cleanup_environ` can restore its original value.

        """
        if name not in self.environ_undos:
            self.environ_undos[name] = os.environ.get(name)
        os.environ[name] = value

    def cleanup_environ(self):
        """Undo all the changes made by `set_environ`."""
        for name, value in self.environ_undos.items():
            if value is None:
                del os.environ[name]
            else:
                os.environ[name] = value

class StdStreamCapturingMixin(TestCase):
    """A test case mixin that captures stdout and stderr."""

    def setUp(self):
        super(StdStreamCapturingMixin, self).setUp()

        # Capture stdout and stderr so we can examine them in tests.
        # nose keeps stdout from littering the screen, so we can safely Tee it,
        # but it doesn't capture stderr, so we don't want to Tee stderr to the
        # real stderr, since it will interfere with our nice field of dots.
        old_stdout = sys.stdout
        self.captured_stdout = StringIO()
        sys.stdout = Tee(sys.stdout, self.captured_stdout)

        old_stderr = sys.stderr
        self.captured_stderr = StringIO()
        sys.stderr = self.captured_stderr

        self.addCleanup(self.cleanup_std_streams, old_stdout, old_stderr)

    def cleanup_std_streams(self, old_stdout, old_stderr):
        """Restore stdout and stderr."""
        sys.stdout = old_stdout
        sys.stderr = old_stderr

    def stdout(self):
        """Return the data written to stdout during the test."""
        return self.captured_stdout.getvalue()

    def stderr(self):
        """Return the data written to stderr during the test."""
        return self.captured_stderr.getvalue()

class DelayedAssertionMixin(TestCase):
    """A test case mixin that provides a `delayed_assertions` context manager.

    Use it like this::

        with self.delayed_assertions():
            self.assertEqual(x, y)
            self.assertEqual(z, w)

    All of the assertions will run.  The failures will be displayed at the end
    of the with-statement.

    NOTE: only works with some assert methods, I'm not sure which!

    """
    def __init__(self, *args, **kwargs):
        super(DelayedAssertionMixin, self).__init__(*args, **kwargs)
        # This mixin only works with assert methods that call `self.fail`.  In
        # Python 2.7, `assertEqual` didn't, but we can do what Python 3 does,
        # and use `assertMultiLineEqual` for comparing strings.
        self.addTypeEqualityFunc(str, 'assertMultiLineEqual')
        self._delayed_assertions = None

    @contextlib.contextmanager
    def delayed_assertions(self):
        """The context manager: assert that we didn't collect any assertions."""
        self._delayed_assertions = []
        old_fail = self.fail
        self.fail = self._delayed_fail
        try:
            yield
        finally:
            self.fail = old_fail
        if self._delayed_assertions:
            if len(self._delayed_assertions) == 1:
                self.fail(self._delayed_assertions[0])
            else:
                self.fail(
                    "{0} failed assertions:\n{1}".format(
                        len(self._delayed_assertions),
                        "\n".join(self._delayed_assertions),
                    )
                )

    def _delayed_fail(self, msg=None):
        """The stand-in for TestCase.fail during delayed_assertions."""
        self._delayed_assertions.append(msg)

class TempDirMixin(SysPathAwareMixin, ModuleAwareMixin, TestCase):
    """A test case mixin that creates a temp directory and files in it.

    Includes SysPathAwareMixin and ModuleAwareMixin, because making and using
    temp directories like this will also need that kind of isolation.

    """

    # Our own setting: most of these tests run in their own temp directory.
    # Set this to False in your subclass if you don't want a temp directory
    # created.
    run_in_temp_dir = True

    # Set this if you aren't creating any files with make_file, but still want
    # the temp directory.  This will stop the test behavior checker from
    # complaining.
    no_files_in_temp_dir = False

    def setUp(self):
        super(TempDirMixin, self).setUp()

        if self.run_in_temp_dir:
            # Create a temporary directory.
            self.temp_dir = self.make_temp_dir("test_cover")
            self.chdir(self.temp_dir)

            # Modules should be importable from this temp directory.  We don't
            # use '' because we make lots of different temp directories and
            # nose's caching importer can get confused.  The full path prevents
            # problems.
            sys.path.insert(0, os.getcwd())

        class_behavior = self.class_behavior()
        class_behavior.tests += 1
        class_behavior.temp_dir = self.run_in_temp_dir
        class_behavior.no_files_ok = self.no_files_in_temp_dir

        self.addCleanup(self.check_behavior)

    def make_temp_dir(self, slug="test_cover"):
        """Make a temp directory that is cleaned up when the test is done."""
        name = "%s_%08d" % (slug, random.randint(0, 99999999))
        temp_dir = os.path.join(tempfile.gettempdir(), name)
        os.makedirs(temp_dir)
        self.addCleanup(shutil.rmtree, temp_dir)
        return temp_dir

    def chdir(self, new_dir):
        """Change directory, and change back when the test is done."""
        old_dir = os.getcwd()
        os.chdir(new_dir)
        self.addCleanup(os.chdir, old_dir)

    def check_behavior(self):
        """Check that we did the right things."""

        class_behavior = self.class_behavior()
        if class_behavior.test_method_made_any_files:
            class_behavior.tests_making_files += 1

    def make_file(self, filename, text="", newline=None):
        """Create a file for testing.

        `filename` is the relative path to the file, including directories if
        desired, which will be created if need be.

        `text` is the content to create in the file, a native string (bytes in
        Python 2, unicode in Python 3).

        If `newline` is provided, it is a string that will be used as the line
        endings in the created file, otherwise the line endings are as provided
        in `text`.

        Returns `filename`.

        """
        # Tests that call `make_file` should be run in a temp environment.
        assert self.run_in_temp_dir
        self.class_behavior().test_method_made_any_files = True

        text = textwrap.dedent(text)
        if newline:
            text = text.replace("\n", newline)

        # Make sure the directories are available.
        dirs, _ = os.path.split(filename)
        if dirs and not os.path.exists(dirs):
            os.makedirs(dirs)

        # Create the file.
        with open(filename, 'wb') as f:
            f.write(to_bytes(text))

        return filename

    # We run some tests in temporary directories, because they may need to make
    # files for the tests. But this is expensive, so we can change per-class
    # whether a temp directory is used or not.  It's easy to forget to set that
    # option properly, so we track information about what the tests did, and
    # then report at the end of the process on test classes that were set
    # wrong.

    class ClassBehavior(object):
        """A value object to store per-class."""
        def __init__(self):
            self.tests = 0
            self.skipped = 0
            self.temp_dir = True
            self.no_files_ok = False
            self.tests_making_files = 0
            self.test_method_made_any_files = False

    # Map from class to info about how it ran.
    class_behaviors = collections.defaultdict(ClassBehavior)

    @classmethod
    def report_on_class_behavior(cls):
        """Called at process exit to report on class behavior."""
        for test_class, behavior in cls.class_behaviors.items():
            bad = ""
            if behavior.tests <= behavior.skipped:
                bad = ""
            elif behavior.temp_dir and behavior.tests_making_files == 0:
                if not behavior.no_files_ok:
                    bad = "Inefficient"
            elif not behavior.temp_dir and behavior.tests_making_files > 0:
                bad = "Unsafe"

            if bad:
                if behavior.temp_dir:
                    where = "in a temp directory"
                else:
                    where = "without a temp directory"
                print(
                    "%s: %s ran %d tests, %d made files %s" % (
                        bad,
                        test_class.__name__,
                        behavior.tests,
                        behavior.tests_making_files,
                        where,
                    )
                )

    def class_behavior(self):
        """Get the ClassBehavior instance for this test."""
        return self.class_behaviors[self.__class__]

# When the process ends, find out about bad classes.
atexit.register(TempDirMixin.report_on_class_behavior)
<|endoftext|>


<commit_before>import os
dirname, filename = os.path.split(os.path.abspath(__file__))
for f in os.listdir(dirname):
    #dirname, filename = os.path.split(f)
    if f.endswith('.py'):
        if f not in ['__init__.py', 'run_crawlers.py', 'socrata_dataset_catalog.py']:
            os.system('python %s &' % (os.path.join(dirname, f)))<commit_msg>auto commit<commit_after>import os
dirname, filename = os.path.split(os.path.abspath(__file__))
for f in os.listdir(dirname):
    #dirname, filename = os.path.split(f)
    if f.endswith('.py'):
        if f not in ['__init__.py', 'run_crawlers.py']:
            os.system('python %s &' % (os.path.join(dirname, f)))<|endoftext|>


<commit_before>from django import forms

from crits.core.handlers import get_source_names
from crits.core.user_tools import get_user_organization

class UploadStandardsForm(forms.Form):
    """
    Django form for uploading a standards document.
    """

    error_css_class = 'error'
    required_css_class = 'required'
    filedata = forms.FileField()
    source = forms.ChoiceField(required=True)
    reference = forms.ChoiceField(required=False)
    make_event = forms.BooleanField(required=False, label="Create event", initial=True)

    def __init__(self, username, *args, **kwargs):
        super(UploadStandardsForm, self).__init__(*args, **kwargs)
        self.fields['source'].choices = [(c.name,
                                          c.name) for c in get_source_names(True,
                                                                            True,
                                                                            username)]
        self.fields['source'].initial = get_user_organization(username)
<commit_msg>Fix error in field type.<commit_after>from django import forms

from crits.core.handlers import get_source_names
from crits.core.user_tools import get_user_organization

class UploadStandardsForm(forms.Form):
    """
    Django form for uploading a standards document.
    """

    error_css_class = 'error'
    required_css_class = 'required'
    filedata = forms.FileField()
    source = forms.ChoiceField(required=True)
    reference = forms.CharField(required=False)
    make_event = forms.BooleanField(required=False, label="Create event", initial=True)

    def __init__(self, username, *args, **kwargs):
        super(UploadStandardsForm, self).__init__(*args, **kwargs)
        self.fields['source'].choices = [(c.name,
                                          c.name) for c in get_source_names(True,
                                                                            True,
                                                                            username)]
        self.fields['source'].initial = get_user_organization(username)
<|endoftext|>


<commit_before>#from pysb.generator.bng import BngGenerator
import os
import subprocess
import random
import sympy

pkg_path = None

generate_network_code = """
begin actions
generate_network({overwrite=>1});
end actions
"""

def generate_equations(content):
    if pkg_path == None:
        raise Exception('must set pysb.bng.pkg_path to BNG directory')

    #gen = BngGenerator(model)

    bng_filename = '%d_%d_temp.bngl' % (os.getpid(), random.randint(0, 10000))
    net_filename = bng_filename.replace('.bngl', '.net')
    try:
        bng_file = open(bng_filename, 'w')
        bng_file.write(content)
        bng_file.write(generate_network_code)
        bng_file.close()

        subprocess.call(['/usr/bin/perl', pkg_path+'/Perl2/BNG2.pl', bng_filename],
                        stdout=subprocess.PIPE)

        net_file = open(net_filename, 'r')
        while net_file.readline().strip() != 'begin reactions':
            pass
        ydot = {}
        while True:
            line = net_file.readline()
            if line == 'end reactions\n' or line == '': break
            (number, reactants, products, rate, rule) = line.strip().split()
            reactants = reactants.split(',')
            products = products.split(',')
            rate = rate.rsplit('*')
            for p in products:
                p = int(p)
                ydot.setdefault(p, sympy.S(0))
                r_names = ['s' + r for r in reactants]
                ydot[p] += sympy.Mul(*[sympy.S(t) for t in r_names + rate])
        net_file.close()
    except Exception as e:
        print "problem running BNG:\n"
        print e
        print "\n"
    finally:
        os.unlink(bng_filename)
        os.unlink(net_filename)

    for i in range(1, max(ydot) + 1):
        ydot.setdefault(i, sympy.S(0))

    return ydot
<commit_msg>fixed math to account for consumption of reactants (!)<commit_after>#from pysb.generator.bng import BngGenerator
import os
import subprocess
import random
import sympy

pkg_path = None

generate_network_code = """
begin actions
generate_network({overwrite=>1});
end actions
"""

def generate_equations(content):
    if pkg_path == None:
        raise Exception('must set pysb.bng.pkg_path to BNG directory')

    #gen = BngGenerator(model)

    bng_filename = '%d_%d_temp.bngl' % (os.getpid(), random.randint(0, 10000))
    net_filename = bng_filename.replace('.bngl', '.net')
    try:
        bng_file = open(bng_filename, 'w')
        bng_file.write(content)
        bng_file.write(generate_network_code)
        bng_file.close()

        subprocess.call(['/usr/bin/perl', pkg_path+'/Perl2/BNG2.pl', bng_filename],
                        stdout=subprocess.PIPE)

        net_file = open(net_filename, 'r')
        while net_file.readline().strip() != 'begin reactions':
            pass
        ydot = {}
        while True:
            line = net_file.readline()
            if line == 'end reactions\n' or line == '': break
            (number, reactants, products, rate, rule) = line.strip().split()
            reactants = reactants.split(',')
            products = products.split(',')
            rate = rate.rsplit('*')
            r_names = ['s' + r for r in reactants]
            combined_rate = sympy.Mul(*[sympy.S(t) for t in r_names + rate])
            for p in products:
                p = int(p)
                ydot.setdefault(p, sympy.S(0))
                ydot[p] += combined_rate
            for r in reactants:
                r = int(r)
                ydot.setdefault(r, sympy.S(0))
                ydot[r] -= combined_rate
        net_file.close()
    except Exception as e:
        print "problem running BNG:\n"
        print e
        print "\n"
    finally:
        os.unlink(bng_filename)
        os.unlink(net_filename)

    for i in range(1, max(ydot) + 1):
        ydot.setdefault(i, sympy.S(0))

    return ydot
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__title__ = 'pyselect'
__version__ = '0.1.0'
__author__ = 'Matthew Behrens'
__license__ = 'MIT'
__copyright__ = 'Copyright 2013 Matthew Behrens'

import sys

def select(options=None):
    if not options:
        return None
    width = len(str(len(options)))
    for x,option in enumerate(options):
        print '{:{width}}) {}'.format(x+1,option, width=width )

    response = raw_input('{:>{width}} '.format('#?', width=width+1)).strip()
    try:
        response = int(response) - 1
    except ValueError:
        return None
    if response < 0:
        return None
    if response > len(options):
        return None
    return options[response]

def main(args=None):
    if args is None:
        args = sys.argv[1:]
    response = select(args)
    if response:
        print response

if __name__ == '__main__':
    main()
    # print select(xrange(10))
    # print select('apple orange banana'.split())
    # print select('foo bar bar'.split())
<commit_msg>lets call it 0.1<commit_after># -*- coding: utf-8 -*-

__title__     = 'pyselect'
__version__   = '0.1.0'
__author__    = 'Matthew Behrens'
__license__   = 'MIT'
__copyright__ = 'Copyright 2013 Matthew Behrens'

import sys

def select(options=None):
    """ pass in a list of options, promt the user to select one, and return the selected option or None """
    if not options:
        return None
    width = len(str(len(options)))
    for x,option in enumerate(options):
        sys.stdout.write('{:{width}}) {}\n'.format(x+1,option, width=width))

    sys.stdout.write('{:>{width}} '.format('#?', width=width+1))
    sys.stdout.flush()
    if sys.stdin.isatty():
        # regular prompt
        try:
            response = raw_input().strip()
        except (EOFError, KeyboardInterrupt):
            # handle ctrl-d, ctrl-c
            response = ''
    else:
        # try connecting to current tty, when using pipes
        sys.stdin = open("/dev/tty")
        try:
            response = ''
            while True:
                response += sys.stdin.read(1)
                if response.endswith('\n'):
                    break
        except (EOFError, KeyboardInterrupt):
            sys.stdout.flush()
            pass
    try:
        response = int(response) - 1
    except ValueError:
        return None
    if response < 0 or response >= len(options):
        return None
    return options[response]

def main(args=None):
    if args is None:
        args = sys.argv[1:]
    response = select(args)
    if response:
        print response

if __name__ == '__main__':
    main()
<|endoftext|>