<commit_before>import enum

from collections import namedtuple, OrderedDict
from itertools import zip_longest

__version__ = "1.3.0"

class Enumap(enum.Enum):
    """An Enum that maps data to its ordered, named members.
    Produces OrderedDicts and namedtuples while ensuring that the
    keys/fields match the names of the Enum members."""

    @classmethod
    def names(cls):
        try:
            names = cls.__names
        except AttributeError:
            names = cls.__names = tuple(cls.__members__)
        return names

    @classmethod
    def map(cls, *values, **named_values):
        """Returns an OrderedDict from `values` & `named_values`, whose
        keys match this Enum's members and their ordering

        >>> Fruit = Enumap("Fruit", names="apple orange papaya")
        >>> Fruit.map("heart-shaped", "spherical", papaya="ellipsoid")
        OrderedDict([('apple', 'heart-shaped'), ('orange', ...), ...])
        """
        mapping = cls._make_checked_mapping(*values, **named_values)
        return OrderedDict(((k, mapping[k]) for k in cls.names()))

    @classmethod
    def map_casted(cls, *values, **named_values):
        """Like `map`, but values are converted with the `types`
        mapping. Useful for deserializing ordered and named values.

        >>> class Order(str, Enumap):
        ...    index: int = "Order ID"
        ...    cost: Decimal = "Total pretax cost"
        ...    due_on: arrow.get = "Delivery date"
        ...
        >>> Order.map_casted("342 32342.23 2017-09-01".split())
        OrderedDict(('index', 342), ('cost', Decimal("3242.23")), ...)
        """
        mapping = cls._make_casted_mapping(*values, **named_values)
        return OrderedDict(((k, mapping[k]) for k in cls.names()))

    @classmethod
    def tuple(cls, *values, **named_values):
        """Returns a namedtuple from `values` & `named_values`, whose
        fields match this Enum's members and their ordering

        >>> Tool = Enumap("Tool", names="hammer mallet forehead")
        >>> Tool.tuple("small", "heavy", forehead="unwieldy")
        Tool_tuple(hammer='small', mallet='heavy', forehead='unwieldy')
        """
        tuple_class = cls.tuple_class()
        try:
            return tuple_class(*values, **named_values)
        except TypeError:
            mapping = cls._make_checked_mapping(*values, **named_values)
            return tuple_class(**mapping)

    @classmethod
    def tuple_casted(cls, *values, **named_values):
        """Like `tuple`, but values are converted with the `types`
        mapping. Useful for deserializing ordered and named values."""
        mapping = cls._make_casted_mapping(*values, **named_values)
        return cls.tuple_class()(**mapping)

    @classmethod
    def tuple_class(cls):
        """`namedtuple` class with fields that match this Enum's
        members and their ordering"""
        try:
            tuple_class = cls.__tuple_class
        except AttributeError:
            tuple_class = namedtuple(cls.__name__ + "_tuple", cls.names())
            cls.__tuple_class = tuple_class
        return tuple_class

    @classmethod
    def set_types(cls, *types, **named_types):
        """Set `types` mapping for `map/tuple_casted` methods.

        >>> Pastry = Enumap("Pastry", names="croissant donut muffin")
        >>> Pastry.set_types(int, int, int, donut=float)
        >>> Pastry.types()  # donut kwarg overrides donut arg
        {'croissant': int, 'donut': float, 'muffin': int}
        """
        # type mappings are allowed to be a subset of the member keys
        # in other words, not all members have to have a type
        sparse_types = SparseEnumap("temporary_types", cls.names())
        sparse_type_map = sparse_types.map(*types, **named_types)
        non_null_types = {k: v for k, v in sparse_type_map.items()
                          if v is not None}
        type_subset = Enumap(f"{cls.__name__}_types",
                             tuple(non_null_types.keys()))
        cls.__member_types = type_subset.map(*types, **named_types)

    @classmethod
    def types(cls):
        """Mapping like `{member_name: callable}` for `map/tuple_casted`.
        This can either come from type annotations or `set_types`."""
        try:
            return cls.__member_types
        except AttributeError:
            types = dict(getattr(cls, "__annotations__", {}))
            cls.__member_types = types
            return cls.__member_types

    @classmethod
    def _make_checked_mapping(cls, *values, **named_values):
        """Generate key-value pairs where keys are strictly the names
        of the members of this Enum. Raises `KeyError` for both
        missing and invalid keys."""
        names = cls.names()
        mapping = dict(zip(names, values), **named_values)
        if set(mapping) == set(names) and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _make_casted_mapping(cls, *values, **named_values):
        """Like `_make_checked_mapping`, but values are casted based
        on the `types()` mapping"""
        mapping = cls._make_checked_mapping(*values, **named_values)
        types = cls.types()
        mapping.update(((k, types[k](mapping[k])) for k, v in types.items()))
        return mapping

    @classmethod
    def _raise_invalid_args(cls, values, mapping, names):
        missing = (set(names) - set(mapping)) or {}
        invalid = (set(mapping) - set(names)) or {}
        if len(values) > len(names):
            n_args = len(values)
            n_expected = len(names)
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"expected {n_expected} arguments, got {n_args}")
        else:
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"missing keys {missing}; invalid keys {invalid}")

class default(enum.auto):
    """A subclass of enum.auto that

    1. behaves as a unqiue enum member because enum members that aren't unique
       effectively become aliases
    2. gives the user a way of signaling that an enum value should be used as
       a default in the collections created by SparseEnumap.map() or .tuple()

    Sample usage:

        >>> class Pets(SparseEnumap):
        ...    dogs: int = default(3)
        ...    cats: int = default(44)
        ...    squirrels: float = 3  # this isn't a default at all

        >>> Pets.tuple()
        Pets_tuple(dogs=3, cats=44, squirrels=None)
    """

    def __init__(self, default_value=None):
        self._value = (enum.auto.value, default_value)

    @property
    def value(self):
        return self

    @value.setter
    def value(self, new_value):
        actual_default = self._value[-1]
        self._value = (new_value, actual_default)

    @property
    def default(self):
        return self._value[1]

def _iter_member_defaults(members):
    """Iterates through Enum members and teases out the default value
    the user selected with `default(<user's special value>)` from the
    `default` object.
    """
    for k, v in members.items():
        if isinstance(v.value, default):
            yield k, v.value.default

        # By not yielding k, v for non-default() objects, we avoid using
        # things like auto() as defaults in our .tuple()/.map() collections.
        # This makes it explicit when a user is using an enum value
        # as a default while ALSO allowing SparseEnumaps to adhere to the
        # rules of Enums. Each value of an Enum must be unique, and those that
        # aren't are basically just aliases

class SparseEnumap(Enumap):
    """A less strict Enumap that provides default values
    for unspecified keys. Invalid keys are still prohibited."""

    @classmethod
    def set_defaults(cls, *values, **named_values):
        cls.__member_defaults = cls.map(*values, **named_values)

    @classmethod
    def defaults(cls):
        try:
            return cls.__member_defaults
        except AttributeError:
            members = cls.__members__
            member_defaults = dict(_iter_member_defaults(members))
            cls.__member_defaults = member_defaults
            return cls.__member_defaults

    @classmethod
    def _make_checked_mapping(cls, *values, **named_values):
        """Generate key-value pairs where keys are strictly the names
        of the members of this Enum. Raises `KeyError` for both
        missing and invalid keys."""
        names = cls.names()
        names_set = set(names)
        defaults = cls.defaults()

        # Create a mapping which will be a subset of the final,
        # sparse mapping. As we go, record which values are present
        # in the mapping and which are missing.
        if defaults:
            mapping = dict(zip(names, values), **named_values)
            missing = names_set - set(mapping)
            mapping.update(((k, defaults[k]) for k in missing))
        else:
            mapping = dict(zip_longest(names, values), **named_values)

        # If we haven't been passed invalid keys and we haven't been
        # passed too many positional arguments, return the mapping
        if set(mapping) == names_set and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _make_casted_mapping(cls, *values, **named_values):
        """Like `_make_checked_mapping`, but values are casted based
        on the `types()` mapping"""
        names = cls.names()
        names_set = set(names)
        defaults = cls.defaults()

        # Create a mapping which will be a subset of the final,
        # sparse mapping. As we go, record which values are present
        # in the mapping and which are missing.
        if defaults:
            mapping = dict(zip(names, values), **named_values)
            present = set(mapping)
            missing = names_set - present
            mapping.update(((k, defaults[k]) for k in missing))
        else:
            mapping = dict(zip(names, values), **named_values)
            present = set(mapping)
            missing = names_set - present

        # Cast the values of our mapping with the the type function
        # corresponding to their keys. We use the `missing` set of keys
        # as a guide here because we don't want to cast missing or default
        # values.
        types = cls.types()
        if types:
            present_typed = present & set(types)
            mapping.update(((k, types[k](mapping[k]))
                            for k in present_typed))

        # Handle default values to create a sparse mapping.
        # Missing values will either be filled in with what's in the
        # `defaults` mapping or with None if the user hasn't set defaults.
        temp = dict(defaults) or {}.fromkeys(names)
        temp.update(mapping)
        mapping = temp

        # If we haven't been passed invalid keys and we haven't been
        # passed too many positional arguments, return the mapping
        if not present - names_set and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _raise_invalid_args(cls, values, mapping, names):
        if len(values) > len(names):
            n_args = len(values)
            n_expected = len(names)
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"expected {n_expected} arguments, got {n_args}")
        else:
            invalid = set(mapping) - set(names)
            raise KeyError(f"{cls.__name__} requires keys {names}; "
                           f"invalid keys {invalid}")
<commit_msg>added descriptive type casting exception<commit_after>import enum

from collections import namedtuple, OrderedDict
from itertools import zip_longest

__version__ = "1.3.0"

class Enumap(enum.Enum):
    """An Enum that maps data to its ordered, named members.
    Produces OrderedDicts and namedtuples while ensuring that the
    keys/fields match the names of the Enum members."""

    @classmethod
    def names(cls):
        try:
            names = cls.__names
        except AttributeError:
            names = cls.__names = tuple(cls.__members__)
        return names

    @classmethod
    def map(cls, *values, **named_values):
        """Returns an OrderedDict from `values` & `named_values`, whose
        keys match this Enum's members and their ordering

        >>> Fruit = Enumap("Fruit", names="apple orange papaya")
        >>> Fruit.map("heart-shaped", "spherical", papaya="ellipsoid")
        OrderedDict([('apple', 'heart-shaped'), ('orange', ...), ...])
        """
        mapping = cls._make_checked_mapping(*values, **named_values)
        return OrderedDict(((k, mapping[k]) for k in cls.names()))

    @classmethod
    def map_casted(cls, *values, **named_values):
        """Like `map`, but values are converted with the `types`
        mapping. Useful for deserializing ordered and named values.

        >>> class Order(str, Enumap):
        ...    index: int = "Order ID"
        ...    cost: Decimal = "Total pretax cost"
        ...    due_on: arrow.get = "Delivery date"
        ...
        >>> Order.map_casted("342 32342.23 2017-09-01".split())
        OrderedDict(('index', 342), ('cost', Decimal("3242.23")), ...)
        """
        mapping = cls._make_casted_mapping(*values, **named_values)
        return OrderedDict(((k, mapping[k]) for k in cls.names()))

    @classmethod
    def tuple(cls, *values, **named_values):
        """Returns a namedtuple from `values` & `named_values`, whose
        fields match this Enum's members and their ordering

        >>> Tool = Enumap("Tool", names="hammer mallet forehead")
        >>> Tool.tuple("small", "heavy", forehead="unwieldy")
        Tool_tuple(hammer='small', mallet='heavy', forehead='unwieldy')
        """
        tuple_class = cls.tuple_class()
        try:
            return tuple_class(*values, **named_values)
        except TypeError:
            mapping = cls._make_checked_mapping(*values, **named_values)
            return tuple_class(**mapping)

    @classmethod
    def tuple_casted(cls, *values, **named_values):
        """Like `tuple`, but values are converted with the `types`
        mapping. Useful for deserializing ordered and named values."""
        mapping = cls._make_casted_mapping(*values, **named_values)
        return cls.tuple_class()(**mapping)

    @classmethod
    def tuple_class(cls):
        """`namedtuple` class with fields that match this Enum's
        members and their ordering"""
        try:
            tuple_class = cls.__tuple_class
        except AttributeError:
            tuple_class = namedtuple(cls.__name__ + "_tuple", cls.names())
            cls.__tuple_class = tuple_class
        return tuple_class

    @classmethod
    def set_types(cls, *types, **named_types):
        """Set `types` mapping for `map/tuple_casted` methods.

        >>> Pastry = Enumap("Pastry", names="croissant donut muffin")
        >>> Pastry.set_types(int, int, int, donut=float)
        >>> Pastry.types()  # donut kwarg overrides donut arg
        {'croissant': int, 'donut': float, 'muffin': int}
        """
        # type mappings are allowed to be a subset of the member keys
        # in other words, not all members have to have a type
        sparse_types = SparseEnumap("temporary_types", cls.names())
        sparse_type_map = sparse_types.map(*types, **named_types)
        non_null_types = {k: v for k, v in sparse_type_map.items()
                          if v is not None}
        type_subset = Enumap(f"{cls.__name__}_types",
                             tuple(non_null_types.keys()))
        cls.__member_types = type_subset.map(*types, **named_types)

    @classmethod
    def types(cls):
        """Mapping like `{member_name: callable}` for `map/tuple_casted`.
        This can either come from type annotations or `set_types`."""
        try:
            return cls.__member_types
        except AttributeError:
            types = dict(getattr(cls, "__annotations__", {}))
            cls.__member_types = types
            return cls.__member_types

    @classmethod
    def _make_checked_mapping(cls, *values, **named_values):
        """Generate key-value pairs where keys are strictly the names
        of the members of this Enum. Raises `KeyError` for both
        missing and invalid keys."""
        names = cls.names()
        mapping = dict(zip(names, values), **named_values)
        if set(mapping) == set(names) and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _make_casted_mapping(cls, *values, **named_values):
        """Like `_make_checked_mapping`, but values are casted based
        on the `types()` mapping"""
        mapping = cls._make_checked_mapping(*values, **named_values)
        mapping.update(_type_cast_items(mapping, cls.types()))
        return mapping

    @classmethod
    def _raise_invalid_args(cls, values, mapping, names):
        missing = (set(names) - set(mapping)) or {}
        invalid = (set(mapping) - set(names)) or {}
        if len(values) > len(names):
            n_args = len(values)
            n_expected = len(names)
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"expected {n_expected} arguments, got {n_args}")
        else:
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"missing keys {missing}; invalid keys {invalid}")

def _type_cast_items(mapping, types):
    """Generates key/value pairs for which each
    value is casted with the callable in the `types` mapping.
    """
    key = value = None
    try:
        for key, type_callable in types.items():
            yield key, type_callable(mapping[key])
    except Exception as e:
        value_type = type(value)
        raise TypeCastError(f"Key '{key}' got invalid value '{value}' "
                            f"of type {value_type} (error: '{e}')", key)

class TypeCastError(TypeError):
    """Raised when an Enumap field raises an exception
    during type casting for Enumap.tuple_casted or Enumap.map_casted

    Attributes
        key: key or field name for which a value could not be
             successfully type casted
    """

    def __init__(self, message, key):
        super().__init__(message)
        self.key = key

class default(enum.auto):
    """A subclass of enum.auto that

    1. behaves as a unqiue enum member because enum members that aren't unique
       effectively become aliases
    2. gives the user a way of signaling that an enum value should be used as
       a default in the collections created by SparseEnumap.map() or .tuple()

    Sample usage:

        >>> class Pets(SparseEnumap):
        ...    dogs: int = default(3)
        ...    cats: int = default(44)
        ...    squirrels: float = 3  # this isn't a default at all

        >>> Pets.tuple()
        Pets_tuple(dogs=3, cats=44, squirrels=None)
    """

    def __init__(self, default_value=None):
        self._value = (enum.auto.value, default_value)

    @property
    def value(self):
        return self

    @value.setter
    def value(self, new_value):
        actual_default = self._value[-1]
        self._value = (new_value, actual_default)

    @property
    def default(self):
        return self._value[1]

def _iter_member_defaults(members):
    """Iterates through Enum members and teases out the default value
    the user selected with `default(<user's special value>)` from the
    `default` object.
    """
    for k, v in members.items():
        if isinstance(v.value, default):
            yield k, v.value.default

        # By not yielding k, v for non-default() objects, we avoid using
        # things like auto() as defaults in our .tuple()/.map() collections.
        # This makes it explicit when a user is using an enum value
        # as a default while ALSO allowing SparseEnumaps to adhere to the
        # rules of Enums. Each value of an Enum must be unique, and those that
        # aren't are basically just aliases

class SparseEnumap(Enumap):
    """A less strict Enumap that provides default values
    for unspecified keys. Invalid keys are still prohibited."""

    @classmethod
    def set_defaults(cls, *values, **named_values):
        cls.__member_defaults = cls.map(*values, **named_values)

    @classmethod
    def defaults(cls):
        try:
            return cls.__member_defaults
        except AttributeError:
            members = cls.__members__
            member_defaults = dict(_iter_member_defaults(members))
            cls.__member_defaults = member_defaults
            return cls.__member_defaults

    @classmethod
    def _make_checked_mapping(cls, *values, **named_values):
        """Generate key-value pairs where keys are strictly the names
        of the members of this Enum. Raises `KeyError` for both
        missing and invalid keys."""
        names = cls.names()
        names_set = set(names)
        defaults = cls.defaults()

        # Create a mapping which will be a subset of the final,
        # sparse mapping. As we go, record which values are present
        # in the mapping and which are missing.
        if defaults:
            mapping = dict(zip(names, values), **named_values)
            missing = names_set - set(mapping)
            mapping.update(((k, defaults[k]) for k in missing))
        else:
            mapping = dict(zip_longest(names, values), **named_values)

        # If we haven't been passed invalid keys and we haven't been
        # passed too many positional arguments, return the mapping
        if set(mapping) == names_set and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _make_casted_mapping(cls, *values, **named_values):
        """Like `_make_checked_mapping`, but values are casted based
        on the `types()` mapping"""
        names = cls.names()
        names_set = set(names)
        defaults = cls.defaults()

        # Create a mapping which will be a subset of the final,
        # sparse mapping. As we go, record which values are present
        # in the mapping and which are missing.
        if defaults:
            mapping = dict(zip(names, values), **named_values)
            present = set(mapping)
            missing = names_set - present
            mapping.update(((k, defaults[k]) for k in missing))
        else:
            mapping = dict(zip(names, values), **named_values)
            present = set(mapping)
            missing = names_set - present

        # Cast the values of our mapping with the the type function
        # corresponding to their keys. We use the `missing` set of keys
        # as a guide here because we don't want to cast missing or default
        # values.
        types = cls.types()
        if types:
            present_typed = present & set(types)
            mapping.update(((k, types[k](mapping[k]))
                            for k in present_typed))

        # Handle default values to create a sparse mapping.
        # Missing values will either be filled in with what's in the
        # `defaults` mapping or with None if the user hasn't set defaults.
        temp = dict(defaults) or {}.fromkeys(names)
        temp.update(mapping)
        mapping = temp

        # If we haven't been passed invalid keys and we haven't been
        # passed too many positional arguments, return the mapping
        if not present - names_set and len(values) <= len(names):
            return mapping
        else:
            cls._raise_invalid_args(values, mapping, names)

    @classmethod
    def _raise_invalid_args(cls, values, mapping, names):
        if len(values) > len(names):
            n_args = len(values)
            n_expected = len(names)
            raise KeyError(
                f"{cls.__name__} requires keys {names}; "
                f"expected {n_expected} arguments, got {n_args}")
        else:
            invalid = set(mapping) - set(names)
            raise KeyError(f"{cls.__name__} requires keys {names}; "
                           f"invalid keys {invalid}")
<|endoftext|>


<commit_before>from dimagi.utils.chunked import chunked
from django.core.management import BaseCommand

from corehq.form_processor.backends.sql.dbaccessors import *
from corehq.pillows.xform import *

def reindex_sql_forms_in_domain(domain):
    reindexer = get_sql_form_reindexer()

    for state, _ in XFormInstanceSQL.STATES:
        all_doc_ids = FormAccessorSQL.get_form_ids_in_domain_by_state(domain, state)
        for doc_ids in chunked(all_doc_ids, 100):
            print 'Reindexing doc_ids: {}'.format(','.join(doc_ids))
            reindexer.doc_processor.process_bulk_docs([
                transform_xform_for_elasticsearch(form.to_json(include_attachments=True))
                for form in FormAccessorSQL.get_forms(list(doc_ids))
            ])

class Command(BaseCommand):
    args = 'domain'
    help = 'Reindex a pillowtop index'

    def handle(self, domain, *args, **options):
        reindex_sql_forms_in_domain(domain)
<commit_msg>remove double transform in reindex_sql_forms_in_domain<commit_after>from dimagi.utils.chunked import chunked
from django.core.management import BaseCommand

from corehq.form_processor.backends.sql.dbaccessors import *
from corehq.pillows.xform import *

def reindex_sql_forms_in_domain(domain):
    reindexer = get_sql_form_reindexer()

    for state, _ in XFormInstanceSQL.STATES:
        all_doc_ids = FormAccessorSQL.get_form_ids_in_domain_by_state(domain, state)
        for doc_ids in chunked(all_doc_ids, 100):
            print 'Reindexing doc_ids: {}'.format(','.join(doc_ids))
            reindexer.doc_processor.process_bulk_docs([
                reindexer.reindex_accessor.doc_to_json(form)
                for form in FormAccessorSQL.get_forms(list(doc_ids))
            ])

class Command(BaseCommand):
    args = 'domain'
    help = 'Reindex a pillowtop index'

    def handle(self, domain, *args, **options):
        reindex_sql_forms_in_domain(domain)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import ConfigParser
import os
from pprint import pprint
import shutil

from jinja2 import Environment, FileSystemLoader, meta, StrictUndefined,\
    TemplateNotFound

# Configuration variables file path is relative to projectpath directory.
VARIABLES_FILENAME = 'variables.ini'

# Backup file extension.
from datetime import datetime
now_time = datetime.now()
BACKUP_EXT = '.bak.%s' % now_time.strftime("%Y%m%d_%H%M%S")

config_files = {
    # semantic_name: (template_location, output_location)
    'apache': ('extras/apache/atmo.conf.j2', 'extras/apache/atmo.conf'),
    'celeryd.default': ('extras/init.d/celeryd.default.dist.j2',
                        'extras/init.d/celeryd.default.dist'),
    'local.py': ('atmosphere/settings/local.py.j2',
                 'atmosphere/settings/local.py'),
    'nginx': ('extras/nginx/Makefile.j2', 'extras/nginx/Makefile'),
    'nginx-site': ('extras/nginx/site.conf.j2',
                   'extras/nginx/site.conf'),
    'nginx-atmo': ('extras/nginx/locations/atmo.conf.j2',
                   'extras/nginx/locations/atmo.conf'),
    'nginx-flower': ('extras/nginx/locations/flower.conf.j2',
                     'extras/nginx/locations/flower.conf'),
    'nginx-jenkins': ('extras/nginx/locations/jenkins.conf.j2',
                      'extras/nginx/locations/jenkins.conf'),
    'nginx-lb': ('extras/nginx/locations/lb.conf.j2',
                 'extras/nginx/locations/lb.conf'),
    'secrets.py': ('atmosphere/settings/secrets.py.j2',
                   'atmosphere/settings/secrets.py')}

projectpath = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '..'))

loader = FileSystemLoader(projectpath)

env = Environment(loader=loader,
                  undefined=StrictUndefined)

variables_path = os.path.join(projectpath, VARIABLES_FILENAME)

def generate_new_key():
    import string, random
    new_key = ''.join(random.SystemRandom().choice(
        string.ascii_lowercase +
        string.digits +
        "!@#$%^&*(-_=+)") for _ in range(50))
    return new_key

def _get_variables():
    try:
        parser = ConfigParser.ConfigParser()
        parser.readfp(open(variables_path))
        variables = {}
        for section in parser.sections():
            # Ensure the variable names are upper case
            for option, value in parser.items(section):
                variables[option.upper()] = value
                if option.upper() == 'SECRET_KEY' and not value:
                    variables['SECRET_KEY'] = generate_new_key()
        return (variables, [])
    except Exception as e:
        return (False,
                ['Unable to get or parse '
                 'variables from %s' % (variables_path)])

def _get_filtered_config_files(configs):
    c_files = []
    messages = []
    success = True
    config_names = config_files.keys()
    if configs:
        for name in configs:
            if name in config_names:
                c_files.append(config_files[name])
            else:
                success = False
                messages.append('%s is not a valid key in'
                                ' config_files.' % (name))
        if not success:
            return (False, messages)
        return (c_files, [])
    else:
        return (config_files.values(), [])

def _handle_preconditions(configs):
    success = True
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        return (False, messages)
    variables, messages = _get_variables()
    if not variables:
        return (False, messages)
    for file_location, _ in c_files:
        try:
            file_path = os.path.join(projectpath,
                                     file_location)
            source = loader.get_source(env, file_location)
            ast = env.parse(loader.get_source(env, file_location))
            used_vars = meta.find_undeclared_variables(ast)
            ud_vars = set()
            for v in used_vars:
                if not v in variables:
                    ud_vars.add(v)
            if ud_vars:
                messages.append('Undeclared variables '
                                'found in %s: %s' % (file_path,
                                                     ud_vars))
                success = False
        except TemplateNotFound:
            messages.append('Template not found: %s' % (file_path))
            success = False
    return (success, messages)

def _backup_file(path):
    """
    Backup path if it's a file. Use the BACKUP_EXT extension.
    Return the backup location.
    """
    if os.path.isfile(path):
        shutil.copyfile(path, path + BACKUP_EXT)
        return path + BACKUP_EXT

def _generate_configs(configs, backup):
    success = True
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        return (False, messages)
    variables, messages = _get_variables()
    if not variables:
        return (False, messages)
    for template_location, output_location in c_files:
        try:
            output_path = os.path.join(projectpath,
                                       output_location)
            template = env.get_template(template_location)
            rendered = template.render(variables)
            if backup:
                backup_path = _backup_file(output_path)
                if backup_path:
                    messages.append('Backed up %s '\
                                    'as %s\n' % (output_location,
                                                 backup_path))
            # Write to the output file.
            with open(output_path, 'wb') as fh:
                fh.write(rendered)
            messages.append('From %s '\
                            'generated %s\n' % (template_location,
                                                output_location))
        except Exception as e:
            messages.append('Exception %s from template '\
                            'location %s and output location '\
                            '%s' % (e.message,
                                    template_location,
                                    output_location))
            success = False
    return (success, messages)

def generate_configs(configs, backup=True):
    print 'Testing for preconditions...\n'
    success, messages = _handle_preconditions(configs)
    print_messages(messages)
    if not success:
        exit(1)
    print 'Generating configs...\n'
    success, messages = _generate_configs(configs, backup)
    print_messages(messages)
    if not success:
        exit(2)

def print_messages(messages):
    for m in messages:
        print m

def print_configs(configs):
    """
    Print config name, template and output file information.
    """
    print 'Config Variables from %s' % (variables_path)
    variables, messages = _get_variables()
    if not variables:
        print_messages(messages)
    else:
        pprint(variables)
    print
    print 'Config Name:\n\tTemplate => Output'
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        print_messages(messages)
    for name in configs:
        template, output = config_files[name]
        print name + ':'
        print '\t %s => %s' % (template, output)
    print

def print_test(configs):
    """
    Print results of testing configs preconditions. If there are no
    problems acknowledge otherwise handle_precondition will print
    the error(s).
    """
    print 'Testing for preconditions...\n'
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        print_messages(messages)
        return
    for name in configs:
        success, messages = _handle_preconditions([name])
        if success:
            print '%s looks good.' % (name)
        else:
            print_messages(messages)
        print

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--show', action='store_true',
                        help='Show a list of availabe configs')
    parser.add_argument('-c', '--configs', nargs='*',
                        help='A list of configs to generate.')
    parser.add_argument('-t', '--test', action='store_true',
                        help='Test configs for preconditions.')
    print 'Project Path => %s\n' % (projectpath)
    args = parser.parse_args()
    if not args.configs:
        args.configs = config_files.keys()
    if args.show:
        print_configs(args.configs)

    if args.test:
        print_test(args.configs)

    # If testing or showing information, exit.
    if args.test or args.show:
        exit(0)
    generate_configs(args.configs)
    print 'Successfully generated configs %s.' % (', '.join(args.configs))
    exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Fix path extension<commit_after>#!/usr/bin/env python
import argparse
import ConfigParser
import os
from pprint import pprint
import shutil

from jinja2 import Environment, FileSystemLoader, meta, StrictUndefined,\
    TemplateNotFound

# Configuration variables file path is relative to projectpath directory.
VARIABLES_FILENAME = 'variables.ini'

# Backup file extension.
from datetime import datetime
now_time = datetime.now()
BACKUP_EXT = '.bak.%s' % now_time.strftime("%Y%m%d_%H%M%S")

config_files = {
    # semantic_name: (template_location, output_location)
    'apache': ('extras/apache/atmo.conf.j2', 'extras/apache/atmo.conf'),
    'celeryd.default': ('extras/init.d/celeryd.default.dist.j2',
                        'extras/init.d/celeryd.default'),
    'local.py': ('atmosphere/settings/local.py.j2',
                 'atmosphere/settings/local.py'),
    'nginx': ('extras/nginx/Makefile.j2', 'extras/nginx/Makefile'),
    'nginx-site': ('extras/nginx/site.conf.j2',
                   'extras/nginx/site.conf'),
    'nginx-atmo': ('extras/nginx/locations/atmo.conf.j2',
                   'extras/nginx/locations/atmo.conf'),
    'nginx-flower': ('extras/nginx/locations/flower.conf.j2',
                     'extras/nginx/locations/flower.conf'),
    'nginx-jenkins': ('extras/nginx/locations/jenkins.conf.j2',
                      'extras/nginx/locations/jenkins.conf'),
    'nginx-lb': ('extras/nginx/locations/lb.conf.j2',
                 'extras/nginx/locations/lb.conf'),
    'secrets.py': ('atmosphere/settings/secrets.py.j2',
                   'atmosphere/settings/secrets.py')}

projectpath = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '..'))

loader = FileSystemLoader(projectpath)

env = Environment(loader=loader,
                  undefined=StrictUndefined)

variables_path = os.path.join(projectpath, VARIABLES_FILENAME)

def generate_new_key():
    import string, random
    new_key = ''.join(random.SystemRandom().choice(
        string.ascii_lowercase +
        string.digits +
        "!@#$%^&*(-_=+)") for _ in range(50))
    return new_key

def _get_variables():
    try:
        parser = ConfigParser.ConfigParser()
        parser.readfp(open(variables_path))
        variables = {}
        for section in parser.sections():
            # Ensure the variable names are upper case
            for option, value in parser.items(section):
                variables[option.upper()] = value
                if option.upper() == 'SECRET_KEY' and not value:
                    variables['SECRET_KEY'] = generate_new_key()
        return (variables, [])
    except Exception as e:
        return (False,
                ['Unable to get or parse '
                 'variables from %s' % (variables_path)])

def _get_filtered_config_files(configs):
    c_files = []
    messages = []
    success = True
    config_names = config_files.keys()
    if configs:
        for name in configs:
            if name in config_names:
                c_files.append(config_files[name])
            else:
                success = False
                messages.append('%s is not a valid key in'
                                ' config_files.' % (name))
        if not success:
            return (False, messages)
        return (c_files, [])
    else:
        return (config_files.values(), [])

def _handle_preconditions(configs):
    success = True
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        return (False, messages)
    variables, messages = _get_variables()
    if not variables:
        return (False, messages)
    for file_location, _ in c_files:
        try:
            file_path = os.path.join(projectpath,
                                     file_location)
            source = loader.get_source(env, file_location)
            ast = env.parse(loader.get_source(env, file_location))
            used_vars = meta.find_undeclared_variables(ast)
            ud_vars = set()
            for v in used_vars:
                if not v in variables:
                    ud_vars.add(v)
            if ud_vars:
                messages.append('Undeclared variables '
                                'found in %s: %s' % (file_path,
                                                     ud_vars))
                success = False
        except TemplateNotFound:
            messages.append('Template not found: %s' % (file_path))
            success = False
    return (success, messages)

def _backup_file(path):
    """
    Backup path if it's a file. Use the BACKUP_EXT extension.
    Return the backup location.
    """
    if os.path.isfile(path):
        shutil.copyfile(path, path + BACKUP_EXT)
        return path + BACKUP_EXT

def _generate_configs(configs, backup):
    success = True
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        return (False, messages)
    variables, messages = _get_variables()
    if not variables:
        return (False, messages)
    for template_location, output_location in c_files:
        try:
            output_path = os.path.join(projectpath,
                                       output_location)
            template = env.get_template(template_location)
            rendered = template.render(variables)
            if backup:
                backup_path = _backup_file(output_path)
                if backup_path:
                    messages.append('Backed up %s '\
                                    'as %s\n' % (output_location,
                                                 backup_path))
            # Write to the output file.
            with open(output_path, 'wb') as fh:
                fh.write(rendered)
            messages.append('From %s '\
                            'generated %s\n' % (template_location,
                                                output_location))
        except Exception as e:
            messages.append('Exception %s from template '\
                            'location %s and output location '\
                            '%s' % (e.message,
                                    template_location,
                                    output_location))
            success = False
    return (success, messages)

def generate_configs(configs, backup=True):
    print 'Testing for preconditions...\n'
    success, messages = _handle_preconditions(configs)
    print_messages(messages)
    if not success:
        exit(1)
    print 'Generating configs...\n'
    success, messages = _generate_configs(configs, backup)
    print_messages(messages)
    if not success:
        exit(2)

def print_messages(messages):
    for m in messages:
        print m

def print_configs(configs):
    """
    Print config name, template and output file information.
    """
    print 'Config Variables from %s' % (variables_path)
    variables, messages = _get_variables()
    if not variables:
        print_messages(messages)
    else:
        pprint(variables)
    print
    print 'Config Name:\n\tTemplate => Output'
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        print_messages(messages)
    for name in configs:
        template, output = config_files[name]
        print name + ':'
        print '\t %s => %s' % (template, output)
    print

def print_test(configs):
    """
    Print results of testing configs preconditions. If there are no
    problems acknowledge otherwise handle_precondition will print
    the error(s).
    """
    print 'Testing for preconditions...\n'
    c_files, messages = _get_filtered_config_files(configs)
    if not c_files:
        print_messages(messages)
        return
    for name in configs:
        success, messages = _handle_preconditions([name])
        if success:
            print '%s looks good.' % (name)
        else:
            print_messages(messages)
        print

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--show', action='store_true',
                        help='Show a list of availabe configs')
    parser.add_argument('-c', '--configs', nargs='*',
                        help='A list of configs to generate.')
    parser.add_argument('-t', '--test', action='store_true',
                        help='Test configs for preconditions.')
    print 'Project Path => %s\n' % (projectpath)
    args = parser.parse_args()
    if not args.configs:
        args.configs = config_files.keys()
    if args.show:
        print_configs(args.configs)

    if args.test:
        print_test(args.configs)

    # If testing or showing information, exit.
    if args.test or args.show:
        exit(0)
    generate_configs(args.configs)
    print 'Successfully generated configs %s.' % (', '.join(args.configs))
    exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup

def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())

def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')
    entries_to_show = properties.get('entries_to_show', 10)

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_and_category_pages(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt,
                            entries_to_show)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)

def extract_site_wide_metadata(all_pages, key='index_and_tags',
                               ignore_threshold_percentage=40):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for tag in page.tags:
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1

            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:
                    timestamp = arrow.now()
                add_to_chronology_dict(
                    chronology, category, timestamp, (page.title, link))

            tags_only = [tag.lower().strip() for tag in page.tags if tag != ""]
            idx_words = [idx_word for idx_word in page.index if
                         idx_word[0].isdigit() is False and len(idx_word) > 2
                         and '_' not in idx_word]
            search_page_type = {
                'index_only': idx_words,
                'tags_only': tags_only,
                'index_and_tags': idx_words + tags_only}

            for idx in sorted(search_page_type[key]):
                word = idx.strip()
                timestamped_entry = chronology[timestamp].get(category)
                if timestamped_entry is None:
                    continue
                link = timestamped_entry[1]
                if word not in index:
                    index[word] = [link]
                else:
                    if link not in index[word]:
                        index[word].append(link)

    # Exclude links that feature in more than a certain percentage of entries:
    threshold = (len(all_pages) / 100) * ignore_threshold_percentage
    exclude_from_index = []
    for word, entries in index.items():
        if len(entries) > threshold:
            exclude_from_index.append(word)
    for common_word in exclude_from_index:
        index.pop(common_word)
    print("Excluding the following common words from search index:\n{}"
          .format(", ".join(sorted(exclude_from_index))))
    words_used = [(idx, len(index[idx])) for idx in index]
    word_cloud = sorted(words_used, key=lambda x: x[1], reverse=True)

    return categories, tags, chronology, index, word_cloud

def add_to_chronology_dict(chronology, category, timestamp, value):
    if timestamp in chronology:
        for num in range(1, 3600):
            one_second_later = timestamp.replace(seconds =+ num)
            if one_second_later not in chronology:
                timestamp = one_second_later
                break
    if timestamp not in chronology:
        chronology[timestamp] = {}
    chronology[timestamp][category] = value

def generate_front_and_category_pages(site_title, homepage, searchpage,
                                      categories, tags, chronology, word_cloud,
                                      proj_root, css, ts_frmt,
                                      entries_to_show):
    generate_front_or_cat_page(site_title, homepage, searchpage, categories,
                               tags, chronology, word_cloud, proj_root, css,
                               ts_frmt, entries_to_show)

    for cat in categories:
        page_title = site_title + ' - ' + cat
        generate_front_or_cat_page(page_title, homepage, searchpage, [cat],
                                   tags, chronology, word_cloud, proj_root, css,
                                   ts_frmt, cat_page=True)

def generate_front_or_cat_page(site_title, homepage, searchpage, categories, tags,
                        chronology, word_cloud, proj_root, css, ts_frmt,
                        entries_to_show=10, cat_page=False):
    this_page = categories[0] + '/index.html' if cat_page else homepage
    with open(this_page, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>{}</title>'.format(site_title), file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        for cat in sorted(categories, reverse=True):
            show_more_link = True
            category = cat.lower()
            print('
{}
'.format(category), file=op_file)
            print('
', file=op_file)

            count = 0
            for timestamp, cat_dict in sorted(chronology.items(),
                                                   reverse=True):
                for _, (title, link) in cat_dict.items():
                    if category == link.split('/')[0]:
                        if (cat_page is True) or (count < int(entries_to_show)):
                            if cat_page is True:
                                link = link.split('/')[1]
                            print('
{}
'
                                  .format(link, title), file=op_file)
                        elif show_more_link:
                            show_more_link = False
                            print('
more...
'
                                  .format(category), file=op_file)
                        count += 1
            print('
', file=op_file)

        # TODO: Add some kind of javascript tag-cloud, word-cloud and category
        # list here.

        print('    ', file=op_file)
        print('', file=op_file)

    print(this_page + ' generated.')

def generate_search_page(site_title, homepage, searchpage, index, css,
                         proj_root):
    with open(searchpage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>Search</title>', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        print('
', file=op_file)
        for word, links in sorted(index.items()):
            print('
{0}'.format(word),
                  file=op_file)
            print('
', file=op_file)
            for link in links:
                print('
{0}
'.format(link),
                      file=op_file)
            print('
', file=op_file)
        print('
', file=op_file)

        print('    ', file=op_file)
        print('', file=op_file)

        # TODO: Add some javascript wizardry so this displays a search box and
        # only presents the list of pages when that word is entered into the
        # box (everything else should be hidden).

    print(searchpage + ' generated.')

def generate_static_page(site_title, homepage, searchpage, meta, css, ts_frmt,
                         proj_root, media_dir="../media"):
    for cat in meta.categories:
        category = cat.lower()
        create_dir_if_absent(category)

        with open(os.path.join(category, meta.slug) + '.html', 'w') as op_file:
            print('', file=op_file)
            print('    <title>{} ({})</title>'.format(
                  meta.title.title(), category.title()), file=op_file)
            print('    ', file=op_file)
            print('    ', file=op_file)
            print('
{} - {}
'.format(
                  site_title.title(), category.title()), file=op_file)
            print('        ', file=op_file)
            print('        <link rel="stylesheet" type="text/css"' +
                  ' media="screen" href="{}" />'.format(css), file=op_file)
            print('        <title>{}</title>'.format(meta.title), file=op_file)
            link_bar = 'Home | '
            link_bar += '{3} | '
            link_bar += 'Search'
            print(link_bar.format(proj_root, homepage, searchpage,
                  category.title()), file=op_file)
            print('        ', file=op_file)
            print('    ', file=op_file)
            print('
', file=op_file)
            if meta.header_image != "":
                print('
', file=op_file)
                img_path = os.path.join(media_dir, meta.header_image)
                print(''.format(img_path), file=op_file)
                print('
', file=op_file)
            print('
{}
'.format(meta.title), file=op_file)
            print(    meta.content, file=op_file)
            print('
', file=op_file)
            print('

', file=op_file)
            date = \
                arrow.get(meta.date).format(ts_frmt) if meta.date != '' else ''

            print('
{}, {}
'.format(meta.authors, date),
                  file=op_file)
            if meta.tags not in [[], ['']]:
                tags = []
                for tag in meta.tags:
                    linked_tag = '{tag}'
                    tags.append(linked_tag.format(proj_root=proj_root,
                                                  search=searchpage,
                                                  tag=tag.strip().lower()))
                print('    <tiny>Tags: {}</tiny>'.format(", ".join(tags)),
                      file=op_file)
                print('

', file=op_file)
            print('', file=op_file)

def slugify(value):
    """
    Converts to lowercase, removes non-word characters (alphanumerics and
    underscores) and converts spaces to hyphens. Also strips leading and
    trailing whitespace.
    """
    normalized_value = unicodedata.normalize(
        'NFKD', value).encode('ascii', 'ignore').decode('ascii')
    stripped_value = re.sub('[^\w\s-]', '', normalized_value).strip().lower()
    return re.sub('[-\s]+', '-', stripped_value)

def create_dir_if_absent(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def main():
    time_now = arrow.now().strftime('%d-%b-%y %H:%M:%S')
    print("{}: Generating pages".format(time_now))
    properties = get_properties("properties.json")
    generate_pages(properties=properties)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>tag exception for common words<commit_after>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup

def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())

def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')
    entries_to_show = properties.get('entries_to_show', 10)

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_and_category_pages(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt,
                            entries_to_show)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)

def extract_site_wide_metadata(all_pages, key='index_and_tags',
                               ignore_threshold_percentage=40):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for tag in page.tags:
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1

            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:
                    timestamp = arrow.now()
                add_to_chronology_dict(
                    chronology, category, timestamp, (page.title, link))

            tags_only = [tag.lower().strip() for tag in page.tags if tag != ""]
            idx_words = [idx_word for idx_word in page.index if
                         idx_word[0].isdigit() is False and len(idx_word) > 2
                         and '_' not in idx_word]
            search_page_type = {
                'index_only': idx_words,
                'tags_only': tags_only,
                'index_and_tags': idx_words + tags_only}

            for idx in sorted(search_page_type[key]):
                word = idx.strip()
                timestamped_entry = chronology[timestamp].get(category)
                if timestamped_entry is None:
                    continue
                link = timestamped_entry[1]
                if word not in index:
                    index[word] = [link]
                else:
                    if link not in index[word]:
                        index[word].append(link)

    # Exclude links that feature in more than a certain percentage of entries:
    threshold = (len(all_pages) / 100) * ignore_threshold_percentage
    exclude_from_index = []
    for word, entries in index.items():
        if (len(entries) > threshold) and (word not in tags):
            exclude_from_index.append(word)
    for common_word in exclude_from_index:
        index.pop(common_word)
    print("Excluding the following common words from search index:\n{}"
          .format(", ".join(sorted(exclude_from_index))))
    words_used = [(idx, len(index[idx])) for idx in index]
    word_cloud = sorted(words_used, key=lambda x: x[1], reverse=True)

    return categories, tags, chronology, index, word_cloud

def add_to_chronology_dict(chronology, category, timestamp, value):
    if timestamp in chronology:
        for num in range(1, 3600):
            one_second_later = timestamp.replace(seconds =+ num)
            if one_second_later not in chronology:
                timestamp = one_second_later
                break
    if timestamp not in chronology:
        chronology[timestamp] = {}
    chronology[timestamp][category] = value

def generate_front_and_category_pages(site_title, homepage, searchpage,
                                      categories, tags, chronology, word_cloud,
                                      proj_root, css, ts_frmt,
                                      entries_to_show):
    generate_front_or_cat_page(site_title, homepage, searchpage, categories,
                               tags, chronology, word_cloud, proj_root, css,
                               ts_frmt, entries_to_show)

    for cat in categories:
        page_title = site_title + ' - ' + cat
        generate_front_or_cat_page(page_title, homepage, searchpage, [cat],
                                   tags, chronology, word_cloud, proj_root, css,
                                   ts_frmt, cat_page=True)

def generate_front_or_cat_page(site_title, homepage, searchpage, categories, tags,
                        chronology, word_cloud, proj_root, css, ts_frmt,
                        entries_to_show=10, cat_page=False):
    this_page = categories[0] + '/index.html' if cat_page else homepage
    with open(this_page, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>{}</title>'.format(site_title), file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        for cat in sorted(categories, reverse=True):
            show_more_link = True
            category = cat.lower()
            print('
{}
'.format(category), file=op_file)
            print('
', file=op_file)

            count = 0
            for timestamp, cat_dict in sorted(chronology.items(),
                                                   reverse=True):
                for _, (title, link) in cat_dict.items():
                    if category == link.split('/')[0]:
                        if (cat_page is True) or (count < int(entries_to_show)):
                            if cat_page is True:
                                link = link.split('/')[1]
                            print('
{}
'
                                  .format(link, title), file=op_file)
                        elif show_more_link:
                            show_more_link = False
                            print('
more...
'
                                  .format(category), file=op_file)
                        count += 1
            print('
', file=op_file)

        # TODO: Add some kind of javascript tag-cloud, word-cloud and category
        # list here.

        print('    ', file=op_file)
        print('', file=op_file)

    print(this_page + ' generated.')

def generate_search_page(site_title, homepage, searchpage, index, css,
                         proj_root):
    with open(searchpage, 'w') as op_file:
        print('', file=op_file)
        print('    ', file=op_file)
        print('
{}
'.format(site_title), file=op_file)
        print('        ', file=op_file)
        print('        <link rel="stylesheet" type="text/css" media="screen"' +
              ' href="{}" />'.format(css), file=op_file)
        print('        <title>Search</title>', file=op_file)
        print('Home | Search'
              .format(proj_root, homepage, searchpage), file=op_file)
        print('        ', file=op_file)
        print('    ', file=op_file)
        print('    ', file=op_file)

        print('
', file=op_file)
        for word, links in sorted(index.items()):
            print('
{0}'.format(word),
                  file=op_file)
            print('
', file=op_file)
            for link in links:
                print('
{0}
'.format(link),
                      file=op_file)
            print('
', file=op_file)
        print('
', file=op_file)

        print('    ', file=op_file)
        print('', file=op_file)

        # TODO: Add some javascript wizardry so this displays a search box and
        # only presents the list of pages when that word is entered into the
        # box (everything else should be hidden).

    print(searchpage + ' generated.')

def generate_static_page(site_title, homepage, searchpage, meta, css, ts_frmt,
                         proj_root, media_dir="../media"):
    for cat in meta.categories:
        category = cat.lower()
        create_dir_if_absent(category)

        with open(os.path.join(category, meta.slug) + '.html', 'w') as op_file:
            print('', file=op_file)
            print('    <title>{} ({})</title>'.format(
                  meta.title.title(), category.title()), file=op_file)
            print('    ', file=op_file)
            print('    ', file=op_file)
            print('
{} - {}
'.format(
                  site_title.title(), category.title()), file=op_file)
            print('        ', file=op_file)
            print('        <link rel="stylesheet" type="text/css"' +
                  ' media="screen" href="{}" />'.format(css), file=op_file)
            print('        <title>{}</title>'.format(meta.title), file=op_file)
            link_bar = 'Home | '
            link_bar += '{3} | '
            link_bar += 'Search'
            print(link_bar.format(proj_root, homepage, searchpage,
                  category.title()), file=op_file)
            print('        ', file=op_file)
            print('    ', file=op_file)
            print('
', file=op_file)
            if meta.header_image != "":
                print('
', file=op_file)
                img_path = os.path.join(media_dir, meta.header_image)
                print(''.format(img_path), file=op_file)
                print('
', file=op_file)
            print('
{}
'.format(meta.title), file=op_file)
            print(    meta.content, file=op_file)
            print('
', file=op_file)
            print('

', file=op_file)
            date = \
                arrow.get(meta.date).format(ts_frmt) if meta.date != '' else ''

            print('
{}, {}
'.format(meta.authors, date),
                  file=op_file)
            if meta.tags not in [[], ['']]:
                tags = []
                for tag in meta.tags:
                    linked_tag = '{tag}'
                    tags.append(linked_tag.format(proj_root=proj_root,
                                                  search=searchpage,
                                                  tag=tag.strip().lower()))
                print('    <tiny>Tags: {}</tiny>'.format(", ".join(tags)),
                      file=op_file)
                print('

', file=op_file)
            print('', file=op_file)

def slugify(value):
    """
    Converts to lowercase, removes non-word characters (alphanumerics and
    underscores) and converts spaces to hyphens. Also strips leading and
    trailing whitespace.
    """
    normalized_value = unicodedata.normalize(
        'NFKD', value).encode('ascii', 'ignore').decode('ascii')
    stripped_value = re.sub('[^\w\s-]', '', normalized_value).strip().lower()
    return re.sub('[-\s]+', '-', stripped_value)

def create_dir_if_absent(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)

def main():
    time_now = arrow.now().strftime('%d-%b-%y %H:%M:%S')
    print("{}: Generating pages".format(time_now))
    properties = get_properties("properties.json")
    generate_pages(properties=properties)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>from django.test import TestCase
from django.utils import translation
from storybase.utils import slugify
from models import (Organization, OrganizationTranslation, Project,
    ProjectTranslation)

class OrganizationModelTest(TestCase):
    def _create_organization(self, name, language):
        org = Organization()
        org.save()
        trans = OrganizationTranslation(name=name, language=language,
            organization=org)
        trans.save()
        return org

    def test_get_translated_field(self):
        """
        Tests that you can get a translated field as if it were the model's own
        """
        org_name_en = "Mile High Connects"
        org = self._create_organization(org_name_en, "en")
        self.assertEqual(org.name, org_name_en)

    def test_get_translated_field_default_fallback(self):
        """
        Tests that if a translated field doesn't exist in the current
        language, it falls back to the default language.
        """
        org_name_en = "Mile High Connects"
        org = self._create_organization(org_name_en, "en")
        translation.activate('es')
        self.assertEqual(org.name, org_name_en)

    def test_get_translated_field_first_fallback(self):
        """
        Tests that if a translated field doesn't exist in the current
        language, and it doesn't exist in the default language, the
        first available language is used
        """
        org_name_es = "Mile High Conecta"
        org = self._create_organization(org_name_es, 'es')
        translation.activate('en')
        self.assertEqual(org.name, org_name_es)

    def test_auto_slug(self):
        name = 'Mile High Connects'
        organization = Organization()
        organization.save()
        organization_translation = OrganizationTranslation(name=name, organization=organization)
        self.assertEqual(organization_translation.slug, '')
        organization_translation.save()
        self.assertEqual(organization_translation.slug, slugify(name))

class OrganizationApiTest(TestCase):
    def test_create_organization(self):
        from storybase_user.models import create_organization, Organization

        name = "Mile High Connects"
        website_url = "http://www.urbanlandc.org/collaboratives/mile-high-connects/"
        description = 'Mile High Connects (formerly know as the Mile High Transit Opportunity Collaborative) is an emerging collaborative of nonprofit and philanthropic organizations working together to ensure the creation of the region’s $6.7 billion FasTracks transit system benefits all communities in the region, including low-income populations.'
        with self.assertRaises(Organization.DoesNotExist):
            Organization.objects.get(organizationtranslation__name=name)
        org = create_organization(name=name, description=description, website_url=website_url)
        self.assertEqual(org.name, name)
        self.assertEqual(org.description, description)
        self.assertEqual(org.website_url, website_url)
        retrieved_org = Organization.objects.get(pk=org.pk)
        self.assertEqual(retrieved_org.name, name)
        self.assertEqual(retrieved_org.description, description)
        self.assertEqual(retrieved_org.website_url, website_url)

class ProjectModelTest(TestCase):
    def test_auto_slug(self):
        name = 'The Metro Denver Regional Equity Atlas'
        project = Project()
        project.save()
        project_translation = ProjectTranslation(name=name, project=project)
        self.assertEqual(project_translation.slug, '')
        project_translation.save()
        self.assertEqual(project_translation.slug, slugify(name))

class ProjectApiTest(TestCase):
    def test_create_project(self):
        from storybase_user.models import create_project, Project

        name = "The Metro Denver Regional Equity Atlas"
        description = """
            The Denver Regional Equity Atlas is a product of Mile High
            Connects (MHC), which came together in 2011 to ensure that
            the region’s significant investment in new rail and bus
            service will provide greater access to opportunity and a
            higher quality of life for all of the region’s residents, but
            especially for economically disadvantaged populations who
            would benefit the most from safe, convenient transit service.

            The Atlas visually documents the Metro Denver region’s
            demographic, educational, employment, health and housing
            characteristics in relation to transit, with the goal of
            identifying areas of opportunity as well as challenges to
            creating and preserving quality communities near transit.
            """
        with self.assertRaises(Project.DoesNotExist):
            Project.objects.get(projecttranslation__name=name)
        project = create_project(name=name, description=description)
        self.assertEqual(project.name, name)
        self.assertEqual(project.description, description)
        retrieved_project = Project.objects.get(pk=project.pk)
        self.assertEqual(retrieved_project.name, name)
        self.assertEqual(retrieved_project.description, description)
<commit_msg>Removed unicode characters<commit_after>from django.test import TestCase
from django.utils import translation
from storybase.utils import slugify
from models import (Organization, OrganizationTranslation, Project,
    ProjectTranslation)

class OrganizationModelTest(TestCase):
    def _create_organization(self, name, language):
        org = Organization()
        org.save()
        trans = OrganizationTranslation(name=name, language=language,
            organization=org)
        trans.save()
        return org

    def test_get_translated_field(self):
        """
        Tests that you can get a translated field as if it were the model's own
        """
        org_name_en = "Mile High Connects"
        org = self._create_organization(org_name_en, "en")
        self.assertEqual(org.name, org_name_en)

    def test_get_translated_field_default_fallback(self):
        """
        Tests that if a translated field doesn't exist in the current
        language, it falls back to the default language.
        """
        org_name_en = "Mile High Connects"
        org = self._create_organization(org_name_en, "en")
        translation.activate('es')
        self.assertEqual(org.name, org_name_en)

    def test_get_translated_field_first_fallback(self):
        """
        Tests that if a translated field doesn't exist in the current
        language, and it doesn't exist in the default language, the
        first available language is used
        """
        org_name_es = "Mile High Conecta"
        org = self._create_organization(org_name_es, 'es')
        translation.activate('en')
        self.assertEqual(org.name, org_name_es)

    def test_auto_slug(self):
        name = 'Mile High Connects'
        organization = Organization()
        organization.save()
        organization_translation = OrganizationTranslation(name=name, organization=organization)
        self.assertEqual(organization_translation.slug, '')
        organization_translation.save()
        self.assertEqual(organization_translation.slug, slugify(name))

class OrganizationApiTest(TestCase):
    def test_create_organization(self):
        from storybase_user.models import create_organization, Organization

        name = "Mile High Connects"
        website_url = "http://www.urbanlandc.org/collaboratives/mile-high-connects/"
        description = 'Mile High Connects (formerly know as the Mile High Transit Opportunity Collaborative) is an emerging collaborative of nonprofit and philanthropic organizations working together to ensure the creation of the region\'s $6.7 billion FasTracks transit system benefits all communities in the region, including low-income populations.'
        with self.assertRaises(Organization.DoesNotExist):
            Organization.objects.get(organizationtranslation__name=name)
        org = create_organization(name=name, description=description, website_url=website_url)
        self.assertEqual(org.name, name)
        self.assertEqual(org.description, description)
        self.assertEqual(org.website_url, website_url)
        retrieved_org = Organization.objects.get(pk=org.pk)
        self.assertEqual(retrieved_org.name, name)
        self.assertEqual(retrieved_org.description, description)
        self.assertEqual(retrieved_org.website_url, website_url)

class ProjectModelTest(TestCase):
    def test_auto_slug(self):
        name = 'The Metro Denver Regional Equity Atlas'
        project = Project()
        project.save()
        project_translation = ProjectTranslation(name=name, project=project)
        self.assertEqual(project_translation.slug, '')
        project_translation.save()
        self.assertEqual(project_translation.slug, slugify(name))

class ProjectApiTest(TestCase):
    def test_create_project(self):
        from storybase_user.models import create_project, Project

        name = "The Metro Denver Regional Equity Atlas"
        description = """
            The Denver Regional Equity Atlas is a product of Mile High
            Connects (MHC), which came together in 2011 to ensure that
            the region\'s significant investment in new rail and bus
            service will provide greater access to opportunity and a
            higher quality of life for all of the region\'s residents, but
            especially for economically disadvantaged populations who
            would benefit the most from safe, convenient transit service.

            The Atlas visually documents the Metro Denver region\'s
            demographic, educational, employment, health and housing
            characteristics in relation to transit, with the goal of
            identifying areas of opportunity as well as challenges to
            creating and preserving quality communities near transit.
            """
        with self.assertRaises(Project.DoesNotExist):
            Project.objects.get(projecttranslation__name=name)
        project = create_project(name=name, description=description)
        self.assertEqual(project.name, name)
        self.assertEqual(project.description, description)
        retrieved_project = Project.objects.get(pk=project.pk)
        self.assertEqual(retrieved_project.name, name)
        self.assertEqual(retrieved_project.description, description)
<|endoftext|>