<commit_before># Copyright (C) 2010,2011  Chris Lalancette <clalance@redhat.com>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Template Description Language (TDL)
"""

import libxml2
import base64
import urlparse

import oz.ozutil
import oz.OzException

def get_value(doc, xmlstring, component, optional=False):
    """
    Function to get the contents from an XML node.  It takes 4 arguments:

    doc       - The libxml2 document to get a value from.
    xmlstring - The XPath string to use.
    component - A string representing which TDL component is being
                looked for (used in error reporting).
    optional  - A boolean describing whether this XML node is allowed to be
                absent or not.  If optional is True and the node is absent,
                None is returned.  If optional is False and the node is
                absent, an exception is raised.  (default: False)

    Returns the content of the XML node if found, None if the node is not
    found and optional is True.
    """
    res = doc.xpathEval(xmlstring)
    if len(res) == 1:
        return res[0].getContent()
    elif len(res) == 0:
        if optional:
            return None
        else:
            raise oz.OzException.OzException("Failed to find %s in TDL" % (component))
    else:
        raise oz.OzException.OzException("Expected 0 or 1 %s in TDL, saw %d" % (component, len(res)))

class Repository(object):
    """
    Class that represents a single repository to be used for installing
    packages.  Objects of this type contain 3 pieces of information:

    name   - The name of this repository.
    url    - The URL of this repository.
    signed - Whether this repository is signed (optional).
    """
    def __init__(self, name, url, signed):
        self.name = name
        self.url = url
        self.signed = signed

class Package(object):
    """
    Class that represents a single package to be installed.
    Objects of this type contain 4 pieces of information:

    name     - The name of the package.
    repo     - The repository that this package comes from (optional).
    filename - The filename that contains this package (optional).
    args     - Arguments necessary to install this package (optional).
    """
    def __init__(self, name, repo, filename, args):
        self.name = name
        self.repo = repo
        self.filename = filename
        self.args = args

class TDL(object):
    """
    Class that represents a parsed piece of TDL XML.  Objects of this kind
    contain 10 pieces of information:

    name         - The name assigned to this TDL.
    distro       - The type of operating system this TDL represents.
    update       - The version of the operating system this TDL represents.
    arch         - The architecture of the operating system this TDL
                   represents. Currently this must be one of "i386" or
                   "x86_64".
    key          - The installation key necessary to install this operating
                   system (optional).
    description  - A free-form description of this TDL (optional).
    installtype  - The method to be used to install this operating system.
                   Currently this must be one of "url" or "iso".
    packages     - A list of Package objects describing the packages to be
                   installed on the operating system.  This list may be
                   empty.
    repositories - A dictionary of Repository objects describing the
                   repositories to be searched to find packages.  The
                   dictionary is indexed by repository name.  This
                   dictionary may be empty.
    files        - A dictionary of file contents to be added to the
                   operating system.  The dictionary is indexed by filename.
    commands     - A dictionary of commands to run inside the guest VM.  The
                   dictionary is indexed by commands.  This dictionary may be
                   empty.
    """
    def __init__(self, xmlstring, rootpw_required=False):
        self.doc = None

        self.doc = libxml2.parseDoc(xmlstring)

        self.name = get_value(self.doc, '/template/name', 'template name')

        self.distro = get_value(self.doc, '/template/os/name', 'OS name')

        self.update = get_value(self.doc, '/template/os/version', 'OS version')

        self.arch = get_value(self.doc, '/template/os/arch', 'OS architecture')
        if self.arch != "i386" and self.arch != "x86_64":
            raise oz.OzException.OzException("Architecture must be one of 'i386' or 'x86_64'")

        self.key = get_value(self.doc, '/template/os/key', 'OS key',
                             optional=True)
        # key is not required, so it is not fatal if it is None

        self.description = get_value(self.doc, '/template/description',
                                     'description', optional=True)
        # description is not required, so it is not fatal if it is None

        install = self.doc.xpathEval('/template/os/install')
        if len(install) != 1:
            raise oz.OzException.OzException("Expected 1 OS install section in TDL, saw %d" % (len(install)))
        self.installtype = install[0].prop('type')
        if self.installtype is None:
            raise oz.OzException.OzException("Failed to find OS install type in TDL")

        # we only support md5/sha1/sha256 sums for ISO install types.  However,
        # we make sure the instance variables are set to None for both types
        # so code lower down in the stack doesn't have to care about the ISO
        # vs. URL install type distinction, and can just check whether or not
        # these URLs are None
        self.iso_md5_url = None
        self.iso_sha1_url = None
        self.iso_sha256_url = None

        if self.installtype == "url":
            self.url = get_value(self.doc, '/template/os/install/url',
                                 'OS install URL')
        elif self.installtype == "iso":
            self.iso = get_value(self.doc, '/template/os/install/iso',
                                 'OS install ISO')
            self.iso_md5_url = get_value(self.doc,
                                         '/template/os/install/md5sum',
                                         'OS install ISO MD5SUM', optional=True)
            self.iso_sha1_url = get_value(self.doc,
                                          '/template/os/install/sha1sum',
                                          'OS install ISO SHA1SUM',
                                          optional=True)
            self.iso_sha256_url = get_value(self.doc,
                                            '/template/os/install/sha256sum',
                                            'OS install ISO SHA256SUM',
                                            optional=True)
            # only one of md5, sha1, or sha256 can be specified; raise an error
            # if multiple are
            if (self.iso_md5_url and self.iso_sha1_url) or (self.iso_md5_url and self.iso_sha256_url) or (self.iso_sha1_url and self.iso_sha256_url):
                raise oz.OzException.OzException("Only one of <md5sum>, <sha1sum>, and <sha256sum> can be specified")
        else:
            raise oz.OzException.OzException("Unknown install type " + self.installtype + " in TDL")

        self.rootpw = get_value(self.doc, '/template/os/rootpw',
                                "root/Administrator password",
                                optional=not rootpw_required)

        self.packages = []
        for package in self.doc.xpathEval('/template/packages/package'):
            # package name
            name = package.prop('name')
            if name is None:
                raise oz.OzException.OzException("Package without a name was given")

            # repository that the package lives in (optional)
            repo = get_value(package, 'repository',
                             "package repository section", optional=True)

            # filename of the package (optional)
            filename = get_value(package, 'file', "package filename",
                                 optional=True)

            # arguments to install package (optional)
            args = get_value(package, 'arguments', "package arguments",
                             optional=True)

            self.packages.append(Package(name, repo, filename, args))

        self.files = {}
        for afile in self.doc.xpathEval('/template/files/file'):
            name = afile.prop('name')
            if name is None:
                raise oz.OzException.OzException("File without a name was given")
            contenttype = afile.prop('type')
            if contenttype is None:
                contenttype = 'raw'

            content = afile.getContent().strip()
            if contenttype == 'raw':
                self.files[name] = content
            elif contenttype == 'base64':
                if len(content) == 0:
                    self.files[name] = ""
                else:
                    self.files[name] = base64.b64decode(content)
            else:
                raise oz.OzException.OzException("File type for %s must be 'raw' or 'base64'" % (name))

        self.repositories = {}
        for repo in self.doc.xpathEval('/template/repositories/repository'):
            name = repo.prop('name')
            if name is None:
                raise oz.OzException.OzException("Repository without a name was given")
            url = get_value(repo, 'url', 'repository url')

            if urlparse.urlparse(url)[1] in ["localhost", "127.0.0.1",
                                             "localhost.localdomain"]:
                raise oz.OzException.OzException("Repositories cannot be localhost, since they must be reachable from the guest operating system")

            signstr = get_value(repo, 'signed', 'signed', optional=True)
            if signstr is None:
                signstr = 'no'

            signed = oz.ozutil.string_to_bool(signstr)
            if signed is None:
                raise oz.OzException.OzException("Repository signed property must be 'true', 'yes', 'false', or 'no'")

            self.repositories[name] = Repository(name, url, signed)

        self.commands = {}
        for command in self.doc.xpathEval('/template/commands/command'):
            name = command.prop('name')
            if name is None:
                raise oz.OzException.OzException("Command without a name was given")
            contenttype = command.prop('type')
            if contenttype is None:
                contenttype = 'raw'

            content = command.getContent().strip()
            if contenttype == 'raw':
                self.commands[name] = content
            elif contenttype == 'base64':
                if len(content) == 0:
                    self.commands[name] = ""
                else:
                    self.commands[name] = base64.b64decode(content)
            else:
                raise oz.OzException.OzException("File type for %s must be 'raw' or 'base64'" % (name))

    def __del__(self):
        if self.doc is not None:
            self.doc.freeDoc()
<commit_msg>Add methods to TDL class to assist in merging additional packages and repos<commit_after># Copyright (C) 2010,2011  Chris Lalancette <clalance@redhat.com>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""
Template Description Language (TDL)
"""

import libxml2
import base64
import urlparse

import oz.ozutil
import oz.OzException

def get_value(doc, xmlstring, component, optional=False):
    """
    Function to get the contents from an XML node.  It takes 4 arguments:

    doc       - The libxml2 document to get a value from.
    xmlstring - The XPath string to use.
    component - A string representing which TDL component is being
                looked for (used in error reporting).
    optional  - A boolean describing whether this XML node is allowed to be
                absent or not.  If optional is True and the node is absent,
                None is returned.  If optional is False and the node is
                absent, an exception is raised.  (default: False)

    Returns the content of the XML node if found, None if the node is not
    found and optional is True.
    """
    res = doc.xpathEval(xmlstring)
    if len(res) == 1:
        return res[0].getContent()
    elif len(res) == 0:
        if optional:
            return None
        else:
            raise oz.OzException.OzException("Failed to find %s in TDL" % (component))
    else:
        raise oz.OzException.OzException("Expected 0 or 1 %s in TDL, saw %d" % (component, len(res)))

class Repository(object):
    """
    Class that represents a single repository to be used for installing
    packages.  Objects of this type contain 3 pieces of information:

    name   - The name of this repository.
    url    - The URL of this repository.
    signed - Whether this repository is signed (optional).
    """
    def __init__(self, name, url, signed):
        self.name = name
        self.url = url
        self.signed = signed

class Package(object):
    """
    Class that represents a single package to be installed.
    Objects of this type contain 4 pieces of information:

    name     - The name of the package.
    repo     - The repository that this package comes from (optional).
    filename - The filename that contains this package (optional).
    args     - Arguments necessary to install this package (optional).
    """
    def __init__(self, name, repo, filename, args):
        self.name = name
        self.repo = repo
        self.filename = filename
        self.args = args

class TDL(object):
    """
    Class that represents a parsed piece of TDL XML.  Objects of this kind
    contain 10 pieces of information:

    name         - The name assigned to this TDL.
    distro       - The type of operating system this TDL represents.
    update       - The version of the operating system this TDL represents.
    arch         - The architecture of the operating system this TDL
                   represents. Currently this must be one of "i386" or
                   "x86_64".
    key          - The installation key necessary to install this operating
                   system (optional).
    description  - A free-form description of this TDL (optional).
    installtype  - The method to be used to install this operating system.
                   Currently this must be one of "url" or "iso".
    packages     - A list of Package objects describing the packages to be
                   installed on the operating system.  This list may be
                   empty.
    repositories - A dictionary of Repository objects describing the
                   repositories to be searched to find packages.  The
                   dictionary is indexed by repository name.  This
                   dictionary may be empty.
    files        - A dictionary of file contents to be added to the
                   operating system.  The dictionary is indexed by filename.
    commands     - A dictionary of commands to run inside the guest VM.  The
                   dictionary is indexed by commands.  This dictionary may be
                   empty.
    """
    def __init__(self, xmlstring, rootpw_required=False):
        self.doc = None

        self.doc = libxml2.parseDoc(xmlstring)

        self.name = get_value(self.doc, '/template/name', 'template name')

        self.distro = get_value(self.doc, '/template/os/name', 'OS name')

        self.update = get_value(self.doc, '/template/os/version', 'OS version')

        self.arch = get_value(self.doc, '/template/os/arch', 'OS architecture')
        if self.arch != "i386" and self.arch != "x86_64":
            raise oz.OzException.OzException("Architecture must be one of 'i386' or 'x86_64'")

        self.key = get_value(self.doc, '/template/os/key', 'OS key',
                             optional=True)
        # key is not required, so it is not fatal if it is None

        self.description = get_value(self.doc, '/template/description',
                                     'description', optional=True)
        # description is not required, so it is not fatal if it is None

        install = self.doc.xpathEval('/template/os/install')
        if len(install) != 1:
            raise oz.OzException.OzException("Expected 1 OS install section in TDL, saw %d" % (len(install)))
        self.installtype = install[0].prop('type')
        if self.installtype is None:
            raise oz.OzException.OzException("Failed to find OS install type in TDL")

        # we only support md5/sha1/sha256 sums for ISO install types.  However,
        # we make sure the instance variables are set to None for both types
        # so code lower down in the stack doesn't have to care about the ISO
        # vs. URL install type distinction, and can just check whether or not
        # these URLs are None
        self.iso_md5_url = None
        self.iso_sha1_url = None
        self.iso_sha256_url = None

        if self.installtype == "url":
            self.url = get_value(self.doc, '/template/os/install/url',
                                 'OS install URL')
        elif self.installtype == "iso":
            self.iso = get_value(self.doc, '/template/os/install/iso',
                                 'OS install ISO')
            self.iso_md5_url = get_value(self.doc,
                                         '/template/os/install/md5sum',
                                         'OS install ISO MD5SUM', optional=True)
            self.iso_sha1_url = get_value(self.doc,
                                          '/template/os/install/sha1sum',
                                          'OS install ISO SHA1SUM',
                                          optional=True)
            self.iso_sha256_url = get_value(self.doc,
                                            '/template/os/install/sha256sum',
                                            'OS install ISO SHA256SUM',
                                            optional=True)
            # only one of md5, sha1, or sha256 can be specified; raise an error
            # if multiple are
            if (self.iso_md5_url and self.iso_sha1_url) or (self.iso_md5_url and self.iso_sha256_url) or (self.iso_sha1_url and self.iso_sha256_url):
                raise oz.OzException.OzException("Only one of <md5sum>, <sha1sum>, and <sha256sum> can be specified")
        else:
            raise oz.OzException.OzException("Unknown install type " + self.installtype + " in TDL")

        self.rootpw = get_value(self.doc, '/template/os/rootpw',
                                "root/Administrator password",
                                optional=not rootpw_required)

        self.packages = []
        packageslist = self.doc.xpathEval('/template/packages/package')
        self._add_packages(packageslist)

        self.files = {}
        for afile in self.doc.xpathEval('/template/files/file'):
            name = afile.prop('name')
            if name is None:
                raise oz.OzException.OzException("File without a name was given")
            contenttype = afile.prop('type')
            if contenttype is None:
                contenttype = 'raw'

            content = afile.getContent().strip()
            if contenttype == 'raw':
                self.files[name] = content
            elif contenttype == 'base64':
                if len(content) == 0:
                    self.files[name] = ""
                else:
                    self.files[name] = base64.b64decode(content)
            else:
                raise oz.OzException.OzException("File type for %s must be 'raw' or 'base64'" % (name))

        self.repositories = {}
        repositorieslist = self.doc.xpathEval('/template/repositories/repository')
        self._add_repositories(repositorieslist)

        self.commands = {}
        for command in self.doc.xpathEval('/template/commands/command'):
            name = command.prop('name')
            if name is None:
                raise oz.OzException.OzException("Command without a name was given")
            contenttype = command.prop('type')
            if contenttype is None:
                contenttype = 'raw'

            content = command.getContent().strip()
            if contenttype == 'raw':
                self.commands[name] = content
            elif contenttype == 'base64':
                if len(content) == 0:
                    self.commands[name] = ""
                else:
                    self.commands[name] = base64.b64decode(content)
            else:
                raise oz.OzException.OzException("File type for %s must be 'raw' or 'base64'" % (name))

    def merge_packages(self, packages):
        """
        Method to merge additional packages into an existing TDL.  The packages
        argument should be a properly structured <packages/> string as explained
        in the TDL documentation.  If a package with the same name is in the
        existing TDL and in packages, the value in packages overrides.
        """
        packsdoc = libxml2.parseDoc(packages)
        packslist = packsdoc.xpathEval('/packages/package')
        self._add_packages(packslist, True)

    def _add_packages(self, packslist, remove_duplicates = False):
        """
        Internal method to add the list of libxml2 nodes from packslist into
        the self.packages array.  If remove_duplicates is False (the default),
        then a package that is listed both in packslist and in the initial
        TDL is listed twice.  If it is set to True, then a package that is
        listed both in packslist and the initial TDL is listed only once,
        from the packslist.
        """
        for package in packslist:
            # package name
            name = package.prop('name')

            if name is None:
                raise oz.OzException.OzException("Package without a name was given")

            # repository that the package lives in (optional)
            repo = get_value(package, 'repository',
                             "package repository section", optional=True)

            # filename of the package (optional)
            filename = get_value(package, 'file', "package filename",
                                 optional=True)

            # arguments to install package (optional)
            args = get_value(package, 'arguments', "package arguments",
                             optional=True)

            if remove_duplicates:
                # delete any existing packages with this name
                for package in filter(lambda package: package.name == name,
                                      self.packages):
                    self.packages.remove(package)

            # now add in our new package def
            self.packages.append(Package(name, repo, filename, args))

    def merge_repositories(self, repos):
        """
        Method to merge additional repositories into an existing TDL.  The
        repos argument should be a properly structured <repositories/> string
        as explained in the TDL documentation.  If a repository with the same
        name is in the existing TDL and in repos, the value in repos overrides.
        """
        reposdoc = libxml2.parseDoc(repos)
        reposlist = reposdoc.xpathEval('/repositories/repository')
        self._add_repositories(reposlist)

    def _add_repositories(self, reposlist):
        """
        Internal method to add the list of libxml2 nodes from reposlist into
        the self.repositories dictionary.
        """
        for repo in reposlist:
            name = repo.prop('name')
            if name is None:
                raise oz.OzException.OzException("Repository without a name was given")
            url = get_value(repo, 'url', 'repository url')

            if urlparse.urlparse(url)[1] in ["localhost", "127.0.0.1",
                                             "localhost.localdomain"]:
                raise oz.OzException.OzException("Repositories cannot be localhost, since they must be reachable from the guest operating system")

            signstr = get_value(repo, 'signed', 'signed', optional=True)
            if signstr is None:
                signstr = 'no'

            signed = oz.ozutil.string_to_bool(signstr)
            if signed is None:
                raise oz.OzException.OzException("Repository signed property must be 'true', 'yes', 'false', or 'no'")

            # no need to delete - if the name matches we just overwrite here
            self.repositories[name] = Repository(name, url, signed)

    def __del__(self):
        if self.doc is not None:
            self.doc.freeDoc()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Sentiment classifiers
"""

import collections
import datetime
import json
import time
import os
from pkg_resources import resource_string
from textblob import TextBlob

class classifier(object):
    """MetaClass for classifier objects"""
    def __init__(self):
        self.data = {}

    __type__ = 'meta'
    now = datetime.datetime.now()
    city = 'Pythopolis'
    items = 0
    terms = 0

    def write(self, filepath):
        if not os.path.isfile(filepath):
            with open(filepath, 'w') as f:
                f.write(','.join([
                'city', 'year', 'month', 'mday', 'wday', 'hour', 'source', 'type', 'variable', 'value', 'n_items', 'n_terms'
                ]))
        for variable in self.data:
            with open(filepath, 'a') as f:
                f.write('\n' + ','.join([str(item) for item in [
                self.city,
                self.now.year,
                self.now.month,
                self.now.day,
                self.now.weekday(),
                self.now.hour,
                self.__class__,
                self.type,
                variable,
                self.data[variable],
                self.items,
                self.terms
                ]]))

class count_dict(classifier):
    """A simple dictionary method for mood analysis"""
    def __init__(self):
        f = resource_string(__name__, 'data/emo_dict.json')
        lookup = json.loads(f.decode('utf-8'))
        self.data = {key:0 for key in lookup}
        self.lookup = lookup
        self.type = 'count'

    def classify(self, text):
        """Count number of matches for emotion categories"""
        if type(text) == str:
            text = [text]
        if type(text) == tuple:
            text = list(text)
        for item in text:
            self.items += 1
            self.terms += len(item.split(' '))
            for key in self.data:
                self.data[key] += len(set(item.lower().split()) & set(self.lookup[key]))
        return self

class polar_summary(classifier):
    """
    A summary of sentiment and subjectivity using pattern's classifier (via TextBlob)
    """
    def __init__(self):
        self.data = {'polarity':0, 'subjectivity':0}
        self.type = 'polarity'

    def classify(self, text):
        """Calculate sentiment summary"""
        if type(text) == str:
            text = [text]
        if type(text) == tuple:
            text = list(text)
        for item in text:
            self.items += 1
            self.terms += len(item.split(' '))
            item = TextBlob(item)
            self.data['polarity'] = self.data['polarity'] * self.items/(self.items+1) + item.sentiment.polarity / (self.items+1)
            self.data['subjectivity'] = self.data['subjectivity'] * self.items/(self.items+1) + item.sentiment.subjectivity / (self.items+1)
        return self
<commit_msg>naming conventions<commit_after>#!/usr/bin/env python
"""
Sentiment classifiers
"""

import collections
import datetime
import json
import time
import os
from pkg_resources import resource_string
from textblob import TextBlob

class Classifier(object):
    """MetaClass for classifier objects"""
    def __init__(self):
        self.data = {}

    __type__ = 'meta'
    now = datetime.datetime.now()
    city = 'Pythopolis'
    items = 0
    terms = 0

    def write(self, filepath):
        if not os.path.isfile(filepath):
            with open(filepath, 'w') as f:
                f.write(','.join([
                'city', 'year', 'month', 'mday', 'wday', 'hour', 'source', 'type', 'variable', 'value', 'n_items', 'n_terms'
                ]))
        for variable in self.data:
            with open(filepath, 'a') as f:
                f.write('\n' + ','.join([str(item) for item in [
                self.city,
                self.now.year,
                self.now.month,
                self.now.day,
                self.now.weekday(),
                self.now.hour,
                self.__class__,
                self.type,
                variable,
                self.data[variable],
                self.items,
                self.terms
                ]]))

class CountDict(Classifier):
    """A simple dictionary method for mood analysis"""
    def __init__(self):
        f = resource_string(__name__, 'data/emo_dict.json')
        lookup = json.loads(f.decode('utf-8'))
        self.data = {key:0 for key in lookup}
        self.lookup = lookup
        self.type = 'count'

    def classify(self, text):
        """Count number of matches for emotion categories"""
        if type(text) == str:
            text = [text]
        if type(text) == tuple:
            text = list(text)
        for item in text:
            self.items += 1
            self.terms += len(item.split(' '))
            for key in self.data:
                self.data[key] += len(set(item.lower().split()) & set(self.lookup[key]))
        return self

class PolarSummary(Classifier):
    """
    A summary of sentiment and subjectivity using pattern's classifier (via TextBlob)
    """
    def __init__(self):
        self.data = {'polarity':0, 'subjectivity':0}
        self.type = 'polarity'

    def classify(self, text):
        """Calculate sentiment summary"""
        if type(text) == str:
            text = [text]
        if type(text) == tuple:
            text = list(text)
        for item in text:
            self.items += 1
            self.terms += len(item.split(' '))
            item = TextBlob(item)
            self.data['polarity'] = self.data['polarity'] * self.items/(self.items+1) + item.sentiment.polarity / (self.items+1)
            self.data['subjectivity'] = self.data['subjectivity'] * self.items/(self.items+1) + item.sentiment.subjectivity / (self.items+1)
        return self
<|endoftext|>


<commit_before>import os
import sys
import time
import subprocess
import threading
from itertools import chain

from werkzeug._internal import _log
from werkzeug._compat import PY2, iteritems, text_type

def _iter_module_files():
    """This iterates over all relevant Python files.  It goes through all
    loaded files from modules, all files in folders of already loaded modules
    as well as all files reachable through a package.
    """
    found = set()
    entered = set()

    def _verify_file(filename):
        if not filename:
            return
        filename = os.path.abspath(filename)
        old = None
        while not os.path.isfile(filename):
            old = filename
            filename = os.path.dirname(filename)
            if filename == old:
                break
        else:
            if filename[-4:] in ('.pyc', '.pyo'):
                filename = filename[:-1]
            if filename not in found:
                found.add(filename)
                return filename

    def _recursive_walk(path_entry):
        if path_entry in entered:
            return
        entered.add(path_entry)
        try:
            for filename in os.listdir(path_entry):
                path = os.path.join(path_entry, filename)
                if os.path.isdir(path):
                    for filename in _recursive_walk(path):
                        yield filename
                else:
                    if not filename.endswith(('.py', '.pyc', '.pyo')):
                        continue
                    filename = _verify_file(path)
                    if filename:
                        yield filename
        except OSError:
            pass

    # The list call is necessary on Python 3 in case the module
    # dictionary modifies during iteration.
    for path_entry in list(sys.path):
        for filename in _recursive_walk(os.path.abspath(path_entry)):
            yield filename

    for module in list(sys.modules.values()):
        if module is None:
            continue
        filename = _verify_file(getattr(module, '__file__', None))
        if filename:
            yield filename
            for filename in _recursive_walk(os.path.dirname(filename)):
                yield filename
        for package_path in getattr(module, '__path__', ()):
            for filename in _recursive_walk(os.path.abspath(package_path)):
                yield filename

def _find_observable_paths(extra_files=None):
    """Finds all paths that should be observed."""
    rv = set(os.path.abspath(x) for x in sys.path)
    for filename in extra_files or ():
        rv.append(os.path.dirname(os.path.abspath(filename)))
    for module in list(sys.modules.values()):
        fn = getattr(module, '__file__', None)
        if fn is None:
            continue
        fn = os.path.abspath(fn)
        rv.add(os.path.dirname(fn))
    return rv

def _find_common_roots(paths):
    """Out of some paths it finds the common roots that need monitoring."""
    paths = [x.split(os.path.sep) for x in paths]
    root = {}
    for chunks in sorted(paths, key=len, reverse=True):
        node = root
        for chunk in chunks:
            node = node.setdefault(chunk, {})
        node.clear()

    rv = set()
    def _walk(node, path):
        for prefix, child in iteritems(node):
            _walk(child, path + (prefix,))
        if not node:
            rv.add('/'.join(path))
    _walk(root, ())
    return rv

class ReloaderLoop(object):
    name = None

    # monkeypatched by testsuite. this used to just say `_sleep = time.sleep`,
    # but because c functions behave differently to python functions when
    # attached to classes, this breaks if sleep has been already replaced with
    # a python function (e.g. by eventlet.monkey_patch). python functions are
    # descriptors and become methods when attahed to classes, whereas c
    # functions remain as functions (thus behaving like staticmethods)
    @staticmethod
    def _sleep(secs):
        time.sleep(secs)

    def __init__(self, extra_files=None, interval=1):
        self.extra_files = set(os.path.abspath(x)
                               for x in extra_files or ())
        self.interval = interval

    def run(self):
        pass

    def restart_with_reloader(self):
        """Spawn a new Python interpreter with the same arguments as this one,
        but running the reloader thread.
        """
        while 1:
            _log('info', ' * Restarting with %s' % self.name)
            args = [sys.executable] + sys.argv
            new_environ = os.environ.copy()
            new_environ['WERKZEUG_RUN_MAIN'] = 'true'

            # a weird bug on windows. sometimes unicode strings end up in the
            # environment and subprocess.call does not like this, encode them
            # to latin1 and continue.
            if os.name == 'nt' and PY2:
                for key, value in iteritems(new_environ):
                    if isinstance(value, text_type):
                        new_environ[key] = value.encode('iso-8859-1')

            exit_code = subprocess.call(args, env=new_environ)
            if exit_code != 3:
                return exit_code

    def trigger_reload(self, filename):
        filename = os.path.abspath(filename)
        _log('info', ' * Detected change in %r, reloading' % filename)
        sys.exit(3)

class StatReloaderLoop(ReloaderLoop):
    name = 'stat'

    def run(self):
        mtimes = {}
        while 1:
            for filename in chain(_iter_module_files(), self.extra_files):
                try:
                    mtime = os.stat(filename).st_mtime
                except OSError:
                    continue

                old_time = mtimes.get(filename)
                if old_time is None:
                    mtimes[filename] = mtime
                    continue
                elif mtime > old_time:
                    self.trigger_reload(filename)
            self._sleep(self.interval)

class WatchdogReloaderLoop(ReloaderLoop):

    def __init__(self, *args, **kwargs):
        ReloaderLoop.__init__(self, *args, **kwargs)
        from watchdog.observers import Observer
        from watchdog.events import FileSystemEventHandler
        self.observable_paths = set()

        def _check_modification(filename):
            if filename in self.extra_files:
                self.trigger_reload(filename)
            dirname = os.path.dirname(filename)
            if dirname.startswith(tuple(self.observable_paths)):
                if filename.endswith(('.pyc', '.pyo')):
                    self.trigger_reload(filename[:-1])
                elif filename.endswith('.py'):
                    self.trigger_reload(filename)

        class _CustomHandler(FileSystemEventHandler):
            def on_created(self, event):
                _check_modification(event.src_path)
            def on_modified(self, event):
                _check_modification(event.src_path)

        reloader_name = Observer.__name__.lower()
        if reloader_name.endswith('observer'):
            reloader_name = reloader_name[:-8]
        reloader_name += ' reloader'

        self.name = reloader_name

        self.observer_class = Observer
        self.event_handler = _CustomHandler()
        self.should_reload = False

    def trigger_reload(self, filename):
        # This is called inside an event handler, which means we can't throw
        # SystemExit here. https://github.com/gorakhargosh/watchdog/issues/294
        self.should_reload = True
        ReloaderLoop.trigger_reload(self, filename)

    def run(self):
        watches = {}
        observer = self.observer_class()
        observer.start()

        while not self.should_reload:
            to_delete = set(watches)
            paths = _find_common_roots(
                _find_observable_paths(self.extra_files))
            for path in paths:
                if path not in watches:
                    try:
                        watches[path] = observer.schedule(
                            self.event_handler, path, recursive=True)
                    except OSError:
                        # "Path is not a directory". We could filter out
                        # those paths beforehand, but that would cause
                        # additional stat calls.
                        watches[path] = None
                to_delete.discard(path)
            for path in to_delete:
                watch = watches.pop(path, None)
                if watch is not None:
                    observer.unschedule(watch)
            self.observable_paths = paths
            self._sleep(self.interval)

        sys.exit(3)

reloader_loops = {
    'stat': StatReloaderLoop,
    'watchdog': WatchdogReloaderLoop,
}

try:
    __import__('watchdog.observers')
except ImportError:
    reloader_loops['auto'] = reloader_loops['stat']
else:
    reloader_loops['auto'] = reloader_loops['watchdog']

def run_with_reloader(main_func, extra_files=None, interval=1,
                      reloader_type='auto'):
    """Run the given function in an independent python interpreter."""
    import signal
    reloader = reloader_loops[reloader_type](extra_files, interval)
    signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
    try:
        if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
            t = threading.Thread(target=main_func, args=())
            t.setDaemon(True)
            t.start()
            reloader.run()
        else:
            sys.exit(reloader.restart_with_reloader())
    except KeyboardInterrupt:
        pass
<commit_msg>Fix typo<commit_after>import os
import sys
import time
import subprocess
import threading
from itertools import chain

from werkzeug._internal import _log
from werkzeug._compat import PY2, iteritems, text_type

def _iter_module_files():
    """This iterates over all relevant Python files.  It goes through all
    loaded files from modules, all files in folders of already loaded modules
    as well as all files reachable through a package.
    """
    found = set()
    entered = set()

    def _verify_file(filename):
        if not filename:
            return
        filename = os.path.abspath(filename)
        old = None
        while not os.path.isfile(filename):
            old = filename
            filename = os.path.dirname(filename)
            if filename == old:
                break
        else:
            if filename[-4:] in ('.pyc', '.pyo'):
                filename = filename[:-1]
            if filename not in found:
                found.add(filename)
                return filename

    def _recursive_walk(path_entry):
        if path_entry in entered:
            return
        entered.add(path_entry)
        try:
            for filename in os.listdir(path_entry):
                path = os.path.join(path_entry, filename)
                if os.path.isdir(path):
                    for filename in _recursive_walk(path):
                        yield filename
                else:
                    if not filename.endswith(('.py', '.pyc', '.pyo')):
                        continue
                    filename = _verify_file(path)
                    if filename:
                        yield filename
        except OSError:
            pass

    # The list call is necessary on Python 3 in case the module
    # dictionary modifies during iteration.
    for path_entry in list(sys.path):
        for filename in _recursive_walk(os.path.abspath(path_entry)):
            yield filename

    for module in list(sys.modules.values()):
        if module is None:
            continue
        filename = _verify_file(getattr(module, '__file__', None))
        if filename:
            yield filename
            for filename in _recursive_walk(os.path.dirname(filename)):
                yield filename
        for package_path in getattr(module, '__path__', ()):
            for filename in _recursive_walk(os.path.abspath(package_path)):
                yield filename

def _find_observable_paths(extra_files=None):
    """Finds all paths that should be observed."""
    rv = set(os.path.abspath(x) for x in sys.path)
    for filename in extra_files or ():
        rv.append(os.path.dirname(os.path.abspath(filename)))
    for module in list(sys.modules.values()):
        fn = getattr(module, '__file__', None)
        if fn is None:
            continue
        fn = os.path.abspath(fn)
        rv.add(os.path.dirname(fn))
    return rv

def _find_common_roots(paths):
    """Out of some paths it finds the common roots that need monitoring."""
    paths = [x.split(os.path.sep) for x in paths]
    root = {}
    for chunks in sorted(paths, key=len, reverse=True):
        node = root
        for chunk in chunks:
            node = node.setdefault(chunk, {})
        node.clear()

    rv = set()
    def _walk(node, path):
        for prefix, child in iteritems(node):
            _walk(child, path + (prefix,))
        if not node:
            rv.add('/'.join(path))
    _walk(root, ())
    return rv

class ReloaderLoop(object):
    name = None

    # monkeypatched by testsuite. this used to just say `_sleep = time.sleep`,
    # but because c functions behave differently to python functions when
    # attached to classes, this breaks if sleep has been already replaced with
    # a python function (e.g. by eventlet.monkey_patch). python functions are
    # descriptors and become methods when attached to classes, whereas c
    # functions remain as functions (thus behaving like staticmethods)
    @staticmethod
    def _sleep(secs):
        time.sleep(secs)

    def __init__(self, extra_files=None, interval=1):
        self.extra_files = set(os.path.abspath(x)
                               for x in extra_files or ())
        self.interval = interval

    def run(self):
        pass

    def restart_with_reloader(self):
        """Spawn a new Python interpreter with the same arguments as this one,
        but running the reloader thread.
        """
        while 1:
            _log('info', ' * Restarting with %s' % self.name)
            args = [sys.executable] + sys.argv
            new_environ = os.environ.copy()
            new_environ['WERKZEUG_RUN_MAIN'] = 'true'

            # a weird bug on windows. sometimes unicode strings end up in the
            # environment and subprocess.call does not like this, encode them
            # to latin1 and continue.
            if os.name == 'nt' and PY2:
                for key, value in iteritems(new_environ):
                    if isinstance(value, text_type):
                        new_environ[key] = value.encode('iso-8859-1')

            exit_code = subprocess.call(args, env=new_environ)
            if exit_code != 3:
                return exit_code

    def trigger_reload(self, filename):
        filename = os.path.abspath(filename)
        _log('info', ' * Detected change in %r, reloading' % filename)
        sys.exit(3)

class StatReloaderLoop(ReloaderLoop):
    name = 'stat'

    def run(self):
        mtimes = {}
        while 1:
            for filename in chain(_iter_module_files(), self.extra_files):
                try:
                    mtime = os.stat(filename).st_mtime
                except OSError:
                    continue

                old_time = mtimes.get(filename)
                if old_time is None:
                    mtimes[filename] = mtime
                    continue
                elif mtime > old_time:
                    self.trigger_reload(filename)
            self._sleep(self.interval)

class WatchdogReloaderLoop(ReloaderLoop):

    def __init__(self, *args, **kwargs):
        ReloaderLoop.__init__(self, *args, **kwargs)
        from watchdog.observers import Observer
        from watchdog.events import FileSystemEventHandler
        self.observable_paths = set()

        def _check_modification(filename):
            if filename in self.extra_files:
                self.trigger_reload(filename)
            dirname = os.path.dirname(filename)
            if dirname.startswith(tuple(self.observable_paths)):
                if filename.endswith(('.pyc', '.pyo')):
                    self.trigger_reload(filename[:-1])
                elif filename.endswith('.py'):
                    self.trigger_reload(filename)

        class _CustomHandler(FileSystemEventHandler):
            def on_created(self, event):
                _check_modification(event.src_path)
            def on_modified(self, event):
                _check_modification(event.src_path)

        reloader_name = Observer.__name__.lower()
        if reloader_name.endswith('observer'):
            reloader_name = reloader_name[:-8]
        reloader_name += ' reloader'

        self.name = reloader_name

        self.observer_class = Observer
        self.event_handler = _CustomHandler()
        self.should_reload = False

    def trigger_reload(self, filename):
        # This is called inside an event handler, which means we can't throw
        # SystemExit here. https://github.com/gorakhargosh/watchdog/issues/294
        self.should_reload = True
        ReloaderLoop.trigger_reload(self, filename)

    def run(self):
        watches = {}
        observer = self.observer_class()
        observer.start()

        while not self.should_reload:
            to_delete = set(watches)
            paths = _find_common_roots(
                _find_observable_paths(self.extra_files))
            for path in paths:
                if path not in watches:
                    try:
                        watches[path] = observer.schedule(
                            self.event_handler, path, recursive=True)
                    except OSError:
                        # "Path is not a directory". We could filter out
                        # those paths beforehand, but that would cause
                        # additional stat calls.
                        watches[path] = None
                to_delete.discard(path)
            for path in to_delete:
                watch = watches.pop(path, None)
                if watch is not None:
                    observer.unschedule(watch)
            self.observable_paths = paths
            self._sleep(self.interval)

        sys.exit(3)

reloader_loops = {
    'stat': StatReloaderLoop,
    'watchdog': WatchdogReloaderLoop,
}

try:
    __import__('watchdog.observers')
except ImportError:
    reloader_loops['auto'] = reloader_loops['stat']
else:
    reloader_loops['auto'] = reloader_loops['watchdog']

def run_with_reloader(main_func, extra_files=None, interval=1,
                      reloader_type='auto'):
    """Run the given function in an independent python interpreter."""
    import signal
    reloader = reloader_loops[reloader_type](extra_files, interval)
    signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
    try:
        if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
            t = threading.Thread(target=main_func, args=())
            t.setDaemon(True)
            t.start()
            reloader.run()
        else:
            sys.exit(reloader.restart_with_reloader())
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>import sys
import os
from wufoo_model import Wufoo_Survey_Model
from flask.ext.restful import Resource, reqparse
from flask import Flask, request, make_response
from flask.ext.sqlalchemy import SQLAlchemy
from database import db
from json import dumps
from flask.ext.security import current_user
import traceback

class Wufoo_DAO(object):
    def __init__(self, username):
        self.username = username

class Wufoo_resource(Resource):

    def get(self, **kwargs):
        print 'inside wufoo  get'
        if current_user.is_anonymous():
            return make_response(dumps([{"status":400}]))
        else:
            return make_response(dumps({"status":200}))

    def post(self, **kwargs):
        """
        Get wufoo data from webhook
        """
        print 'inside wufoo post'
        # get json data
        data = request.json
        print data
        create = data.get("create")
        # if creating/registering survey to user
        if create:
            if current_user.is_anonymous():
                return dumps([{"status":400}])
            else:
                url = data.get("url")
                if not url:
                    return jsonify(emsg="No url given")
                else:
                    print 'attempting to add survey'
                    try:
                        new_survey = Wufoo_Survey_Model(username=current_user.email, url=url)
                        db.session.add(new_survey)
                        db.session.commit()
                        return make_response(dumps({"status":200, "msg":"Survey successfully added"}))
                    except:
                        traceback.print_exc()
                        return jsonify(status=500)
        # if webhook and not the user registering the survey for the first time
        if not create:
            print 'not create'
            # parse json load
            created_by = data.get("CreatedBy")
            entry_id = data.get("EntryId")
            form_structure = data.get("FormStructure")
            url = form_structure.get("Url")
            field_structure = data.get("FieldStructure")
            fields = field_structure.get("Fields")
            form_url = "%s.wufoo.com/forms/%surl" %(created_by, url)
            # get survey
            survey = Wufoo_Survey_Model.query.filter_by(url=form_url).last()
            if not survey:
                # if survey doesn't exist yet, pass
                return jsonify(status="This survey does not exist yet.")
            # create new entry
            new_entry = Wufoo_Entry_Model(entry_id=entry_id)
            survey_values = survey.values

            if survey_values:
                values = True
            else:
                values = False

            survey_values = []
            new_fields = []
            field_ids = [field.id for field in survey.fields]
            for field in fields:
                # keep track if any modifications
                modified_field = True
                new_field = True
                field_id = field.get("ID")
                title = field.get("Title")
                field_type = field.get("Type")
                subfields = field.get("Subfields")
                field_value = data.get("field_id")

                # check if field_id, field_type, and field_title are still the same
                if field_id in field_ids:
                    new_field = False
                    old_field_index = field_ids.index('field_id')
                    old_field = survey.fields[old_field_index]
                    if old_field.field_id == field_id and old_field.title == title and old_field.field_type == field_type:
                        modified_field = False

                    if new_field or modified_field:
                        new_field = Wufoo_Field_Model(field_id=field_id, title = title, field_type = field_type)
                        survey.fields.append(new_field)
                # make new value
                new_value = Wufoo_Value(field_value)
                db.session.add(new_value)
                # add new value to new entry
                new_entry.values.append(new_value)

        db.session.add(new_entry)
        # add new entries to survey
        survey.entries.append(new_entry)
        db.session.add(survey)
        db.session.commit()
<commit_msg>wufoo edits<commit_after>import sys
import os
from wufoo_model import Wufoo_Survey_Model
from flask.ext.restful import Resource, reqparse
from flask import Flask, request, make_response, jsonify
from flask.ext.sqlalchemy import SQLAlchemy
from database import db
from json import dumps
from flask.ext.security import current_user
import traceback

class Wufoo_DAO(object):
    def __init__(self, username):
        self.username = username

class Wufoo_resource(Resource):

    def get(self, **kwargs):
        print 'inside wufoo  get'
        try:
            print request.args
            print request.data
            print dir(request)
        except:
            print traceback.print_exc
        return jsonify(status=200)

    def post(self, **kwargs):
        """
        Get wufoo data from webhook
        """
        print 'inside wufoo post'
        # get json data
        data = request.json
        print data
        create = data.get("create")
        # if creating/registering survey to user
        if create:
            if current_user.is_anonymous():
                return dumps([{"status":400}])
            else:
                url = data.get("url")
                if not url:
                    return jsonify(emsg="No url given")
                else:
                    print 'attempting to add survey'
                    try:
                        new_survey = Wufoo_Survey_Model(username=current_user.email, url=url)
                        db.session.add(new_survey)
                        db.session.commit()
                        return make_response(dumps({"status":200, "msg":"Survey successfully added"}))
                    except:
                        traceback.print_exc()
                        return jsonify(status=500)
        # if webhook and not the user registering the survey for the first time
        if not create:
            print 'not create'
            # parse json load
            created_by = data.get("CreatedBy")
            entry_id = data.get("EntryId")
            form_structure = data.get("FormStructure")
            url = form_structure.get("Url")
            field_structure = data.get("FieldStructure")
            fields = field_structure.get("Fields")
            form_url = "%s.wufoo.com/forms/%surl" %(created_by, url)
            # get survey
            survey = Wufoo_Survey_Model.query.filter_by(url=form_url).last()
            if not survey:
                # if survey doesn't exist yet, pass
                return jsonify(status="This survey does not exist yet.")
            # create new entry
            new_entry = Wufoo_Entry_Model(entry_id=entry_id)
            survey_values = survey.values

            if survey_values:
                values = True
            else:
                values = False

            survey_values = []
            new_fields = []
            field_ids = [field.id for field in survey.fields]
            for field in fields:
                # keep track if any modifications
                modified_field = True
                new_field = True
                field_id = field.get("ID")
                title = field.get("Title")
                field_type = field.get("Type")
                subfields = field.get("Subfields")
                field_value = data.get("field_id")

                # check if field_id, field_type, and field_title are still the same
                if field_id in field_ids:
                    new_field = False
                    old_field_index = field_ids.index('field_id')
                    old_field = survey.fields[old_field_index]
                    if old_field.field_id == field_id and old_field.title == title and old_field.field_type == field_type:
                        modified_field = False

                    if new_field or modified_field:
                        new_field = Wufoo_Field_Model(field_id=field_id, title = title, field_type = field_type)
                        survey.fields.append(new_field)
                # make new value
                new_value = Wufoo_Value(field_value)
                db.session.add(new_value)
                # add new value to new entry
                new_entry.values.append(new_value)

        db.session.add(new_entry)
        # add new entries to survey
        survey.entries.append(new_entry)
        db.session.add(survey)
        db.session.commit()
<|endoftext|>


<commit_before>import pywikibot, json, os, requests, argparse, logging, time, json, sys
import mwparserfromhell, datetime, math
from pywikibot.data import api
from pywikibot import pagegenerators

#get values from CENSUS API.  Return response from first year with valid response starting with
#current year and ending with 2013
def get_census_values(api_url, get_var, for_var, api_key, year=datetime.datetime.today().year):
    try:
        year = datetime.datetime.today().year
        while year >= 2013:
            payload = {'get': get_var, 'for': for_var, 'key': api_key}
            r = requests.get(api_url.replace('XXXX', str(year)), params=payload)
            if r.status_code == 200:
                return r.json()
            else:
                logging.info('No API Results for year: {}'.format(year))
                year = year - 1
        else:
            return
    except requests.exceptions.RequestException as e:
        logging.error('General Exception: {}'.format(e))
    except IOError as err:
        logging.error('IOError: {}'.format(err))

def search_for_page_items(template, infobox_keys):
    template_values = {}
    for item, item_keys in infobox_keys.items():
        for key in item_keys:
            if template.has(key):
                template_values[item] = str(template.get(key))
                break
    return template_values

#sort items by population (exluding PR and DC)
def population_rank_sort(pop_list):
    non_states = []
    for i, val in enumerate(pop_list):
        if val[2] in ['11', '72']:
            non_states.append(pop_list.pop(i))
    pop_list = sorted(pop_list, key=lambda x: int(x[1]), reverse=True)
    ordinal = lambda n: "%d%s" % (n,"tsnrhtdd"[(n//10%10!=1)*(n%10<4)*n%10::4])
    for i,val in enumerate(pop_list):
        val.append(ordinal(i+1))
    pop_list.extend(non_states)
    return pop_list

def update_page_items(page, text, api_values, page_values):
    edits = 0
    for key, val in template_values.items():
        pos = int(key.split(' - ')[1])
        new_value = api_values[pos]
        comparison_val = val.split('=', 1)[1].strip()
        if key == 'total_pop - 1':
            comparison_val = comparison_val[:comparison_val.find('<ref')].replace(',','')
        if comparison_val == new_value:
            print('Value for {} is correct already'.format(key))
        else:
            edits += 1
            if key == 'total_pop - 1':
                new_value = '{:,}'.format(int(new_value))+'<ref name=PopEstUS>{{cite web|url=https://www.census.gov/programs-surveys/popest.html |title=Population and Housing Unit Estimates| publisher=[[U.S. Census Bureau]]}}</ref>'
            new_value = val.split('=', 1)[0] + '= ' + new_value + '\n'
            print('OLD:{}\nNEW:{}'.format(val, new_value))
            text = text.replace(val, new_value)
    if edits > 0:
        page.text = text
        page.save(u'Updating population estimate and associated population rank (when applicable)\
                    with latest value from Census Bureau')

if __name__ == '__main__':
    get_var = 'GEONAME,POP'
    for_var = 'state:*'
    api_url = 'http://api.census.gov/data/XXXX/pep/population'
    api_key = os.environ['CENSUS']
    infobox_keys = {'total_pop - 1': ['population_total', '2010Pop', '2000Pop', 'population_estimate'],
            'rank - 3': ['PopRank']
            }
    site = pywikibot.Site('en', 'wikipedia')
    repo = site.data_repository()

    #metric_values = get_census_values(api_url, get_var, for_var, api_key)
    metric_values = [['sfddsa','2222222'], ['User:Sasan-CDS/sandbox', '2302030', '21']]
    if metric_values:
        #remove header
        metric_values.pop(0)
        metric_values = population_rank_sort(metric_values)
        print('Number of items in API Response: {}'.format(len(metric_values)))
        for i, val in enumerate(metric_values):
            key = val[0].split(',')[0]
            print('[STATE: {}]'.format(key))
            if key in ['Kansas', 'North Carolina', 'Georgia']:
                key = key + ', United States'
            elif key == 'Washington':
                key = 'Washington (state)'
            page = pywikibot.Page(site, key)
            if page.exists():
                if page.isRedirectPage():
                    page = page.getRedirectTarget()
                text = page.get(get_redirect=True)
                code = mwparserfromhell.parse(text)
                template_values = {}
                for template in code.filter_templates():
                    if template_values:
                        break
                    else:
                        template_values = search_for_page_items(template, infobox_keys)
                if template_values:
                    update_page_items(page, text, val, template_values)
                else:
                    print('No items were found in this page!!!')
            else:
                print('NO RESULTS FOR: {}'.format(key))
    else:
        sys.exit('NO RESULTS FROM THE CENSUS API FOR ANY YEARS.  EXAMINE FOR OTHER ISSUES!')
<commit_msg>moved value comparison to separate dedicated function, added additional text to value and reference that gets written, removed DC and PR from Census API response, return year from Census API call function to pass around to other functions<commit_after>import pywikibot, json, os, requests, argparse, logging, time, json, sys
import mwparserfromhell, datetime, math
from pywikibot.data import api
from pywikibot import pagegenerators

#get values from CENSUS API.  Return response from first year with valid response starting with
#current year and ending with 2013
def get_census_values(api_url, get_var, for_var, api_key, year=datetime.datetime.today().year):
    try:
        year = datetime.datetime.today().year
        while year >= 2013:
            payload = {'get': get_var, 'for': for_var, 'key': api_key}
            r = requests.get(api_url.replace('XXXX', str(year)), params=payload)
            if r.status_code == 200:
                return r.json(), str(year)
            else:
                logging.info('No API Results for year: {}'.format(year))
                year = year - 1
        else:
            return
    except requests.exceptions.RequestException as e:
        logging.error('General Exception: {}'.format(e))
    except IOError as err:
        logging.error('IOError: {}'.format(err))

def search_for_page_items(template, infobox_keys):
    template_values = {}
    for item, item_keys in infobox_keys.items():
        for key in item_keys:
            if template.has(key):
                template_values[item] = str(template.get(key))
                break
    return template_values

#sort items by population (exluding PR and DC)
def population_rank_sort(pop_list):
    non_states = []
    for i, val in enumerate(pop_list):
        #11 and 72 are the FIPS 5-2 codes for PR and DC
        if val[2] in ['11', '72']:
            non_states.append(pop_list.pop(i))
    pop_list = sorted(pop_list, key=lambda x: int(x[1]), reverse=True)
    ordinal = lambda n: "%d%s" % (n,"tsnrhtdd"[(n//10%10!=1)*(n%10<4)*n%10::4])
    for i,val in enumerate(pop_list):
        val.append(ordinal(i+1))
    #leaving this commented out for now because we decided to exclude these from
    #state processing
    #pop_list.extend(non_states)
    return pop_list

#remove value from dict and return copy rather than mutated dict
def removekey(d, key):
    r = dict(d)
    del r[key]
    return r

def compare_page_items(api_values, page_values, year):
    for key, val in template_values.items():
        pos = int(key.split(' - ')[1])
        new_value = api_values[pos] + ' ('+year+' est)'
        comparison_val = val.split('=', 1)[1].strip()
        if key == 'total_pop - 1':
            comparison_val = comparison_val[:comparison_val.find('<ref')].replace(',', '')
        if comparison_val == new_value:
            print('value for {} is correct already - {}'.format(key.split(' - ')[0], new_value))
            page_values = removekey(page_values, key)
    return page_values

def update_page_items(page, text, api_values, page_values, year):
    edits = 0
    for key, val in template_values.items():
        pos = int(key.split(' - ')[1])
        new_value = api_values[pos]
        #comparison_val = val.split('=', 1)[1].strip()
        #if key == 'total_pop - 1':
            #comparison_val = comparison_val[:comparison_val.find('<ref')].replace(',','')
        #if comparison_val == new_value:
            #print('Value for {} is correct already - {}'.format(key.split(' - ')[0], new_value))
        #else:
            #edits += 1
        if key == 'total_pop - 1':
            new_value = '{:,}'.format(int(new_value))+' ('+year+' est)<ref name=PopEstUS>{{cite web|'\
                        'url=https://www.census.gov/programs-surveys/popest.html |title=Population'\
                        ' and Housing Unit Estimates |date={} |accessdate={} |publisher=[[U.S. Census Bureau]]}}</ref>'\
                        .format(datetime.datetime.today().strftime('%B %d, %Y'), datetime.datetime.today().strftime('%B %d, %Y'))
        new_value = val.split('=', 1)[0] + '= ' + new_value + '\n'
        print('OLD:{}\nNEW:{}'.format(val, new_value))
        text = text.replace(val, new_value)
    #if edits > 0:
    #    page.text = text
    #    page.save(u'Updating population estimate and associated population rank (when applicable)'\
    #                'with latest value from Census Bureau')

if __name__ == '__main__':
    get_var = 'GEONAME,POP'
    for_var = 'state:*'
    api_url = 'http://api.census.gov/data/XXXX/pep/population'
    api_key = os.environ['CENSUS']
    infobox_keys = {'total_pop - 1': ['population_total', '2010Pop', '2000Pop', 'population_estimate'],
            'rank - 3': ['PopRank']
            }
    site = pywikibot.Site('en', 'wikipedia')
    repo = site.data_repository()

    metric_values, year = get_census_values(api_url, get_var, for_var, api_key)
    #metric_values = [['sfddsa','2222222'], ['User:Sasan-CDS/sandbox', '2302030', '21']]
    if metric_values:
        #remove header
        metric_values.pop(0)
        metric_values = population_rank_sort(metric_values)
        print('Number of items in API Response: {}'.format(len(metric_values)))
        for i, val in enumerate(metric_values):
            key = val[0].split(',')[0]
            print('[STATE: {}]'.format(key))
            if key in ['Kansas', 'North Carolina', 'Georgia']:
                key = key + ', United States'
            elif key == 'Washington':
                key = 'Washington (state)'
            #remove DC and PR from Census API Response
            if val[2] in ['11', '72']:
                metric_values.pop(i)
            page = pywikibot.Page(site, key)
            if page.exists():
                if page.isRedirectPage():
                    page = page.getRedirectTarget()
                text = page.get(get_redirect=True)
                code = mwparserfromhell.parse(text)
                template_values = {}
                for template in code.filter_templates():
                    if template_values:
                        break
                    else:
                        template_values = search_for_page_items(template, infobox_keys)
                #compare page items
                if template_values:
                    template_value = compare_page_items(val, template_values, year)
                if template_values:
                    update_page_items(page, text, val, template_values, year)
                else:
                    print('No items were found in this page!!!')
            else:
                print('NO RESULTS FOR: {}'.format(key))
    else:
        sys.exit('NO RESULTS FROM THE CENSUS API FOR ANY YEARS.  EXAMINE FOR OTHER ISSUES!')
<|endoftext|>


<commit_before># Daniel Borchmann, 2010
# This code is a mixture from the sage interfaces of
#  - Lisp
#  - Octave
# with some specials for conexp-clj. The copyright is held by William
# Stein, see the corresponding files for more information.

# TODO:
#  - conversion from clojure to python data (and vice versa) is very slow

from sage.interfaces.expect import Expect, ExpectElement, ExpectFunction, FunctionElement, gc_disabled
from sage.misc.sage_eval import sage_eval

###

class ConexpCLJ(Expect):
    def __init__(self,
                 maxread=100000,
                 script_subdirectory=None,
                 logfile=None,
                 server=None,
                 server_tmpdir=None):
        Expect.__init__(self,
                        name = 'conexp-clj',
                        prompt = 'user=> ', # fix this!
                        command = 'conexp-clj.sh',
                        maxread = maxread,
                        server = server,
                        server_tmpdir = server_tmpdir,
                        script_subdirectory = script_subdirectory,
                        restart_on_ctrlc = True,
                        verbose_start = False,
                        logfile = logfile,
                        eval_using_file_cutoff=1024)

    def _read_in_file_command(self, filename):
        return '(load-file "%s")'%filename

    def _start(self, *args, **kwds):
        Expect._start(self, *args, **kwds)

    def _quit_string(self):
        return "(System/exit 0)"

    def console(self):
        conexp_clj_console()

    def _synchronize(self):
        E = self._expect
        if E is None:
            self._start()
            E = self._expect
        r = random.randrange(2147483647)
        s = str(r+1)
        cmd = "(+ 1 %s)\n"%r
        E.sendline(cmd)
        E.expect(s)
        E.expect(self._prompt)

    def eval(self, code, *args, **kwds):
#        print "Evaluating %s"%code
        return Expect.eval(self,code,args,kwds)

    def _an_element_impl(self):
        return self(0)

    def set(self, var, value):
        cmd = "(def %s %s)"%(var,value)
        out = self.eval(cmd)

    def get(self, var):
        return self.eval('%s'%var)

    def clear(self, var):
        self.eval("(ns-unmap *ns* '%s)"%var)

    def _repr_(self):
        return "conexp-clj"

    def __reduce__(self):
        return reduce_load_conexp_clj, tuple([])

    def version(self):
        return conexp_clj_version()

    def _object_class(self):
        return ConexpCLJElement

    def _function_class(self):
        return ConexpCLJFunction

    def _function_element_class(self):
        return ConexpCLJFunctionElement

    def _true_symbol(self):
        return "true"

    def _false_symbol(self):
        return "false"

    def _equality_symbol(self):
        return "="

    def _inequality_symbol(self):
        return "not="

    def _assign_symbol(self):
        raise NotImplementedError

    def _left_list_delim(self):
        return "["

    def _right_list_delim(self):
        return "]"

    def help(self, command):
        print self.eval("(doc %s)"%str(command))

    def function_call(self, function, args=None, kwds=None):
        args, kwds = self._convert_args_kwds(args, kwds)
        self._check_valid_function_name(function)
        if function[0:3] == "is_":
            function = function[3:] + "?"
        function = function.replace("_","-")
        arg_string = " ".join([s.name() for s in args])
        kwd_string = " ".join([":" + key + " " + value.name() for key, value in kwds.iteritems()])
        return self.new("(%s %s %s)"%(function,arg_string,kwd_string))

    def _coerce_impl(self, x, **kwds):
        if x == None or x == False:
            return "nil"
        elif x == True:
            return "true"
        elif isinstance(x, (set, frozenset)):
            return "#{" + str(map(lambda y:self._coerce_impl(y,kwds), x))[1:-1] + "}"
        elif isinstance(x, (list, tuple)):
            return "[" + str(map(lambda y:self._coerce_impl(y,kwds), x))[1:-1] + "]"
        elif isinstance(x, dict):
            string = "{"
            for key in x:
                string += str(self._coerce_impl(key,kwds)) + " " + str(self._coerce_impl(x[key],kwds)) + ", "
            string += "}"
            return string
        else:
            return str(x)

class ConexpCLJElement(ExpectElement):
    def __cmp__(self, other):
        P = self._check_valid()
        if not hasattr(other, 'parent') or P is not other.parent():
            other = P(other)

        # THIS IS DAMN WRONG! (but it works, sometimes...)
        return int(P.eval("(compare (hash %s) (hash %s))"%(self.name(), other.name())))

    def __eq__(self, other):
        P = self._check_valid()
        if not hasattr(other, 'parent') or P is not other.parent():
            other = P(other)

        return "true" == P.eval("(= %s %s)"%(self.name(), other.name()))

    def _operation(self, operation, right):
        P = self._check_valid()
        if not hasattr(right, 'parent') or P is not right.parent():
            right = P(right)

        try:
            return P("(%s %s %s)"%(operation, self._name, right._name))
        except Exception, msg:
            raise TypeError, msg

    def bool(self):
        P = self.parent()
        cmd = '(not (not %s))'%self._name
        return P.eval(cmd) == "true"

    def gen(self, n):
        P = self.parent()
        return P("(nth %s %s)"%(self._name, n))

    def _sage_repr(self):
        raise NotImplementedError

    def sage(self):
        P = self._check_valid()

        if hasattr(self, "__conexp_value__"):
            return self.__conexp_value__

        name = self._name
        val = None

        if "true" == P.eval("(nil? %s)"%name):
            val = None
        elif "true" == P.eval("(set? %s)"%name):
            val = frozenset(self)
        elif "true" == P.eval("(sequential? %s)"%name):
            val = tuple(self)
        elif "true" == P.eval("(instance? conexp.fca.lattices.Lattice %s)"%name):
            edges = map(lambda x: (str(x[0]), str(x[1])),
                        P("(conexp.layouts.util/edges %s)"%name))
            val = LatticePoset([[], edges])
        elif "true" == P.eval("(map? %s)"%name):
            dit = {}
            for pair in [x for x in self]:
                dit[pair[0]] = pair[1]
            val = dit
        else:
            val = sage_eval(str(self))

        self.__conexp_value__ = val
        return val

    def attribute(self, attrname):
        P = self._check_valid()
        return P("(%s %s)"%(attrname, self._name))

    def __pow__(self, n):
        P = self._check_valid()
        if not hasattr(n, 'parent') or P is not n.parent():
            n = P(n)
        return self._operation("expt", n)

    def _matrix_(self, n):
        raise NotImplementedError

    def _vector_(self, n):
        raise NotImplementedError

    def __len__(self):
        P = self._check_valid()
        return int(P.eval("(count %s)"%self._name))

    def __iter__(self):
        self_seq = self.seq()
        for i in xrange(0, len(self)):
            yield self_seq.gen(i)

    def _contains(self, x):
        P = self._check_valid()
        if not hasattr(x, 'parent') or P is not x.parent():
            x = P(x)
        return "true" == P.eval("(contains? %s %s)"%(self._name, x))

    def __getitem__(self, key):
        P = self._check_valid()
        return P("(get %s %s)"%(self._name, key))

class ConexpCLJFunction(ExpectFunction):
    def _sage_doc_(self):
        M = self._obj.parent()
        return M.help(self._name)

class ConexpCLJFunctionElement(FunctionElement):
    def _sage_doc_(self):
        M = self._obj.parent()
        return M.help(self._name)

###

conexp_clj = ConexpCLJ()

def reduce_load_conexp_clj():
    return conexp_clj

import os
def conexp_clj_console():
    if "nt" == os.name():
        os.system("conexp-clj.bat")
    else:
        os.system("conexp-clj.sh")

def conexp_clj_version():
    return str(conexp_clj('(conexp-version)')).strip()

###
<commit_msg>Using eval implementation from lisp interface<commit_after># Daniel Borchmann, 2010
# This code is a mixture from the sage interfaces of
#  - Lisp
#  - Octave
# with some specials for conexp-clj. The copyright is held by William
# Stein, see the corresponding files for more information.

# TODO:
#  - conversion from clojure to python data (and vice versa) is very slow

from sage.interfaces.expect import Expect, ExpectElement, ExpectFunction, FunctionElement, gc_disabled
from sage.misc.sage_eval import sage_eval

###

class ConexpCLJ(Expect):
    def __init__(self,
                 maxread=100000,
                 script_subdirectory=None,
                 logfile=None,
                 server=None,
                 server_tmpdir=None):
        Expect.__init__(self,
                        name = 'conexp-clj',
                        prompt = 'user=> ', # fix this!
                        command = 'conexp-clj.sh',
                        maxread = maxread,
                        server = server,
                        server_tmpdir = server_tmpdir,
                        script_subdirectory = script_subdirectory,
                        restart_on_ctrlc = True,
                        verbose_start = False,
                        logfile = logfile,
                        eval_using_file_cutoff=1024)

    def _read_in_file_command(self, filename):
        return '(load-file "%s")'%filename

    def _start(self, *args, **kwds):
        Expect._start(self, *args, **kwds)

    def _quit_string(self):
        return "(System/exit 0)"

    def console(self):
        conexp_clj_console()

    # from lisp interface
    def _synchronize(self):
        E = self._expect
        if E is None:
            self._start()
            E = self._expect
        r = random.randrange(2147483647)
        s = str(r+1)
        cmd = "(+ 1 %s)\n"%r
        E.sendline(cmd)
        E.expect(s)
        E.expect(self._prompt)

    # from lisp interface
    def eval(self, code, strip=True, **kwds):
#        print "Evaluating %s"%code
        with gc_disabled():
            self._synchronize()
            code = str(code)
            code = code.strip()
            code = code.replace('\n',' ')
            x = []
            for L in code.split('\n'):
                if L != '':
                    try:
                        s = self.__in_seq + 1
                        M = self._eval_line(L, wait_for_prompt=self._prompt)
                        if M.startswith(L + "\n"):
                            M = M[len(L):]      # skip L in case it was echoed
                        x.append(M.strip())
                        self.__in_seq = s
                    except KeyboardInterrupt:
                        # DO NOT CATCH KeyboardInterrupt, as it is being caught
                        # by _eval_line
                        # In particular, do NOT call self._keyboard_interrupt()
                        raise
                    except TypeError, s:
                        return 'error evaluating "%s":\n%s'%(code,s)
            return '\n'.join(x)

    def _an_element_impl(self):
        return self(0)

    def set(self, var, value):
        cmd = "(def %s %s)"%(var,value)
        out = self.eval(cmd)

    def get(self, var):
        return self.eval('%s'%var)

    def clear(self, var):
        self.eval("(ns-unmap *ns* '%s)"%var)

    def _repr_(self):
        return "conexp-clj"

    def __reduce__(self):
        return reduce_load_conexp_clj, tuple([])

    def version(self):
        return conexp_clj_version()

    def _object_class(self):
        return ConexpCLJElement

    def _function_class(self):
        return ConexpCLJFunction

    def _function_element_class(self):
        return ConexpCLJFunctionElement

    def _true_symbol(self):
        return "true"

    def _false_symbol(self):
        return "false"

    def _equality_symbol(self):
        return "="

    def _inequality_symbol(self):
        return "not="

    def _assign_symbol(self):
        raise NotImplementedError

    def _left_list_delim(self):
        return "["

    def _right_list_delim(self):
        return "]"

    def help(self, command):
        print self.eval("(doc %s)"%str(command))

    def function_call(self, function, args=None, kwds=None):
        args, kwds = self._convert_args_kwds(args, kwds)
        self._check_valid_function_name(function)
        if function[0:3] == "is_":
            function = function[3:] + "?"
        function = function.replace("_","-")
        arg_string = " ".join([s.name() for s in args])
        kwd_string = " ".join([":" + key + " " + value.name() for key, value in kwds.iteritems()])
        return self.new("(%s %s %s)"%(function,arg_string,kwd_string))

    def _coerce_impl(self, x, **kwds):
        if x == None or x == False:
            return "nil"
        elif x == True:
            return "true"
        elif isinstance(x, (set, frozenset)):
            return "#{" + str(map(lambda y:self._coerce_impl(y,kwds), x))[1:-1] + "}"
        elif isinstance(x, (list, tuple)):
            return "[" + str(map(lambda y:self._coerce_impl(y,kwds), x))[1:-1] + "]"
        elif isinstance(x, dict):
            string = "{"
            for key in x:
                string += str(self._coerce_impl(key,kwds)) + " " + str(self._coerce_impl(x[key],kwds)) + ", "
            string += "}"
            return string
        else:
            return str(x)

class ConexpCLJElement(ExpectElement):
    def __cmp__(self, other):
        P = self._check_valid()
        if not hasattr(other, 'parent') or P is not other.parent():
            other = P(other)

        # THIS IS DAMN WRONG! (but it works, sometimes...)
        return int(P.eval("(compare (hash %s) (hash %s))"%(self.name(), other.name())))

    def __eq__(self, other):
        P = self._check_valid()
        if not hasattr(other, 'parent') or P is not other.parent():
            other = P(other)

        return "true" == P.eval("(= %s %s)"%(self.name(), other.name()))

    def _operation(self, operation, right):
        P = self._check_valid()
        if not hasattr(right, 'parent') or P is not right.parent():
            right = P(right)

        try:
            return P("(%s %s %s)"%(operation, self._name, right._name))
        except Exception, msg:
            raise TypeError, msg

    def bool(self):
        P = self.parent()
        cmd = '(not (not %s))'%self._name
        return P.eval(cmd) == "true"

    def gen(self, n):
        P = self.parent()
        return P("(nth %s %s)"%(self._name, n))

    def _sage_repr(self):
        raise NotImplementedError

    def sage(self):
        P = self._check_valid()

        if hasattr(self, "__conexp_value__"):
            return self.__conexp_value__

        name = self._name
        val = None

        if "true" == P.eval("(nil? %s)"%name):
            val = None
        elif "true" == P.eval("(set? %s)"%name):
            val = frozenset(self)
        elif "true" == P.eval("(sequential? %s)"%name):
            val = tuple(self)
        elif "true" == P.eval("(instance? conexp.fca.lattices.Lattice %s)"%name):
            edges = map(lambda x: (str(x[0]), str(x[1])),
                        P("(conexp.layouts.util/edges %s)"%name))
            val = LatticePoset([[], edges])
        elif "true" == P.eval("(map? %s)"%name):
            dit = {}
            for pair in [x for x in self]:
                dit[pair[0]] = pair[1]
            val = dit
        else:
            val = sage_eval(str(self))

        self.__conexp_value__ = val
        return val

    def attribute(self, attrname):
        P = self._check_valid()
        return P("(%s %s)"%(attrname, self._name))

    def __pow__(self, n):
        P = self._check_valid()
        if not hasattr(n, 'parent') or P is not n.parent():
            n = P(n)
        return self._operation("expt", n)

    def _matrix_(self, n):
        raise NotImplementedError

    def _vector_(self, n):
        raise NotImplementedError

    def __len__(self):
        P = self._check_valid()
        return int(P.eval("(count %s)"%self._name))

    def __iter__(self):
        self_seq = self.seq()
        for i in xrange(0, len(self)):
            yield self_seq.gen(i)

    def _contains(self, x):
        P = self._check_valid()
        if not hasattr(x, 'parent') or P is not x.parent():
            x = P(x)
        return "true" == P.eval("(contains? %s %s)"%(self._name, x))

    def __getitem__(self, key):
        P = self._check_valid()
        return P("(get %s %s)"%(self._name, key))

class ConexpCLJFunction(ExpectFunction):
    def _sage_doc_(self):
        M = self._obj.parent()
        return M.help(self._name)

class ConexpCLJFunctionElement(FunctionElement):
    def _sage_doc_(self):
        M = self._obj.parent()
        return M.help(self._name)

###

conexp_clj = ConexpCLJ()

def reduce_load_conexp_clj():
    return conexp_clj

import os
def conexp_clj_console():
    if "nt" == os.name():
        os.system("conexp-clj.bat")
    else:
        os.system("conexp-clj.sh")

def conexp_clj_version():
    return str(conexp_clj('(conexp-version)')).strip()

###
<|endoftext|>


<commit_before>from app import flask_app
from app.models import kaizen_user, mapping
from flask import g, render_template, redirect, flash, url_for
from flask.ext.login import login_required

@flask_app.route('/me', endpoint='me')
@login_required
def owner_profile():
    mappings = view_mappings(g.user)
    allmaps = summarize_maps(mappings)
    allmaps['notes'] = view_notes(g.user, 5)
    return render_template('user_profile.html', user=g.user, mappings=mappings, allmaps=allmaps)

@flask_app.route('/user/<name>')
@login_required
def user_profile(name):
    if g.user and g.user.name == name:
        return redirect(url_for('user_profile_owner'))

    viewed = kaizen_user.user_with_name(name)
    if viewed:
        return render_template('user_profile.html', user=viewed, mappings=view_mappings(viewed), notes=view_notes(viewed, 5))
    flash('No user with name %s, should have 404ed' % name)
    #TODO return no user with name 404 page

def view_mappings(u):
    return [{'name':str(m.name), 'notes':m.get_all_notes(), 'sites':m.get_all_sites()} for m in u.get_all_mappings_in_notes_order()]

def view_notes(u, n):
    return u.recent_notes(n)

def summarize_maps(mappings):
    ret = {'sumSites':0, 'sumNotes':0}
    for m in mappings:
        ret['sumSites'] += len(m['sites'])
        ret['sumNotes'] += len(m['notes'])
    return ret
<commit_msg>user_profile: name_route support. incomplete, so when we allow people to see other profiles, need to fix this by running a migrate<commit_after>from app import flask_app
from app.models import kaizen_user, mapping
from flask import g, render_template, redirect, flash, url_for
from flask.ext.login import login_required

@flask_app.route('/me', endpoint='me')
@login_required
def owner_profile():
    mappings = view_mappings(g.user)
    allmaps = summarize_maps(mappings)
    allmaps['notes'] = view_notes(g.user, 5)
    return render_template('user_profile.html', user=g.user, mappings=mappings, allmaps=allmaps)

@flask_app.route('/user/<name_route>')
@login_required
def user_profile(name):
    if g.user and g.user.name_route == name_route:
        return redirect(url_for('user_profile_owner'))

    viewed = kaizen_user.user_with_name_route(name_route)
    if viewed:
        return render_template('user_profile.html', user=viewed, mappings=view_mappings(viewed), notes=view_notes(viewed, 5))
    flash('No user with name %s, should have 404ed' % name)
    return redirect(url_for('404'))

def view_mappings(u):
    return [{'name':str(m.name), 'notes':m.get_all_notes(), 'sites':m.get_all_sites()} for m in u.get_all_mappings_in_notes_order()]

def view_notes(u, n):
    return u.recent_notes(n)

def summarize_maps(mappings):
    ret = {'sumSites':0, 'sumNotes':0}
    for m in mappings:
        ret['sumSites'] += len(m['sites'])
        ret['sumNotes'] += len(m['notes'])
    return ret
<|endoftext|>