<commit_before>from linode.api import api_call
from linode import config
from linode import mappings

from datetime import datetime

class Property:
    def __init__(self, mutable=False, identifier=False, volatile=False, relationship=False):
        self.mutable = mutable
        self.identifier = identifier
        self.volatile = volatile
        self.relationship = relationship

class MappedObject:
    def __init__(self, **vals):
        print("Making a mapped object with {}".format(vals))
        MappedObject._expand_vals(self.__dict__, **vals)

    def _expand_vals(target, **vals):
        for v in vals:
            if type(vals[v]) is dict:
                vals[v] = MappedObject(**vals[v])
            elif type(vals[v]) is list:
                # oh mama
                vals[v] = [ MappedObject(**i) if type(i) is dict else i for i in vals[v] ]
        target.update(vals)

    def __repr__(self):
        return "Mapping containing {}".format(vars(self).keys())

class Base(object):
    """
    The Base class knows how to look up api properties of a model, and lazy-load them.
    """
    properties = ()

    def __init__(self):
        self._set('_populated', False)
        self._set('_last_updated', datetime.min)

        for prop in type(self).properties:
            self._set(prop, None)

    def __getattribute__(self, name):
        if name in type(self).properties.keys():
            if (object.__getattribute__(self, name) is None and not self._populated) \
                or (type(self).properties[name].volatile \
                and object.__getattribute__(self, '_last_updated')
                + config.volatile_refresh_timeout < datetime.now()):
                self._api_get()

        return object.__getattribute__(self, name)

    def __setattr__(self, name, value):
        if name in type(self).properties.keys() and not type(self).properties[name].mutable:
            raise AttributeError("'{}' is not a mutable field of '{}'"
                .format(name, type(self).__name__))
        self._set(name, value)

    def save(self):
        resp = api_call(type(self).api_endpoint, model=self, method="PUT",
            data=self._serialize())

        if 'error' in resp:
            return False
        return True

    def invalidate(self):
        for key in (k for k in type(self).properties.keys()
            if not type(self).properties[k].identifier):
            self._set(key, None)

        self._populated = False

    def _serialize(self):
       return {a: getattr(self, a) for a in type(self).properties}

    def _api_get(self):
        json = api_call(type(self).api_endpoint, model=self)
        self._populate(json)

    def _populate(self, json):
        if not json:
            return

        for key in json:
            if key in (k for k in type(self).properties.keys()
                if not type(self).properties[k].identifier):

                if type(self).properties[key].relationship  \
                    and not json[key] is None:
                    if not 'id' in json[key]:
                        continue
                    obj = mappings.make(json[key]['id'])
                    if obj:
                        obj._populate(json[key])
                    self._set(key, obj)
                elif type(json[key]) is dict:
                    self._set(key, MappedObject(**json[key]))
                else:
                    self._set(key, json[key])

        self._set('_last_updated', datetime.now())

    def _set(self, name, value):
        object.__setattr__(self, name, value)
<commit_msg>Removed print<commit_after>from linode.api import api_call
from linode import config
from linode import mappings

from datetime import datetime

class Property:
    def __init__(self, mutable=False, identifier=False, volatile=False, relationship=False):
        self.mutable = mutable
        self.identifier = identifier
        self.volatile = volatile
        self.relationship = relationship

class MappedObject:
    def __init__(self, **vals):
        MappedObject._expand_vals(self.__dict__, **vals)

    def _expand_vals(target, **vals):
        for v in vals:
            if type(vals[v]) is dict:
                vals[v] = MappedObject(**vals[v])
            elif type(vals[v]) is list:
                # oh mama
                vals[v] = [ MappedObject(**i) if type(i) is dict else i for i in vals[v] ]
        target.update(vals)

    def __repr__(self):
        return "Mapping containing {}".format(vars(self).keys())

class Base(object):
    """
    The Base class knows how to look up api properties of a model, and lazy-load them.
    """
    properties = ()

    def __init__(self):
        self._set('_populated', False)
        self._set('_last_updated', datetime.min)

        for prop in type(self).properties:
            self._set(prop, None)

    def __getattribute__(self, name):
        if name in type(self).properties.keys():
            if (object.__getattribute__(self, name) is None and not self._populated) \
                or (type(self).properties[name].volatile \
                and object.__getattribute__(self, '_last_updated')
                + config.volatile_refresh_timeout < datetime.now()):
                self._api_get()

        return object.__getattribute__(self, name)

    def __setattr__(self, name, value):
        if name in type(self).properties.keys() and not type(self).properties[name].mutable:
            raise AttributeError("'{}' is not a mutable field of '{}'"
                .format(name, type(self).__name__))
        self._set(name, value)

    def save(self):
        resp = api_call(type(self).api_endpoint, model=self, method="PUT",
            data=self._serialize())

        if 'error' in resp:
            return False
        return True

    def invalidate(self):
        for key in (k for k in type(self).properties.keys()
            if not type(self).properties[k].identifier):
            self._set(key, None)

        self._populated = False

    def _serialize(self):
       return {a: getattr(self, a) for a in type(self).properties}

    def _api_get(self):
        json = api_call(type(self).api_endpoint, model=self)
        self._populate(json)

    def _populate(self, json):
        if not json:
            return

        for key in json:
            if key in (k for k in type(self).properties.keys()
                if not type(self).properties[k].identifier):

                if type(self).properties[key].relationship  \
                    and not json[key] is None:
                    if not 'id' in json[key]:
                        continue
                    obj = mappings.make(json[key]['id'])
                    if obj:
                        obj._populate(json[key])
                    self._set(key, obj)
                elif type(json[key]) is dict:
                    self._set(key, MappedObject(**json[key]))
                else:
                    self._set(key, json[key])

        self._set('_last_updated', datetime.now())

    def _set(self, name, value):
        object.__setattr__(self, name, value)
<|endoftext|>


<commit_before>#ipc_lista1.8
#Professor: Jucimar Junior
#Any Mendes Carvalho - 1615310044
#
#
#
#
#Faça um programa que pergunte quanto você ganha por hora e o número de horas trabalhadas no mês.
#Calcule e mostre o total do seu salário no referido mês.

QntHora = input("Entre com
<commit_msg>Update ipc_lista1.8.py<commit_after>#ipc_lista1.8
#Professor: Jucimar Junior
#Any Mendes Carvalho - 1615310044
#
#
#
#
#Faça um programa que pergunte quanto você ganha por hora e o número de horas trabalhadas no mês.
#Calcule e mostre o total do seu salário no referido mês.

QntHora = input("Entre com o valor
<|endoftext|>


<commit_before>"""django-livereload"""
__version__ = '1.3'
__license__ = 'BSD License'

__author__ = 'Fantomas42'
__email__ = 'fantomas42@gmail.com'

__url__ = 'https://github.com/Fantomas42/django-livereload'
<commit_msg>Bumping to version 1.4<commit_after>"""django-livereload"""
__version__ = '1.4'
__license__ = 'BSD License'

__author__ = 'Fantomas42'
__email__ = 'fantomas42@gmail.com'

__url__ = 'https://github.com/Fantomas42/django-livereload'
<|endoftext|>


<commit_before>from datetime import datetime
from elasticsearch import Elasticsearch
import json
import unicodecsv

es = Elasticsearch()

def load_data(index):
  if es.indices.exists(index=index):
    print es.indices.delete(index=index)
  print es.indices.create(index=index)

  with open('MergedArtworks.csv', 'rb') as in_csv:
    for m in unicodecsv.DictReader(in_csv, encoding='utf-8'):
      id = m['ObjectID']
      if not m['DateAcquired']:
        m['DateAcquired'] = '2015-01-01'
      try:
        es.index(index=index, doc_type='moma_art_collection', id=id, body=m)
      except KeyboardInterrupt:
        raise
      except:
        print m

load_data('moma')
<commit_msg>added counting<commit_after>from datetime import datetime
from elasticsearch import Elasticsearch
import json
import unicodecsv

es = Elasticsearch()

def load_data(index):
  if es.indices.exists(index=index):
    print es.indices.delete(index=index)
  print es.indices.create(index=index)

  count = 0

  with open('MergedArtworks.csv', 'rb') as in_csv:
    for m in unicodecsv.DictReader(in_csv, encoding='utf-8'):
      count += 1
      if count % 1000 == 0:
        print ".",
      id = m['ObjectID']
      if not m['DateAcquired']:
        m['DateAcquired'] = '2015-01-01'
      try:
        es.index(index=index, doc_type='moma_art_collection', id=id, body=m)
      except KeyboardInterrupt:
        raise
      except:
        print m

  print "loaded", count, "records"

load_data('moma')
<|endoftext|>


<commit_before>import pocket
import config
import time
import pymongo
import requests
import json
import unidecode
import arrow
requests.packages.urllib3.disable_warnings()

# disable warning about HTTPS
try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

# initialize the connection to Mongo
client = pymongo.MongoClient()
db = client.nachios
timedb = db.time

def updateTimeDb(rtime):
    timedb.drop()
    timedb.insert_one(dict(
        _id="lastTime",
        lastTime=time.time()
    ))
    print "{0}: time updated.".format(rtime)

def makeStep(titleString):
    url = 'https://nachapp.com/api/users/' + str(config.nachUser) + '/nodes'
    newStep = requests.post(url,
                            auth=(config.nachApiKey, ''),
                            verify=False,
                            data={
                                  "parent": config.nachPTRNode,
                                  "type": "Step",
                                  "name": titleString
                            })
    # pause - completion doesn't work if we do not do this
    response = json.loads(newStep.text)
    nodeId = response['id']
    time.sleep(8)
    url = 'https://nachapp.com/api/todos/' + str(nodeId)
    newCompletion = requests.patch(url,
                                   auth=(config.nachApiKey, ''),
                                   verify=False,
                                   data={"status": "completed"})

# when did we last check for new articles?
lastTime = timedb.find_one({"_id": "lastTime"})['lastTime']
readableTime = arrow.utcnow().to(
    'US/Pacific').format("YYYY-MM-DD HH:mm:ss ZZ")

# initialize pocket
p = pocket.Pocket(config.consumerKey, config.accessToken)

# get all archived articles modified since last check
articleList = p.get(detailType="simple", state="archive", since=lastTime)

# this goes through the raw Pocket data for each item and finds a nonblank
# string to use as a title. The IFTTT -> Hook version of this was failing
# I think because sometimes the only usable title is resolved_url and
# IFTTT sends a blank title in such cases.

# also IFTTT is really slow compared to running this on a 5 minute cron.
try:
    for each in articleList[0]['list'].itervalues():
        ert = None
        egt = None
        eru = None
        try:
            ert = each['resolved_title']
            egt = each['given_title']
            eru = each['resolved_url'] # not Iluvatar
        except:
            print "{0}: unknown title: {1}".format(
                readableTime,
                each
            )
        for title in(ert, egt, eru, "unknown title"):
            if not title:
                pass
            elif len(title) == 0:
                pass
            else:
                makeStep(title)
                print "{1}: made step for {0}".format(
                    unidecode.unidecode(title),
                    readableTime)
                break
except AttributeError:
    print "{0}: no articles found".format(readableTime)

# set last checked time to now

updateTimeDb(readableTime)
<commit_msg>minor fix<commit_after>import pocket
import config
import time
import pymongo
import requests
import json
import unidecode
import arrow

# disable warning about HTTPS
try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

# initialize the connection to Mongo
client = pymongo.MongoClient()
db = client.nachios
timedb = db.time

def updateTimeDb(rtime):
    timedb.drop()
    timedb.insert_one(dict(
        _id="lastTime",
        lastTime=time.time()
    ))
    print "{0}: time updated.".format(rtime)

def makeStep(titleString):
    url = 'https://nachapp.com/api/users/' + str(config.nachUser) + '/nodes'
    newStep = requests.post(url,
                            auth=(config.nachApiKey, ''),
                            verify=False,
                            data={
                                  "parent": config.nachPTRNode,
                                  "type": "Step",
                                  "name": titleString
                            })
    # pause - completion doesn't work if we do not do this
    response = json.loads(newStep.text)
    nodeId = response['id']
    time.sleep(8)
    url = 'https://nachapp.com/api/todos/' + str(nodeId)
    newCompletion = requests.patch(url,
                                   auth=(config.nachApiKey, ''),
                                   verify=False,
                                   data={"status": "completed"})

# when did we last check for new articles?
lastTime = timedb.find_one({"_id": "lastTime"})['lastTime']
readableTime = arrow.utcnow().to(
    'US/Pacific').format("YYYY-MM-DD HH:mm:ss ZZ")

# initialize pocket
p = pocket.Pocket(config.consumerKey, config.accessToken)

# get all archived articles modified since last check
articleList = p.get(detailType="simple", state="archive", since=lastTime)

# this goes through the raw Pocket data for each item and finds a nonblank
# string to use as a title. The IFTTT -> Hook version of this was failing
# I think because sometimes the only usable title is resolved_url and
# IFTTT sends a blank title in such cases.

# also IFTTT is really slow compared to running this on a 5 minute cron.
try:
    for each in articleList[0]['list'].itervalues():
        ert = None
        egt = None
        eru = None
        try:
            ert = each['resolved_title']
            egt = each['given_title']
            eru = each['resolved_url'] # not Iluvatar
        except:
            print "{0}: unknown title: {1}".format(
                readableTime,
                each
            )
        for title in(ert, egt, eru, "unknown title"):
            if not title:
                pass
            elif len(title) == 0:
                pass
            else:
                makeStep(title)
                print "{1}: made step for {0}".format(
                    unidecode.unidecode(title),
                    readableTime)
                break
except AttributeError:
    print "{0}: no articles found".format(readableTime)

# set last checked time to now

updateTimeDb(readableTime)
<|endoftext|>


<commit_before># lpbm/modules/articles.py - Loads articles and treats them.
# Author: Franck Michea < franck.michea@gmail.com >
# License: New BSD License (See LICENSE)

import PyRSS2Gen
import codecs
import datetime
import jinja2
import markdown
import math
import os
import sys
import tempfile

import lpbm.module_loader
import lpbm.tools as ltools

_ENV = jinja2.Environment(loader=jinja2.FileSystemLoader(
    ltools.join(ltools.ROOT, 'medias', 'templates')
))

def _get_template(*args):
    return _ENV.get_template(os.path.join(*args))

# Miscenalleous filters for jinja2
def do_markdown(value, code=True):
    if not code:
        return markdown.markdown(value)
    return markdown.markdown(value, ['codehilite(force_linenos=True)'])

def do_sorted(value):
    return sorted(value)

def do_authors_list(value, mod):
    res, template = [], _get_template('authors', 'link.html')
    for author_id in value:
        try:
            res.append(template.render({'author': mod[int(author_id)]}))
        except (lpbm.exceptions.ModelDoesNotExistError, ValueError):
            pass
    return ltools.join_names(sorted(res))

class Render(lpbm.module_loader.Module):
    def name(self): return 'render'
    def abstract(self): return 'Blog generation module.'

    def init(self):
        self.needed_modules = ['authors', 'articles', 'categories']

        self.parser.add_argument('--drafts', action='store_true', default=False,
                                 help='also render drafts.')

    def load(self, modules, args):
        self.build_dir = tempfile.mkdtemp(prefix='lpbm_')
        self.output_dir = ltools.join(args.exec_path, 'result')

        if not os.path.exists(self.output_dir):
            sys.exit('I didn\'t find directory/symbolic link named `result`'
                     ' where to put the blog.')

        # Menu header.
        menu_header = None
        menu_path = lpbm.tools.join(args.exec_path, 'menu.markdown')
        if os.path.exists(menu_path):
            with codecs.open(menu_path, 'r', 'utf-8') as f:
                menu_header = markdown.markdown(f.read())

        # Jinja2 Environment Globals
        _ENV.filters.update({
            'authors_list': do_authors_list,
            'markdown': do_markdown,
            'sorted': do_sorted,
        })
        _ENV.globals.update({
            'authors_mod': self.modules['authors'],
            'categories_mod': self.modules['categories'],
            'config_mod': self.modules['config'],
            'menu_header': menu_header,
        })

    def process(self, modules, args):
        # Last update of environment before beginning of page generation,
        _ENV.globals.update({
            'static_files':  self.copy_static_files(),
        })

        self.render_articles()
        self.render_rss()
        self.render_categories()

        # If full rendering completed (we are still alive), then we copy the
        # temporary directory to the output directory.
        self._copy_all()

    # Functions for internal use.
    def _build_path(self, *args):
        lpbm.tools.mkdir_p(ltools.join(self.build_dir, *(args[:-1])))
        return ltools.join(self.build_dir, *args)

    def _get_articles(self, limit=None, filter=None):
        articles = sorted(self.modules['articles'].objects)
        articles = [a for a in articles if a.published or self.args.drafts]
        articles = list(reversed(articles))
        if filter is not None:
            articles = [a for a in articles if filter(a)]
        if limit is not None:
            articles = articles[:limit]
        return articles

    def _copy_all(self):
        # First big clean up of all the files.
        ltools.empty_directory(self.output_dir)
        ltools.move_content(self.build_dir, self.output_dir)
        os.rmdir(self.build_dir)

    def _copy_static_dir(self, statics, fltr, *subdirs):
        out_root = self._build_path(*subdirs)
        def sub(root):
            res = []
            for root, filename in ltools.filter_files(fltr, root, *subdirs):
                res.append(ltools.join('/', *(subdirs + (filename,))))
                ltools.copy(ltools.join(root, filename),
                            ltools.join(out_root, filename))
            return res
        statics.extend(sub(ltools.ROOT))
        statics.extend(sub(self.args.exec_path))

    def copy_static_files(self):
        static_files = {'css': []}
        self._copy_static_dir(static_files['css'], lambda a: a.endswith('.css'),
                              'medias', 'css')
        return static_files

    # Public functions.
    def render_articles(self):
        template = _get_template('articles', 'base.html')
        for article in self._get_articles():
            path = self._build_path('articles', article.html_filename())
            with codecs.open(path, 'w', 'utf-8') as f:
                print('Writing article to', path)
                f.write(template.render({
                    'articles': [article],
                }))
        self.render_index([], self._get_articles())
        self.render_pages(['pages'], self._get_articles())

    def render_index(self, directory, articles):
        template = _get_template('articles', 'base.html')
        limit = self.modules['config']['paginate.nb_articles'] or 5
        articles_ = articles[:limit]
        with codecs.open(self._build_path(*(directory + ['index.html'])), 'w', 'utf-8') as f:
            print('Writing index file.')
            f.write(template.render({
                'show_more': limit < len(articles),
                'articles': articles_,
            }))

    def render_pages(self, directory, articles):
        app = self.modules['config']['paginate.nb_articles'] or 5
        pwidth = self.modules['config']['paginate.width'] or 5
        pages = int(math.ceil(len(articles) / float(app)))
        def left_stone(page):
            return min(max(0, page - pwidth / 2), max(0, pages - pwidth))
        def right_stone(page):
            return max(min(pages, page + pwidth / 2), min(pages, pwidth - 1))
        template = _get_template('articles', 'base.html')
        display_pages = range(1, pages + 1)
        for page in range(pages):
            display_page = page + 1
            tmp = 'page-{}.html'.format(display_page)
            with codecs.open(self._build_path(*(directory + [tmp])), 'w', 'utf-8') as f:
                f.write(template.render({
                    'articles': articles[page * app: (page + 1) * app],
                    'cur_page': display_page,
                    'last_page': pages,
                    'pages': display_pages[left_stone(page):right_stone(page)],
                    'paginate': True,
                }))

    def render_rss(self):
        def rss_item(article):
            def rss_aut(author_id):
                try:
                    author = self.modules['authors'][author_id]
                    return '{email} ({full_name})'.format(
                        email = author.email,
                        full_name = author.full_name(),
                    )
                except lpbm.exceptions.ModelDoesNotExistError:
                    return '[deleted]'
            authors = ltools.join_names([rss_aut(a) for a in article.authors])
            return PyRSS2Gen.RSSItem(
                title = article.title,
                link = '{base_url}{html_filename}'.format(
                    base_url = self.modules['config']['general.url'],
                    html_filename = article.html_filename(),
                ),
                author = authors,
                guid = str(article.id),
                description = do_markdown(article.content),
                pubDate = article.date,
            )
        print('Generating RSS file.')
        articles = self._get_articles(self.modules['config']['rss.nb_articles'] or 10)
        rss = PyRSS2Gen.RSS2(
            title = self.modules['config']['general.title'],
            link = self.modules['config']['general.url'],
            description = self.modules['config']['general.subtitle'],
            lastBuildDate = datetime.datetime.now(),
            items = [rss_item(a) for a in articles],
        )
        rss_path = ltools.join(self.build_dir, 'rssfeed.xml')
        with codecs.open(rss_path, 'w', 'utf-8') as f:
            rss.write_xml(f, encoding='utf-8')

    def render_categories(self):
        for cat in self.modules['categories'].objects:
            dirs = list(os.path.split(os.path.dirname(cat.html_filename())))
            articles = self._get_articles(filter=lambda a: cat.id in a.categories)
            self.render_pages(dirs, articles)
            self.render_index(dirs, articles)
<commit_msg>fix index message.<commit_after># lpbm/modules/articles.py - Loads articles and treats them.
# Author: Franck Michea < franck.michea@gmail.com >
# License: New BSD License (See LICENSE)

import PyRSS2Gen
import codecs
import datetime
import jinja2
import markdown
import math
import os
import sys
import tempfile

import lpbm.module_loader
import lpbm.tools as ltools

_ENV = jinja2.Environment(loader=jinja2.FileSystemLoader(
    ltools.join(ltools.ROOT, 'medias', 'templates')
))

def _get_template(*args):
    return _ENV.get_template(os.path.join(*args))

# Miscenalleous filters for jinja2
def do_markdown(value, code=True):
    if not code:
        return markdown.markdown(value)
    return markdown.markdown(value, ['codehilite(force_linenos=True)'])

def do_sorted(value):
    return sorted(value)

def do_authors_list(value, mod):
    res, template = [], _get_template('authors', 'link.html')
    for author_id in value:
        try:
            res.append(template.render({'author': mod[int(author_id)]}))
        except (lpbm.exceptions.ModelDoesNotExistError, ValueError):
            pass
    return ltools.join_names(sorted(res))

class Render(lpbm.module_loader.Module):
    def name(self): return 'render'
    def abstract(self): return 'Blog generation module.'

    def init(self):
        self.needed_modules = ['authors', 'articles', 'categories']

        self.parser.add_argument('--drafts', action='store_true', default=False,
                                 help='also render drafts.')

    def load(self, modules, args):
        self.build_dir = tempfile.mkdtemp(prefix='lpbm_')
        self.output_dir = ltools.join(args.exec_path, 'result')

        if not os.path.exists(self.output_dir):
            sys.exit('I didn\'t find directory/symbolic link named `result`'
                     ' where to put the blog.')

        # Menu header.
        menu_header = None
        menu_path = lpbm.tools.join(args.exec_path, 'menu.markdown')
        if os.path.exists(menu_path):
            with codecs.open(menu_path, 'r', 'utf-8') as f:
                menu_header = markdown.markdown(f.read())

        # Jinja2 Environment Globals
        _ENV.filters.update({
            'authors_list': do_authors_list,
            'markdown': do_markdown,
            'sorted': do_sorted,
        })
        _ENV.globals.update({
            'authors_mod': self.modules['authors'],
            'categories_mod': self.modules['categories'],
            'config_mod': self.modules['config'],
            'menu_header': menu_header,
        })

    def process(self, modules, args):
        # Last update of environment before beginning of page generation,
        _ENV.globals.update({
            'static_files':  self.copy_static_files(),
        })

        self.render_articles()
        self.render_rss()
        self.render_categories()

        # If full rendering completed (we are still alive), then we copy the
        # temporary directory to the output directory.
        self._copy_all()

    # Functions for internal use.
    def _build_path(self, *args):
        lpbm.tools.mkdir_p(ltools.join(self.build_dir, *(args[:-1])))
        return ltools.join(self.build_dir, *args)

    def _get_articles(self, limit=None, filter=None):
        articles = sorted(self.modules['articles'].objects)
        articles = [a for a in articles if a.published or self.args.drafts]
        articles = list(reversed(articles))
        if filter is not None:
            articles = [a for a in articles if filter(a)]
        if limit is not None:
            articles = articles[:limit]
        return articles

    def _copy_all(self):
        # First big clean up of all the files.
        ltools.empty_directory(self.output_dir)
        ltools.move_content(self.build_dir, self.output_dir)
        os.rmdir(self.build_dir)

    def _copy_static_dir(self, statics, fltr, *subdirs):
        out_root = self._build_path(*subdirs)
        def sub(root):
            res = []
            for root, filename in ltools.filter_files(fltr, root, *subdirs):
                res.append(ltools.join('/', *(subdirs + (filename,))))
                ltools.copy(ltools.join(root, filename),
                            ltools.join(out_root, filename))
            return res
        statics.extend(sub(ltools.ROOT))
        statics.extend(sub(self.args.exec_path))

    def copy_static_files(self):
        static_files = {'css': []}
        self._copy_static_dir(static_files['css'], lambda a: a.endswith('.css'),
                              'medias', 'css')
        return static_files

    # Public functions.
    def render_articles(self):
        template = _get_template('articles', 'base.html')
        for article in self._get_articles():
            path = self._build_path('articles', article.html_filename())
            with codecs.open(path, 'w', 'utf-8') as f:
                print('Writing article to', path)
                f.write(template.render({
                    'articles': [article],
                }))
        self.render_index([], self._get_articles())
        self.render_pages(['pages'], self._get_articles())

    def render_index(self, directory, articles):
        template = _get_template('articles', 'base.html')
        limit = self.modules['config']['paginate.nb_articles'] or 5
        articles_ = articles[:limit]
        with codecs.open(self._build_path(*(directory + ['index.html'])), 'w', 'utf-8') as f:
            print('Writing index file for {}.'.format(os.path.join('/', *directory)))
            f.write(template.render({
                'show_more': limit < len(articles),
                'articles': articles_,
            }))

    def render_pages(self, directory, articles):
        app = self.modules['config']['paginate.nb_articles'] or 5
        pwidth = self.modules['config']['paginate.width'] or 5
        pages = int(math.ceil(len(articles) / float(app)))
        def left_stone(page):
            return min(max(0, page - pwidth / 2), max(0, pages - pwidth))
        def right_stone(page):
            return max(min(pages, page + pwidth / 2), min(pages, pwidth - 1))
        template = _get_template('articles', 'base.html')
        display_pages = range(1, pages + 1)
        for page in range(pages):
            display_page = page + 1
            tmp = 'page-{}.html'.format(display_page)
            with codecs.open(self._build_path(*(directory + [tmp])), 'w', 'utf-8') as f:
                f.write(template.render({
                    'articles': articles[page * app: (page + 1) * app],
                    'cur_page': display_page,
                    'last_page': pages,
                    'pages': display_pages[left_stone(page):right_stone(page)],
                    'paginate': True,
                }))

    def render_rss(self):
        def rss_item(article):
            def rss_aut(author_id):
                try:
                    author = self.modules['authors'][author_id]
                    return '{email} ({full_name})'.format(
                        email = author.email,
                        full_name = author.full_name(),
                    )
                except lpbm.exceptions.ModelDoesNotExistError:
                    return '[deleted]'
            authors = ltools.join_names([rss_aut(a) for a in article.authors])
            return PyRSS2Gen.RSSItem(
                title = article.title,
                link = '{base_url}{html_filename}'.format(
                    base_url = self.modules['config']['general.url'],
                    html_filename = article.html_filename(),
                ),
                author = authors,
                guid = str(article.id),
                description = do_markdown(article.content),
                pubDate = article.date,
            )
        print('Generating RSS file.')
        articles = self._get_articles(self.modules['config']['rss.nb_articles'] or 10)
        rss = PyRSS2Gen.RSS2(
            title = self.modules['config']['general.title'],
            link = self.modules['config']['general.url'],
            description = self.modules['config']['general.subtitle'],
            lastBuildDate = datetime.datetime.now(),
            items = [rss_item(a) for a in articles],
        )
        rss_path = ltools.join(self.build_dir, 'rssfeed.xml')
        with codecs.open(rss_path, 'w', 'utf-8') as f:
            rss.write_xml(f, encoding='utf-8')

    def render_categories(self):
        for cat in self.modules['categories'].objects:
            dirs = list(os.path.split(os.path.dirname(cat.html_filename())))
            articles = self._get_articles(filter=lambda a: cat.id in a.categories)
            self.render_pages(dirs, articles)
            self.render_index(dirs, articles)
<|endoftext|>


<commit_before>"""
Provides the LuminosoClient object, a wrapper for making
properly-authenticated requests to the Luminoso REST API.
"""
import json
import logging
import os
import requests
import time
from urllib.parse import urlparse

from .constants import URL_BASE
from .errors import (LuminosoError, LuminosoAuthError, LuminosoClientError,
                     LuminosoServerError)

logger = logging.getLogger(__name__)

class LuminosoClient(object):
    """
    A tool for making authenticated requests to the Luminoso API version 5.

    A LuminosoClient is a thin wrapper around the API documented at
    https://analytics.luminoso.com/api/v5/. As such, you interact with it by
    calling its methods that correspond to HTTP methods: `.get(url)`,
    `.post(url)`, `.put(url)`, `.patch(url)`, and `.delete(url)`.

    These URLs are relative to a 'base URL' for the LuminosoClient. For
    example, you can make requests for a specific project by creating a
    LuminosoClient for
    `https://analytics.luminoso.com/api/v5/projects/<project_id>`.

    Methods take parameters as keyword arguments, and encode them in the
    appropriate way for the request, which is described in the individual
    documentation for each method.

    The easiest way to create a LuminosoClient is using the
    `LuminosoClient.connect()` static method.

    In addition to the base URL, the LuminosoClient has a `root_url`,
    pointing to the root of the API, such as
    https://analytics.luminoso.com/api/v5. This is used, for example, as a
    starting point for the `change_path` method: when it gets a path starting
    with `/`, it will go back to the `root_url` instead of adding to the
    existing URL.
    """
    def __init__(self, session, url):
        """
        Create a LuminosoClient given an existing Session object that has a
        _TokenAuth object as its .auth attribute.

        It is probably easier to call LuminosoClient.connect() to handle
        the authentication for you.
        """
        self.session = session
        self.url = ensure_trailing_slash(url)
        # Don't warn this time; warning happened in connect()
        self.root_url = get_root_url(url, warn=False)

    def __repr__(self):
        return '<LuminosoClient for %s>' % self.url

    @classmethod
    def connect(cls, url=None, token_file=None, token=None):
        """
        Returns an object that makes requests to the API, authenticated
        with a saved or specified long-lived token, at URLs beginning with
        `url`.

        If no URL is specified, or if the specified URL is a path such as
        '/projects' without a scheme and domain, the client will default to
        https://analytics.luminoso.com/api/v5/.

        If neither token nor token_file are specified, the client will look
        for a token in $HOME/.luminoso/tokens.json. The file should contain
        a single json dictionary of the format
        `{'root_url': 'token', 'root_url2': 'token2', ...}`.
        """
        if url is None:
            url = '/'

        if url.startswith('http'):
            root_url = get_root_url(url)
        else:
            url = URL_BASE + '/' + url.lstrip('/')
            root_url = URL_BASE

        if token is None:
            token_file = token_file or get_token_filename()
            with open(token_file) as tf:
                token_dict = json.load(tf)
            try:
                token = token_dict[root_url]
            except KeyError:
                raise LuminosoAuthError('No token stored for %s' % root_url)

        session = requests.session()
        session.auth = _TokenAuth(token)
        return cls(session, url)

    @staticmethod
    def save_token(token, domain='analytics.luminoso.com', token_file=None):
        """
        Take a long-lived API token and store it to a local file.  Long-lived
        tokens can be retrieved through the UI.  Optional arguments are the
        domain for which the token is valid and the file in which to store the
        token.
        """
        token_file = token_file or get_token_filename()
        if os.path.exists(token_file):
            saved_tokens = json.load(open(token_file))
        else:
            saved_tokens = {}
        saved_tokens[domain] = token
        directory, filename = os.path.split(token_file)
        if directory and not os.path.exists(directory):
            os.makedirs(directory)
        with open(token_file, 'w') as f:
            json.dump(saved_tokens, f)

    def _request(self, req_type, url, **kwargs):
        """
        Make a request via the `requests` module. If the result has an HTTP
        error status, convert that to a Python exception.
        """
        logger.debug('%s %s' % (req_type, url))
        func = getattr(self.session, req_type)
        result = func(url, **kwargs)
        try:
            result.raise_for_status()
        except requests.HTTPError:
            error = result.text
            try:
                error = json.loads(error)
            except ValueError:
                pass
            if result.status_code in (401, 403):
                error_class = LuminosoAuthError
            elif result.status_code in (400, 404, 405):
                error_class = LuminosoClientError
            elif result.status_code >= 500:
                error_class = LuminosoServerError
            else:
                error_class = LuminosoError
            raise error_class(error)
        return result

    def _json_request(self, req_type, url, **kwargs):
        """
        Make a request of the specified type and expect a JSON object in
        response.

        If the result has an 'error' value, raise a LuminosoAPIError with
        its contents. Otherwise, return the contents of the 'result' value.
        """
        response = self._request(req_type, url, **kwargs)
        try:
            json_response = response.json()
        except ValueError:
            logger.error("Received response with no JSON: %s %s" %
                         (response, response.content))
            raise LuminosoError('Response body contained no JSON. '
                                'Perhaps you meant to use get_raw?')
        return json_response

    # Simple REST operations
    def get(self, path='', **params):
        """
        Make a GET request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to URL parameters.

        GET requests are requests that retrieve information without changing
        anything on the server.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('get', url, params=params)

    def post(self, path='', **params):
        """
        Make a POST request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the POST.

        POST requests are requests that cause a change on the server,
        especially those that ask to create and return an object of some kind.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('post', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def put(self, path='', **params):
        """
        Make a PUT request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the PUT.

        PUT requests are usually requests to *update* the object represented by
        this URL. Unlike POST requests, PUT requests can be safely duplicated.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('put', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def patch(self, path='', **params):
        """
        Make a PATCH request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the PATCH.

        PATCH requests are usually requests to make *small fixes* to the
        object represented by this URL.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('patch', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def delete(self, path='', **params):
        """
        Make a DELETE request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to URL parameters.

        DELETE requests ask to delete the object represented by this URL.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('delete', url, params=params)

    # Useful abstractions
    def change_path(self, path):
        """
        Change the client's path. Paths with leading slashes are appended to
        the root url, otherwise paths are set relative to the current path.
        Returns a copy of the client to avoid breaking old code.
        """
        if path.startswith('/'):
            self.url = self.root_url + path
        else:
            self.url = self.url + path
        return self

    def upload(self, path, docs, **params):
        """
        A deprecated alias for post(path, docs=docs), included only for
        backward compatibility.
        """
        logger.warning('The upload method is deprecated; use post instead.')
        return self.post(path, docs=docs)

    def wait_for_build(self, interval=5, path=None):
        """
        A convenience method designed to inform you when a project build has
        completed.  It polls the API every `interval` seconds until there is
        not a build running.  At that point, it returns the "last_build_info"
        field of the project record if the build succeeded, and raises a
        LuminosoError with the field as its message if the build failed.

        If a `path` is not specified, this method will assume that its URL is
        the URL for the project.  Otherwise, it will use the specified path
        (which should be "/projects/<project_id>/").
        """
        path = path or ''
        start = time.time()
        next_log = 0
        while True:
            response = self.get(path)['last_build_info']
            if not response:
                raise ValueError('This project is not building!')
            if response['stop_time']:
                if response['success']:
                    return response
                else:
                    raise LuminosoError(response)
            elapsed = time.time() - start
            if elapsed > next_log:
                logger.info('Still waiting (%d seconds elapsed).', next_log)
                next_log += 120
            time.sleep(interval)

    def get_raw(self, path, **params):
        """
        Get the raw text of a response.

        This is only generally useful for specific URLs, such as documentation.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._request('get', url, params=params).text

    def save_to_file(self, path, filename, **params):
        """
        Saves binary content to a file with name filename. filename should
        include the appropriate file extension, such as .xlsx or .txt, e.g.,
        filename = 'sample.xlsx'.

        Useful for downloading .xlsx files.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        content = self._request('get', url, params=params).content
        with open(filename, 'wb') as f:
            f.write(content)

class _TokenAuth(requests.auth.AuthBase):
    """
    An object designed to attach to a requests.Session object to handle
    Luminoso API authentication.
    """
    def __init__(self, token):
        self.token = token

    def __call__(self, request):
        request.headers['Authorization'] = 'Token ' + self.token
        return request

def get_token_filename():
    """
    Return the default filename for storing API tokens.
    """
    return os.path.join(os.path.expanduser('~'), '.luminoso', 'tokens.json')

def get_root_url(url, warn=True):
    """
    Get the "root URL" for a URL, as described in the LuminosoClient
    documentation.
    """
    parsed_url = urlparse(url)

    # Make sure it's a complete URL, not a relative one
    if not parsed_url.scheme:
        raise ValueError('Please supply a full URL, beginning with http:// '
                         'or https:// .')

    # Issue a warning if the path didn't already start with /api/v5
    root_url = '%s://%s/api/v5' % (parsed_url.scheme, parsed_url.netloc)
    if warn and not parsed_url.path.startswith('/api/v5'):
        logger.warning('Using %s as the root url' % root_url)
    return root_url

def ensure_trailing_slash(url):
    """
    Ensure that a URL has a slash at the end, which helps us avoid HTTP
    redirects.
    """
    return url.rstrip('/') + '/'

def jsonify_parameters(params):
    """
    When sent in an authorized REST request, only strings and integers can be
    transmitted accurately. Other types of data need to be encoded into JSON.
    """
    result = {}
    for param, value in params.items():
        if isinstance(value, (int, str)):
            result[param] = value
        else:
            result[param] = json.dumps(value)
    return result
<commit_msg>connect(): restore urlparse().netloc functionality for token lookups<commit_after>"""
Provides the LuminosoClient object, a wrapper for making
properly-authenticated requests to the Luminoso REST API.
"""
import json
import logging
import os
import requests
import time
from urllib.parse import urlparse

from .constants import URL_BASE
from .errors import (LuminosoError, LuminosoAuthError, LuminosoClientError,
                     LuminosoServerError)

logger = logging.getLogger(__name__)

class LuminosoClient(object):
    """
    A tool for making authenticated requests to the Luminoso API version 5.

    A LuminosoClient is a thin wrapper around the API documented at
    https://analytics.luminoso.com/api/v5/. As such, you interact with it by
    calling its methods that correspond to HTTP methods: `.get(url)`,
    `.post(url)`, `.put(url)`, `.patch(url)`, and `.delete(url)`.

    These URLs are relative to a 'base URL' for the LuminosoClient. For
    example, you can make requests for a specific project by creating a
    LuminosoClient for
    `https://analytics.luminoso.com/api/v5/projects/<project_id>`.

    Methods take parameters as keyword arguments, and encode them in the
    appropriate way for the request, which is described in the individual
    documentation for each method.

    The easiest way to create a LuminosoClient is using the
    `LuminosoClient.connect()` static method.

    In addition to the base URL, the LuminosoClient has a `root_url`,
    pointing to the root of the API, such as
    https://analytics.luminoso.com/api/v5. This is used, for example, as a
    starting point for the `change_path` method: when it gets a path starting
    with `/`, it will go back to the `root_url` instead of adding to the
    existing URL.
    """
    def __init__(self, session, url):
        """
        Create a LuminosoClient given an existing Session object that has a
        _TokenAuth object as its .auth attribute.

        It is probably easier to call LuminosoClient.connect() to handle
        the authentication for you.
        """
        self.session = session
        self.url = ensure_trailing_slash(url)
        # Don't warn this time; warning happened in connect()
        self.root_url = get_root_url(url, warn=False)

    def __repr__(self):
        return '<LuminosoClient for %s>' % self.url

    @classmethod
    def connect(cls, url=None, token_file=None, token=None):
        """
        Returns an object that makes requests to the API, authenticated
        with a saved or specified long-lived token, at URLs beginning with
        `url`.

        If no URL is specified, or if the specified URL is a path such as
        '/projects' without a scheme and domain, the client will default to
        https://analytics.luminoso.com/api/v5/.

        If neither token nor token_file are specified, the client will look
        for a token in $HOME/.luminoso/tokens.json. The file should contain
        a single json dictionary of the format
        `{'root_url': 'token', 'root_url2': 'token2', ...}`.
        """
        if url is None:
            url = '/'

        if url.startswith('http'):
            root_url = get_root_url(url)
        else:
            url = URL_BASE + '/' + url.lstrip('/')
            root_url = URL_BASE

        if token is None:
            token_file = token_file or get_token_filename()
            with open(token_file) as tf:
                token_dict = json.load(tf)
            try:
                token = token_dict[urlparse(root_url).netloc]
            except KeyError:
                raise LuminosoAuthError('No token stored for %s' % root_url)

        session = requests.session()
        session.auth = _TokenAuth(token)
        return cls(session, url)

    @staticmethod
    def save_token(token, domain='analytics.luminoso.com', token_file=None):
        """
        Take a long-lived API token and store it to a local file.  Long-lived
        tokens can be retrieved through the UI.  Optional arguments are the
        domain for which the token is valid and the file in which to store the
        token.
        """
        token_file = token_file or get_token_filename()
        if os.path.exists(token_file):
            saved_tokens = json.load(open(token_file))
        else:
            saved_tokens = {}
        saved_tokens[domain] = token
        directory, filename = os.path.split(token_file)
        if directory and not os.path.exists(directory):
            os.makedirs(directory)
        with open(token_file, 'w') as f:
            json.dump(saved_tokens, f)

    def _request(self, req_type, url, **kwargs):
        """
        Make a request via the `requests` module. If the result has an HTTP
        error status, convert that to a Python exception.
        """
        logger.debug('%s %s' % (req_type, url))
        func = getattr(self.session, req_type)
        result = func(url, **kwargs)
        try:
            result.raise_for_status()
        except requests.HTTPError:
            error = result.text
            try:
                error = json.loads(error)
            except ValueError:
                pass
            if result.status_code in (401, 403):
                error_class = LuminosoAuthError
            elif result.status_code in (400, 404, 405):
                error_class = LuminosoClientError
            elif result.status_code >= 500:
                error_class = LuminosoServerError
            else:
                error_class = LuminosoError
            raise error_class(error)
        return result

    def _json_request(self, req_type, url, **kwargs):
        """
        Make a request of the specified type and expect a JSON object in
        response.

        If the result has an 'error' value, raise a LuminosoAPIError with
        its contents. Otherwise, return the contents of the 'result' value.
        """
        response = self._request(req_type, url, **kwargs)
        try:
            json_response = response.json()
        except ValueError:
            logger.error("Received response with no JSON: %s %s" %
                         (response, response.content))
            raise LuminosoError('Response body contained no JSON. '
                                'Perhaps you meant to use get_raw?')
        return json_response

    # Simple REST operations
    def get(self, path='', **params):
        """
        Make a GET request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to URL parameters.

        GET requests are requests that retrieve information without changing
        anything on the server.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('get', url, params=params)

    def post(self, path='', **params):
        """
        Make a POST request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the POST.

        POST requests are requests that cause a change on the server,
        especially those that ask to create and return an object of some kind.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('post', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def put(self, path='', **params):
        """
        Make a PUT request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the PUT.

        PUT requests are usually requests to *update* the object represented by
        this URL. Unlike POST requests, PUT requests can be safely duplicated.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('put', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def patch(self, path='', **params):
        """
        Make a PATCH request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to form values, sent in the body
        of the PATCH.

        PATCH requests are usually requests to make *small fixes* to the
        object represented by this URL.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('patch', url, data=json.dumps(params),
                                  headers={'Content-Type': 'application/json'})

    def delete(self, path='', **params):
        """
        Make a DELETE request to the given path, and return the JSON-decoded
        result.

        Keyword parameters will be converted to URL parameters.

        DELETE requests ask to delete the object represented by this URL.
        """
        params = jsonify_parameters(params)
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._json_request('delete', url, params=params)

    # Useful abstractions
    def change_path(self, path):
        """
        Change the client's path. Paths with leading slashes are appended to
        the root url, otherwise paths are set relative to the current path.
        Returns a copy of the client to avoid breaking old code.
        """
        if path.startswith('/'):
            self.url = self.root_url + path
        else:
            self.url = self.url + path
        return self

    def upload(self, path, docs, **params):
        """
        A deprecated alias for post(path, docs=docs), included only for
        backward compatibility.
        """
        logger.warning('The upload method is deprecated; use post instead.')
        return self.post(path, docs=docs)

    def wait_for_build(self, interval=5, path=None):
        """
        A convenience method designed to inform you when a project build has
        completed.  It polls the API every `interval` seconds until there is
        not a build running.  At that point, it returns the "last_build_info"
        field of the project record if the build succeeded, and raises a
        LuminosoError with the field as its message if the build failed.

        If a `path` is not specified, this method will assume that its URL is
        the URL for the project.  Otherwise, it will use the specified path
        (which should be "/projects/<project_id>/").
        """
        path = path or ''
        start = time.time()
        next_log = 0
        while True:
            response = self.get(path)['last_build_info']
            if not response:
                raise ValueError('This project is not building!')
            if response['stop_time']:
                if response['success']:
                    return response
                else:
                    raise LuminosoError(response)
            elapsed = time.time() - start
            if elapsed > next_log:
                logger.info('Still waiting (%d seconds elapsed).', next_log)
                next_log += 120
            time.sleep(interval)

    def get_raw(self, path, **params):
        """
        Get the raw text of a response.

        This is only generally useful for specific URLs, such as documentation.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        return self._request('get', url, params=params).text

    def save_to_file(self, path, filename, **params):
        """
        Saves binary content to a file with name filename. filename should
        include the appropriate file extension, such as .xlsx or .txt, e.g.,
        filename = 'sample.xlsx'.

        Useful for downloading .xlsx files.
        """
        url = ensure_trailing_slash(self.url + path.lstrip('/'))
        content = self._request('get', url, params=params).content
        with open(filename, 'wb') as f:
            f.write(content)

class _TokenAuth(requests.auth.AuthBase):
    """
    An object designed to attach to a requests.Session object to handle
    Luminoso API authentication.
    """
    def __init__(self, token):
        self.token = token

    def __call__(self, request):
        request.headers['Authorization'] = 'Token ' + self.token
        return request

def get_token_filename():
    """
    Return the default filename for storing API tokens.
    """
    return os.path.join(os.path.expanduser('~'), '.luminoso', 'tokens.json')

def get_root_url(url, warn=True):
    """
    Get the "root URL" for a URL, as described in the LuminosoClient
    documentation.
    """
    parsed_url = urlparse(url)

    # Make sure it's a complete URL, not a relative one
    if not parsed_url.scheme:
        raise ValueError('Please supply a full URL, beginning with http:// '
                         'or https:// .')

    # Issue a warning if the path didn't already start with /api/v5
    root_url = '%s://%s/api/v5' % (parsed_url.scheme, parsed_url.netloc)
    if warn and not parsed_url.path.startswith('/api/v5'):
        logger.warning('Using %s as the root url' % root_url)
    return root_url

def ensure_trailing_slash(url):
    """
    Ensure that a URL has a slash at the end, which helps us avoid HTTP
    redirects.
    """
    return url.rstrip('/') + '/'

def jsonify_parameters(params):
    """
    When sent in an authorized REST request, only strings and integers can be
    transmitted accurately. Other types of data need to be encoded into JSON.
    """
    result = {}
    for param, value in params.items():
        if isinstance(value, (int, str)):
            result[param] = value
        else:
            result[param] = json.dumps(value)
    return result
<|endoftext|>


<commit_before>#
# Copyright (c) 2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import re
import types

from conary.build import policy

class TagLocale(policy.PackagePolicy):
    """
    NAME
    ====

    B{C{r.TagLocale()}} - Tags locale files based on path

    SYNOPSIS
    ========

    C{r.TagLocale([I{filterexp}] || [I{exceptions=filterexp}] || localeExp=localExpression)}

    DESCRIPTION
    ===========

    The C{r.TagLocale()} policy tags files according to their
    locale as C{locale(I{localename})}, choosing C{I{localname}}
    from the file path.  Exceptions to the policy are passed in via
    C{r.TagLocale(I{exceptions=filterexpression})} and are
    expressed in terms of file names.

    Additional expressions for tagging locale-specific files can be
    passed in with the C{localeExp} keyword argument.  This is a
    string or list or tuple of strings, where each string is a
    regular expression which has at least one non-optional match
    group which is the first match group, and which specifies the
    name of the locale.

    EXAMPLES
    ========

    C{r.TagLocale(exceptions='/')}

    Effectively disables the C{TagLocale} policy.

    C{r.TagLocale(exceptions='/usr/share/local/zzz/broken')}

    Do not tag the file C{/usr/share/local/zzz/broken} as C{locale(zzz)}.

    C{r.TagLocale(localeExp=r'/some/path/(?P<locale>.*).locale.messages')

    Tags the file /some/path/pt.locale.messages as locale(pt)
    and the file /some/path/en.locale.messages as locale(en).
    There must be exactly one named group named "C{locale}"; do
    do this specify the locale using C{(?P<locale>...)} in the
    regular expression.
    """

    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    # changing (or adding) locale tagging might be *the* reason for a
    # derived package
    processUnmodified = True
    invariantexceptions = [ '(%(debuglibdir)s|%(debugsrcdir)s)/', ]
    filetree = policy.PACKAGE
    localeExpressions = [
        re.compile(r'.*/locale(?:s)?/(?P<locale>[a-z]{2,3}(?:_[a-zA-Z]{2,3})?)(?:(?:\@|\.).*)?/.*/'),
        re.compile(r'.*/locale/man/(?P<locale>[a-z]{2,3})'),
        re.compile(r'/usr/share/man/(?P<locale>(?!man|cat|whatis|web)[a-zA-Z]{2,3}(?:_[a-z]{2,3})?).*/.*'),
    ]
    legalLocaleName = re.compile(r'^[a-zA-Z_]*$')

    def addLocaleExpression(self, exp):
        if '(?P<locale>' in exp:
            self.localeExpressions.append(re.compile(exp))
        else:
            self.warn('localeExp "%s" missing named group "locale";'
                      ' use "(?P<locale>" to introduce named group', exp)

    def updateArgs(self, *args, **keywords):
        localeExp = keywords.pop('localeExp', None)
        if isinstance(localeExp, (list, tuple, set)):
            for localeInstance in localeExp:
                self.addLocaleExpression(localeInstance)
        elif isinstance(localeExp, types.StringTypes):
            self.addLocaleExpression(localeExp)
        policy.PackagePolicy.updateArgs(self, *args, **keywords)

    def doFile(self, filename):
        for localeExp in self.localeExpressions:
            m = localeExp.match(filename)
            if m is not None:
                groupdict = m.groupdict()
                if 'locale' in groupdict:
                    locale = m.groupdict()['locale']
                    self._tagLocale(filename, locale)

    def _tagLocale(self, filename, locale):
        if self.legalLocaleName.match(locale) is None:
            self.warn('locale "%s" for file %s includes disallowed characters',
                      locale, filename)
            return
        componentMap = self.recipe.autopkg.componentMap
        if filename not in componentMap:
            return
        pkg = componentMap[filename]
        componentName = pkg.name.split(':')[1]
        if componentName not in set(('locale', 'doc', 'supdoc')):
            self.warn('locale "%s" for file %s in non-locale :%s component,'
                      ' not adding "locale(%s)" tag',
                      locale, filename, componentName, locale)
            return
        f = pkg.getFile(filename)
        f.tags.set('locale(%s)' %locale)
<commit_msg>The TagLocale policy is now capsule-aware (PFM-506)<commit_after>#
# Copyright (c) 2008 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import re
import types

from conary.build import policy

class TagLocale(policy.PackagePolicy):
    """
    NAME
    ====

    B{C{r.TagLocale()}} - Tags locale files based on path

    SYNOPSIS
    ========

    C{r.TagLocale([I{filterexp}] || [I{exceptions=filterexp}] || localeExp=localExpression)}

    DESCRIPTION
    ===========

    The C{r.TagLocale()} policy tags files according to their
    locale as C{locale(I{localename})}, choosing C{I{localname}}
    from the file path.  Exceptions to the policy are passed in via
    C{r.TagLocale(I{exceptions=filterexpression})} and are
    expressed in terms of file names.

    Additional expressions for tagging locale-specific files can be
    passed in with the C{localeExp} keyword argument.  This is a
    string or list or tuple of strings, where each string is a
    regular expression which has at least one non-optional match
    group which is the first match group, and which specifies the
    name of the locale.

    EXAMPLES
    ========

    C{r.TagLocale(exceptions='/')}

    Effectively disables the C{TagLocale} policy.

    C{r.TagLocale(exceptions='/usr/share/local/zzz/broken')}

    Do not tag the file C{/usr/share/local/zzz/broken} as C{locale(zzz)}.

    C{r.TagLocale(localeExp=r'/some/path/(?P<locale>.*).locale.messages')

    Tags the file /some/path/pt.locale.messages as locale(pt)
    and the file /some/path/en.locale.messages as locale(en).
    There must be exactly one named group named "C{locale}"; do
    do this specify the locale using C{(?P<locale>...)} in the
    regular expression.
    """

    requires = (
        ('PackageSpec', policy.REQUIRED_PRIOR),
    )
    # changing (or adding) locale tagging might be *the* reason for a
    # derived package
    processUnmodified = True
    invariantexceptions = [ '(%(debuglibdir)s|%(debugsrcdir)s)/', ]
    filetree = policy.PACKAGE
    localeExpressions = [
        re.compile(r'.*/locale(?:s)?/(?P<locale>[a-z]{2,3}(?:_[a-zA-Z]{2,3})?)(?:(?:\@|\.).*)?/.*/'),
        re.compile(r'.*/locale/man/(?P<locale>[a-z]{2,3})'),
        re.compile(r'/usr/share/man/(?P<locale>(?!man|cat|whatis|web)[a-zA-Z]{2,3}(?:_[a-z]{2,3})?).*/.*'),
    ]
    legalLocaleName = re.compile(r'^[a-zA-Z_]*$')

    def addLocaleExpression(self, exp):
        if '(?P<locale>' in exp:
            self.localeExpressions.append(re.compile(exp))
        else:
            self.warn('localeExp "%s" missing named group "locale";'
                      ' use "(?P<locale>" to introduce named group', exp)

    def updateArgs(self, *args, **keywords):
        localeExp = keywords.pop('localeExp', None)
        if isinstance(localeExp, (list, tuple, set)):
            for localeInstance in localeExp:
                self.addLocaleExpression(localeInstance)
        elif isinstance(localeExp, types.StringTypes):
            self.addLocaleExpression(localeExp)
        policy.PackagePolicy.updateArgs(self, *args, **keywords)

    def doFile(self, filename):
        if hasattr(self.recipe, '_getCapsulePathsForFile'):
            if self.recipe._getCapsulePathsForFile(filename):
                # even if a capsule were called :locale we still could
                # not do locale filtering on it
                return

        for localeExp in self.localeExpressions:
            m = localeExp.match(filename)
            if m is not None:
                groupdict = m.groupdict()
                if 'locale' in groupdict:
                    locale = m.groupdict()['locale']
                    self._tagLocale(filename, locale)

    def _tagLocale(self, filename, locale):
        if self.legalLocaleName.match(locale) is None:
            self.warn('locale "%s" for file %s includes disallowed characters',
                      locale, filename)
            return
        componentMap = self.recipe.autopkg.componentMap
        if filename not in componentMap:
            return
        pkg = componentMap[filename]
        componentName = pkg.name.split(':')[1]
        if componentName not in set(('locale', 'doc', 'supdoc')):
            self.warn('locale "%s" for file %s in non-locale :%s component,'
                      ' not adding "locale(%s)" tag',
                      locale, filename, componentName, locale)
            return
        f = pkg.getFile(filename)
        f.tags.set('locale(%s)' %locale)
<|endoftext|>


<commit_before>import sys
import argparse
import dbus
from subprocess import call
from dbus.mainloop.glib import DBusGMainLoop
from gi.repository import GLib

from lyricify import lyrics
from lyricify.view import LyricifyUI

class SpotifyCtl():

    def __init__(self, ui=None):
        self.connect_to_spotify()
        self.set_properties()
        self.ui = ui

        if self.ui:
            self.win = LyricifyUI()
            self.win.connect('delete-event', self.on_exit)

        self.lyricify()

        GLib.MainLoop().run()

    def connect_to_spotify(self):
        try:
            dbus_loop = DBusGMainLoop(set_as_default=True)
            self.spotify_bus = dbus.SessionBus(dbus_loop).get_object(
                "org.mpris.MediaPlayer2.spotify",
                "/org/mpris/MediaPlayer2")
            self.spotify_bus.connect_to_signal("PropertiesChanged",
                                               self.on_change)
        except:
            print("\nCould not connect to Spotify, exiting...")
            sys.exit()

    def set_properties(self):
        metadata = self.spotify_bus.Get(
            "org.mpris.MediaPlayer2.Player",
            "Metadata",
            dbus_interface="org.freedesktop.DBus.Properties")

        self.song = metadata['xesam:title']
        self.artist = metadata['xesam:artist'][0]

    def on_change(self, interface, changed_properties, invalidated_properties):
        metadata = changed_properties.get("Metadata", {})
        if(self.song == metadata['xesam:title'] and
           self.artist == metadata['xesam:artist'][0]):
            return

        self.song = metadata['xesam:title']
        self.artist = metadata['xesam:artist'][0]

        self.lyricify()

    def on_exit(self, window, event):
        self.win.hide_on_delete()
        self.ui = False

        return True

    def lyricify(self):
        song_url = lyrics.create_song_url(self.artist, self.song)
        try:
            song_lyrics = lyrics.get_lyrics(song_url)
            if self.ui:
                lyrics.get_album_image(song_url)
        except Exception as e:
            print(e)
            print("\nUnable to find lyrics/album art for the song, sorry!")
            return

        if self.ui:
            self.win.update_view(self.artist, self.song, song_lyrics)
        else:
            call("clear")
            print("{} - {}\n\n{}".format(self.artist, self.song, song_lyrics))

def main():
    parser = argparse.ArgumentParser(description='Get lyrics for Spotify song')
    parser.add_argument('-ui', action='store_true',
                        help='Open a GUI for lyrics')
    args = parser.parse_args()

    try:
        SpotifyCtl(ui=args.ui)
    except KeyboardInterrupt:
        sys.exit()

if __name__ == "__main__":
        main()
<commit_msg>Clean up<commit_after>import sys
import argparse
import dbus
from subprocess import call
from dbus.mainloop.glib import DBusGMainLoop
from gi.repository import GLib

from lyricify import lyrics
from lyricify.view import LyricifyUI

class SpotifyCtl():

    def __init__(self, ui=None):
        self.connect_to_spotify()
        self.set_properties()
        self.ui = ui

        if self.ui:
            self.win = LyricifyUI()
            self.win.connect('delete-event', self.on_exit)

        self.lyricify()

        GLib.MainLoop().run()

    def connect_to_spotify(self):
        try:
            dbus_loop = DBusGMainLoop(set_as_default=True)
            self.spotify_bus = dbus.SessionBus(dbus_loop).get_object(
                "org.mpris.MediaPlayer2.spotify",
                "/org/mpris/MediaPlayer2")
            self.spotify_bus.connect_to_signal("PropertiesChanged",
                                               self.on_change)
        except:
            print("\nCould not connect to Spotify, exiting...")
            sys.exit()

    def set_properties(self):
        metadata = self.spotify_bus.Get(
            "org.mpris.MediaPlayer2.Player",
            "Metadata",
            dbus_interface="org.freedesktop.DBus.Properties")

        self.song = metadata['xesam:title']
        self.artist = metadata['xesam:artist'][0]

    def on_change(self, interface, changed_properties, invalidated_properties):
        metadata = changed_properties.get("Metadata", {})
        if(self.song == metadata['xesam:title'] and
           self.artist == metadata['xesam:artist'][0]):
            return

        self.song = metadata['xesam:title']
        self.artist = metadata['xesam:artist'][0]

        self.lyricify()

    def on_exit(self, window, event):
        self.win.hide_on_delete()
        self.ui = False

        return True

    def lyricify(self):
        song_url = lyrics.create_song_url(self.artist, self.song)
        try:
            song_lyrics = lyrics.get_lyrics(song_url)
            if self.ui:
                lyrics.get_album_image(song_url)
        except Exception:
            print("\nCannot find lyrics/album art for the song, sorry!")
            return

        if self.ui:
            self.win.update_view(self.artist, self.song, song_lyrics)
        else:
            call("clear")
            print("{} - {}\n\n{}".format(self.artist, self.song, song_lyrics))

def main():
    parser = argparse.ArgumentParser(description='Get lyrics for Spotify song')
    parser.add_argument('-ui', action='store_true',
                        help='Open a GUI for lyrics')
    args = parser.parse_args()

    try:
        SpotifyCtl(ui=args.ui)
    except KeyboardInterrupt:
        sys.exit()

if __name__ == "__main__":
        main()
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import absolute_import

import flask

import auth
import config
import model
import util

from main import app

microsoft_config = dict(
  access_token_method='POST',
  access_token_url='https://login.microsoftonline.com/common/oauth2/v2.0/token',
  api_base_url='https://graph.microsoft.com/v1.0/',
  authorize_url='https://login.microsoftonline.com/common/oauth2/v2.0/authorize',
  client_id=config.CONFIG_DB.microsoft_client_id,
  client_secret=config.CONFIG_DB.microsoft_client_secret,
  client_kwargs={'scope': 'user.read'},
)

microsoft = auth.create_oauth_app(microsoft_config, 'microsoft')

@app.route('/api/auth/callback/microsoft/')
def microsoft_authorized():
  id_token = microsoft.authorize_access_token()
  if id_token is None:
    flask.flash('You denied the request to sign in.')
    return flask.redirect(util.get_next_url())
  me = microsoft.get('me')
  user_db = retrieve_user_from_microsoft(me.json())
  return auth.signin_user_db(user_db)

@app.route('/signin/microsoft/')
def signin_microsoft():
  return auth.signin_oauth(microsoft)

def retrieve_user_from_microsoft(response):
  auth_id = 'microsoft_%s' % response['id']
  user_db = model.User.get_by('auth_ids', auth_id)
  if user_db:
    return user_db
  email = response['userPrincipalName']
  name = response.get('displayName', '')
  return auth.create_user_db(
    auth_id=auth_id,
    name=name,
    username=name or email,
    email=email,
    verified=bool(email),
  )
<commit_msg>Handle cancel of access rights in Microsoft OAuth2 sign-up (#1452)<commit_after># coding: utf-8

from __future__ import absolute_import

import flask

import auth
import config
import model
import util

from main import app

microsoft_config = dict(
  access_token_method='POST',
  access_token_url='https://login.microsoftonline.com/common/oauth2/v2.0/token',
  api_base_url='https://graph.microsoft.com/v1.0/',
  authorize_url='https://login.microsoftonline.com/common/oauth2/v2.0/authorize',
  client_id=config.CONFIG_DB.microsoft_client_id,
  client_secret=config.CONFIG_DB.microsoft_client_secret,
  client_kwargs={'scope': 'user.read'},
)

microsoft = auth.create_oauth_app(microsoft_config, 'microsoft')

@app.route('/api/auth/callback/microsoft/')
def microsoft_authorized():
  err = flask.request.args.get('error')
  if err in ['access_denied']:
    flask.flash('You denied the request to sign in.')
    return flask.redirect(util.get_next_url())
  id_token = microsoft.authorize_access_token()
  if id_token is None:
    flask.flash('You denied the request to sign in.')
    return flask.redirect(util.get_next_url())
  me = microsoft.get('me')
  user_db = retrieve_user_from_microsoft(me.json())
  return auth.signin_user_db(user_db)

@app.route('/signin/microsoft/')
def signin_microsoft():
  return auth.signin_oauth(microsoft)

def retrieve_user_from_microsoft(response):
  auth_id = 'microsoft_%s' % response['id']
  user_db = model.User.get_by('auth_ids', auth_id)
  if user_db:
    return user_db
  email = response['userPrincipalName']
  name = response.get('displayName', '')
  return auth.create_user_db(
    auth_id=auth_id,
    name=name,
    username=name or email,
    email=email,
    verified=bool(email),
  )
<|endoftext|>


<commit_before># vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import os

from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response
from django.template import RequestContext
from django import forms
from django.db import IntegrityError

from pyxform.errors import PyXFormError
from odk_viewer.models import DataDictionary

class QuickConverter(forms.Form):
    xls_file = forms.FileField(label="XLS File")

    def publish(self, user):
        if self.is_valid():
            return DataDictionary.objects.create(
                user=user,
                xls=self.cleaned_data['xls_file']
                )

@login_required
def dashboard(request):
    context = RequestContext(request)
    context.form = QuickConverter()
    context.odk_url = request.build_absolute_uri("/%s" % request.user.username)

    if request.method == 'POST':
        try:
            form = QuickConverter(request.POST, request.FILES)
            survey = form.publish(request.user).survey
            context.message = {
                'type': 'success',
                'text': 'Successfully published %s.' % survey.id_string,
                }
        except PyXFormError as e:
            context.message = {
                'type': 'error',
                'text': unicode(e),
                }
        except IntegrityError as e:
            context.message = {
                'type': 'error',
                'text': 'Form with this id already exists.',
                }
    return render_to_response("dashboard.html", context_instance=context)

def tutorial(request):
    context = RequestContext(request)
    context.template = 'tutorial.html'
    username = request.user.username if request.user.username else \
        'your-user-name'
    context.odk_url = request.build_absolute_uri("/%s" % username)
    return render_to_response('base.html', context_instance=context)

from google_doc import GoogleDoc

def syntax(request):
    url = 'https://docs.google.com/document/pub?id=1Dze4IZGr0IoIFuFAI_ohKR5mYUt4IAn5Y-uCJmnv1FQ'
    doc = GoogleDoc(url)
    context = RequestContext(request)
    context.content = doc.to_html()
    return render_to_response('base.html', context_instance=context)

def gallery(request):
    """
    Return a list of urls for all the shared xls files. This could be
    made a lot prettier.
    """
    context = RequestContext(request)
    context.shared_forms = DataDictionary.objects.filter(shared=True)
    return render_to_response('gallery.html', context_instance=context)
<commit_msg>support page fix<commit_after># vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import os

from django.contrib.auth.decorators import login_required
from django.shortcuts import render_to_response
from django.template import RequestContext
from django import forms
from django.db import IntegrityError

from pyxform.errors import PyXFormError
from odk_viewer.models import DataDictionary

class QuickConverter(forms.Form):
    xls_file = forms.FileField(label="XLS File")

    def publish(self, user):
        if self.is_valid():
            return DataDictionary.objects.create(
                user=user,
                xls=self.cleaned_data['xls_file']
                )

@login_required
def dashboard(request):
    context = RequestContext(request)
    context.form = QuickConverter()
    context.odk_url = request.build_absolute_uri("/%s" % request.user.username)

    if request.method == 'POST':
        try:
            form = QuickConverter(request.POST, request.FILES)
            survey = form.publish(request.user).survey
            context.message = {
                'type': 'success',
                'text': 'Successfully published %s.' % survey.id_string,
                }
        except PyXFormError as e:
            context.message = {
                'type': 'error',
                'text': unicode(e),
                }
        except IntegrityError as e:
            context.message = {
                'type': 'error',
                'text': 'Form with this id already exists.',
                }
    return render_to_response("dashboard.html", context_instance=context)

def public_profile(request):
    context = RequestContext(request)
    context.form = QuickConverter()
    context.odk_url = request.build_absolute_uri("/%s" % request.user.username)

    return render_to_response("public_profile.html", context_instance=context)

def tutorial(request):
    context = RequestContext(request)
    context.template = 'tutorial.html'
    username = request.user.username if request.user.username else \
        'your-user-name'
    context.odk_url = request.build_absolute_uri("/%s" % username)
    return render_to_response('base.html', context_instance=context)

def support(request):
    context = RequestContext(request)
    context.template = 'support.html'
    return render_to_response('base.html', context_instance=context)

from google_doc import GoogleDoc

def syntax(request):
    url = 'https://docs.google.com/document/pub?id=1Dze4IZGr0IoIFuFAI_ohKR5mYUt4IAn5Y-uCJmnv1FQ'
    doc = GoogleDoc(url)
    context = RequestContext(request)
    context.content = doc.to_html()
    return render_to_response('base.html', context_instance=context)

def gallery(request):
    """
    Return a list of urls for all the shared xls files. This could be
    made a lot prettier.
    """
    context = RequestContext(request)
    context.shared_forms = DataDictionary.objects.filter(shared=True)
    return render_to_response('gallery.html', context_instance=context)
<|endoftext|>