<commit_before># -*- coding: UTF-8 -*-
# Copyright 2009-2018 Luc Saffre
# License: BSD (see file COPYING for details)

# python setup.py test -s tests.test_misc.PackagesTests

from __future__ import unicode_literals
import sys
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3

SETUP_INFO = dict(
    name='lino_book',
    version='18.1.0',
    install_requires=[
        'lino', 'selenium',
        'django-iban', 'metafone', 'channels',
        'djangorestframework', 'bleach', 'radicale==1.1.2',
        'icalendar',
        'vobject',
        'social-auth-app-django',
        'lino_cosi',
        'lino_noi',
        'lino_voga',
        # 'lino_welfare',
        'requests_mock',
        'lino_care',
        'lino_vilma',
        'lino_avanti',
        'lino_tera',
        'lino_amici',
        'commondata', 'commondata.be', 'commondata.ee', 'commondata.eg'],
    dependency_links=[
        'git+https://github.com/cylonoven/django-mailbox.git#egg=django_mailbox'],
    # tests_require=['requests_mock'],

    description="The documentation for Lino",
    license='BSD License',
    include_package_data=True,
    zip_safe=False,
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    #~ test_suite = 'lino_book.projects',
    test_suite='tests',
    classifiers="""\
  Programming Language :: Python
  Programming Language :: Python :: 2
  Programming Language :: Python :: 3
  Development Status :: 5 - Production/Stable
  Environment :: Web Environment
  Framework :: Django
  Intended Audience :: Developers
  Intended Audience :: System Administrators
  License :: OSI Approved :: BSD License
  Natural Language :: English
  Natural Language :: French
  Natural Language :: German
  Operating System :: OS Independent
  Topic :: Database :: Front-Ends
  Topic :: Home Automation
  Topic :: Office/Business
  Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

# if PY2:
#     SETUP_INFO['install_requires'].append('reportlab<2.7')
# else:
#     SETUP_INFO['install_requires'].append('reportlab')

SETUP_INFO.update(long_description="""

.. image:: https://readthedocs.org/projects/lino/badge/?version=latest
   :target: http://lino.readthedocs.io/en/latest/?badge=latest
.. image:: https://coveralls.io/repos/github/lino-framework/book/badge.svg?branch=master
   :target: https://coveralls.io/github/lino-framework/book?branch=master
.. image:: https://travis-ci.org/lino-framework/book.svg?branch=stable
   :target: https://travis-ci.org/lino-framework/book?branch=stable
.. image:: https://img.shields.io/pypi/v/lino.svg
   :target: https://pypi.python.org/pypi/lino/
.. image:: https://img.shields.io/pypi/l/lino.svg
   :target: https://pypi.python.org/pypi/lino/

This is the code repository that contains (1) the Sphinx source files
of the Lino Book, (2) the ``lino_book`` Python package and (3) a test
suite with doctest-based tests for the Lino framework.

The **Lino Book** is the central documentation tree of the Lino
framework.  It is visible on `www.lino-framework.org
<http://www.lino-framework.org>`__ and on `lino.readthedocs.io
<http://lino.readthedocs.io>`__.

The ``lino_book`` Python package is a collection of small example Lino
applications used for educational and testing purposes.

The code repositories for the ``lino`` and ``lino_xl`` Python packages
have no documentation tree on their own and almost no unit tests, they
are tested and documented here.

""")

SETUP_INFO.update(packages=[str(n) for n in """
lino_book
lino_book.projects
lino_book.projects.babel_tutorial
lino_book.projects.babel_tutorial.fixtures
lino_book.projects.dumps
lino_book.projects.dumps.fixtures
lino_book.projects.dumps.settings
lino_book.projects.belref
lino_book.projects.belref.fixtures
lino_book.projects.belref.settings
lino_book.projects.docs
lino_book.projects.docs.settings
lino_book.projects.estref
lino_book.projects.estref.settings
lino_book.projects.estref.tests
lino_book.projects.events
lino_book.projects.polls
lino_book.projects.polls.polls
lino_book.projects.polls.polls.fixtures
lino_book.projects.polls.mysite
lino_book.projects.polly
lino_book.projects.polly.settings
lino_book.projects.polly.tests
lino_book.projects.20090714
lino_book.projects.20090717
lino_book.projects.20100126
lino_book.projects.20100127
lino_book.projects.20100206
lino_book.projects.20100212
lino_book.projects.20121124
lino_book.projects.chooser
lino_book.projects.example
lino_book.projects.properties
lino_book.projects.quantityfield
lino_book.projects.cms
lino_book.projects.cms.settings
lino_book.projects.crl
lino_book.projects.crl.fixtures
lino_book.projects.eric
lino_book.projects.eric.settings
lino_book.projects.eric.settings.fixtures
lino_book.projects.eric.tests
lino_book.projects.homeworkschool
lino_book.projects.homeworkschool.fixtures
lino_book.projects.homeworkschool.settings
lino_book.projects.i18n
lino_book.projects.igen
lino_book.projects.igen.tests
lino_book.projects.max
lino_book.projects.max.settings
lino_book.projects.chatter
lino_book.projects.chatter.settings
lino_book.projects.chatter.tests
lino_book.projects.min1
lino_book.projects.min1.settings
lino_book.projects.min2
lino_book.projects.min2.settings
lino_book.projects.min2.tests
lino_book.projects.min3
lino_book.projects.min3.lib
lino_book.projects.min3.lib.contacts
lino_book.projects.min3.lib.contacts.fixtures
lino_book.projects.min3.settings
lino_book.projects.min3.tests
lino_book.projects.min9
lino_book.projects.min9.modlib
lino_book.projects.min9.modlib.contacts
lino_book.projects.min9.modlib.contacts.fixtures
lino_book.projects.min9.modlib.contacts.management
lino_book.projects.min9.modlib.contacts.management.commands
lino_book.projects.min9.settings
lino_book.projects.min9.tests
lino_book.projects.apc
lino_book.projects.apc.settings
lino_book.projects.apc.tests
lino_book.projects.adg
lino_book.projects.adg.settings
lino_book.projects.adg.settings.fixtures
lino_book.projects.adg.tests
lino_book.projects.cosi_ee
lino_book.projects.cosi_ee.settings
lino_book.projects.pierre
lino_book.projects.pierre.settings
lino_book.projects.bs3
lino_book.projects.bs3.settings
lino_book.projects.bs3.tests
lino_book.projects.team
lino_book.projects.team.tests
lino_book.projects.team.settings
lino_book.projects.team.settings.fixtures
lino_book.projects.liina
lino_book.projects.liina.tests
lino_book.projects.liina.settings
lino_book.projects.liina.settings.fixtures
lino_book.projects.lydia
lino_book.projects.lydia.tests
lino_book.projects.lydia.settings
lino_book.projects.lydia.settings.fixtures
lino_book.projects.public
lino_book.projects.anna
lino_book.projects.anna.settings
lino_book.projects.anna.tests
lino_book.projects.anna.lib
lino_book.projects.anna.lib.tickets
lino_book.projects.public.settings
lino_book.projects.public.tests
lino_book.projects.lets1
lino_book.projects.lets1.lets
lino_book.projects.lets1.fixtures
lino_book.projects.lets2
lino_book.projects.lets2.lets
lino_book.projects.lets2.fixtures
lino_book.projects.mti
lino_book.projects.mti.app
lino_book.projects.mti.fixtures
lino_book.projects.mti.tests
lino_book.projects.nomti
lino_book.projects.nomti.app
lino_book.projects.nomti.fixtures
lino_book.projects.watch
lino_book.projects.watch.entries
lino_book.projects.watch.fixtures
lino_book.projects.watch.tests
lino_book.projects.mldbc
lino_book.projects.mldbc.fixtures
lino_book.projects.actions
lino_book.projects.actors
lino_book.projects.actors.fixtures
lino_book.projects.addrloc
lino_book.projects.addrloc.fixtures
lino_book.projects.myroles
lino_book.projects.auto_create
lino_book.projects.de_BE
lino_book.projects.de_BE.fixtures
lino_book.projects.diamond
lino_book.projects.diamond.main
lino_book.projects.diamond2
lino_book.projects.diamond2.main
lino_book.projects.float2decimal
lino_book.projects.float2decimal.lib
lino_book.projects.float2decimal.lib.float2decimal
lino_book.projects.integer_pk
lino_book.projects.integer_pk.lib
lino_book.projects.integer_pk.lib.integer_pk
lino_book.projects.human
lino_book.projects.gfktest
lino_book.projects.gfktest.settings
lino_book.projects.gfktest.lib
lino_book.projects.gfktest.lib.gfktest
lino_book.projects.pisa
lino_book.projects.sendchanges
lino_book.projects.tables
lino_book.projects.tables.fixtures
lino_book.projects.ui5
lino_book.projects.ui5.settings
lino_book.projects.ui5.tests
lino_book.projects.vtables
lino_book.projects.edmund
lino_book.projects.edmund.settings
lino_book.projects.edmund.settings.fixtures
lino_book.projects.roger
lino_book.projects.roger.settings
lino_book.projects.roger.settings.fixtures
lino_book.projects.roger.tests
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**',        'ignore', None),
        ('**/cache/**',          'ignore', None),
        ('**.py',                'python', None),
        ('**/linoweb.js',        'jinja2', None),
        #~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        #~ ('**/templates/**.txt',  'genshi', {
        #~ 'template_class': 'genshi.template:TextTemplate'
        #~ })
    ],
})

<commit_msg>http://luc.lino-framework.org/blog/2018/0131.html<commit_after># -*- coding: UTF-8 -*-
# Copyright 2009-2018 Luc Saffre
# License: BSD (see file COPYING for details)

# python setup.py test -s tests.test_misc.PackagesTests

from __future__ import unicode_literals
import sys
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3

SETUP_INFO = dict(
    name='lino_book',
    version='18.1.0',
    install_requires=[
        'lino', 'selenium',
        'django-iban', 'metafone', 'channels',
        'djangorestframework', 'bleach', 'radicale==1.1.2',
        'icalendar',
        'vobject',
        'social-auth-app-django',
        'lino_cosi',
        'lino_noi',
        'lino_voga',
        # 'lino_welfare',
        'requests_mock',
        'lino_care',
        'lino_vilma',
        'lino_avanti',
        'lino_tera',
        'lino_amici',
        'commondata', 'commondata.be', 'commondata.ee', 'commondata.eg'],
    dependency_links=[
        'git+https://github.com/cylonoven/django-mailbox.git#egg=django_mailbox'],
    # tests_require=['requests_mock'],

    description="The documentation for Lino",
    license='BSD License',
    include_package_data=True,
    zip_safe=False,
    author='Luc Saffre',
    author_email='luc@lino-framework.org',
    url="http://www.lino-framework.org",
    #~ test_suite = 'lino_book.projects',
    test_suite='tests',
    classifiers="""\
  Programming Language :: Python
  Programming Language :: Python :: 2
  Programming Language :: Python :: 3
  Development Status :: 5 - Production/Stable
  Environment :: Web Environment
  Framework :: Django
  Intended Audience :: Developers
  Intended Audience :: System Administrators
  License :: OSI Approved :: BSD License
  Natural Language :: English
  Natural Language :: French
  Natural Language :: German
  Operating System :: OS Independent
  Topic :: Database :: Front-Ends
  Topic :: Home Automation
  Topic :: Office/Business
  Topic :: Software Development :: Libraries :: Application Frameworks""".splitlines())

# if PY2:
#     SETUP_INFO['install_requires'].append('reportlab<2.7')
# else:
#     SETUP_INFO['install_requires'].append('reportlab')

SETUP_INFO.update(long_description="""

.. image:: https://readthedocs.org/projects/lino/badge/?version=latest
   :target: http://lino.readthedocs.io/en/latest/?badge=latest
.. image:: https://coveralls.io/repos/github/lino-framework/book/badge.svg?branch=master
   :target: https://coveralls.io/github/lino-framework/book?branch=master
.. image:: https://travis-ci.org/lino-framework/book.svg?branch=stable
   :target: https://travis-ci.org/lino-framework/book?branch=stable
.. image:: https://img.shields.io/pypi/v/lino.svg
   :target: https://pypi.python.org/pypi/lino/
.. image:: https://img.shields.io/pypi/l/lino.svg
   :target: https://pypi.python.org/pypi/lino/

This is the code repository that contains (1) the Sphinx source files
of the Lino Book, (2) the ``lino_book`` Python package and (3) a test
suite with doctest-based tests for the Lino framework.

The **Lino Book** is the central documentation tree of the Lino
framework.  It is visible on `www.lino-framework.org
<http://www.lino-framework.org>`__ and on `lino.readthedocs.io
<http://lino.readthedocs.io>`__.

The ``lino_book`` Python package is a collection of small example Lino
applications used for educational and testing purposes.

The code repositories for the ``lino`` and ``lino_xl`` Python packages
have no documentation tree on their own and almost no unit tests, they
are tested and documented here.

""")

SETUP_INFO.update(packages=[str(n) for n in """
lino_book
lino_book.projects
lino_book.projects.babel_tutorial
lino_book.projects.babel_tutorial.fixtures
lino_book.projects.dumps
lino_book.projects.dumps.fixtures
lino_book.projects.dumps.settings
lino_book.projects.belref
lino_book.projects.belref.fixtures
lino_book.projects.belref.settings
lino_book.projects.combo
lino_book.projects.combo.fixtures
lino_book.projects.docs
lino_book.projects.docs.settings
lino_book.projects.estref
lino_book.projects.estref.settings
lino_book.projects.estref.tests
lino_book.projects.events
lino_book.projects.polls
lino_book.projects.polls.polls
lino_book.projects.polls.polls.fixtures
lino_book.projects.polls.mysite
lino_book.projects.polly
lino_book.projects.polly.settings
lino_book.projects.polly.tests
lino_book.projects.20090714
lino_book.projects.20090717
lino_book.projects.20100126
lino_book.projects.20100127
lino_book.projects.20100206
lino_book.projects.20100212
lino_book.projects.20121124
lino_book.projects.chooser
lino_book.projects.example
lino_book.projects.properties
lino_book.projects.quantityfield
lino_book.projects.cms
lino_book.projects.cms.settings
lino_book.projects.crl
lino_book.projects.crl.fixtures
lino_book.projects.eric
lino_book.projects.eric.settings
lino_book.projects.eric.settings.fixtures
lino_book.projects.eric.tests
lino_book.projects.homeworkschool
lino_book.projects.homeworkschool.fixtures
lino_book.projects.homeworkschool.settings
lino_book.projects.i18n
lino_book.projects.igen
lino_book.projects.igen.tests
lino_book.projects.max
lino_book.projects.max.settings
lino_book.projects.chatter
lino_book.projects.chatter.settings
lino_book.projects.chatter.tests
lino_book.projects.min1
lino_book.projects.min1.settings
lino_book.projects.min2
lino_book.projects.min2.settings
lino_book.projects.min2.tests
lino_book.projects.min3
lino_book.projects.min3.lib
lino_book.projects.min3.lib.contacts
lino_book.projects.min3.lib.contacts.fixtures
lino_book.projects.min3.settings
lino_book.projects.min3.tests
lino_book.projects.min9
lino_book.projects.min9.modlib
lino_book.projects.min9.modlib.contacts
lino_book.projects.min9.modlib.contacts.fixtures
lino_book.projects.min9.modlib.contacts.management
lino_book.projects.min9.modlib.contacts.management.commands
lino_book.projects.min9.settings
lino_book.projects.min9.tests
lino_book.projects.apc
lino_book.projects.apc.settings
lino_book.projects.apc.tests
lino_book.projects.adg
lino_book.projects.adg.settings
lino_book.projects.adg.settings.fixtures
lino_book.projects.adg.tests
lino_book.projects.cosi_ee
lino_book.projects.cosi_ee.settings
lino_book.projects.pierre
lino_book.projects.pierre.settings
lino_book.projects.bs3
lino_book.projects.bs3.settings
lino_book.projects.bs3.tests
lino_book.projects.team
lino_book.projects.team.tests
lino_book.projects.team.settings
lino_book.projects.team.settings.fixtures
lino_book.projects.liina
lino_book.projects.liina.tests
lino_book.projects.liina.settings
lino_book.projects.liina.settings.fixtures
lino_book.projects.lydia
lino_book.projects.lydia.tests
lino_book.projects.lydia.settings
lino_book.projects.lydia.settings.fixtures
lino_book.projects.public
lino_book.projects.anna
lino_book.projects.anna.settings
lino_book.projects.anna.tests
lino_book.projects.anna.lib
lino_book.projects.anna.lib.tickets
lino_book.projects.public.settings
lino_book.projects.public.tests
lino_book.projects.lets1
lino_book.projects.lets1.lets
lino_book.projects.lets1.fixtures
lino_book.projects.lets2
lino_book.projects.lets2.lets
lino_book.projects.lets2.fixtures
lino_book.projects.mti
lino_book.projects.mti.app
lino_book.projects.mti.fixtures
lino_book.projects.mti.tests
lino_book.projects.nomti
lino_book.projects.nomti.app
lino_book.projects.nomti.fixtures
lino_book.projects.watch
lino_book.projects.watch.entries
lino_book.projects.watch.fixtures
lino_book.projects.watch.tests
lino_book.projects.mldbc
lino_book.projects.mldbc.fixtures
lino_book.projects.actions
lino_book.projects.actors
lino_book.projects.actors.fixtures
lino_book.projects.addrloc
lino_book.projects.addrloc.fixtures
lino_book.projects.myroles
lino_book.projects.auto_create
lino_book.projects.de_BE
lino_book.projects.de_BE.fixtures
lino_book.projects.diamond
lino_book.projects.diamond.main
lino_book.projects.diamond2
lino_book.projects.diamond2.main
lino_book.projects.float2decimal
lino_book.projects.float2decimal.lib
lino_book.projects.float2decimal.lib.float2decimal
lino_book.projects.integer_pk
lino_book.projects.integer_pk.lib
lino_book.projects.integer_pk.lib.integer_pk
lino_book.projects.human
lino_book.projects.gfktest
lino_book.projects.gfktest.settings
lino_book.projects.gfktest.lib
lino_book.projects.gfktest.lib.gfktest
lino_book.projects.pisa
lino_book.projects.sendchanges
lino_book.projects.tables
lino_book.projects.tables.fixtures
lino_book.projects.ui5
lino_book.projects.ui5.settings
lino_book.projects.ui5.tests
lino_book.projects.vtables
lino_book.projects.edmund
lino_book.projects.edmund.settings
lino_book.projects.edmund.settings.fixtures
lino_book.projects.roger
lino_book.projects.roger.settings
lino_book.projects.roger.settings.fixtures
lino_book.projects.roger.tests
""".splitlines() if n])

SETUP_INFO.update(message_extractors={
    'lino': [
        ('**/sandbox/**',        'ignore', None),
        ('**/cache/**',          'ignore', None),
        ('**.py',                'python', None),
        ('**/linoweb.js',        'jinja2', None),
        #~ ('**.js',                'javascript', None),
        ('**/config/**.html', 'jinja2', None),
        #~ ('**/templates/**.txt',  'genshi', {
        #~ 'template_class': 'genshi.template:TextTemplate'
        #~ })
    ],
})

<|endoftext|>


<commit_before>from .base import Base
from .. import errors

class DerivedBase(Base):
    """
    The DerivedBase class holds information about an object who belongs to another object
    (for example, a disk belongs to a linode).  These objects have their own endpoints,
    but they are below another object in the hierarchy (i.e. /linodes/lnde_123/disks/disk_123)
    """
    derived_url_path = '' #override in child classes
    parent_id_name = 'parent_id' #override in child classes

    def __init__(self, client, id, parent_id, json={}):
        Base.__init__(self, client, id, json=json)

        self._set(type(self).parent_id_name, parent_id)

    @classmethod
    def _api_get_derived(cls, parent, client):
        base_url = "{}/{}".format(type(parent).api_endpoint, cls.derived_url_path)

        return client._get_objects(base_url, cls, model=parent, parent_id=parent.id)
<commit_msg>objects/ import cleanup<commit_after>from .base import Base

class DerivedBase(Base):
    """
    The DerivedBase class holds information about an object who belongs to another object
    (for example, a disk belongs to a linode).  These objects have their own endpoints,
    but they are below another object in the hierarchy (i.e. /linodes/lnde_123/disks/disk_123)
    """
    derived_url_path = '' #override in child classes
    parent_id_name = 'parent_id' #override in child classes

    def __init__(self, client, id, parent_id, json={}):
        Base.__init__(self, client, id, json=json)

        self._set(type(self).parent_id_name, parent_id)

    @classmethod
    def _api_get_derived(cls, parent, client):
        base_url = "{}/{}".format(type(parent).api_endpoint, cls.derived_url_path)

        return client._get_objects(base_url, cls, model=parent, parent_id=parent.id)
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ...tests.helper import pytest, assert_quantity_allclose, remote_data
from ... import units as u
from .. import Longitude, Latitude, EarthLocation
from ..sites import get_builtin_sites, get_downloaded_sites, SiteRegistry

def test_builtin_sites():
    reg = get_builtin_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    keck = reg['ctio']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('70.815', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('-30.16527778', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 2215*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        reg['nonexistent site']

@remote_data
def test_online_stes():
    reg = get_downloaded_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        reg['nonexistent site']

@remote_data
# this will *try* the online so we have to make it remote_data, even though it
# falls back on the non-remote version
def test_EarthLocation_basic():
    keckel = EarthLocation.of_site('keck')
    lon, lat, el = keckel.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = EarthLocation.get_site_names()
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        EarthLocation.of_site('nonexistent site')

def test_EarthLocation_state_offline():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_builtin=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_builtin=True)
    assert oldreg is not newreg

@remote_data
def test_EarthLocation_state_online():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_download=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_download=True)
    assert oldreg is not newreg

def test_registry():
    reg = SiteRegistry()

    assert len(reg.names) == 0

    names = ['sitea', 'site A']
    loc = EarthLocation.from_geodetic(lat=1*u.deg, lon=2*u.deg,height=3*u.km)
    reg.add_site(names, loc)

    assert len(reg.names) == 2

    loc1 = reg['SIteA']
    assert loc1 is loc

    loc2 = reg['sIte a']
    assert loc2 is loc

@remote_data
def test_non_EarthLocation():
    """
    A regression test for a typo bug pointed out at the bottom of
    https://github.com/astropy/astropy/pull/4042
    """
    class EarthLocation2(EarthLocation):
        pass

    el2 = EarthLocation2.of_site('keck')
    assert type(el2) is EarthLocation2
    assert el2.info.name == 'W. M. Keck Observatory'
<commit_msg>adjust test_non_EarthLocation to not need to be remote<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ...tests.helper import pytest, assert_quantity_allclose, remote_data
from ... import units as u
from .. import Longitude, Latitude, EarthLocation
from ..sites import get_builtin_sites, get_downloaded_sites, SiteRegistry

def test_builtin_sites():
    reg = get_builtin_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    keck = reg['ctio']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('70.815', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('-30.16527778', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 2215*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        reg['nonexistent site']

@remote_data
def test_online_stes():
    reg = get_downloaded_sites()

    keck = reg['keck']
    lon, lat, el = keck.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = reg.names
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        reg['nonexistent site']

@remote_data
# this will *try* the online so we have to make it remote_data, even though it
# falls back on the non-remote version
def test_EarthLocation_basic():
    keckel = EarthLocation.of_site('keck')
    lon, lat, el = keckel.to_geodetic()
    assert_quantity_allclose(lon, -1*Longitude('155:28.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(lat, Latitude('19:49.7', unit=u.deg),
                             atol=0.001*u.deg)
    assert_quantity_allclose(el, 4160*u.m, atol=1*u.m)

    names = EarthLocation.get_site_names()
    assert 'keck' in names
    assert 'ctio' in names

    with pytest.raises(KeyError):
        EarthLocation.of_site('nonexistent site')

def test_EarthLocation_state_offline():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_builtin=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_builtin=True)
    assert oldreg is not newreg

@remote_data
def test_EarthLocation_state_online():
    EarthLocation._site_registry = None
    EarthLocation._get_site_registry(force_download=True)
    assert EarthLocation._site_registry is not None

    oldreg = EarthLocation._site_registry
    newreg = EarthLocation._get_site_registry()
    assert oldreg is newreg
    newreg = EarthLocation._get_site_registry(force_download=True)
    assert oldreg is not newreg

def test_registry():
    reg = SiteRegistry()

    assert len(reg.names) == 0

    names = ['sitea', 'site A']
    loc = EarthLocation.from_geodetic(lat=1*u.deg, lon=2*u.deg,height=3*u.km)
    reg.add_site(names, loc)

    assert len(reg.names) == 2

    loc1 = reg['SIteA']
    assert loc1 is loc

    loc2 = reg['sIte a']
    assert loc2 is loc

def test_non_EarthLocation():
    """
    A regression test for a typo bug pointed out at the bottom of
    https://github.com/astropy/astropy/pull/4042
    """
    class EarthLocation2(EarthLocation):
        pass

    # This lets keeps us from needing to do remote_data
    # note that this does *not* mess up the registry for EarthLocation because
    # registry is cached on a per-class basis
    EarthLocation2._get_site_registry(force_builtin=True)

    el2 = EarthLocation2.of_site('keck')
    assert type(el2) is EarthLocation2
    assert el2.info.name == 'W. M. Keck Observatory'
<|endoftext|>


<commit_before># Get the list of images for a list of pages
import re
from pprint import pprint

import requests

base_url = "https://en.wikipedia.org/w/api.php"
payload = {
    'action': 'query',
    'prop': 'images',
    'pageids': None,
    'format': 'json',
    'imlimit': '5',
    'imcontinue': None,
    'imdir': 'ascending'
}

def __strip_file(text):
    return re.sub("File:", "", text, count=1)

def get_images(*pageids):
    image_names = {}
    payload['pageids'] = '|'.join(str(x) for x in list(pageids))
    res = requests.get(base_url, params=payload).json()
    for key, pages in res['query']['pages'].items():
        if 'images' not in pages:
            continue
        image_names[key] = [__strip_file(images['title']) for images in pages['images']]
    while 'continue' in res:
        payload['imcontinue'] = res['continue']['imcontinue']
        res = requests.get(base_url, params=payload).json()
        for key, pages in res['query']['pages'].items():
            if 'images' not in pages:
                continue
            image_names[key] = image_names.get(key, []) + [__strip_file(images['title']) for images in pages['images']]
    payload['imcontinue'] = None
    payload['pageids'] = None
    return image_names

pprint(get_images(843158, 20715044))
<commit_msg>Refactored variables!<commit_after># Get the list of images for a list of pages
import re
from pprint import pprint

import requests

base_url = "https://en.wikipedia.org/w/api.php"
payload = {
    'action': 'query',
    'prop': 'images',
    'pageids': None,
    'format': 'json',
    'imlimit': 'max',
    'imcontinue': None,
    'imdir': 'ascending'
}

def __strip_file(text):
    return re.sub("File:", "", text, count=1)

def get_images(*pageids):
    image_names = {}
    payload['pageids'] = '|'.join(str(x) for x in list(pageids))
    res = requests.get(base_url, params=payload).json()
    for pageid, page_content in res['query']['pages'].items():
        if 'images' not in page_content:
            continue
        image_names[pageid] = [__strip_file(image['title']) for image in page_content['images']]
    while 'continue' in res:
        payload['imcontinue'] = res['continue']['imcontinue']
        res = requests.get(base_url, params=payload).json()
        for pageid, page_content in res['query']['pages'].items():
            if 'images' not in page_content:
                continue
            image_names[pageid] = image_names.get(pageid, []) + [__strip_file(image['title']) for image in
                                                                 page_content['images']]
    payload['imcontinue'] = None
    payload['pageids'] = None
    return image_names

pprint(get_images(843158, 20715044))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from distutils.core import setup
import py2exe

setup(service=["OpenERPServerService"],
      options={"py2exe":{"excludes":["Tkconstants","Tkinter","tcl",
                                     "_imagingtk","PIL._imagingtk",
                                     "ImageTk", "PIL.ImageTk",
                                     "FixTk"],
                         "skip_archive": 1,
                         "optimize": 2,}}
      )

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[FIX] win32: also package vs2008 runtime under win32 so that service start correctly<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import os
import glob
from distutils.core import setup
import py2exe

def datas():
    r = []
    if os.name == 'nt':
        r.append(("Microsoft.VC90.CRT", glob.glob('C:\Microsoft.VC90.CRT\*.*')))
    return r

setup(service=["OpenERPServerService"],
      options={"py2exe":{"excludes":["Tkconstants","Tkinter","tcl",
                                     "_imagingtk","PIL._imagingtk",
                                     "ImageTk", "PIL.ImageTk",
                                     "FixTk"],
                         "skip_archive": 1,
                         "optimize": 2,}},
      data_files=datas(),
      )

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

import time, signal, sys, os
import threading
import argparse
import codecs, locale
from kitchen.text.converters import getwriter
import pkg_resources

from witica.site import Site
from witica.log import *
from witica.metadata import extractor
from witica.source import Source, ItemChanged, ItemRemoved
from witica.targets import target, web, statichtml
from witica.check import IntegrityChecker
from witica.util import sstr, suni, throw

VERSION = pkg_resources.get_distribution("witica").version

currentsite = None

def shutdown():
	log("Shutting down...", Logtype.INFO)
	try:
		if currentsite:
			currentsite.shutdown()
	except Exception, e:
		log_exception("Shutdown failed.", Logtype.ERROR)
	finally:
		seconds_left = 20
		while seconds_left > 0 and threading.active_count() > 2: #main + logging thread
			time.sleep(1)
			seconds_left -= 1
		if threading.active_count() > 2: #main + logging thread
			print("Hanging threads:")
			for t in threading.enumerate():
				if not(t == threading.current_thread()) and t.isAlive():
					print("* " + t.name)
			print("Force quit.")
			Logger.stop()
			sys.exit(0) #TODO: does this really kill all threads?
		Logger.stop()

def signal_handler(signal, frame):
	log("Shutdown requested by user.", Logtype.WARNING)
	shutdown()

def log(msg, logtype = Logtype.NONE):
	Logger.log("witica", msg, logtype)

def log_exception(msg, logtype = Logtype.NONE, exc_info=None):
	Logger.log_exception("witica", msg, logtype, exc_info)

def create_source(args):
	if args.source:
		return Source.construct_from_file(args.source)
	else:
		return Source.construct_from_working_dir()

def init_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)
	try:
		source = create_source(args)
	except Exception, e:
		log_exception("Source could not be initialized.", Logtype.ERROR)
		shutdown()

	folder = source.get_absolute_path("")
	if os.listdir(folder) == []: #if dir is empty
		print("do init")
	else:
		log("Init is only possible when the source is empty.", Logtype.ERROR)

def update_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)
	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = args.targets)
		currentsite.source.start_update(continuous = args.deamon)
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()

def rebuild_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)
	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = args.targets)
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)

	for item in items:
		try:
			for path in item.files:
				currentsite.source.changeEvent(currentsite.source, ItemChanged(currentsite.source, item.item_id, path))
		except Exception, e:
			log_exception("Item '" + item.item_id + "'' could not be enqued for rebuilding.", Logtype.ERROR)
	currentsite.source.stoppedEvent(currentsite.source, None)

def check_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)

	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = [])
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)

	numberfaults = 0
	ic = IntegrityChecker(currentsite.source)
	for item in items:
		#log("Checking item " + item.item_id + "...")
		for fault in ic.check(item):
			log(sstr(fault), Logtype.WARNING)
			numberfaults += 1
	log("Checked " + str(len(items)) + " items. " + str(numberfaults) + " integrity fault" + (" was" if numberfaults==1 else "s were") + " found.", Logtype.WARNING)
	currentsite.source.stoppedEvent(currentsite.source, None)

def list_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)

	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = [])
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)
	s = u"\n"
	count = 0
	for item in items:
		s += suni(item.item_id) + u"\n"
		count += 1
		if count == 100:
			log(s, Logtype.WARNING)
			s = u"\n"
			count = 0
	if count > 0:
		log(s, Logtype.WARNING)

	log("Source contains " + str(len(items)) + (" matching" if (len(args.item) > 0) else "") + " items.", Logtype.WARNING)
	currentsite.source.stoppedEvent(currentsite.source, None)

def get_matching_items(source, args):
	items = []
	if len(args.item) > 0:
		#return matching items
		for idpattern in args.item:
			items.extend(currentsite.source.items.get_items(idpattern))
	elif len(args.item) == 0:
		items = source.items
	return items

def main():
	#initialize witica

	signal.signal(signal.SIGINT, signal_handler) #abort on CTRL-C

	UTF8Writer = getwriter('utf8')
	sys.stdout = UTF8Writer(sys.stdout)

	extractor.register("item", extractor.JSONExtractor)
	extractor.register("json", extractor.JSONExtractor)
	extractor.register("md", extractor.MDExtractor)
	extractor.register("txt", extractor.MDExtractor)
	extractor.register("jpg", extractor.ImageExtractor)
	extractor.register("jpeg", extractor.ImageExtractor)

	target.register("WebTarget", web.WebTarget)
	target.register("StaticHtmlTarget", statichtml.StaticHtmlTarget)

	parser = argparse.ArgumentParser(description="Reads contents from a source, converts them and publishes to one or more targets.")
	parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + VERSION)

	subparsers = parser.add_subparsers(title='sub-commands', help='sub-commands')

	#init command parser
	parser_init = subparsers.add_parser('init', help='inits a source with an example web site (WARNING: modifies the source)')
	parser_init.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_init.add_argument('-s', '--source', help="the source configuration file to use")
	parser_init.set_defaults(func=init_command)

	#update command parser
	parser_update = subparsers.add_parser('update', help='fetch changes and update targets')
	parser_update.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_update.add_argument('-s', '--source', help="the source configuration file to use")
	parser_update.add_argument('-d', '--deamon', action='store_true', help="keep running in background and process incoming events")
	parser_update.add_argument('-t', '--targets', nargs='+', help="list of ids of targets that should be used for the conversion")
	parser_update.set_defaults(func=update_command)

	#rebuild command parser
	parser_rebuild = subparsers.add_parser('rebuild', help='update single items or indicies')
	parser_rebuild.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_rebuild.add_argument('-s', '--source', help="the source configuration file to use")
	parser_rebuild.add_argument('-t', '--targets', nargs='+', help="list of ids of targets that should be used for the conversion, default: all")
	parser_rebuild.add_argument('item', nargs='*', help="list of ids of items or indicies that should be updated")
	parser_rebuild.set_defaults(func=rebuild_command)

	#check command parser
	parser_check = subparsers.add_parser('check', help='checks the integrity of the source')
	parser_check.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_check.add_argument('-s', '--source', help="the source configuration file to use")
	parser_check.add_argument('item', nargs='*', help="list of ids of items or indicies that should be checked")
	parser_check.set_defaults(func=check_command)

	#items command parser
	parser_list = subparsers.add_parser('list', help='lists available item ids')
	parser_list.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_list.add_argument('-s', '--source', help="the source configuration file to use")
	parser_list.add_argument('item', nargs='*', help="list of ids of items or indicies that should be included")
	parser_list.set_defaults(func=list_command)

	args = parser.parse_args()
	args.func(args)

	#to receive sigint, continue program until all threads stopped
	while threading.active_count() > 1:
		try:
			# Join all threads to receive sigint
			[t.join(1) for t in threading.enumerate() if t.isAlive() and not(t == threading.current_thread() or t == Logger.get_thread())]

			#if only the main and logging threads are running stop program
			working = False
			for t in threading.enumerate():
				if t.isAlive() and not(t == threading.current_thread() or t == Logger.get_thread()):
					working = True

			if not working:
				shutdown()
				break

			#print("Running threads:")
			#for t in threading.enumerate():
			#	if not(t == threading.current_thread()) and t.isAlive():
			#		print("* " + t.name)
		except KeyboardInterrupt:
			signal_handler(signal.SIGINT, None)

if __name__ == '__main__':
	main()<commit_msg>implement witica init<commit_after>#!/usr/bin/env python
# coding=utf-8

import time, signal, sys, os, shutil
import threading
import argparse
import codecs, locale
from kitchen.text.converters import getwriter
import pkg_resources

from witica.site import Site
from witica.log import *
from witica.metadata import extractor
from witica.source import Source, ItemChanged, ItemRemoved
from witica.targets import target, web, statichtml
from witica.check import IntegrityChecker
from witica.util import sstr, suni, throw

VERSION = pkg_resources.get_distribution("witica").version

currentsite = None

def shutdown():
	log("Shutting down...", Logtype.INFO)
	try:
		if currentsite:
			currentsite.shutdown()
	except Exception, e:
		log_exception("Shutdown failed.", Logtype.ERROR)
	finally:
		seconds_left = 20
		while seconds_left > 0 and threading.active_count() > 2: #main + logging thread
			time.sleep(1)
			seconds_left -= 1
		if threading.active_count() > 2: #main + logging thread
			print("Hanging threads:")
			for t in threading.enumerate():
				if not(t == threading.current_thread()) and t.isAlive():
					print("* " + t.name)
			print("Force quit.")
			Logger.stop()
			sys.exit(0) #TODO: does this really kill all threads?
		Logger.stop()
		pkg_resources.cleanup_resources(force=False)

def signal_handler(signal, frame):
	log("Shutdown requested by user.", Logtype.WARNING)
	shutdown()

def log(msg, logtype = Logtype.NONE):
	Logger.log("witica", msg, logtype)

def log_exception(msg, logtype = Logtype.NONE, exc_info=None):
	Logger.log_exception("witica", msg, logtype, exc_info)

def create_source(args):
	if args.source:
		return Source.construct_from_file(args.source)
	else:
		return Source.construct_from_working_dir()

def init_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)

	target_id = "web"

	cwd = os.getcwd()
	if cwd.find(os.sep + "Dropbox" + os.sep) > -1:
		if os.listdir(cwd) == []: #if dir is empty
			try: #copy sample source
				for fn in os.listdir(pkg_resources.resource_filename("witica","client")):
					abs_fn = pkg_resources.resource_filename("witica","client") + os.sep + fn
					if os.path.isdir(abs_fn):
						shutil.copytree(abs_fn, cwd + os.sep + fn)
					else:
						shutil.copy2(abs_fn, cwd)
			except Exception, e:
				log_exception("Init finished with errors.", Logtype.ERROR)
		else:
			log("Init is only possible when the current folder is empty.", Logtype.ERROR)
	else:
		log("Working directory is not a valid source. Must be a folder inside your Dropbox.", Logtype.ERROR)

def update_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)
	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = args.targets)
		currentsite.source.start_update(continuous = args.deamon)
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()

def rebuild_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)
	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = args.targets)
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)

	for item in items:
		try:
			for path in item.files:
				currentsite.source.changeEvent(currentsite.source, ItemChanged(currentsite.source, item.item_id, path))
		except Exception, e:
			log_exception("Item '" + item.item_id + "'' could not be enqued for rebuilding.", Logtype.ERROR)
	currentsite.source.stoppedEvent(currentsite.source, None)

def check_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)

	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = [])
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)

	numberfaults = 0
	ic = IntegrityChecker(currentsite.source)
	for item in items:
		#log("Checking item " + item.item_id + "...")
		for fault in ic.check(item):
			log(sstr(fault), Logtype.WARNING)
			numberfaults += 1
	log("Checked " + str(len(items)) + " items. " + str(numberfaults) + " integrity fault" + (" was" if numberfaults==1 else "s were") + " found.", Logtype.WARNING)
	currentsite.source.stoppedEvent(currentsite.source, None)

def list_command(args):
	global currentsite
	Logger.start(verbose=args.verbose)

	try:
		source = create_source(args)
		currentsite = Site(source, target_ids = [])
	except Exception, e:
		log_exception("Site could not be initialized.", Logtype.ERROR)
		shutdown()
		return

	items = get_matching_items(currentsite.source, args)
	s = u"\n"
	count = 0
	for item in items:
		s += suni(item.item_id) + u"\n"
		count += 1
		if count == 100:
			log(s, Logtype.WARNING)
			s = u"\n"
			count = 0
	if count > 0:
		log(s, Logtype.WARNING)

	log("Source contains " + str(len(items)) + (" matching" if (len(args.item) > 0) else "") + " items.", Logtype.WARNING)
	currentsite.source.stoppedEvent(currentsite.source, None)

def get_matching_items(source, args):
	items = []
	if len(args.item) > 0:
		#return matching items
		for idpattern in args.item:
			items.extend(currentsite.source.items.get_items(idpattern))
	elif len(args.item) == 0:
		items = source.items
	return items

def main():
	#initialize witica

	signal.signal(signal.SIGINT, signal_handler) #abort on CTRL-C

	UTF8Writer = getwriter('utf8')
	sys.stdout = UTF8Writer(sys.stdout)

	extractor.register("item", extractor.JSONExtractor)
	extractor.register("json", extractor.JSONExtractor)
	extractor.register("md", extractor.MDExtractor)
	extractor.register("txt", extractor.MDExtractor)
	extractor.register("jpg", extractor.ImageExtractor)
	extractor.register("jpeg", extractor.ImageExtractor)

	target.register("WebTarget", web.WebTarget)
	target.register("StaticHtmlTarget", statichtml.StaticHtmlTarget)

	parser = argparse.ArgumentParser(description="Reads contents from a source, converts them and publishes to one or more targets.")
	parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + VERSION)

	subparsers = parser.add_subparsers(title='sub-commands', help='sub-commands')

	#init command parser
	parser_init = subparsers.add_parser('init', help='inits a source with an example web site (WARNING: modifies the current working dir)')
	parser_init.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_init.set_defaults(func=init_command)

	#update command parser
	parser_update = subparsers.add_parser('update', help='fetch changes and update targets')
	parser_update.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_update.add_argument('-s', '--source', help="the source configuration file to use")
	parser_update.add_argument('-d', '--deamon', action='store_true', help="keep running in background and process incoming events")
	parser_update.add_argument('-t', '--targets', nargs='+', help="list of ids of targets that should be used for the conversion")
	parser_update.set_defaults(func=update_command)

	#rebuild command parser
	parser_rebuild = subparsers.add_parser('rebuild', help='update single items or indicies')
	parser_rebuild.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_rebuild.add_argument('-s', '--source', help="the source configuration file to use")
	parser_rebuild.add_argument('-t', '--targets', nargs='+', help="list of ids of targets that should be used for the conversion, default: all")
	parser_rebuild.add_argument('item', nargs='*', help="list of ids of items or indicies that should be updated")
	parser_rebuild.set_defaults(func=rebuild_command)

	#check command parser
	parser_check = subparsers.add_parser('check', help='checks the integrity of the source')
	parser_check.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_check.add_argument('-s', '--source', help="the source configuration file to use")
	parser_check.add_argument('item', nargs='*', help="list of ids of items or indicies that should be checked")
	parser_check.set_defaults(func=check_command)

	#items command parser
	parser_list = subparsers.add_parser('list', help='lists available item ids')
	parser_list.add_argument('-V', '--verbose', action='store_true', help="show also info messages and debbuging info")
	parser_list.add_argument('-s', '--source', help="the source configuration file to use")
	parser_list.add_argument('item', nargs='*', help="list of ids of items or indicies that should be included")
	parser_list.set_defaults(func=list_command)

	args = parser.parse_args()
	args.func(args)

	#to receive sigint, continue program until all threads stopped
	while threading.active_count() > 1:
		try:
			# Join all threads to receive sigint
			[t.join(1) for t in threading.enumerate() if t.isAlive() and not(t == threading.current_thread() or t == Logger.get_thread())]

			#if only the main and logging threads are running stop program
			working = False
			for t in threading.enumerate():
				if t.isAlive() and not(t == threading.current_thread() or t == Logger.get_thread()):
					working = True

			if not working:
				shutdown()
				break

			#print("Running threads:")
			#for t in threading.enumerate():
			#	if not(t == threading.current_thread()) and t.isAlive():
			#		print("* " + t.name)
		except KeyboardInterrupt:
			signal_handler(signal.SIGINT, None)

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before>#!/usr/bin/env python2

import requests
from requests_toolbelt.multipart.encoder import MultipartEncoder
import time
import random
import threading
import time
import sys
import signal

NUM_THREADS = 10
MEAN_SLEEP_SEC = 5

class ThreadInfo:
    def __init__(self, numthreads):
        self.mutex = threading.Lock()
        self.numthreads = numthreads
        self.status = [ "STARTUP" for i in range(numthreads)]
        self.count = [ 0 for i in range(numthreads)]
        self.exiting = False

    def setStatus(self, idx, status):
        self.mutex.acquire()
        self.status[idx] = status
        self.mutex.release()
        return status

    def incCount(self, idx):
        self.mutex.acquire()
        cnt = self.count[idx] + 1
        self.count[idx] = cnt
        self.mutex.release()
        return cnt

    def getStatus(self, idx, status):
        self.mutex.acquire()
        status = self.status[idx]
        self.mutex.release()
        return status

    def pleaseDie(self, signum, frame):
        print("Please die")
        self.mutex.acquire()
        self.exiting = True
        self.status = [ "CLOSING" for i in range(self.numthreads) ]
        self.mutex.release()
        self.printInfo()

    def printInfo(self):
        info.mutex.acquire()
        str = ""
        for i in range(NUM_THREADS):
            str += "\033[95mThread {} {:15s} Done: {:3d}\033[0m\n".format(
                    i, info.status[i], info.count[1])
            str += "\033[{}F".format(NUM_THREADS+1)
        print(str)
        info.mutex.release()

info = ThreadInfo(NUM_THREADS)

def genWL(base, tidx):
    global info

    st = random.expovariate(MEAN_SLEEP_SEC)
    time.sleep(st)
    while (not info.exiting):
          genWL1(base, tidx, info)
          st = random.randint(8,10)
          time.sleep(st)

def genWL1(base, tidx, info):
    info.setStatus(tidx, "UPLOADING")

    m = MultipartEncoder(fields={ 'file' :
        ('filename', open('video.mp4', 'rb'), 'video/mp4')})
    queueTime = time.time()
    r = requests.post(base,data = m,
            headers={'Content-Type': m.content_type})
    if r.status_code == 500:
        info.setStatus(tids, "SERVER ERROR")
        info.exiting = True
        return
    file_url = r.headers["Location"]
    r2 = requests.get(file_url + "/status")

    info.setStatus(tidx, r2.json()["status"])
    while ( info.setStatus(tidx, r2.json()["status"]) == "QUEUED" ):
        try:
            time.sleep(1)
        except Exception:
            pass
        if info.exiting:
            return
        r2 = requests.get(file_url + "/status")

    startTime = time.time()
    while ( info.setStatus(tidx, r2.json()["status"]) != "DONE" ):
        try:
            time.sleep(1)
        except Exception:
            pass
        if info.exiting:
            return
        r2 = requests.get(file_url + "/status")

    t = time.time()
    info.mutex.acquire()
    sys.stderr.write(file_url + "," +
            str(startTime-queueTime) + "," +
            str(t-startTime) + "," +
            str(t-queueTime) + "\n")
    info.mutex.release()
    info.incCount(tidx)

if __name__ == '__main__':
    print("Workload Generator started with %d threads and %d mean sleep time." \
            % (NUM_THREADS, MEAN_SLEEP_SEC))
    print("CSV is printed to stderr and info to stdout.")
    print("Press CTRL+C to quit.")

    sys.stderr.write("URL,QueueTime,ProcessingTime,TotalTime\n")

    signal.signal(signal.SIGINT, info.pleaseDie)

    for i in range(NUM_THREADS):
        t = threading.Thread(target = genWL ,
                args = ["http://transcode.thefuturenow.se:5000/", i])
        t.start()

    while not info.exiting:

        time.sleep(1)
<commit_msg>added shutdown<commit_after>#!/usr/bin/env python2

import requests
from requests_toolbelt.multipart.encoder import MultipartEncoder
import time
import random
import threading
import time
import sys
import signal

NUM_THREADS = 10
MEAN_SLEEP_SEC = 5

class ThreadInfo:
    def __init__(self, numthreads):
        self.mutex = threading.Lock()
        self.numthreads = numthreads
        self.status = [ "STARTUP" for i in range(numthreads)]
        self.count = [ 0 for i in range(numthreads)]
        self.exiting = False

    def setStatus(self, idx, status):
        self.mutex.acquire()
        self.status[idx] = status
        self.mutex.release()
        return status

    def incCount(self, idx):
        self.mutex.acquire()
        cnt = self.count[idx] + 1
        self.count[idx] = cnt
        self.mutex.release()
        return cnt

    def getStatus(self, idx, status):
        self.mutex.acquire()
        status = self.status[idx]
        self.mutex.release()
        return status

    def pleaseDie(self, signum, frame):
        print("Please die")
        self.mutex.acquire()
        self.exiting = True
        self.status = [ "CLOSING" for i in range(self.numthreads) ]
        self.mutex.release()
        self.printInfo()

    def printInfo(self):
        info.mutex.acquire()
        str = ""
        for i in range(NUM_THREADS):
            str += "\033[95mThread {} {:15s} Done: {:3d}\033[0m\n".format(
                    i, info.status[i], info.count[1])
            str += "\033[{}F".format(NUM_THREADS+1)
        print(str)
        info.mutex.release()

info = ThreadInfo(NUM_THREADS)

def genWL(base, tidx):
    global info

    st = random.expovariate(MEAN_SLEEP_SEC)
    time.sleep(st)
    while (not info.exiting):
          genWL1(base, tidx, info)
          st = random.randint(8,10)
          time.sleep(st)

def genWL1(base, tidx, info):
    info.setStatus(tidx, "UPLOADING")

    m = MultipartEncoder(fields={ 'file' :
        ('filename', open('video.mp4', 'rb'), 'video/mp4')})
    queueTime = time.time()
    r = requests.post(base,data = m,
            headers={'Content-Type': m.content_type})
    if r.status_code == 500:
        info.setStatus(tids, "SERVER ERROR")
        info.exiting = True
        return
    file_url = r.headers["Location"]
    r2 = requests.get(file_url + "/status")

    info.setStatus(tidx, r2.json()["status"])
    while ( info.setStatus(tidx, r2.json()["status"]) == "QUEUED" ):
        try:
            time.sleep(1)
        except Exception:
            pass
        if info.exiting:
            return
        r2 = requests.get(file_url + "/status")

    startTime = time.time()
    while ( info.setStatus(tidx, r2.json()["status"]) != "DONE" ):
        try:
            time.sleep(1)
        except Exception:
            pass
        if info.exiting:
            return
        r2 = requests.get(file_url + "/status")

    t = time.time()
    info.mutex.acquire()
    sys.stderr.write(file_url + "," +
            str(startTime-queueTime) + "," +
            str(t-startTime) + "," +
            str(t-queueTime) + "\n")
    info.mutex.release()
    info.incCount(tidx)

if __name__ == '__main__':
    print("Workload Generator started with %d threads and %d mean sleep time." \
            % (NUM_THREADS, MEAN_SLEEP_SEC))
    print("CSV is printed to stderr and info to stdout.")
    print("Press CTRL+C to quit.")

    sys.stderr.write("URL,QueueTime,ProcessingTime,TotalTime\n")

    signal.signal(signal.SIGINT, info.pleaseDie)

    for i in range(NUM_THREADS):
        t = threading.Thread(target = genWL ,
                args = ["http://transcode.thefuturenow.se:5000/", i])
        t.start()

    while not info.exiting:
        info.printInfo()
        time.sleep(1)
<|endoftext|>


<commit_before>import os.path
import re

def sudo(user, cmd, **kwargs):
    import subprocess

    run = ['sudo', '-u', user]
    run.extend(cmd)
    if 'stdout_io' in kwargs:
        # XXX unfinished
        stdout_io = kwargs['stdout_io']
        del kwargs['stdout_io']
        kwargs['stdout'] = subprocess.PIPE
        no_check = kwargs.get('no_check')
        p = Popen(cmd, **kwargs)
        out, err = p.communicate()
    if 'no_check' in kwargs:
        fn = subprocess.call
        del kwargs['no_check']
    else:
        fn = subprocess.check_call
    fn(run, **kwargs)

def sudo_php(cmd):
    sudo('php', cmd)

def sudo_rvm(cmd, **kwargs):
    run = ['-iH']
    run.extend(cmd)
    sudo('rvm', run, **kwargs)

def sudo_chmod(path, mode):
    sudo_php(['chmod', oct(mode), str(path)])

def rsync(src, dest, delete=False, exclude=None):
    import subprocess

    cmd = ['rsync', '-a', '--exclude', '.git']
    if delete:
        cmd.append('--delete')
    if exclude:
        if isinstance(exclude, basestring):
            cmd.extend(['--exclude', exclude])
        else:
            for path in exclucde:
                cmd.extend(['--exclude', path])
    cmd.extend([src, dest])
    subprocess.check_call(cmd)

def casper(path, pre=None):
    path = os.path.realpath(path)
    # workaround for https://github.com/n1k0/casperjs/issues/343 -
    # pass through coffeescript first
    with open('/dev/null', 'wb') as f:
        sudo_rvm(['coffee', '-cp', path], stdout=f)
    casperjs_wrapper = os.path.join(os.path.dirname(__file__), '../script/casperjs-wrapper')
    cmd = [casperjs_wrapper, 'test', path]
    if pre is not None:
        cmd.append('--pre=%s' % pre)
    sudo_rvm(cmd)

def git_in_dir(dir, *args):
    import subprocess

    cmd = ['git',
        '--git-dir', os.path.join(dir, '.git'),
        '--work-tree', dir]
    cmd.extend(args)
    subprocess.check_call(cmd)

def yaml_to_json(input_text=None, input_file=None, output_file=None):
    import yaml
    import json

    if input_text is not None:
        data = yaml.load(input_text)
    elif input_file is not None:
        if isinstance(input_file, basestring):
            with open(input_file, 'rb') as f:
                data = yaml.load(f)
        else:
            data = yaml.load(input_file)
    else:
        raise ArgumentError('Neither input_text nor input_file given')

    if output_file is not None:
        if isinstance(output_file, basestring):
            with open(output_file, 'wb') as f:
                json.dump(data, f)
        else:
            json.dump(data, output_file)
    else:
        return json.dumps(data)

def naive_strip_html(text):
    # http://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
    return re.sub('<[^<]+?>', '', text)

class PhpbbVersion(object):
    def __init__(self, conf):
        if os.path.exists(os.path.join(conf.test_root_phpbb, 'includes/extension/manager.php')):
            self.version = (3, 1, 0)
        else:
            self.version = (3, 0, 0)

    def __cmp__(self, other):
        return cmp(self.version, tuple(other))

    def __str__(self):
        return 'phpBB version %d.%d.%d' % self.version

    def matches(self, spec):
        assert spec.startswith('>=')
        target_version = spec[2:].split('.')
        return self >= target_version

class Current(object):
    phpbb_version = None

current = Current()
<commit_msg>Fix phpbb version comparison<commit_after>import os.path
import re

def sudo(user, cmd, **kwargs):
    import subprocess

    run = ['sudo', '-u', user]
    run.extend(cmd)
    if 'stdout_io' in kwargs:
        # XXX unfinished
        stdout_io = kwargs['stdout_io']
        del kwargs['stdout_io']
        kwargs['stdout'] = subprocess.PIPE
        no_check = kwargs.get('no_check')
        p = Popen(cmd, **kwargs)
        out, err = p.communicate()
    if 'no_check' in kwargs:
        fn = subprocess.call
        del kwargs['no_check']
    else:
        fn = subprocess.check_call
    fn(run, **kwargs)

def sudo_php(cmd):
    sudo('php', cmd)

def sudo_rvm(cmd, **kwargs):
    run = ['-iH']
    run.extend(cmd)
    sudo('rvm', run, **kwargs)

def sudo_chmod(path, mode):
    sudo_php(['chmod', oct(mode), str(path)])

def rsync(src, dest, delete=False, exclude=None):
    import subprocess

    cmd = ['rsync', '-a', '--exclude', '.git']
    if delete:
        cmd.append('--delete')
    if exclude:
        if isinstance(exclude, basestring):
            cmd.extend(['--exclude', exclude])
        else:
            for path in exclucde:
                cmd.extend(['--exclude', path])
    cmd.extend([src, dest])
    subprocess.check_call(cmd)

def casper(path, pre=None):
    path = os.path.realpath(path)
    # workaround for https://github.com/n1k0/casperjs/issues/343 -
    # pass through coffeescript first
    with open('/dev/null', 'wb') as f:
        sudo_rvm(['coffee', '-cp', path], stdout=f)
    casperjs_wrapper = os.path.join(os.path.dirname(__file__), '../script/casperjs-wrapper')
    cmd = [casperjs_wrapper, 'test', path]
    if pre is not None:
        cmd.append('--pre=%s' % pre)
    sudo_rvm(cmd)

def git_in_dir(dir, *args):
    import subprocess

    cmd = ['git',
        '--git-dir', os.path.join(dir, '.git'),
        '--work-tree', dir]
    cmd.extend(args)
    subprocess.check_call(cmd)

def yaml_to_json(input_text=None, input_file=None, output_file=None):
    import yaml
    import json

    if input_text is not None:
        data = yaml.load(input_text)
    elif input_file is not None:
        if isinstance(input_file, basestring):
            with open(input_file, 'rb') as f:
                data = yaml.load(f)
        else:
            data = yaml.load(input_file)
    else:
        raise ArgumentError('Neither input_text nor input_file given')

    if output_file is not None:
        if isinstance(output_file, basestring):
            with open(output_file, 'wb') as f:
                json.dump(data, f)
        else:
            json.dump(data, output_file)
    else:
        return json.dumps(data)

def naive_strip_html(text):
    # http://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
    return re.sub('<[^<]+?>', '', text)

class PhpbbVersion(object):
    def __init__(self, conf):
        if os.path.exists(os.path.join(conf.test_root_phpbb, 'includes/extension/manager.php')):
            self.version = (3, 1, 0)
        else:
            self.version = (3, 0, 0)

    def __cmp__(self, other):
        return cmp(self.version, tuple(other))

    def __str__(self):
        return 'phpBB version %d.%d.%d' % self.version

    def matches(self, spec):
        assert spec.startswith('>=')
        target_version = map(int, spec[2:].split('.'))
        return self >= target_version

class Current(object):
    phpbb_version = None

current = Current()
<|endoftext|>


<commit_before>import asyncio
import discord
import textwrap
import random
import math
from   discord.ext import commands

def setup(bot):
    # Not a cog
    return

class Message:
    def __init__(self, **kwargs):
        # Creates a new message - with an optional setup dictionary
        self.max_chars = 2000
        self.pm_after = kwargs.get("pm_after", 1)
        self.header = kwargs.get("header", "")
        self.footer = kwargs.get("footer", "")
        self.pm_react = kwargs.get("pm_react", "📬")
        self.message = kwargs.get("message", None)
        self.max_pages = 0

    async def _send_message(self, ctx, message, pm = False):
        # Helper method to send embeds to their proper location
        if pm == True and not ctx.channel == ctx.author.dm_channel:
            # More than 2 pages - try to dm
            try:
                message = await ctx.author.send(message)
                await ctx.message.add_reaction(self.pm_react)
                return message
            except discord.Forbidden:
                pass
        return await ctx.send(message)

    async def send(self, ctx):
        if not ctx or not self.message or not len(self.message):
            return
        text_list = textwrap.wrap(
            self.message,
            self.max_chars - len(self.header) - len(self.footer),
            break_long_words=True,
            replace_whitespace=False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(text_list) > self.pm_after if self.pm_after else False

        page_count = 1
        for m in text_list:
            if self.max_pages > 0 and page_count > self.max_pages:
                break
            message = await self._send_message(ctx, self.header + m + self.footer, to_pm)
            page_count += 1
        return message

class Embed:
    def __init__(self, **kwargs):
        # Set defaults
        self.title_max = 256
        self.desc_max = 2048
        self.field_max = 25
        self.fname_max = 256
        self.fval_max = 1024
        self.foot_max = 2048
        self.auth_max = 256
        self.total_max = 6000
        # Creates a new embed - with an option setup dictionary
        self.pm_after = kwargs.get("pm_after", 10)
        self.pm_react = kwargs.get("pm_react", "📬")
        self.title = kwargs.get("title", None)
        self.page_count = kwargs.get("page_count", True)
        self.url = kwargs.get("url", None)
        self.description = kwargs.get("description", None)
        self.image = kwargs.get("image", None)
        self.footer = kwargs.get("footer", None)
        # self.footer_text = kwargs.get("footer_text", discord.Embed.Empty)
        # self.footer_icon = kwargs.get("footer_icon", discord.Embed.Empty)
        self.thumbnail = kwargs.get("thumbnail", None)
        self.author = kwargs.get("author", None)
        self.fields = kwargs.get("fields", [])
        self.colors = [
            discord.Color.teal(),
            discord.Color.dark_teal(),
            discord.Color.green(),
            discord.Color.dark_green(),
            discord.Color.blue(),
            discord.Color.dark_blue(),
            discord.Color.purple(),
            discord.Color.dark_purple(),
            discord.Color.magenta(),
            discord.Color.dark_magenta(),
            discord.Color.gold(),
            discord.Color.dark_gold(),
            discord.Color.orange(),
            discord.Color.dark_orange(),
            discord.Color.red(),
            discord.Color.dark_red(),
            discord.Color.lighter_grey(),
            discord.Color.dark_grey(),
            discord.Color.light_grey(),
            discord.Color.darker_grey(),
            discord.Color.blurple(),
            discord.Color.greyple(),
            discord.Color.default()
        ]
        self.color = kwargs.get("color", None)

    def add_field(self, **kwargs):
        self.fields.append({
            "name" : kwargs.get("name", "None"),
            "value" : kwargs.get("value", "None"),
            "inline" : kwargs.get("inline", False)
        })

    def clear_fields(self):
        self.fields = []

    # Embed stuff!
    async def _send_embed(self, ctx, embed, pm = False):
        # Helper method to send embeds to their proper location
        if pm == True and not ctx.channel == ctx.author.dm_channel:
            # More than 2 pages - try to dm
            try:
                message = await ctx.author.send(embed=embed)
                await ctx.message.add_reaction(self.pm_react)
                return message
            except discord.Forbidden:
                pass
        return await ctx.send(embed=embed)

    def _truncate_string(self, value, max_chars):
        if not type(value) is str:
            return value
        # Truncates the string to the max chars passed
        return (value[:max_chars-3]+"...") if len(value) > max_chars else value

    def _total_chars(self, embed):
        # Returns how many chars are in the embed
        tot = 0
        if embed.title:
            tot += len(embed.title)
        if embed.description:
            tot += len(embed.description)
        if not embed.footer is discord.Embed.Empty:
            tot += len(embed.footer)
        for field in embed.fields:
            tot += len(field.name) + len(field.value)
        return tot

    def _embed_with_self(self):
        if self.color == None:
            self.color = random.choice(self.colors)
        elif type(self.color) is discord.Member:
            self.color = self.color.color
        elif type(self.color) is discord.User:
            self.color = random.choice(self.colors)
        # Sends the current embed
        em = discord.Embed(color=self.color)
        em.title = self._truncate_string(self.title, self.title_max)
        em.url = self.url
        em.description = self._truncate_string(self.description, self.desc_max)
        if self.image:
            em.set_image(url=self.image)
        if self.thumbnail:
            em.set_thumbnail(self.thumbnail)
        if self.author:
            if type(self.author) is discord.Member or type(self.author) is discord.User:
                name = self.author.nick if hasattr(self.author, "nick") and self.author.nick else self.author.name
                em.set_author(
                    name    =self._truncate_string(name, self.auth_max),
                    # Ignore the url here
                    icon_url=self.author.avatar_url
                )
            elif type(self.author) is dict:
                if any(item in self.author for item in ["name", "url", "icon"]):
                    em.set_author(
                        name    =self._truncate_string(self.author.get("name",     discord.Embed.Empty), self.auth_max),
                        url     =self.author.get("url",      discord.Embed.Empty),
                        icon_url=self.author.get("icon_url", discord.Embed.Empty)
                    )
                else:
                    em.set_author(name=self._truncate_string(str(self.author), self.auth_max))
            else:
                # Cast to string and hope for the best
                em.set_author(name=self._truncate_string(str(self.author), self.auth_max))
        return em

    def _get_footer(self):
        # Get our footer if we have one
        footer_text = footer_icon = discord.Embed.Empty
        if type(self.footer) is str:
                footer_text = self.footer
        elif type(self.footer) is dict:
                footer_text = self.footer.get("text", discord.Embed.Empty)
                footer_icon = self.footer.get("icon_url", discord.Embed.Empty)
        elif self.footer == None:
                # Never setup
                pass
        else:
                # Try to cast it
                footer_text = str(self.footer)
        return (footer_text, footer_icon)

    async def edit(self, ctx, message):
        # Edits the passed message - and sends any remaining pages
        # check if we can steal the color from the message
        if self.color == None and len(message.embeds):
            self.color = message.embeds[0].color
        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        to_pm = len(self.fields) > self.pm_after if self.pm_after else False

        if len(self.fields) <= self.pm_after and not to_pm:
            # Edit in place, nothing else needs to happen
            for field in self.fields:
                em.add_field(
                    name=self._truncate_string(field.get("name", "None"), self.fname_max),
                    value=self._truncate_string(field.get("value", "None"), self.fval_max),
                    inline=field.get("inline", False)
                )
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await message.edit(content=None, embed=em)
        # Now we need to edit the first message to just a space - then send the rest
        new_message = await self.send(ctx)
        if new_message.channel == ctx.author.dm_channel and not ctx.channel == ctx.author.dm_channel:
            em = Embed(title=self.title, description="📬 Check your dm's", color=self.color)._embed_with_self()
            await message.edit(content=None, embed=em)
        else:
            await message.edit(content=" ", embed=None)

    async def send(self, ctx):
        if not ctx:
            return

        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        # First check if we have any fields at all - and try to send
        # as one page if not
        if not len(self.fields):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await self._send_embed(ctx, em, False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(self.fields) > self.pm_after if self.pm_after else False

        page_count = 1
        page_total = math.ceil(len(self.fields)/self.field_max)

        if page_total > 1 and self.page_count and self.title:
            add_title = " (Page {:,} of {:,})".format(page_count, page_total)
            em.title = self._truncate_string(self.title, self.title_max - len(add_title)) + add_title
        for field in self.fields:
            em.add_field(
                name=self._truncate_string(field.get("name", "None"), self.fname_max),
                value=self._truncate_string(field.get("value", "None"), self.fval_max),
                inline=field.get("inline", False)
            )
            # 25 field max - send the embed if we get there
            if len(em.fields) >= self.field_max:
                if page_total == page_count:
                    em.set_footer(
                        text=self._truncate_string(footer_text, self.foot_max),
                        icon_url=footer_icon
                    )
                message = await self._send_embed(ctx, em, to_pm)
                em.clear_fields()
                page_count += 1
                if page_total > 1 and self.page_count and self.title:
                    add_title = " (Page {:,} of {:,})".format(page_count, page_total)
                    em.title = self._truncate_string(self.title, self.title_max - len(add_title)) + add_title

        if len(em.fields):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            message = await self._send_embed(ctx, em, to_pm)
        return message

class EmbedText(Embed):
    def __init__(self, **kwargs):
        Embed.__init__(self, **kwargs)
        # Creates a new embed - with an option setup dictionary
        self.pm_after = kwargs.get("pm_after", 1)
        self.max_pages = 0
        self.desc_head = kwargs.get("desc_head", "") # Header for description markdown
        self.desc_foot = kwargs.get("desc_foot", "") # Footer for description markdown

    async def edit(self, ctx, message):
        # Edits the passed message - and sends any remaining pages
        # check if we can steal the color from the message
        if self.color == None and len(message.embeds):
            self.color = message.embeds[0].color
        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        if self.description == None or not len(self.description):
            text_list = []
        else:
            text_list = textwrap.wrap(
                self.description,
                self.desc_max - len(self.desc_head) - len(self.desc_foot),
                break_long_words=True,
                replace_whitespace=False)
        to_pm = len(text_list) > self.pm_after if self.pm_after else False
        if len(text_list) <= 1 and not to_pm:
            # Edit in place, nothing else needs to happen
            if len(text_list):
                em.description = self.desc_head + text_list[0] + self.desc_foot
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await message.edit(content=None, embed=em)
        # Now we need to edit the first message to just a space - then send the rest
        new_message = await self.send(ctx)
        if new_message.channel == ctx.author.dm_channel and not ctx.channel == ctx.author.dm_channel:
            em = Embed(title=self.title, description="📬 Check your dm's", color=self.color)._embed_with_self()
            await message.edit(content=None, embed=em)
        else:
            await message.edit(content=" ", embed=None)

    async def send(self, ctx):
        if not ctx:
            return

        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        # First check if we have any fields at all - and try to send
        # as one page if not
        if self.description == None or not len(self.description):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await self._send_embed(ctx, em, False)

        text_list = textwrap.wrap(
            self.description,
            self.desc_max - len(self.desc_head) - len(self.desc_foot),
            break_long_words=True,
            replace_whitespace=False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(text_list) > self.pm_after if self.pm_after else False

        i = 0
        for i in range(len(text_list)):
            m = text_list[i]
            if self.max_pages > 0 and i >= self.max_pages:
                break
            # Strip the title if not the first page
            if i > 0:
                em.title = None
            if i == len(text_list)-1:
                # Last item - apply footer
                em.set_footer(
                    text=self._truncate_string(footer_text, self.foot_max),
                    icon_url=footer_icon
                )
            em.description = self.desc_head + m + self.desc_foot
            message = await self._send_embed(ctx, em, to_pm)
        return message<commit_msg>Add option for color lists/tuples in embed setup<commit_after>import asyncio
import discord
import textwrap
import random
import math
from   discord.ext import commands

def setup(bot):
    # Not a cog
    return

class Message:
    def __init__(self, **kwargs):
        # Creates a new message - with an optional setup dictionary
        self.max_chars = 2000
        self.pm_after = kwargs.get("pm_after", 1)
        self.header = kwargs.get("header", "")
        self.footer = kwargs.get("footer", "")
        self.pm_react = kwargs.get("pm_react", "📬")
        self.message = kwargs.get("message", None)
        self.max_pages = 0

    async def _send_message(self, ctx, message, pm = False):
        # Helper method to send embeds to their proper location
        if pm == True and not ctx.channel == ctx.author.dm_channel:
            # More than 2 pages - try to dm
            try:
                message = await ctx.author.send(message)
                await ctx.message.add_reaction(self.pm_react)
                return message
            except discord.Forbidden:
                pass
        return await ctx.send(message)

    async def send(self, ctx):
        if not ctx or not self.message or not len(self.message):
            return
        text_list = textwrap.wrap(
            self.message,
            self.max_chars - len(self.header) - len(self.footer),
            break_long_words=True,
            replace_whitespace=False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(text_list) > self.pm_after if self.pm_after else False

        page_count = 1
        for m in text_list:
            if self.max_pages > 0 and page_count > self.max_pages:
                break
            message = await self._send_message(ctx, self.header + m + self.footer, to_pm)
            page_count += 1
        return message

class Embed:
    def __init__(self, **kwargs):
        # Set defaults
        self.title_max = 256
        self.desc_max = 2048
        self.field_max = 25
        self.fname_max = 256
        self.fval_max = 1024
        self.foot_max = 2048
        self.auth_max = 256
        self.total_max = 6000
        # Creates a new embed - with an option setup dictionary
        self.pm_after = kwargs.get("pm_after", 10)
        self.pm_react = kwargs.get("pm_react", "📬")
        self.title = kwargs.get("title", None)
        self.page_count = kwargs.get("page_count", True)
        self.url = kwargs.get("url", None)
        self.description = kwargs.get("description", None)
        self.image = kwargs.get("image", None)
        self.footer = kwargs.get("footer", None)
        # self.footer_text = kwargs.get("footer_text", discord.Embed.Empty)
        # self.footer_icon = kwargs.get("footer_icon", discord.Embed.Empty)
        self.thumbnail = kwargs.get("thumbnail", None)
        self.author = kwargs.get("author", None)
        self.fields = kwargs.get("fields", [])
        self.colors = [
            discord.Color.teal(),
            discord.Color.dark_teal(),
            discord.Color.green(),
            discord.Color.dark_green(),
            discord.Color.blue(),
            discord.Color.dark_blue(),
            discord.Color.purple(),
            discord.Color.dark_purple(),
            discord.Color.magenta(),
            discord.Color.dark_magenta(),
            discord.Color.gold(),
            discord.Color.dark_gold(),
            discord.Color.orange(),
            discord.Color.dark_orange(),
            discord.Color.red(),
            discord.Color.dark_red(),
            discord.Color.lighter_grey(),
            discord.Color.dark_grey(),
            discord.Color.light_grey(),
            discord.Color.darker_grey(),
            discord.Color.blurple(),
            discord.Color.greyple(),
            discord.Color.default()
        ]
        self.color = kwargs.get("color", None)

    def add_field(self, **kwargs):
        self.fields.append({
            "name" : kwargs.get("name", "None"),
            "value" : kwargs.get("value", "None"),
            "inline" : kwargs.get("inline", False)
        })

    def clear_fields(self):
        self.fields = []

    # Embed stuff!
    async def _send_embed(self, ctx, embed, pm = False):
        # Helper method to send embeds to their proper location
        if pm == True and not ctx.channel == ctx.author.dm_channel:
            # More than 2 pages - try to dm
            try:
                message = await ctx.author.send(embed=embed)
                await ctx.message.add_reaction(self.pm_react)
                return message
            except discord.Forbidden:
                pass
        return await ctx.send(embed=embed)

    def _truncate_string(self, value, max_chars):
        if not type(value) is str:
            return value
        # Truncates the string to the max chars passed
        return (value[:max_chars-3]+"...") if len(value) > max_chars else value

    def _total_chars(self, embed):
        # Returns how many chars are in the embed
        tot = 0
        if embed.title:
            tot += len(embed.title)
        if embed.description:
            tot += len(embed.description)
        if not embed.footer is discord.Embed.Empty:
            tot += len(embed.footer)
        for field in embed.fields:
            tot += len(field.name) + len(field.value)
        return tot

    def _embed_with_self(self):
        if self.color == None:
            self.color = random.choice(self.colors)
        elif type(self.color) is discord.Member:
            self.color = self.color.color
        elif type(self.color) is discord.User:
            self.color = random.choice(self.colors)
        elif type(self.color) is tuple or type(self.color) is list:
            if len(self.color) == 3:
                try:
                    r, g, b = [ int(a) for a in self.color ]
                    self.color = discord.Color.from_rgb(r, g, b)
                except:
                    self.color = random.choice(self.colors)
            else:
                self.color = random.choice(self.colors)

        # Sends the current embed
        em = discord.Embed(color=self.color)
        em.title = self._truncate_string(self.title, self.title_max)
        em.url = self.url
        em.description = self._truncate_string(self.description, self.desc_max)
        if self.image:
            em.set_image(url=self.image)
        if self.thumbnail:
            em.set_thumbnail(self.thumbnail)
        if self.author:
            if type(self.author) is discord.Member or type(self.author) is discord.User:
                name = self.author.nick if hasattr(self.author, "nick") and self.author.nick else self.author.name
                em.set_author(
                    name    =self._truncate_string(name, self.auth_max),
                    # Ignore the url here
                    icon_url=self.author.avatar_url
                )
            elif type(self.author) is dict:
                if any(item in self.author for item in ["name", "url", "icon"]):
                    em.set_author(
                        name    =self._truncate_string(self.author.get("name",     discord.Embed.Empty), self.auth_max),
                        url     =self.author.get("url",      discord.Embed.Empty),
                        icon_url=self.author.get("icon_url", discord.Embed.Empty)
                    )
                else:
                    em.set_author(name=self._truncate_string(str(self.author), self.auth_max))
            else:
                # Cast to string and hope for the best
                em.set_author(name=self._truncate_string(str(self.author), self.auth_max))
        return em

    def _get_footer(self):
        # Get our footer if we have one
        footer_text = footer_icon = discord.Embed.Empty
        if type(self.footer) is str:
                footer_text = self.footer
        elif type(self.footer) is dict:
                footer_text = self.footer.get("text", discord.Embed.Empty)
                footer_icon = self.footer.get("icon_url", discord.Embed.Empty)
        elif self.footer == None:
                # Never setup
                pass
        else:
                # Try to cast it
                footer_text = str(self.footer)
        return (footer_text, footer_icon)

    async def edit(self, ctx, message):
        # Edits the passed message - and sends any remaining pages
        # check if we can steal the color from the message
        if self.color == None and len(message.embeds):
            self.color = message.embeds[0].color
        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        to_pm = len(self.fields) > self.pm_after if self.pm_after else False

        if len(self.fields) <= self.pm_after and not to_pm:
            # Edit in place, nothing else needs to happen
            for field in self.fields:
                em.add_field(
                    name=self._truncate_string(field.get("name", "None"), self.fname_max),
                    value=self._truncate_string(field.get("value", "None"), self.fval_max),
                    inline=field.get("inline", False)
                )
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await message.edit(content=None, embed=em)
        # Now we need to edit the first message to just a space - then send the rest
        new_message = await self.send(ctx)
        if new_message.channel == ctx.author.dm_channel and not ctx.channel == ctx.author.dm_channel:
            em = Embed(title=self.title, description="📬 Check your dm's", color=self.color)._embed_with_self()
            await message.edit(content=None, embed=em)
        else:
            await message.edit(content=" ", embed=None)

    async def send(self, ctx):
        if not ctx:
            return

        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        # First check if we have any fields at all - and try to send
        # as one page if not
        if not len(self.fields):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await self._send_embed(ctx, em, False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(self.fields) > self.pm_after if self.pm_after else False

        page_count = 1
        page_total = math.ceil(len(self.fields)/self.field_max)

        if page_total > 1 and self.page_count and self.title:
            add_title = " (Page {:,} of {:,})".format(page_count, page_total)
            em.title = self._truncate_string(self.title, self.title_max - len(add_title)) + add_title
        for field in self.fields:
            em.add_field(
                name=self._truncate_string(field.get("name", "None"), self.fname_max),
                value=self._truncate_string(field.get("value", "None"), self.fval_max),
                inline=field.get("inline", False)
            )
            # 25 field max - send the embed if we get there
            if len(em.fields) >= self.field_max:
                if page_total == page_count:
                    em.set_footer(
                        text=self._truncate_string(footer_text, self.foot_max),
                        icon_url=footer_icon
                    )
                message = await self._send_embed(ctx, em, to_pm)
                em.clear_fields()
                page_count += 1
                if page_total > 1 and self.page_count and self.title:
                    add_title = " (Page {:,} of {:,})".format(page_count, page_total)
                    em.title = self._truncate_string(self.title, self.title_max - len(add_title)) + add_title

        if len(em.fields):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            message = await self._send_embed(ctx, em, to_pm)
        return message

class EmbedText(Embed):
    def __init__(self, **kwargs):
        Embed.__init__(self, **kwargs)
        # Creates a new embed - with an option setup dictionary
        self.pm_after = kwargs.get("pm_after", 1)
        self.max_pages = 0
        self.desc_head = kwargs.get("desc_head", "") # Header for description markdown
        self.desc_foot = kwargs.get("desc_foot", "") # Footer for description markdown

    async def edit(self, ctx, message):
        # Edits the passed message - and sends any remaining pages
        # check if we can steal the color from the message
        if self.color == None and len(message.embeds):
            self.color = message.embeds[0].color
        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        if self.description == None or not len(self.description):
            text_list = []
        else:
            text_list = textwrap.wrap(
                self.description,
                self.desc_max - len(self.desc_head) - len(self.desc_foot),
                break_long_words=True,
                replace_whitespace=False)
        to_pm = len(text_list) > self.pm_after if self.pm_after else False
        if len(text_list) <= 1 and not to_pm:
            # Edit in place, nothing else needs to happen
            if len(text_list):
                em.description = self.desc_head + text_list[0] + self.desc_foot
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await message.edit(content=None, embed=em)
        # Now we need to edit the first message to just a space - then send the rest
        new_message = await self.send(ctx)
        if new_message.channel == ctx.author.dm_channel and not ctx.channel == ctx.author.dm_channel:
            em = Embed(title=self.title, description="📬 Check your dm's", color=self.color)._embed_with_self()
            await message.edit(content=None, embed=em)
        else:
            await message.edit(content=" ", embed=None)

    async def send(self, ctx):
        if not ctx:
            return

        em = self._embed_with_self()
        footer_text, footer_icon = self._get_footer()

        # First check if we have any fields at all - and try to send
        # as one page if not
        if self.description == None or not len(self.description):
            em.set_footer(
                text=self._truncate_string(footer_text, self.foot_max),
                icon_url=footer_icon
            )
            return await self._send_embed(ctx, em, False)

        text_list = textwrap.wrap(
            self.description,
            self.desc_max - len(self.desc_head) - len(self.desc_foot),
            break_long_words=True,
            replace_whitespace=False)

        # Only pm if our self.pm_after is above 0
        to_pm = len(text_list) > self.pm_after if self.pm_after else False

        i = 0
        for i in range(len(text_list)):
            m = text_list[i]
            if self.max_pages > 0 and i >= self.max_pages:
                break
            # Strip the title if not the first page
            if i > 0:
                em.title = None
            if i == len(text_list)-1:
                # Last item - apply footer
                em.set_footer(
                    text=self._truncate_string(footer_text, self.foot_max),
                    icon_url=footer_icon
                )
            em.description = self.desc_head + m + self.desc_foot
            message = await self._send_embed(ctx, em, to_pm)
        return message<|endoftext|>