<commit_before>#!/usr/bin/env python3
"""
resources:
- https://pymotw.com/3/asyncio/subprocesses.html
- http://stackoverflow.com/questions/24435987/how-to-stream-stdout-stderr-from-a-child-process-using-asyncio-and-obtain-its-e
- http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python/20697159#20697159

How it should work

class Project:
    pass

class Task:
    pass

class Slave:
    pass

async for task in project.tasks:

    await result = slave.do(task)

    await task.complete(result)

"""

import sys
import signal
import functools
import asyncio
from asyncio.subprocess import PIPE
import shlex
import base64
import uuid

class PingProtocol(asyncio.SubprocessProtocol):
    FD_NAMES = ['stdin', 'stdout', 'stderr']

    def __init__(self, loop, done):
        self.loop = loop
        self.done = done
        self.buffer = bytearray()
        self.transport = None
        self.stdin = None
        self.stdout = None
        self.stderr = None
        super().__init__()

    def connection_made(self, transport):
        print('process started {}'.format(transport.get_pid()))
        self.transport = transport

        self.stdin = transport.get_pipe_transport(0)
        self.stdout = transport.get_pipe_transport(1)
        self.stderr = transport.get_pipe_transport(2)

        # initial start
        self.stdin.write(b'foo')

    def pipe_connection_lost(self, fd, exc):
        print("connection lost", fd, exc)

    @staticmethod
    def _create_message(data):
        return b':'.join((bytes(uuid.uuid1().hex, 'ascii'), base64.b64encode(data)))

    def send(self, data):
        self.stdin.write(data)
        # self.stdin.write(self._create_message(data))

    def pipe_data_received(self, fd, data):
        print('read {} bytes from {}'.format(len(data),
                                             self.FD_NAMES[fd]))
        print("<", fd, data)
        if fd == 1:

            self.buffer.extend(data)

            if data == b'foo':
                self.send(b'bar')

            elif data == b'bar':
                self.send(b'terminate')

            elif data == b'terminate':
                self.send(b'exit')

    def process_exited(self):
        print('process exited')
        return_code = self.transport.get_returncode()
        print('return code {}'.format(return_code))
        if not return_code:
            cmd_output = bytes(self.buffer).decode()
            # results = self._parse_results(cmd_output)
        else:
            results = []
        self.done.set_result((return_code, 'foo'))
        # self.loop.stop()

processes = set()

async def run_client(loop, done, *command):
    factory = functools.partial(PingProtocol, loop, done)

    asyncio.create_subprocess_exec
    proc = loop.subprocess_exec(
        factory,
        *command,
        stdout=PIPE,
        stdin=PIPE
    )

    try:
        print('launching process')
        transport, protocol = await proc

        print("register process")
        processes.add(asyncio.subprocess.Process(transport, protocol, loop))

        print('waiting for process to complete', transport, protocol)
        await done

    finally:
        transport.close()

def get_python_source(obj):
    import inspect
    return inspect.getsource(obj)

def main():
    import receive

    receive_source = get_python_source(receive).encode('utf-8')
    receive_source_b64 = base64.b64encode(receive_source).decode('utf-8')

    if sys.platform == "win32":
        loop = asyncio.ProactorEventLoop()  # for subprocess' pipes on Windows
        asyncio.set_event_loop(loop)
    else:
        loop = asyncio.get_event_loop()

    done = asyncio.Future(loop=loop)

    def ask_exit(signame):
        print("got signal %s: exit" % signame)
        for proc in processes:
            proc.send_signal(signal.SIGHUP)
            proc.terminate()
            print("Killing: ", proc)
        loop.stop()
        done.cancelled()

    # register shutdowen
    for signame in ('SIGINT', 'SIGTERM'):
        loop.add_signal_handler(
            getattr(signal, signame),
            functools.partial(ask_exit, signame)
        )

    try:

        ssh_command = [
            "ssh", "localhost",
            "/home/olli/.pyenv/versions/debellator3/bin/python",
            "-u",
            "-c",
            """'import base64; c = compile(base64.b64decode(b"{}"), "<string>", "exec"); exec(c)'""".format(receive_source_b64)
        ]
        command = [
            "/home/olli/.pyenv/versions/debellator3/bin/python",
            "-u",
            "-c", """import base64; c = compile(base64.b64decode(b"{}"), "<string>", "exec"); exec(c)""".format(receive_source_b64)
        ]

        returncode = loop.create_task(
            run_client(
                loop,
                done,
                *ssh_command
            )
        )
        loop.run_until_complete(done)
    finally:
        loop.close()

if __name__ == '__main__':
    main()

    # asyncio.StreamReader
    # asyncio.StreamReaderProtocol
<commit_msg>receive now started with module import<commit_after>#!/usr/bin/env python3
"""
resources:
- https://pymotw.com/3/asyncio/subprocesses.html
- http://stackoverflow.com/questions/24435987/how-to-stream-stdout-stderr-from-a-child-process-using-asyncio-and-obtain-its-e
- http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python/20697159#20697159

How it should work

class Project:
    pass

class Task:
    pass

class Slave:
    pass

async for task in project.tasks:

    await result = slave.do(task)

    await task.complete(result)

"""

import sys
import signal
import functools
import asyncio
from asyncio.subprocess import PIPE
import shlex
import base64
import uuid

class PingProtocol(asyncio.SubprocessProtocol):
    FD_NAMES = ['stdin', 'stdout', 'stderr']

    def __init__(self, loop, done):
        self.loop = loop
        self.done = done
        self.buffer = bytearray()
        self.transport = None
        self.stdin = None
        self.stdout = None
        self.stderr = None
        super().__init__()

    def connection_made(self, transport):
        print('process started {}'.format(transport.get_pid()))
        self.transport = transport

        self.stdin = transport.get_pipe_transport(0)
        self.stdout = transport.get_pipe_transport(1)
        self.stderr = transport.get_pipe_transport(2)

        # initial start
        self.stdin.write(b'foo')

    def pipe_connection_lost(self, fd, exc):
        print("connection lost", fd, exc)

    @staticmethod
    def _create_message(data):
        return b':'.join((bytes(uuid.uuid1().hex, 'ascii'), base64.b64encode(data)))

    def send(self, data):
        self.stdin.write(data)
        # self.stdin.write(self._create_message(data))

    def pipe_data_received(self, fd, data):
        print('read {} bytes from {}'.format(len(data),
                                             self.FD_NAMES[fd]))
        print("<", fd, data)
        if fd == 1:

            self.buffer.extend(data)

            if data == b'foo':
                self.send(b'bar')

            elif data == b'bar':
                self.send(b'terminate')

            elif data == b'terminate':
                self.send(b'exit')

    def process_exited(self):
        print('process exited')
        return_code = self.transport.get_returncode()
        print('return code {}'.format(return_code))
        if not return_code:
            cmd_output = bytes(self.buffer).decode()
            # results = self._parse_results(cmd_output)
        else:
            results = []
        self.done.set_result((return_code, 'foo'))
        # self.loop.stop()

processes = set()

async def run_client(loop, done, *command):
    factory = functools.partial(PingProtocol, loop, done)

    asyncio.create_subprocess_exec
    proc = loop.subprocess_exec(
        factory,
        *command,
        stdout=PIPE,
        stdin=PIPE
    )

    try:
        print('launching process')
        transport, protocol = await proc

        print("register process")
        processes.add(asyncio.subprocess.Process(transport, protocol, loop))

        print('waiting for process to complete', transport, protocol)
        await done

    finally:
        transport.close()

def get_python_source(obj):
    import inspect
    return inspect.getsource(obj)

def main():
    import receive

    receive_source = get_python_source(receive).encode('utf-8')
    receive_source_b64 = base64.b64encode(receive_source).decode('utf-8')

    if sys.platform == "win32":
        loop = asyncio.ProactorEventLoop()  # for subprocess' pipes on Windows
        asyncio.set_event_loop(loop)
    else:
        loop = asyncio.get_event_loop()

    done = asyncio.Future(loop=loop)

    def ask_exit(signame):
        print("got signal %s: exit" % signame)
        for proc in processes:
            proc.send_signal(signal.SIGHUP)
            proc.terminate()
            print("Killing: ", proc)
        loop.stop()
        done.cancelled()

    # register shutdowen
    for signame in ('SIGINT', 'SIGTERM'):
        loop.add_signal_handler(
            getattr(signal, signame),
            functools.partial(ask_exit, signame)
        )

    try:

        ssh_command = [
            "ssh", "localhost",
            "/home/olli/.pyenv/versions/debellator3/bin/python",
            "-u",
            "-c",
            "'"
            'import imp, base64; boot = imp.new_module("dbltr.boot");'
            'c = compile(base64.b64decode(b"{}"), "<string>", "exec");'
            'exec(c, boot.__dict__); boot.main(None);'
            "'".format(receive_source_b64)
        ]
        command = [
            "/home/olli/.pyenv/versions/debellator3/bin/python",
            "-u",
            "-c", """import base64; c = compile(base64.b64decode(b"{}"), "<string>", "exec"); exec(c)""".format(receive_source_b64)
        ]

        returncode = loop.create_task(
            run_client(
                loop,
                done,
                *ssh_command
            )
        )
        loop.run_until_complete(done)
    finally:
        loop.close()

if __name__ == '__main__':
    main()

    # asyncio.StreamReader
    # asyncio.StreamReaderProtocol
<|endoftext|>


<commit_before># Copyright 2017 PESOL (http://pesol.es) - Angel Moya (angel.moya@pesol.es)
# Copyright 2020 Tecnativa - Carlos Dauden
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import _, api, fields, models
from odoo.exceptions import ValidationError

class AccountAnalyticDimension(models.Model):
    _name = "account.analytic.dimension"
    _description = "Account Analytic Dimension"

    name = fields.Char(required=True)
    code = fields.Char(required=True)
    analytic_tag_ids = fields.One2many(
        comodel_name="account.analytic.tag",
        inverse_name="analytic_dimension_id",
        string="Analytic Tags",
    )

    @api.constrains("code")
    def _check_code(self):
        for dimension in self:
            if " " in dimension.code:
                raise ValidationError(_("Code can't contain spaces!"))

    @api.model
    def get_model_names(self):
        return [
            "account.move.line",
            "account.analytic.line",
            "account.invoice.report",
        ]

    def get_field_name(self, code=False):
        return "x_dimension_{}".format(code or self.code).lower()

    @api.model
    def create(self, values):
        res = super().create(values)
        _models = self.env["ir.model"].search(
            [("model", "in", self.get_model_names())], order="id"
        )
        _models.write(
            {
                "field_id": [
                    (
                        0,
                        0,
                        {
                            "name": self.get_field_name(values["code"]),
                            "field_description": values.get("name"),
                            "ttype": "many2one",
                            "relation": "account.analytic.tag",
                        },
                    )
                ],
            }
        )
        # Launch this manually for taking the new dimension field
        self.env["account.invoice.report"].init()
        return res

    def write(self, vals):
        field_vals = {}
        if "name" in vals or "code" in vals:
            if "name" in vals:
                field_vals["field_description"] = vals["name"]
            if "code" in vals:
                field_vals["name"] = self.get_field_name(vals["code"])
            for dimension in self:
                fields_to_update = self.env["ir.model.fields"].search(
                    [
                        ("model", "in", self.get_model_names()),
                        ("name", "=", dimension.get_field_name()),
                    ],
                    order="id",
                )
                # To avoid 'Can only rename one field at a time!'
                for field_to_update in fields_to_update:
                    field_to_update.write(field_vals)
        return super().write(vals)

    def unlink(self):
        """Clean created fields before unlinking."""
        models = self.env["ir.model"].search([("model", "in", self.get_model_names())])
        for record in self:
            field_name = self.get_field_name(record.code)
            self.env["ir.model.fields"].search(
                [("model_id", "in", models.ids), ("name", "=", field_name)]
            ).unlink()
        return super().unlink()
<commit_msg>[FIX] update invoice_report direct databbase<commit_after># Copyright 2017 PESOL (http://pesol.es) - Angel Moya (angel.moya@pesol.es)
# Copyright 2020 Tecnativa - Carlos Dauden
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from psycopg2 import sql

from odoo import _, api, fields, models
from odoo.exceptions import ValidationError

class AccountAnalyticDimension(models.Model):
    _name = "account.analytic.dimension"
    _description = "Account Analytic Dimension"

    name = fields.Char(required=True)
    code = fields.Char(required=True)
    analytic_tag_ids = fields.One2many(
        comodel_name="account.analytic.tag",
        inverse_name="analytic_dimension_id",
        string="Analytic Tags",
    )

    @api.constrains("code")
    def _check_code(self):
        for dimension in self:
            if " " in dimension.code:
                raise ValidationError(_("Code can't contain spaces!"))

    @api.model
    def get_model_names(self):
        return [
            "account.move.line",
            "account.analytic.line",
            "account.invoice.report",
        ]

    def get_field_name(self, code=False):
        return "x_dimension_{}".format(code or self.code).lower()

    def _convert_dict_query(self, field_vals):
        val_query = [
            "{} = '{}'".format(key, field_val) for key, field_val in field_vals.items()
        ]
        vals = ", ".join(val_query)
        return vals

    def _update_invoice_report(self, field_to_update, value):
        self._cr.execute(
            sql.SQL(
                """ UPDATE {} SET {} WHERE id={} """.format(
                    field_to_update._table, value, field_to_update.id
                )
            )
        )
        field_to_update.invalidate_cache()

    @api.model
    def create(self, values):
        res = super().create(values)
        _models = self.env["ir.model"].search(
            [("model", "in", self.get_model_names())], order="id"
        )
        _models.write(
            {
                "field_id": [
                    (
                        0,
                        0,
                        {
                            "name": self.get_field_name(values["code"]),
                            "field_description": values.get("name"),
                            "ttype": "many2one",
                            "relation": "account.analytic.tag",
                        },
                    )
                ],
            }
        )
        return res

    def write(self, vals):
        field_vals = {}
        if "name" in vals or "code" in vals:
            if "name" in vals:
                field_vals["field_description"] = vals["name"]
            if "code" in vals:
                field_vals["name"] = self.get_field_name(vals["code"])
            for dimension in self:
                fields_to_update = self.env["ir.model.fields"].search(
                    [
                        ("model", "in", self.get_model_names()),
                        ("name", "=", dimension.get_field_name()),
                    ],
                    order="id",
                )
                # To avoid 'Can only rename one field at a time!'
                for field_to_update in fields_to_update:
                    if field_to_update.model == "account.invoice.report":
                        value = self._convert_dict_query(field_vals)
                        self._update_invoice_report(field_to_update, value)
                    else:
                        field_to_update.write(field_vals)
        return super().write(vals)

    def unlink(self):
        """Clean created fields before unlinking."""
        models = self.env["ir.model"].search([("model", "in", self.get_model_names())])
        for record in self:
            field_name = self.get_field_name(record.code)
            self.env["ir.model.fields"].search(
                [("model_id", "in", models.ids), ("name", "=", field_name)]
            ).unlink()
        return super().unlink()
<|endoftext|>


<commit_before>import copy

from optparse import OptionParser
from auth import auth
from jsonconfig import JsonConfig
from pocket import retrieve, add_tags

def has_long_tag(item):
    return has_tag(item, 'long')

def has_tag(item, tag):
    if "tags" not in item:
        return False
    else:
        if tag not in item["tags"].keys():
            return False
        else:
            return True

def isolate_long_articles(items, long_word_count=5000):
    uids = []

    for uid, item in items.iteritems():
        # get a word count from each item
        word_count = item.get('word_count', None)
        if not word_count:
            continue
        word_count = long(word_count)
        # if the word count is above the long word count
        if word_count > long_word_count:
            if not has_long_tag(item):
                uids.append(uid)

    return uids

def isolate_keyword_articles(items, kword, kword_tag):
    uids = []

    for uid, item in items.iteritems():
        # get a word count from each item
        title = item.get('resolved_title', None)
        if not title:
            continue
        if kword in title.lower():
            if not has_tag(item, kword_tag):
                uids.append(uid)

    return uids

def tag_long_articles(credentials, items):
    uids = isolate_long_articles(items)
    add_tags(credentials, uids, ["long"])
    return len(uids)

def tag_kword_articles(credentials, items):
    jc = JsonConfig(".kwords")
    config = jc.read()
    kwords = config["keywords"]
    total_uids = []
    for kword, kword_tag in kwords:
        uids = isolate_keyword_articles(items, kword.lower(), kword_tag)
        add_tags(credentials, uids, [kword_tag])
        total_uids.extend(uids)

    return len(total_uids)

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option('--key', dest='key',
        help='pocket apps consumer key')
    parser.add_option('--longs', dest='longs', action='store_true',
        default=False, help='tag untagged long articles')
    parser.add_option('--kwords', dest='kwords', action='store_true',
        default=False, help='tag untagged keyword articles')
    (options, args) = parser.parse_args()

    config = auth(options)
    credentials = copy.deepcopy(config)
    items = retrieve(config, verbose=True)

    if options.longs:
        affected = tag_long_articles(credentials, items)
        print "Tagged %d long articles" % (affected)

    if options.kwords:
        affected = tag_kword_articles(credentials, items)
        print "Tagged %d kword articles" % (affected)
<commit_msg>Added ability to match only words with regex support<commit_after>import copy
import re

from optparse import OptionParser
from auth import auth
from jsonconfig import JsonConfig
from pocket import retrieve, add_tags

def has_long_tag(item):
    return has_tag(item, 'long')

def has_tag(item, tag):
    if "tags" not in item:
        return False
    else:
        if tag not in item["tags"].keys():
            return False
        else:
            return True

def isolate_long_articles(items, long_word_count=5000):
    uids = []

    for uid, item in items.iteritems():
        # get a word count from each item
        word_count = item.get('word_count', None)
        if not word_count:
            continue
        word_count = long(word_count)
        # if the word count is above the long word count
        if word_count > long_word_count:
            if not has_long_tag(item):
                uids.append(uid)

    return uids

def isolate_keyword_articles(items, kword, kword_tag):
    uids = []
    for uid, item in items.iteritems():
        # get a word count from each item
        title = item.get('resolved_title', None)
        if not title:
            continue
        word_exp = re.compile(r'\b%s\b' % kword, flags=re.IGNORECASE)
        match = word_exp.search(title.lower())
        if match:
            if not has_tag(item, kword_tag):
                uids.append(uid)

    return uids

def tag_long_articles(credentials, items):
    uids = isolate_long_articles(items)
    add_tags(credentials, uids, ["long"])
    return len(uids)

def tag_kword_articles(credentials, items):
    jc = JsonConfig(".kwords")
    config = jc.read()
    kwords = config["keywords"]
    total_uids = []
    for kword, kword_tag in kwords:
        uids = isolate_keyword_articles(items, kword.lower(), kword_tag)
        add_tags(credentials, uids, [kword_tag])
        total_uids.extend(uids)

    return len(total_uids)

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option('--key', dest='key',
        help='pocket apps consumer key')
    parser.add_option('--longs', dest='longs', action='store_true',
        default=False, help='tag untagged long articles')
    parser.add_option('--kwords', dest='kwords', action='store_true',
        default=False, help='tag untagged keyword articles')
    (options, args) = parser.parse_args()

    config = auth(options)
    credentials = copy.deepcopy(config)
    items = retrieve(config, verbose=True)

    if options.longs:
        affected = tag_long_articles(credentials, items)
        print "Tagged %d long articles" % (affected)

    if options.kwords:
        affected = tag_kword_articles(credentials, items)
        print "Tagged %d kword articles" % (affected)
<|endoftext|>


<commit_before>from urlparse import urljoin, urlparse, urlunparse
from datetime import datetime
import re
import cgi
import urllib

import remoteobjects
from remoteobjects import fields
from remoteobjects.dataobject import find_by_name
from remoteobjects.promise import PromiseError
import typepad
from batchhttp.client import BatchError
import logging

class TypePadObject(remoteobjects.RemoteObject):
    base_url = None
    batch_requests = True

    @classmethod
    def get_response(cls, url, http=None, **kwargs):
        http = typepad.client.http
        return super(TypePadObject, cls).get_response(url, http=http, **kwargs)

    @classmethod
    def get(cls, url, *args, **kwargs):
        if cls.base_url is not None and not urlparse(url)[1]:  # network location
            url = urljoin(cls.base_url, url)

        ret = super(TypePadObject, cls).get(url, *args, **kwargs)
        if cls.batch_requests:
            try:
                typepad.client.add(ret.get_request(), ret.update_from_response)
            except BatchError, ex:
                # We're suppressing an exception here for ListObjects
                # since in some cases we merely want the object to
                # 'post' against.
                if not issubclass(cls, ListObject):
                    raise PromiseError("Cannot get %s %s outside a batch request"
                        % (cls.__name__, url))
        return ret

    def deliver(self):
        if self.batch_requests:
            raise PromiseError("Cannot deliver %s %s except by batch request"
                % (type(self).__name__, self._location))
        return super(TypePadObject, self).deliver()

class Link(TypePadObject):
    rel      = fields.Field()
    href     = fields.Field()
    type     = fields.Field()
    width    = fields.Field()
    height   = fields.Field()
    duration = fields.Field()
    total    = fields.Field()

    def __repr__(self):
        return "<Link %s>" % self.href

class LinkSet(set, TypePadObject):
    def update_from_dict(self, data):
        self.update([Link.from_dict(x) for x in data])

    def to_dict(self):
        return [x.to_dict() for x in self]

    def __getitem__(self, key):
        if isinstance(key, slice):
            raise KeyError('LinkSets cannot be sliced')

        if key.endswith('_set'):
            # Gimme all matching links.
            key = key[:-4]
            return LinkSet([x for x in self if x.rel == key])

        # Gimme the first matching link.
        for x in self:
            if x.rel == key:
                return x

        raise KeyError('No such link %r in this set' % key)

    def link_by_width(self, width=None):
        """ An algorithm for selecting the best appropriate image
        given a specified display width.

        If no width is given, the widest image is selected.

        If a width is given, the first image that meets that width
        or is larger is selected. If none meet that size, the one
        that was closest to the specified width is chosen.
        """
        # TODO: is there a brisk way to do this?
        widest = None
        best = None
        for link in self:
            # Keep track of the widest variant; this will be our failsafe.
            if (not widest) or link.width > widest.width:
                widest = link
            # Width was specified; enclosure is equal or larger than this
            if width and link.width >= width:
                # Assign if nothing was already chosen or if this new
                # enclosure is smaller than the last best one.
                if (not best) or link.width < best.width:
                    best = link

        # use best available image if none was selected as the 'best' fit
        if best is None:
            return widest
        else:
            return best

class SequenceProxy(object):
    def make_sequence_method(methodname):
        def seqmethod(self, *args, **kwargs):
            # Proxy these methods to self.entries.
            return getattr(self.entries, methodname)(*args, **kwargs)
        seqmethod.__name__ = methodname
        return seqmethod

    __len__      = make_sequence_method('__len__')
    __getitem__  = make_sequence_method('__getitem__')
    __setitem__  = make_sequence_method('__setitem__')
    __delitem__  = make_sequence_method('__delitem__')
    __iter__     = make_sequence_method('__iter__')
    __reversed__ = make_sequence_method('__reversed__')
    __contains__ = make_sequence_method('__contains__')

class ListOf(remoteobjects.ListObject.__metaclass__):
    def __new__(cls, name, bases=None, attr=None):
        if attr is None:
            # TODO: memoize me
            entryclass = name
            if callable(entryclass):
                name = cls.__name__ + entryclass.__name__
            else:
                name = cls.__name__ + entryclass
            bases = (ListObject,)
            attr = {'entryclass': entryclass}

        bases = bases + (SequenceProxy,)
        return super(ListOf, cls).__new__(cls, name, bases, attr)

class ListObject(TypePadObject, remoteobjects.ListObject):
    __metaclass__ = ListOf

    total_results = fields.Field(api_name='totalResults')
    start_index   = fields.Field(api_name='startIndex')
    links         = fields.Object(LinkSet)
    entries       = fields.List(fields.Field())

    filterorder = ['following', 'follower', 'friend', 'nonreciprocal',
        'published', 'unpublished', 'spam', 'admin', 'member',
        'by-group', 'by-user', 'photo', 'post', 'video', 'audio', 'comment', 'link']

    def filter(self, **kwargs):
        # Split the list's URL into URL parts, filters, and queryargs.
        parts = list(urlparse(self._location))
        queryargs = cgi.parse_qs(parts[4], keep_blank_values=True)
        queryargs = dict([(k, v[0]) for k, v in queryargs.iteritems()])

        oldpath = parts[2]
        if not oldpath.endswith('.json'):
            raise AssertionError('oldpath %r does not end in %r' % (oldpath, '.json'))
        path = oldpath[:-5].split('/')

        filters = dict()
        newpath = list()
        pathparts = iter(path)
        for x in pathparts:
            if x.startswith('@'):
                x = x[1:]
                if x in ('by-group', 'by-user'):
                    filters[x] = pathparts.next()
                else:
                    filters[x] = True
            else:
                newpath.append(x)

        # Add kwargs into the filters and queryargs as appropriate.
        for k, v in kwargs.iteritems():
            # Convert by_group to by-group.
            k = k.replace('_', '-')
            # Convert by_group=7 to by_group='7'.
            v = str(v)

            if k in self.filterorder:
                filters[k] = v
            else:
                queryargs[k] = v

        # Put the filters back on the URL path in API order.
        keys = filters.keys()
        keys.sort(key=self.filterorder.index)
        for k in keys:
            if filters[k]:
                newpath.append('@' + k)
                if k in ('by-group', 'by-user'):
                    newpath.append(filters[k])

        # Coalesce the URL back into a string and make a new List from it.
        parts[2] = '/'.join(newpath) + '.json'
        parts[4] = urllib.urlencode(queryargs)
        newurl = urlunparse(parts)

        ret = self.get(newurl)
        ret.of_cls = self.of_cls
        return ret

    def __getitem__(self, key):
        if self._delivered or not isinstance(key, slice):
            return self.entries[key]
        args = dict()
        if key.start is not None:
            args['start_index'] = key.start
            if key.stop is not None:
                args['max_results'] = key.stop - key.start
        elif key.stop is not None:
            args['max_results'] = key.stop
        return self.filter(**args)

    def update_from_dict(self, data):
        super(ListObject, self).update_from_dict(data)
        # Post-convert all the "entries" list items to our entry class.
        entryclass = self.entryclass
        if not callable(entryclass):
            entryclass = find_by_name(entryclass)
        self.entries = [entryclass.from_dict(d) for d in self.entries]

class ApiLink(fields.Link):
    def __get__(self, instance, owner):
        try:
            if instance._location is None:
                raise AttributeError('Cannot find URL of %s relative to URL-less %s' % (type(self).__name__, owner.__name__))

            assert instance._location.endswith('.json')
            newurl = instance._location[:-5]
            newurl += '/' + self.api_name
            newurl += '.json'

            ret = self.cls.get(newurl)
            ret.of_cls = self.of_cls
            return ret
        except Exception, e:
            logging.error(str(e))
            raise

class User(TypePadObject):
    # documented fields
    id                 = fields.Field(api_name='urlId')
    atom_id            = fields.Field(api_name='id')
    display_name       = fields.Field(api_name='displayName')
    preferred_username = fields.Field(api_name='preferredUsername')
    about_me           = fields.Field(api_name='aboutMe')
    interests          = fields.List(fields.Field())
    urls               = fields.List(fields.Field())
    accounts           = fields.List(fields.Field())
    links              = fields.Object(LinkSet)
    object_types       = fields.Field(api_name='objectTypes')

    # astropad extras
    email              = fields.Field()

    relationships      = ApiLink(ListOf('UserRelationship'))
    events             = ApiLink(ListOf('Event'))
    comments           = ApiLink(ListOf('Asset'), api_name='comments-sent')
    notifications      = ApiLink(ListOf('Event'))
    memberships        = ApiLink(ListOf('UserRelationship'))
    elsewhere_accounts = ApiLink(ListOf('ElsewhereAccount'))

    @classmethod
    def get_self(cls, **kwargs):
        return cls.get('/users/@self.json', **kwargs)

class UserRelationship(TypePadObject):
    #status = fields.Field()
    source = fields.Object('User')
    target = fields.Object('User')

class PublicationStatus(TypePadObject):
    published = fields.Field()
    spam      = fields.Field()

class AssetRef(TypePadObject):
    ref  = fields.Field()
    href = fields.Field()
    type = fields.Field()
    id   = fields.Field(api_name='urlId')

class Asset(TypePadObject):
    # documented fields
    id           = fields.Field(api_name='urlId')
    atom_id      = fields.Field(api_name='id')
    title        = fields.Field()
    author       = fields.Object('User')
    published    = fields.Datetime()
    updated      = fields.Datetime()
    summary      = fields.Field()
    content      = fields.Field()
    # TODO  categories should be Tags?
    categories   = fields.List(fields.Field())
    object_types = fields.List(fields.Field(), api_name='objectTypes')
    status       = fields.Object('PublicationStatus')
    links        = fields.Object('LinkSet')
    in_reply_to  = fields.Object('AssetRef', api_name='inReplyTo')

    @property
    def actor(self):
        """
        An alias for author to satisify more generic 'actor' name used
        in templates where event/asset are used interchangeably.
        """
        return self.author

    def comment_count(self):
        for l in self.links:
            if l.rel == 'replies':
                return l.total
        return 0

    comments = ApiLink(ListOf('Asset'))

    def favorite_count(self):
        for l in self.links:
            if l.rel == 'favorites':
                return l.total
        return 0

    favorites = ApiLink(ListOf('Asset'))

    @property
    def asset_ref(self):
        # This is also stupid. Why not have in_reply_to just be another asset??
        ref = AssetRef()
        ref.type = 'application/json'
        ref.href = '/assets/%s.json' % self.id
        ref.ref = self.atom_id
        return ref

    def __unicode__(self):
        if self.title:
            return self.title
        if self.summary:
            return self.summary
        return self.content

    '''
    @property
    def author(self):
        try:
            return self.authors[0]
        except IndexError:
            return None
    '''

class Event(TypePadObject):
    id      = fields.Field(api_name='urlId')
    atom_id = fields.Field(api_name='id')
    verbs   = fields.List(fields.Field())
    # TODO: vary these based on verb content? oh boy
    actor   = fields.Object(User)
    object  = fields.Object(Asset)

    def __unicode__(self):
        return unicode(self.object)

class Comment(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Comment",), api_name='objectTypes')

class Post(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Post",), api_name='objectTypes')

class LinkAsset(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Link",), api_name='objectTypes')

class ElsewhereAccount(TypePadObject):
    domain            = fields.Field()
    username          = fields.Field()
    user_id           = fields.Field(api_name='userId')
    url               = fields.Field()
    provider_name     = fields.Field(api_name='providerName')
    provider_url      = fields.Field(api_name='providerURL')
    provider_icon_url = fields.Field(api_name='providerIconURL')

class Group(TypePadObject):
    id           = fields.Field(api_name='urlId')
    atom_id      = fields.Field(api_name='id')
    display_name = fields.Field(api_name='displayName')
    tagline      = fields.Field()
    urls         = fields.List(fields.Field())
    links        = fields.List(fields.Field())
    object_types = fields.List(fields.Field(), api_name='objectTypes')

    # TODO: these aren't really UserRelationships because the target is really a group
    memberships  = ApiLink(ListOf('UserRelationship'))
    assets       = ApiLink(ListOf('Asset'))
    events       = ApiLink(ListOf('Event'))
    comments     = ApiLink(ListOf('Asset'))

    # comments     = ApiLink(ListOf(Asset), api_name='comment-assets')
    post_assets  = ApiLink(ListOf(Post), api_name='post-assets')
    # photo_assets = ApiLink(ListOf(Post), api_name='photo-assets')
    # link_assets  = ApiLink(ListOf(Post), api_name='link-assets')
    # video_assets = ApiLink(ListOf(Post), api_name='video-assets')
    # audio_assets = ApiLink(ListOf(Post), api_name='audio-assets')
    # link_assets  = ApiLink(ListOf(LinkAsset), api_name='assets/@link')

class Application(TypePadObject):
    owner   = fields.Object(Group)
    api_key = fields.Field()
    links   = fields.Object(LinkSet)

    @property
    def oauth_request_token(self):
        return self.links['oauth-request-token-endpoint'].href

    @property
    def oauth_authorization_page(self):
        return self.links['oauth-authorization-page'].href

    @property
    def oauth_access_token_endpoint(self):
        return self.links['oauth-access-token-endpoint'].href

class GroupStatus(TypePadObject):
    #status = fields.Field()
    source = fields.Object('User')
    target = fields.Object('Group')
<commit_msg>spacing<commit_after>from urlparse import urljoin, urlparse, urlunparse
from datetime import datetime
import re
import cgi
import urllib

import remoteobjects
from remoteobjects import fields
from remoteobjects.dataobject import find_by_name
from remoteobjects.promise import PromiseError
import typepad
from batchhttp.client import BatchError
import logging

class TypePadObject(remoteobjects.RemoteObject):
    base_url = None
    batch_requests = True

    @classmethod
    def get_response(cls, url, http=None, **kwargs):
        http = typepad.client.http
        return super(TypePadObject, cls).get_response(url, http=http, **kwargs)

    @classmethod
    def get(cls, url, *args, **kwargs):
        if cls.base_url is not None and not urlparse(url)[1]:  # network location
            url = urljoin(cls.base_url, url)

        ret = super(TypePadObject, cls).get(url, *args, **kwargs)
        if cls.batch_requests:
            try:
                typepad.client.add(ret.get_request(), ret.update_from_response)
            except BatchError, ex:
                # We're suppressing an exception here for ListObjects
                # since in some cases we merely want the object to
                # 'post' against.
                if not issubclass(cls, ListObject):
                    raise PromiseError("Cannot get %s %s outside a batch request"
                        % (cls.__name__, url))
        return ret

    def deliver(self):
        if self.batch_requests:
            raise PromiseError("Cannot deliver %s %s except by batch request"
                % (type(self).__name__, self._location))
        return super(TypePadObject, self).deliver()

class Link(TypePadObject):
    rel      = fields.Field()
    href     = fields.Field()
    type     = fields.Field()
    width    = fields.Field()
    height   = fields.Field()
    duration = fields.Field()
    total    = fields.Field()

    def __repr__(self):
        return "<Link %s>" % self.href

class LinkSet(set, TypePadObject):
    def update_from_dict(self, data):
        self.update([Link.from_dict(x) for x in data])

    def to_dict(self):
        return [x.to_dict() for x in self]

    def __getitem__(self, key):
        if isinstance(key, slice):
            raise KeyError('LinkSets cannot be sliced')

        if key.endswith('_set'):
            # Gimme all matching links.
            key = key[:-4]
            return LinkSet([x for x in self if x.rel == key])

        # Gimme the first matching link.
        for x in self:
            if x.rel == key:
                return x

        raise KeyError('No such link %r in this set' % key)

    def link_by_width(self, width=None):
        """ An algorithm for selecting the best appropriate image
        given a specified display width.

        If no width is given, the widest image is selected.

        If a width is given, the first image that meets that width
        or is larger is selected. If none meet that size, the one
        that was closest to the specified width is chosen.
        """
        # TODO: is there a brisk way to do this?
        widest = None
        best = None
        for link in self:
            # Keep track of the widest variant; this will be our failsafe.
            if (not widest) or link.width > widest.width:
                widest = link
            # Width was specified; enclosure is equal or larger than this
            if width and link.width >= width:
                # Assign if nothing was already chosen or if this new
                # enclosure is smaller than the last best one.
                if (not best) or link.width < best.width:
                    best = link

        # use best available image if none was selected as the 'best' fit
        if best is None:
            return widest
        else:
            return best

class SequenceProxy(object):
    def make_sequence_method(methodname):
        def seqmethod(self, *args, **kwargs):
            # Proxy these methods to self.entries.
            return getattr(self.entries, methodname)(*args, **kwargs)
        seqmethod.__name__ = methodname
        return seqmethod

    __len__      = make_sequence_method('__len__')
    __getitem__  = make_sequence_method('__getitem__')
    __setitem__  = make_sequence_method('__setitem__')
    __delitem__  = make_sequence_method('__delitem__')
    __iter__     = make_sequence_method('__iter__')
    __reversed__ = make_sequence_method('__reversed__')
    __contains__ = make_sequence_method('__contains__')

class ListOf(remoteobjects.ListObject.__metaclass__):
    def __new__(cls, name, bases=None, attr=None):
        if attr is None:
            # TODO: memoize me
            entryclass = name
            if callable(entryclass):
                name = cls.__name__ + entryclass.__name__
            else:
                name = cls.__name__ + entryclass
            bases = (ListObject,)
            attr = {'entryclass': entryclass}

        bases = bases + (SequenceProxy,)
        return super(ListOf, cls).__new__(cls, name, bases, attr)

class ListObject(TypePadObject, remoteobjects.ListObject):
    __metaclass__ = ListOf

    total_results = fields.Field(api_name='totalResults')
    start_index   = fields.Field(api_name='startIndex')
    links         = fields.Object(LinkSet)
    entries       = fields.List(fields.Field())

    filterorder = ['following', 'follower', 'friend', 'nonreciprocal',
        'published', 'unpublished', 'spam', 'admin', 'member',
        'by-group', 'by-user', 'photo', 'post', 'video', 'audio', 'comment', 'link']

    def filter(self, **kwargs):
        # Split the list's URL into URL parts, filters, and queryargs.
        parts = list(urlparse(self._location))
        queryargs = cgi.parse_qs(parts[4], keep_blank_values=True)
        queryargs = dict([(k, v[0]) for k, v in queryargs.iteritems()])

        oldpath = parts[2]
        if not oldpath.endswith('.json'):
            raise AssertionError('oldpath %r does not end in %r' % (oldpath, '.json'))
        path = oldpath[:-5].split('/')

        filters = dict()
        newpath = list()
        pathparts = iter(path)
        for x in pathparts:
            if x.startswith('@'):
                x = x[1:]
                if x in ('by-group', 'by-user'):
                    filters[x] = pathparts.next()
                else:
                    filters[x] = True
            else:
                newpath.append(x)

        # Add kwargs into the filters and queryargs as appropriate.
        for k, v in kwargs.iteritems():
            # Convert by_group to by-group.
            k = k.replace('_', '-')
            # Convert by_group=7 to by_group='7'.
            v = str(v)

            if k in self.filterorder:
                filters[k] = v
            else:
                queryargs[k] = v

        # Put the filters back on the URL path in API order.
        keys = filters.keys()
        keys.sort(key=self.filterorder.index)
        for k in keys:
            if filters[k]:
                newpath.append('@' + k)
                if k in ('by-group', 'by-user'):
                    newpath.append(filters[k])

        # Coalesce the URL back into a string and make a new List from it.
        parts[2] = '/'.join(newpath) + '.json'
        parts[4] = urllib.urlencode(queryargs)
        newurl = urlunparse(parts)

        ret = self.get(newurl)
        ret.of_cls = self.of_cls
        return ret

    def __getitem__(self, key):
        if self._delivered or not isinstance(key, slice):
            return self.entries[key]
        args = dict()
        if key.start is not None:
            args['start_index'] = key.start
            if key.stop is not None:
                args['max_results'] = key.stop - key.start
        elif key.stop is not None:
            args['max_results'] = key.stop
        return self.filter(**args)

    def update_from_dict(self, data):
        super(ListObject, self).update_from_dict(data)
        # Post-convert all the "entries" list items to our entry class.
        entryclass = self.entryclass
        if not callable(entryclass):
            entryclass = find_by_name(entryclass)
        self.entries = [entryclass.from_dict(d) for d in self.entries]

class ApiLink(fields.Link):
    def __get__(self, instance, owner):
        try:
            if instance._location is None:
                raise AttributeError('Cannot find URL of %s relative to URL-less %s' % (type(self).__name__, owner.__name__))

            assert instance._location.endswith('.json')
            newurl = instance._location[:-5]
            newurl += '/' + self.api_name
            newurl += '.json'

            ret = self.cls.get(newurl)
            ret.of_cls = self.of_cls
            return ret
        except Exception, e:
            logging.error(str(e))
            raise

class User(TypePadObject):
    # documented fields
    id                 = fields.Field(api_name='urlId')
    atom_id            = fields.Field(api_name='id')
    display_name       = fields.Field(api_name='displayName')
    preferred_username = fields.Field(api_name='preferredUsername')
    about_me           = fields.Field(api_name='aboutMe')
    interests          = fields.List(fields.Field())
    urls               = fields.List(fields.Field())
    accounts           = fields.List(fields.Field())
    links              = fields.Object(LinkSet)
    object_types       = fields.Field(api_name='objectTypes')

    # astropad extras
    email              = fields.Field()

    relationships      = ApiLink(ListOf('UserRelationship'))
    events             = ApiLink(ListOf('Event'))
    comments           = ApiLink(ListOf('Asset'), api_name='comments-sent')
    notifications      = ApiLink(ListOf('Event'))
    memberships        = ApiLink(ListOf('UserRelationship'))
    elsewhere_accounts = ApiLink(ListOf('ElsewhereAccount'))

    @classmethod
    def get_self(cls, **kwargs):
        return cls.get('/users/@self.json', **kwargs)

class UserRelationship(TypePadObject):
    #status = fields.Field()
    source = fields.Object('User')
    target = fields.Object('User')

class PublicationStatus(TypePadObject):
    published = fields.Field()
    spam      = fields.Field()

class AssetRef(TypePadObject):
    ref  = fields.Field()
    href = fields.Field()
    type = fields.Field()
    id   = fields.Field(api_name='urlId')

class Asset(TypePadObject):
    # documented fields
    id           = fields.Field(api_name='urlId')
    atom_id      = fields.Field(api_name='id')
    title        = fields.Field()
    author       = fields.Object('User')
    published    = fields.Datetime()
    updated      = fields.Datetime()
    summary      = fields.Field()
    content      = fields.Field()
    # TODO  categories should be Tags?
    categories   = fields.List(fields.Field())
    object_types = fields.List(fields.Field(), api_name='objectTypes')
    status       = fields.Object('PublicationStatus')
    links        = fields.Object('LinkSet')
    in_reply_to  = fields.Object('AssetRef', api_name='inReplyTo')

    @property
    def actor(self):
        """
        An alias for author to satisify more generic 'actor' name used
        in templates where event/asset are used interchangeably.
        """
        return self.author

    def comment_count(self):
        for l in self.links:
            if l.rel == 'replies':
                return l.total
        return 0

    comments = ApiLink(ListOf('Asset'))

    def favorite_count(self):
        for l in self.links:
            if l.rel == 'favorites':
                return l.total
        return 0

    favorites = ApiLink(ListOf('Asset'))

    @property
    def asset_ref(self):
        # This is also stupid. Why not have in_reply_to just be another asset??
        ref = AssetRef()
        ref.type = 'application/json'
        ref.href = '/assets/%s.json' % self.id
        ref.ref = self.atom_id
        return ref

    def __unicode__(self):
        if self.title:
            return self.title
        if self.summary:
            return self.summary
        return self.content

    '''
    @property
    def author(self):
        try:
            return self.authors[0]
        except IndexError:
            return None
    '''

class Event(TypePadObject):
    id      = fields.Field(api_name='urlId')
    atom_id = fields.Field(api_name='id')
    verbs   = fields.List(fields.Field())
    # TODO: vary these based on verb content? oh boy
    actor   = fields.Object(User)
    object  = fields.Object(Asset)

    def __unicode__(self):
        return unicode(self.object)

class Comment(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Comment",), api_name='objectTypes')

class Post(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Post",), api_name='objectTypes')

class LinkAsset(Asset):
    object_types = fields.Constant(("tag:api.typepad.com,2009:Link",), api_name='objectTypes')

class ElsewhereAccount(TypePadObject):
    domain            = fields.Field()
    username          = fields.Field()
    user_id           = fields.Field(api_name='userId')
    url               = fields.Field()
    provider_name     = fields.Field(api_name='providerName')
    provider_url      = fields.Field(api_name='providerURL')
    provider_icon_url = fields.Field(api_name='providerIconURL')

class Group(TypePadObject):
    id           = fields.Field(api_name='urlId')
    atom_id      = fields.Field(api_name='id')
    display_name = fields.Field(api_name='displayName')
    tagline      = fields.Field()
    urls         = fields.List(fields.Field())
    links        = fields.List(fields.Field())
    object_types = fields.List(fields.Field(), api_name='objectTypes')

    # TODO: these aren't really UserRelationships because the target is really a group
    memberships  = ApiLink(ListOf('UserRelationship'))
    assets       = ApiLink(ListOf('Asset'))
    events       = ApiLink(ListOf('Event'))
    comments     = ApiLink(ListOf('Asset'))

    # comments     = ApiLink(ListOf(Asset), api_name='comment-assets')
    post_assets  = ApiLink(ListOf(Post), api_name='post-assets')
    # photo_assets = ApiLink(ListOf(Post), api_name='photo-assets')
    # link_assets  = ApiLink(ListOf(Post), api_name='link-assets')
    # video_assets = ApiLink(ListOf(Post), api_name='video-assets')
    # audio_assets = ApiLink(ListOf(Post), api_name='audio-assets')
    # link_assets  = ApiLink(ListOf(LinkAsset), api_name='assets/@link')

class Application(TypePadObject):
    owner   = fields.Object(Group)
    api_key = fields.Field()
    links   = fields.Object(LinkSet)

    @property
    def oauth_request_token(self):
        return self.links['oauth-request-token-endpoint'].href

    @property
    def oauth_authorization_page(self):
        return self.links['oauth-authorization-page'].href

    @property
    def oauth_access_token_endpoint(self):
        return self.links['oauth-access-token-endpoint'].href

class GroupStatus(TypePadObject):
    #status = fields.Field()
    source = fields.Object('User')
    target = fields.Object('Group')
<|endoftext|>


<commit_before>import os
import re
import threading

from . import core

@core.rule
def executable(
    name, sources=None, include=None, define=None, flags=None,
    links=None, compiler=None, warnings_are_errors=False, scan=True
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    include = list(include) if include else []
    define = dict(define) if define else {}
    flags = list(flags) if flags else []

    static = []
    shared = []

    if links:
        for link in links:
            if isinstance(link, str):
                lib = find_static_library(link)
                if lib is None:
                    raise ValueError('lib could not be found: ' + link)
                static.append(lib)
            elif getattr(link, 'type') == 'cpp.static_library':
                include.extend(link.headers)
                static.append(core.resolve(link.output))
            elif getattr(link, 'type') == 'cpp.shared_library':
                include.extend(link.headers)
                if toolchain is _GNU:
                    shared.append(core.resolve(link.output))
                else:
                    shared.append(core.resolve(link.msvc_lib))
            else:
                raise TypeError('invalid entry in links: "{}"'.format(link))

    if toolchain is _MSVC:
        name += '.exe'
    name = core.build(name)

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            include=include,
            define=define,
            flags=flags,
            compiler=compiler,
            error_warnings=warnings_are_errors,
            scan=scan,
        )
        objects.append(core.resolve(obj.output))

    yield core.publish(
        inputs=objects + static + shared,
        message='Link {}'.format(name),
        outputs=[name],
        result={
            'type': 'cpp.executable'
        }
    )

    if toolchain is _GNU:
        command = [compiler, '-o', name]
        command.extend(objects)
        command.extend(static)
        for s in shared:
            command.append(s)
            command.append('-Wl,-rpath,' + os.path.dirname(core.absolute(s)))
        command.append('-lstdc++')
        core.call(command, env=os.environ)
    elif toolchain is _MSVC:
        command = [compiler, '/Fe' + name, '/nologo']
        command.extend(objects + shared + static)
        core.call(command, env=_msvc_get_cl_env(compiler))

@core.rule
def static_library(
    name=None, sources=None, include=None, define=None, flags=None,
    headers=None, compiler=None, warnings_are_errors=False, scan=True
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if headers is None:
        headers = []
    if sources is None:
        sources = []

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            compiler=compiler,
            scan=scan,
            include=include,
            define=define,
            flags=flags,
            error_warnings=warnings_are_errors
        )
        objects.append(obj.output)

    if name is None:
        name = core.intermediate(core.checksum(
            sources, compiler, toolchain, include, define, headers))
    else:
        name = core.build(name)

    if toolchain is _MSVC:
        name += '.lib'
    elif toolchain is _GNU:
        name += '.a'

    yield core.publish(
        inputs=objects,
        message='Static {}'.format(name),
        outputs=[name],
        result={
            'type': 'cpp.static_library',
            'headers': core.absolute(core.resolve(headers))
        }
    )

    if toolchain is _GNU:
        archiver = core.which('ar')
        command = [archiver, 'rs', name]
        command.extend(objects)
        core.call(command)
    elif toolchain is _MSVC:
        archiver = os.path.join(os.path.dirname(compiler), 'lib.exe')
        command = [archiver, '/OUT:' + name]
        command.extend(objects)
        core.call(command)

@core.rule
def shared_library(
    name, sources, include=None, define=None, flags=None, headers=None,
    compiler=None, warnings_are_errors=False, scan=True, msvc_lib=False
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if headers is None:
        headers = []

    if flags is None:
        flags = []
    if toolchain is _GNU:
        flags.append('-fPIC')

    if define is None:
        define = {}
    define['DLL_EXPORT'] = 1

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            compiler=compiler,
            scan=scan,
            include=include,
            define=define,
            flags=flags,
            error_warnings=warnings_are_errors
        )
        objects.append(obj.output)

    if toolchain is _MSVC:
        lib = name + '.lib'
        if msvc_lib:
            lib = core.build(lib)
        else:
            lib = core.intermediate(lib)
        name = core.build(name + '.dll')
    else:
        lib = None
        head, tail = os.path.split(name)
        name = core.build(os.path.join(head, 'lib' + tail + '.so'))

    yield core.publish(
        inputs=objects,
        message='Shared {}'.format(name),
        outputs=[name, lib] if lib else [name],
        result={
            'type': 'cpp.shared_library',
            'msvc_lib': core.absolute(lib),
            'headers': core.absolute(core.resolve(headers)),
            'output': core.absolute(name)
        },
    )

    if toolchain is _GNU:
        command = [compiler, '-shared', '-o', name]
        command.extend(objects)
        command.append('-Wl,-soname,' + os.path.basename(name))
        core.call(command, env=os.environ)
    elif toolchain is _MSVC:
        command = [compiler, '/Fe' + name, '/nologo', '/LD']
        command.extend(objects)
        core.call(command, env=_msvc_get_cl_env(compiler))
        base = os.path.splitext(name)[0]
        if not msvc_lib:
            origin = base + '.lib'
            if os.path.isfile(lib):
                os.remove(lib)
            os.rename(origin, lib)
        os.remove(base + '.exp')
    else:
        raise NotImplementedError

@core.rule
def object(
    name=None, sources=None, include=None, define=None, flags=None,
    compiler=None, error_warnings=False, scan=True
):
    if isinstance(sources, str):
        raise TypeError('sources must not be a string - try to use a list')
    if not sources:
        raise ValueError('sources must not be empty')

    sources = core.resolve(sources)
    include = list(include) if include else []
    define = dict(define) if define else {}
    flags = list(flags) if flags else []

    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if name is None:
        name = core.intermediate(core.checksum(
            core.absolute(sources), compiler)[:16])

    if toolchain is _GNU:
        name += '.o'
    elif toolchain is _MSVC:
        name += '.obj'

    yield core.publish(
        inputs=sources + [compiler],
        message='Compile ' + ', '.join(sources),
        outputs=[name],
        check=[include, define, flags, error_warnings, scan],
        result={
            'type': 'cpp.object',
            'include': include,
            'define': define,
            'flags': flags,
            'compiler': compiler,
        }
    )

    if toolchain is _GNU:
        command = [compiler, '-c', '-o', name, '-x', 'c++', '-std=c++11']
        command.extend(sources)

        for directory in include:
            command.extend(['-I', directory])

        # Enable most warnings. Option to change this?
        command.append('-Wall')
        if error_warnings:
            command.append('-Werror')

        # Generate debug information... Option to turn this off?
        command.append('-g')

        for name, value in define.items():
            command.append('-D{}={}'.format(name, value))

        if scan:
            depfile = core.temporary(core.random('.d'))
            command.extend(['-MMD', '-MF', depfile])
        else:
            depfile = None

        if _gnu_supports_colors(compiler):
            command.append('-fdiagnostics-color')

        command.extend(flags)

        output = core.call(command)

        if scan:
            # TODO: Good parsing.
            with open(depfile) as file:
                content = file.read()
            used = {
                os.path.abspath(x) for x in
                content[content.find(':')+1:].replace('\\\n', '\n').split()
            }
            used.difference_update(sources)  # TODO: No difference?
        else:
            used = None

        yield core.deposit(inputs=used, warnings=output or None)

    elif toolchain is _MSVC:
        command = [compiler, '/c', '/Fo' + name, '/nologo']
        command.extend(sources)
        for directory in include:
            command.extend(['/I' + directory])

        if scan:
            command.append('/showIncludes')
        for name, value in define.items():
            command.append('/D{}={}'.format(name, value))

        # TODO: Option to set c++ standard.
        # command.append('/std:' + standard)

        # TODO: Figure out debug / relase
        # === DEBUG ===
        # command.append('/ZI')  Enable nice debug mode?
        # command.append('/Od')  Disable optimizations for debug
        # command.append('/Gm')  Enable minimal rebuild?
        # command.append('/RTC1') Run-time error checks
        # /MDd
        # === RELEASE ===
        # command.append('/Ox')  Full Optimization or /Oi?
        # /Zi                    Debug information
        # /GL                    Breaks object-linking? Whole prog optimization
        # command.append('/O2')  Optimize for speed

        command.append('/W4')  # Enable most warnings.
        if error_warnings:
            command.append('/WX')  # All warnings as errors.
        command.append('/EHsc')  # Specify exception handling model
        command.append('/sdl')  # Additional security warnings
        command.append('/TP')  # Assume C++ sources
        command.extend(flags)

        try:
            output = core.call(command, env=_msvc_get_cl_env(compiler))
        except core.CallError as exc:
            exc.output = _msvc_strip_includes(exc.output)
            raise

        if scan:
            used = _msvc_extract_includes(output)
        else:
            used = None

        yield core.deposit(
            inputs=used,
            warnings=_msvc_strip_includes(output).strip() or None
        )

def find_static_library(name):
    return _find('lib{}.a'.format(name))

def find_shared_library(name):
    return _find('lib{}.so'.format(name))

_GNU = 'GNU'
_MSVC = 'MSVC'

_compiler = None
_toolchain = None
_prepare_lock = threading.Lock()
_prepared = False

_msvc_envs = {}
_msvc_envs_lock = threading.Lock()
_gnu_color_support = {}
_gnu_color_lock = threading.Lock()

def _find(name):
    architectures = ['x86_64-linux-gnu', 'i386-linux-gnu']
    env_path = os.environ.get('PATH', '').split(os.pathsep)
    for directory in env_path:
        if directory.endswith('bin') or directory.endswith('sbin'):
            directory = os.path.normpath(os.path.dirname(directory))
        for arch in architectures:
            path = os.path.join(directory, 'lib', arch, name)
            if os.path.isfile(path):
                return path
            path = os.path.join(directory, arch, name)
            if os.path.isfile(path):
                return path
        path = os.path.join(directory, 'lib', name)
        if os.path.isfile(path):
            return path
        path = os.path.join(directory, name)
        if os.path.isfile(path):
            return path

def _get_default_compiler():
    if not _prepared:
        with _prepare_lock:
            if not _prepared:
                _do_get_default_compiler()
    return _compiler, _toolchain

def _do_get_default_compiler():
    global _compiler, _toolchain, _prepared

    _compiler = os.environ.get('CXX')
    if _compiler is not None:
        _compiler = core.which(_compiler)
        if _compiler is None:
            raise FileNotFoundError('CXX compiler does not exist')
        _toolchain = _get_toolchain(_compiler)
    elif core.windows:
        _compiler = _msvc_find_cl()
        _toolchain = _get_toolchain(_compiler)
    if _compiler is None:
        _compiler = core.which('c++') or core.which('g++')
        _toolchain = _get_toolchain(_compiler)

    if _compiler is None or _toolchain is None:
        ValueError('compiler could not be determined')

    core.debug('Detected C++ compiler: {} [{}]'.format(_compiler, _toolchain))
    _prepared = True

def _get_toolchain(compiler):
    if compiler is None:
        return None
    if 'g++' in compiler:
        return _GNU
    if 'clang' in compiler:
        return _GNU
    if 'cl.exe' in compiler:
        return _MSVC

def _msvc_get_cl_env(cl):
    with _msvc_envs_lock:
        if cl in _msvc_envs:
            return _msvc_envs[cl]
        else:
            env = _msvc_extract_env(cl)
            _msvc_envs[cl] = env
            return env

def _msvc_extract_env(cl):
    core.debug('Extracting environment for {}'.format(cl))
    bat = os.path.normpath(
        os.path.join(os.path.dirname(cl), '../vcvarsall.bat'))
    if not os.path.isfile(bat):
        return None
    helper = core.temporary(core.random('.bat'))
    with open(helper, 'w') as stream:
        stream.write('\n'.join([
            '@call "{bat}" {mode}',
            '@echo PATH=%PATH%',
            '@echo INCLUDE=%INCLUDE%',
            '@echo LIB=%LIB%;%LIBPATH%'
        ]).format(bat=bat, mode='x86'))
    cmd = core.which('cmd.exe')
    output = core.call([cmd, '/C', helper], env=os.environ)
    env = os.environ.copy()
    for line in output.strip().splitlines():
        key, value = line.split('=', maxsplit=1)
        env[key] = value[:-1]
    return env

def _msvc_find_cl():
    for version in ['120']:
        tools = os.environ.get('VS' + version + 'COMNTOOLS')
        if not tools:
            continue
        cl = os.path.normpath(os.path.join(tools, '../../VC/bin/cl.exe'))
        if os.path.isfile(cl):
            return cl

def _msvc_strip_includes(output):
    regex = re.compile(r'^[^:]+: [^:]+: +(.*)$')
    result = []
    for line in output.splitlines():
        match = regex.match(line)
        if match:
            path = match.group(1)
            if not os.path.isfile(path):
                result.append(line)
    return '\n'.join(result) + '\n'

def _msvc_extract_includes(output):
    regex = re.compile(r'^[^:]+: [^:]+: +(.*)$')
    used = []
    for line in output.splitlines():
        match = regex.match(line)
        if match:
            path = match.group(1)
            if os.path.isfile(path):
                used.append(path)
    return used

def _gnu_supports_colors(compiler):
    if compiler in _gnu_color_support:
        return _gnu_color_support[compiler]
    with _gnu_color_lock:
        if compiler in _gnu_color_support:
            return _gnu_color_support[compiler]
        try:
            core.call([compiler, '-fdiagnostics-color'])
        except core.CallError as exc:
            result = ('unrecognized command line option' not in exc.output
                      and 'unknown argument' not in exc.output)
            _gnu_color_support[compiler] = result
            return result
<commit_msg>Use cache decorator<commit_after>import os
import re
import threading

from . import core

@core.rule
def executable(
    name, sources=None, include=None, define=None, flags=None,
    links=None, compiler=None, warnings_are_errors=False, scan=True
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    include = list(include) if include else []
    define = dict(define) if define else {}
    flags = list(flags) if flags else []

    static = []
    shared = []

    if links:
        for link in links:
            if isinstance(link, str):
                lib = find_static_library(link)
                if lib is None:
                    raise ValueError('lib could not be found: ' + link)
                static.append(lib)
            elif getattr(link, 'type') == 'cpp.static_library':
                include.extend(link.headers)
                static.append(core.resolve(link.output))
            elif getattr(link, 'type') == 'cpp.shared_library':
                include.extend(link.headers)
                if toolchain is _GNU:
                    shared.append(core.resolve(link.output))
                else:
                    shared.append(core.resolve(link.msvc_lib))
            else:
                raise TypeError('invalid entry in links: "{}"'.format(link))

    if toolchain is _MSVC:
        name += '.exe'
    name = core.build(name)

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            include=include,
            define=define,
            flags=flags,
            compiler=compiler,
            error_warnings=warnings_are_errors,
            scan=scan,
        )
        objects.append(core.resolve(obj.output))

    yield core.publish(
        inputs=objects + static + shared,
        message='Link {}'.format(name),
        outputs=[name],
        result={
            'type': 'cpp.executable'
        }
    )

    if toolchain is _GNU:
        command = [compiler, '-o', name]
        command.extend(objects)
        command.extend(static)
        for s in shared:
            command.append(s)
            command.append('-Wl,-rpath,' + os.path.dirname(core.absolute(s)))
        command.append('-lstdc++')
        core.call(command, env=os.environ)
    elif toolchain is _MSVC:
        command = [compiler, '/Fe' + name, '/nologo']
        command.extend(objects + shared + static)
        core.call(command, env=_msvc_get_cl_env(compiler))

@core.rule
def static_library(
    name=None, sources=None, include=None, define=None, flags=None,
    headers=None, compiler=None, warnings_are_errors=False, scan=True
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if headers is None:
        headers = []
    if sources is None:
        sources = []

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            compiler=compiler,
            scan=scan,
            include=include,
            define=define,
            flags=flags,
            error_warnings=warnings_are_errors
        )
        objects.append(obj.output)

    if name is None:
        name = core.intermediate(core.checksum(
            sources, compiler, toolchain, include, define, headers))
    else:
        name = core.build(name)

    if toolchain is _MSVC:
        name += '.lib'
    elif toolchain is _GNU:
        name += '.a'

    yield core.publish(
        inputs=objects,
        message='Static {}'.format(name),
        outputs=[name],
        result={
            'type': 'cpp.static_library',
            'headers': core.absolute(core.resolve(headers))
        }
    )

    if toolchain is _GNU:
        archiver = core.which('ar')
        command = [archiver, 'rs', name]
        command.extend(objects)
        core.call(command)
    elif toolchain is _MSVC:
        archiver = os.path.join(os.path.dirname(compiler), 'lib.exe')
        command = [archiver, '/OUT:' + name]
        command.extend(objects)
        core.call(command)

@core.rule
def shared_library(
    name, sources, include=None, define=None, flags=None, headers=None,
    compiler=None, warnings_are_errors=False, scan=True, msvc_lib=False
):
    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if headers is None:
        headers = []

    if flags is None:
        flags = []
    if toolchain is _GNU:
        flags.append('-fPIC')

    if define is None:
        define = {}
    define['DLL_EXPORT'] = 1

    objects = []
    for source in sources:
        obj = object(
            sources=[source],
            compiler=compiler,
            scan=scan,
            include=include,
            define=define,
            flags=flags,
            error_warnings=warnings_are_errors
        )
        objects.append(obj.output)

    if toolchain is _MSVC:
        lib = name + '.lib'
        if msvc_lib:
            lib = core.build(lib)
        else:
            lib = core.intermediate(lib)
        name = core.build(name + '.dll')
    else:
        lib = None
        head, tail = os.path.split(name)
        name = core.build(os.path.join(head, 'lib' + tail + '.so'))

    yield core.publish(
        inputs=objects,
        message='Shared {}'.format(name),
        outputs=[name, lib] if lib else [name],
        result={
            'type': 'cpp.shared_library',
            'msvc_lib': core.absolute(lib),
            'headers': core.absolute(core.resolve(headers)),
            'output': core.absolute(name)
        },
    )

    if toolchain is _GNU:
        command = [compiler, '-shared', '-o', name]
        command.extend(objects)
        command.append('-Wl,-soname,' + os.path.basename(name))
        core.call(command, env=os.environ)
    elif toolchain is _MSVC:
        command = [compiler, '/Fe' + name, '/nologo', '/LD']
        command.extend(objects)
        core.call(command, env=_msvc_get_cl_env(compiler))
        base = os.path.splitext(name)[0]
        if not msvc_lib:
            origin = base + '.lib'
            if os.path.isfile(lib):
                os.remove(lib)
            os.rename(origin, lib)
        os.remove(base + '.exp')
    else:
        raise NotImplementedError

@core.rule
def object(
    name=None, sources=None, include=None, define=None, flags=None,
    compiler=None, error_warnings=False, scan=True
):
    if isinstance(sources, str):
        raise TypeError('sources must not be a string - try to use a list')
    if not sources:
        raise ValueError('sources must not be empty')

    sources = core.resolve(sources)
    include = list(include) if include else []
    define = dict(define) if define else {}
    flags = list(flags) if flags else []

    if compiler is None:
        compiler, toolchain = _get_default_compiler()
    else:
        toolchain = _get_toolchain(compiler)
        if toolchain is None:
            raise ValueError('toolchain could not be detected')

    if name is None:
        name = core.intermediate(core.checksum(
            core.absolute(sources), compiler)[:16])

    if toolchain is _GNU:
        name += '.o'
    elif toolchain is _MSVC:
        name += '.obj'

    yield core.publish(
        inputs=sources + [compiler],
        message='Compile ' + ', '.join(sources),
        outputs=[name],
        check=[include, define, flags, error_warnings, scan],
        result={
            'type': 'cpp.object',
            'include': include,
            'define': define,
            'flags': flags,
            'compiler': compiler,
        }
    )

    if toolchain is _GNU:
        command = [compiler, '-c', '-o', name, '-x', 'c++', '-std=c++11']
        command.extend(sources)

        for directory in include:
            command.extend(['-I', directory])

        # Enable most warnings. Option to change this?
        command.append('-Wall')
        if error_warnings:
            command.append('-Werror')

        # Generate debug information... Option to turn this off?
        command.append('-g')

        for name, value in define.items():
            command.append('-D{}={}'.format(name, value))

        if scan:
            depfile = core.temporary(core.random('.d'))
            command.extend(['-MMD', '-MF', depfile])
        else:
            depfile = None

        if _gnu_supports_colors(compiler):
            command.append('-fdiagnostics-color')

        command.extend(flags)

        output = core.call(command)

        if scan:
            # TODO: Good parsing.
            with open(depfile) as file:
                content = file.read()
            used = {
                os.path.abspath(x) for x in
                content[content.find(':')+1:].replace('\\\n', '\n').split()
            }
            used.difference_update(sources)  # TODO: No difference?
        else:
            used = None

        yield core.deposit(inputs=used, warnings=output or None)

    elif toolchain is _MSVC:
        command = [compiler, '/c', '/Fo' + name, '/nologo']
        command.extend(sources)
        for directory in include:
            command.extend(['/I' + directory])

        if scan:
            command.append('/showIncludes')
        for name, value in define.items():
            command.append('/D{}={}'.format(name, value))

        # TODO: Option to set c++ standard.
        # command.append('/std:' + standard)

        # TODO: Figure out debug / relase
        # === DEBUG ===
        # command.append('/ZI')  Enable nice debug mode?
        # command.append('/Od')  Disable optimizations for debug
        # command.append('/Gm')  Enable minimal rebuild?
        # command.append('/RTC1') Run-time error checks
        # /MDd
        # === RELEASE ===
        # command.append('/Ox')  Full Optimization or /Oi?
        # /Zi                    Debug information
        # /GL                    Breaks object-linking? Whole prog optimization
        # command.append('/O2')  Optimize for speed

        command.append('/W4')  # Enable most warnings.
        if error_warnings:
            command.append('/WX')  # All warnings as errors.
        command.append('/EHsc')  # Specify exception handling model
        command.append('/sdl')  # Additional security warnings
        command.append('/TP')  # Assume C++ sources
        command.extend(flags)

        try:
            output = core.call(command, env=_msvc_get_cl_env(compiler))
        except core.CallError as exc:
            exc.output = _msvc_strip_includes(exc.output)
            raise

        if scan:
            used = _msvc_extract_includes(output)
        else:
            used = None

        yield core.deposit(
            inputs=used,
            warnings=_msvc_strip_includes(output).strip() or None
        )

def find_static_library(name):
    return _find('lib{}.a'.format(name))

def find_shared_library(name):
    return _find('lib{}.so'.format(name))

_GNU = 'GNU'
_MSVC = 'MSVC'

def _find(name):
    architectures = ['x86_64-linux-gnu', 'i386-linux-gnu']
    env_path = os.environ.get('PATH', '').split(os.pathsep)
    for directory in env_path:
        if directory.endswith('bin') or directory.endswith('sbin'):
            directory = os.path.normpath(os.path.dirname(directory))
        for arch in architectures:
            path = os.path.join(directory, 'lib', arch, name)
            if os.path.isfile(path):
                return path
            path = os.path.join(directory, arch, name)
            if os.path.isfile(path):
                return path
        path = os.path.join(directory, 'lib', name)
        if os.path.isfile(path):
            return path
        path = os.path.join(directory, name)
        if os.path.isfile(path):
            return path

@core.cache
def _get_default_compiler():
    compiler = os.environ.get('CXX')
    if compiler is not None:
        compiler = core.which(compiler)
        if compiler is None:
            raise FileNotFoundError('CXX compiler does not exist')
        toolchain = _get_toolchain(compiler)
    elif core.windows:
        compiler = _msvc_find_cl()
        toolchain = _get_toolchain(compiler)

    if compiler is None:
        compiler = core.which('c++') or core.which('g++')
        toolchain = _get_toolchain(compiler)

    if compiler is None:
        ValueError('compiler could not be determined')
    elif toolchain is None:
        raise ValueError('toolchain could not be detected')

    core.debug('Detected C++ compiler: {} [{}]'.format(compiler, toolchain))
    return compiler, toolchain

def _get_toolchain(compiler):
    if compiler is None:
        return None
    if 'g++' in compiler:
        return _GNU
    if 'clang' in compiler:
        return _GNU
    if 'cl.exe' in compiler:
        return _MSVC

@core.cache
def _msvc_get_cl_env(cl):
    core.debug('Extracting environment for {}'.format(cl))
    bat = os.path.normpath(
        os.path.join(os.path.dirname(cl), '../vcvarsall.bat'))
    if not os.path.isfile(bat):
        return None
    helper = core.temporary(core.random('.bat'))
    with open(helper, 'w') as stream:
        stream.write('\n'.join([
            '@call "{bat}" {mode}',
            '@echo PATH=%PATH%',
            '@echo INCLUDE=%INCLUDE%',
            '@echo LIB=%LIB%;%LIBPATH%'
        ]).format(bat=bat, mode='x86'))
    cmd = core.which('cmd.exe')
    output = core.call([cmd, '/C', helper], env=os.environ)
    env = os.environ.copy()
    for line in output.strip().splitlines():
        key, value = line.split('=', maxsplit=1)
        env[key] = value[:-1]
    return env

def _msvc_find_cl():
    for version in ['120']:
        tools = os.environ.get('VS' + version + 'COMNTOOLS')
        if not tools:
            continue
        cl = os.path.normpath(os.path.join(tools, '../../VC/bin/cl.exe'))
        if os.path.isfile(cl):
            return cl

def _msvc_strip_includes(output):
    regex = re.compile(r'^[^:]+: [^:]+: +(.*)$')
    result = []
    for line in output.splitlines():
        match = regex.match(line)
        if match:
            path = match.group(1)
            if not os.path.isfile(path):
                result.append(line)
    return '\n'.join(result) + '\n'

def _msvc_extract_includes(output):
    regex = re.compile(r'^[^:]+: [^:]+: +(.*)$')
    used = []
    for line in output.splitlines():
        match = regex.match(line)
        if match:
            path = match.group(1)
            if os.path.isfile(path):
                used.append(path)
    return used

@core.cache
def _gnu_supports_colors(compiler):
    try:
        core.call([compiler, '-fdiagnostics-color'])
    except core.CallError as exc:
        return ('unknown argument' not in exc.output and
                'unrecognized command line option' not in exc.output)
<|endoftext|>