<commit_before># -*- coding: utf-8 -*-
#
# © 2014 Ian Eure
# Author: Ian Eure <ian.eure@gmail.com>
#

"""Data I/O serial interface."""

import array
import serial
import sys
import time
from . import format
from yar.cksum import *
import yar.io as io

EOL = "\r"

 # Error handling

ERRS = ((2**31, "Error:"),
        (2**26, "Serial overrun error"),
        (2**25, "Serial framing error"),
        (2**24, "Buffer overflow error"),
        (2**23, "Programming error"),
        (2**22, "Start line not set high"),
        (2**21, "L2 + L3 > DEVICE"),
        (2**20, "Composite DAC error"),
        (2**19, "Device not blank"),
        (2**18, "Illegal bit"),
        (2**17, "Verification failed"),
        (2**16, "Incomplete programming, or no card set"),
        (2**15, "I/O error"),
        (2**12, "Compare error"),
        (2**11, "Checksum error"),
        (2**10, "Record count, address check, or record type error"),
        (2**9, "Address error"),
        (2**8, "Data not hexidecimal, or insufficient data received"),
        (2**7, "RAM error"),
        (2**5, "L2 + L3 > RAM"),
        (2**4, "Invalid center point for split or shuffle"),
        (2**3, "Illegal split or shuffle"),
        (2**2, "No RAM or insufficient RAM"),
        (2**1, "RAM write error, or program memory failure"),
        (2**0, "RAM end not on 1K boundary"))

class ProgrammerError(IOError):

    def __init__(self, code):
        IOError.__init__(self, decode_errors(code))

    def __unicode__(self):
        return self.message

def decode_errors(err):
    """Return errors which err represents."""
    return " ".join([msg for (key, msg) in ERRS if err & key])

def errorp(err):
    """Is this code an error?"""
    return bool(err & 2**31)

class Yar():

    """Class for communicating with Data I/O devices."""

    def __init__(self, *args, **kwargs):
        kwargs['timeout'] = 0
        self.port = serial.Serial(*args, **kwargs)

    def _writeline(self, line, *args):
        """Write a line to the programmer."""
        self.port.write((line + EOL) % args)

    def _readline(self):
        """Read a line from the programmer. Returns None or line.

           The device may be slow to produce a response, so we try a
           few times with a delay in between."""
        tries = 1
        if self.port.inWaiting() == 0 and tries <= 3:
            time.sleep(tries * .25)
            tries += 1

        if self.port.inWaiting() == 0:
            return None

        buf = ""
        ch = ""
        while ch != EOL:
            ch = self.port.read(1)
            buf += ch

        return buf.strip()

    def _readok(self):
        """Await a response, returning a boolean indicating success."""
        line = self._readline()
        if line == None:
            return None
        elif line[-1] == "?" or line[-1] == ">":
            return line[:-1] or True
        return False

    def _resp(self):
        """Return the response, or raise an exception."""
        r = self._readok()
        if r == None:
            return r
        if not r:
            err = self.last_error()
            if not err:
                raise IOError("Failed to get error status!")
            raise ProgrammerError(err)
        return r

    def _await(self, max=60, fd=None):
        """Await a response for long-running operations.
           Does not read the response.

           This will wait up to max seconds for a response.
           Returns True if a response is waiting, False if timed out."""
        tries = 0
        while self.port.inWaiting() == 0 and tries < max:
            time.sleep(1)
            tries += 1
        return tries != max

    def last_error(self):
        """Return the last error, or None"""
        self._writeline("F")
        resp = self._readok()
        return resp and errorp(int(resp, 16)) or None

    def ping(self):
        """Ping the programmer. Returns True if it is responding."""
        self._writeline("H")
        return self._readok()

    def abort(self):
        self.port.write([0x27])
        self.flush()

    def config(self):
        """Return device configuration."""
        self._writeline("G")
        return self._readok()

    def blank_test(self):
        """Ensure the device in the programmer is blank."""
        self._writeline("B")
        self._await()
        return self._readok()

    def illegal_bit_test(self):
        self._writeline("T")
        self._readok()

    def select(self, function):
        """Select an extended function."""
        self._writeline("%02x]" % function)
        return self._readok()

    def set_block_size(self, size):
        self._writeline("%04x;", size)
        return self._readok()

    def set_record_size(self, size):
        self._writeline("%02xH", size)
        return self._readok()

    def await_start(self):
        """Wait for the operator to press START on the programmer."""
        self._writeline('%%')
        r = self._await(6000)
        if not r:
            return r
        return self._readok()

    def set_format(self, format, control_code=0):
        """Set the format used to send and receive data."""
        self._writeline("%x%02xA" % (control_code, format))
        return self._readok()

    def set_device(self, family, pinout):
        """Set the device type to use."""
        self._writeline("%x%x@", family & 0xFF, pinout & 0xFF)
        self._resp()

    def get_device(self):
        """Return the currently selected family and pinout."""
        self._writeline("[")
        resp = self._resp()
        return (int(resp[0:2], 16), int(resp[2:4], 16))

    def load(self):
        """Load a device's contents and return the checksum."""
        self._writeline("L")
        self._await()
        return self._readok()

    def checksum(self):
        """Return the checksum of programmer RAM"""
        self._writeline("S")
        self._await()
        r = self._resp()
        return r and int(r, 16) or r

    def flush(self):
        """Flush any data waiting in the read buffer."""
        if self.port.inWaiting() > 0:
            self.port.flushInput()

    def clear_ram(self):
        """Zero out device memory."""
        self._writeline("^")
        self._await()
        return self._readok()

    def program_device(self):
        """Write programmer RAM to device."""
        self._writeline("P")
        self._await(max=300)
        return self._readok()

    def verify(self):
        """Verify a device against programmer RAM."""
        self._writeline("V")
        self._await(max=300)
        return self._readok()

    def dump_to(self, outp, progress_to=None):
        """Dump programmer RAM to file-like object."""
        out = progress_to or sys.stderr
        self.set_format(format.BINARY)
        self._writeline("O")
        data_buf = array.array("B")
        header = array.array("B")

        # Read 16-byte header
        while len(header) < 16:
            header.fromstring(self.port.read(1))

        if header[:10].tolist() != [0x0D, 0x08, 0x1C, 0x3E, 0x6B, 0x08,
                                    0x00, 0x00, 0x00, 0x00]:
            msg = "Invalid header " + " ".join([r"%02x"] * len(header))
            raise IOError(msg % tuple(header.tolist()))

        sz = int("".join(map(str, header[0x0A:0x0F])), 16)
        p = io.Progress(sz)
        while len(data_buf) < sz:
            # Don't overflow the buffer
            tr = self.port.read(min(sz - len(data_buf), 128))
            if tr != '':
                data_buf.fromstring(tr)
                p.update(len(data_buf))
                out.write(repr(p) + "\r")
                out.flush()
            else:
                time.sleep(.1)

        # Two nulls + checksum
        trailer = array.array("B")
        while len(trailer) < 4:
            trailer.fromstring(self.port.read(1))

        prog_sum = (trailer[2] << 8) + trailer[3]
        pload_sum = checksum(data_buf) & 0xFFFF
        if prog_sum != pload_sum:
            raise IOError("Programmer cksum %04x != data cksum %04x" % (
                prog_sum, pload_sum))
        data_buf.tofile(outp)

    def load_from(self, inp, progress_to=None):
        """Load programmer RAM from file-like object."""
        out = progress_to or sys.stderr
        bytes = io.file_to_bytes(inp)
        p = io.Progress(len(bytes))
        sum = checksum(bytes) & 0xFFFF
        self.set_format(format.BINARY)
        self._writeline("I")

        # Header
        header = [0x08, 0x1C, 0x2A, 0x49, 0x08]
        self.port.write(header)

        # Null
        self.port.write([0x00])

        # Data length
        # FIXME this is stupid but I can't think right now.
        self.port.write([int(n, 16) for n in '%04x' % len(bytes)])

        self.port.write([127])  # Rubout

        # Data
        sent = 0
        while len(bytes) > 0:
            block = min(128, len(bytes))
            assert block > 0, "Can't send empty block"
            loop_bytes = bytes[:block]
            written = self.port.write(loop_bytes)
            assert written == block, "Incomplete write"
            assert self.port.inWaiting() == 0, \
                "Programmer is trying to tell us something"
            self.port.flush()
            sent += written
            p.update(sent)
            out.write(repr(p) + "\r")
            out.flush()
            bytes = bytes[block:]  # Truncate buffer
            time.sleep(.1)

        # Trailer
        self.port.write([0x00, 0x00])

        # Checksum
        self.port.write([sum >> 8, sum & 0xFF])

        self._await()
        if not self._readok():
            raise Exception(self.last_error())
        s = self.checksum()
        if s != sum:
            raise Exception("Checksum error!")
<commit_msg>Use the correct rubout when uploading data.<commit_after># -*- coding: utf-8 -*-
#
# © 2014 Ian Eure
# Author: Ian Eure <ian.eure@gmail.com>
#

"""Data I/O serial interface."""

import array
import serial
import sys
import time
from . import format
from yar.cksum import *
import yar.io as io

EOL = "\r"

 # Error handling

ERRS = ((2**31, "Error:"),
        (2**26, "Serial overrun error"),
        (2**25, "Serial framing error"),
        (2**24, "Buffer overflow error"),
        (2**23, "Programming error"),
        (2**22, "Start line not set high"),
        (2**21, "L2 + L3 > DEVICE"),
        (2**20, "Composite DAC error"),
        (2**19, "Device not blank"),
        (2**18, "Illegal bit"),
        (2**17, "Verification failed"),
        (2**16, "Incomplete programming, or no card set"),
        (2**15, "I/O error"),
        (2**12, "Compare error"),
        (2**11, "Checksum error"),
        (2**10, "Record count, address check, or record type error"),
        (2**9, "Address error"),
        (2**8, "Data not hexidecimal, or insufficient data received"),
        (2**7, "RAM error"),
        (2**5, "L2 + L3 > RAM"),
        (2**4, "Invalid center point for split or shuffle"),
        (2**3, "Illegal split or shuffle"),
        (2**2, "No RAM or insufficient RAM"),
        (2**1, "RAM write error, or program memory failure"),
        (2**0, "RAM end not on 1K boundary"))

class ProgrammerError(IOError):

    def __init__(self, code):
        IOError.__init__(self, decode_errors(code))

    def __unicode__(self):
        return self.message

def decode_errors(err):
    """Return errors which err represents."""
    return " ".join([msg for (key, msg) in ERRS if err & key])

def errorp(err):
    """Is this code an error?"""
    return bool(err & 2**31)

class Yar():

    """Class for communicating with Data I/O devices."""

    def __init__(self, *args, **kwargs):
        kwargs['timeout'] = 0
        self.port = serial.Serial(*args, **kwargs)

    def _writeline(self, line, *args):
        """Write a line to the programmer."""
        self.port.write((line + EOL) % args)

    def _readline(self):
        """Read a line from the programmer. Returns None or line.

           The device may be slow to produce a response, so we try a
           few times with a delay in between."""
        tries = 1
        if self.port.inWaiting() == 0 and tries <= 3:
            time.sleep(tries * .25)
            tries += 1

        if self.port.inWaiting() == 0:
            return None

        buf = ""
        ch = ""
        while ch != EOL:
            ch = self.port.read(1)
            buf += ch

        return buf.strip()

    def _readok(self):
        """Await a response, returning a boolean indicating success."""
        line = self._readline()
        if line == None:
            return None
        elif line[-1] == "?" or line[-1] == ">":
            return line[:-1] or True
        return False

    def _resp(self):
        """Return the response, or raise an exception."""
        r = self._readok()
        if r == None:
            return r
        if not r:
            err = self.last_error()
            if not err:
                raise IOError("Failed to get error status!")
            raise ProgrammerError(err)
        return r

    def _await(self, max=60, fd=None):
        """Await a response for long-running operations.
           Does not read the response.

           This will wait up to max seconds for a response.
           Returns True if a response is waiting, False if timed out."""
        tries = 0
        while self.port.inWaiting() == 0 and tries < max:
            time.sleep(1)
            tries += 1
        return tries != max

    def last_error(self):
        """Return the last error, or None"""
        self._writeline("F")
        resp = self._readok()
        return resp and errorp(int(resp, 16)) or None

    def ping(self):
        """Ping the programmer. Returns True if it is responding."""
        self._writeline("H")
        return self._readok()

    def abort(self):
        self.port.write([0x27])
        self.flush()

    def config(self):
        """Return device configuration."""
        self._writeline("G")
        return self._readok()

    def blank_test(self):
        """Ensure the device in the programmer is blank."""
        self._writeline("B")
        self._await()
        return self._readok()

    def illegal_bit_test(self):
        self._writeline("T")
        self._readok()

    def select(self, function):
        """Select an extended function."""
        self._writeline("%02x]" % function)
        return self._readok()

    def set_block_size(self, size):
        self._writeline("%04x;", size)
        return self._readok()

    def set_record_size(self, size):
        self._writeline("%02xH", size)
        return self._readok()

    def await_start(self):
        """Wait for the operator to press START on the programmer."""
        self._writeline('%%')
        r = self._await(6000)
        if not r:
            return r
        return self._readok()

    def set_format(self, format, control_code=0):
        """Set the format used to send and receive data."""
        self._writeline("%x%02xA" % (control_code, format))
        return self._readok()

    def set_device(self, family, pinout):
        """Set the device type to use."""
        self._writeline("%x%x@", family & 0xFF, pinout & 0xFF)
        self._resp()

    def get_device(self):
        """Return the currently selected family and pinout."""
        self._writeline("[")
        resp = self._resp()
        return (int(resp[0:2], 16), int(resp[2:4], 16))

    def load(self):
        """Load a device's contents and return the checksum."""
        self._writeline("L")
        self._await()
        return self._readok()

    def checksum(self):
        """Return the checksum of programmer RAM"""
        self._writeline("S")
        self._await()
        r = self._resp()
        return r and int(r, 16) or r

    def flush(self):
        """Flush any data waiting in the read buffer."""
        if self.port.inWaiting() > 0:
            self.port.flushInput()

    def clear_ram(self):
        """Zero out device memory."""
        self._writeline("^")
        self._await()
        return self._readok()

    def program_device(self):
        """Write programmer RAM to device."""
        self._writeline("P")
        self._await(max=300)
        return self._readok()

    def verify(self):
        """Verify a device against programmer RAM."""
        self._writeline("V")
        self._await(max=300)
        return self._readok()

    def dump_to(self, outp, progress_to=None):
        """Dump programmer RAM to file-like object."""
        out = progress_to or sys.stderr
        self.set_format(format.BINARY)
        self._writeline("O")
        data_buf = array.array("B")
        header = array.array("B")

        # Read 16-byte header
        while len(header) < 16:
            header.fromstring(self.port.read(1))

        if header[:10].tolist() != [0x0D, 0x08, 0x1C, 0x3E, 0x6B, 0x08,
                                    0x00, 0x00, 0x00, 0x00]:
            msg = "Invalid header " + " ".join([r"%02x"] * len(header))
            raise IOError(msg % tuple(header.tolist()))

        sz = int("".join(map(str, header[0x0A:0x0F])), 16)
        p = io.Progress(sz)
        while len(data_buf) < sz:
            # Don't overflow the buffer
            tr = self.port.read(min(sz - len(data_buf), 128))
            if tr != '':
                data_buf.fromstring(tr)
                p.update(len(data_buf))
                out.write(repr(p) + "\r")
                out.flush()
            else:
                time.sleep(.1)

        # Two nulls + checksum
        trailer = array.array("B")
        while len(trailer) < 4:
            trailer.fromstring(self.port.read(1))

        prog_sum = (trailer[2] << 8) + trailer[3]
        pload_sum = checksum(data_buf) & 0xFFFF
        if prog_sum != pload_sum:
            raise IOError("Programmer cksum %04x != data cksum %04x" % (
                prog_sum, pload_sum))
        data_buf.tofile(outp)

    def load_from(self, inp, progress_to=None):
        """Load programmer RAM from file-like object."""
        out = progress_to or sys.stderr
        bytes = io.file_to_bytes(inp)
        p = io.Progress(len(bytes))
        sum = checksum(bytes) & 0xFFFF
        self.set_format(format.BINARY)
        self._writeline("I")

        # Header
        header = [0x08, 0x1C, 0x2A, 0x49, 0x08]
        self.port.write(header)

        # Null
        self.port.write([0x00])

        # Data length
        # FIXME this is stupid but I can't think right now.
        self.port.write([int(n, 16) for n in '%04x' % len(bytes)])

        self.port.write([0xFF])  # Rubout

        # Data
        sent = 0
        while len(bytes) > 0:
            block = min(128, len(bytes))
            assert block > 0, "Can't send empty block"
            loop_bytes = bytes[:block]
            written = self.port.write(loop_bytes)
            assert written == block, "Incomplete write"
            assert self.port.inWaiting() == 0, \
                "Programmer is trying to tell us something"
            self.port.flush()
            sent += written
            p.update(sent)
            out.write(repr(p) + "\r")
            out.flush()
            bytes = bytes[block:]  # Truncate buffer
            time.sleep(.1)

        # Trailer
        self.port.write([0x00, 0x00])

        # Checksum
        self.port.write([sum >> 8, sum & 0xFF])

        self._await()
        if not self._readok():
            raise Exception(self.last_error())
        s = self.checksum()
        if s != sum:
            raise Exception("Checksum error!")
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from kvirt.config import Kconfig
import json
import os
import argparse

def empty():
    return {'_meta': {'hostvars': {}}}

class KcliInventory(object):

    def __init__(self):
        self.inventory = {}
        self.read_cli_args()
        config = Kconfig(quiet=True)
        self.host = config.host
        self.port = config.port
        self.user = config.user
        self.tunnel = config.tunnel
        self.k = config.k
        self.type = config.type
        if self.k.conn is None:
            os._exit(1)

        # Called with `--list`.
        if self.args.list:
            self.inventory = self.get()
        # Called with `--host [hostname]`.
        elif self.args.host:
            # Not implemented, since we return _meta info `--list`.
            self.inventory = empty()
        # If no groups or vars are present, return an empty inventory.
        else:
            self.inventory = empty()
        print(json.dumps(self.inventory))

    # Read the command line args passed to the script.
    def read_cli_args(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--list', action='store_true')
        parser.add_argument('--host', action='store')
        self.args = parser.parse_args()

    def get(self):
        k = self.k
        tunnel = self.tunnel
        metadata = {'_meta': {'hostvars': {}}}
        hostvalues = metadata['_meta']['hostvars']
        for vm in k.list():
            name = vm[0]
            status = vm[1]
            ip = vm[2]
            template = vm[3]
            description = vm[4]
            profile = vm[5]
            if description == '':
                description = 'kvirt'
            if description not in metadata:
                metadata[description] = {"hosts": [name], "vars": {"plan": description, "profile": profile}}
            else:
                metadata[description]["hosts"].append(name)
            hostvalues[name] = {'status': status}
            if tunnel and self.type in ['kvm', 'kubevirt']:
                hostvalues[name]['ansible_ssh_common_args'] = \
                    "-o ProxyCommand='ssh -p %s -W %%h:%%p %s@%s'" % (self.port, self.user, self.host)
            if ip != '':
                if self.type == 'vbox':
                    hostvalues[name]['ansible_host'] = '127.0.0.1'
                    hostvalues[name]['ansible_port'] = ip
                else:
                    hostvalues[name]['ansible_host'] = ip
                if template != '':
                    user = k._ssh_credentials(name)[0]
                    hostvalues[name]['ansible_user'] = user
        return metadata

KcliInventory()
<commit_msg>klist.py to not rely on ssh_credentials<commit_after>#!/usr/bin/env python3

from kvirt.config import Kconfig
from kvirt.common import get_user
import json
import os
import argparse

def empty():
    return {'_meta': {'hostvars': {}}}

class KcliInventory(object):

    def __init__(self):
        self.inventory = {}
        self.read_cli_args()
        config = Kconfig(quiet=True)
        self.host = config.host
        self.port = config.port
        self.user = config.user
        self.tunnel = config.tunnel
        self.k = config.k
        self.type = config.type
        if self.k.conn is None:
            os._exit(1)

        # Called with `--list`.
        if self.args.list:
            self.inventory = self.get()
        # Called with `--host [hostname]`.
        elif self.args.host:
            # Not implemented, since we return _meta info `--list`.
            self.inventory = empty()
        # If no groups or vars are present, return an empty inventory.
        else:
            self.inventory = empty()
        print(json.dumps(self.inventory))

    # Read the command line args passed to the script.
    def read_cli_args(self):
        parser = argparse.ArgumentParser()
        parser.add_argument('--list', action='store_true')
        parser.add_argument('--host', action='store')
        self.args = parser.parse_args()

    def get(self):
        k = self.k
        tunnel = self.tunnel
        metadata = {'_meta': {'hostvars': {}}}
        hostvalues = metadata['_meta']['hostvars']
        for vm in k.list():
            name = vm[0]
            status = vm[1]
            ip = vm[2]
            template = vm[3]
            description = vm[4]
            profile = vm[5]
            if description == '':
                description = 'kvirt'
            if description not in metadata:
                metadata[description] = {"hosts": [name], "vars": {"plan": description, "profile": profile}}
            else:
                metadata[description]["hosts"].append(name)
            hostvalues[name] = {'status': status}
            if tunnel and self.type in ['kvm', 'kubevirt']:
                hostvalues[name]['ansible_ssh_common_args'] = \
                    "-o ProxyCommand='ssh -p %s -W %%h:%%p %s@%s'" % (self.port, self.user, self.host)
            if ip != '':
                if self.type == 'vbox':
                    hostvalues[name]['ansible_host'] = '127.0.0.1'
                    hostvalues[name]['ansible_port'] = ip
                else:
                    hostvalues[name]['ansible_host'] = ip
                if template != '':
                    user = get_user(template)
                    hostvalues[name]['ansible_user'] = user
        return metadata

KcliInventory()
<|endoftext|>


<commit_before>from __future__ import with_statement

from functools import wraps
import inspect
import sys

from fabric import state
from fabric.utils import abort, warn, error
from fabric.network import to_dict, normalize_to_string, disconnect_all
from fabric.context_managers import settings
from fabric.job_queue import JobQueue
from fabric.task_utils import crawl, merge, parse_kwargs
from fabric.exceptions import NetworkError

def get_task_details(task):
    details = [task.__doc__ if task.__doc__ else 'No docstring provided']
    argspec = inspect.getargspec(task)

    default_args = [] if not argspec.defaults else argspec.defaults
    num_default_args = len(default_args)
    args_without_defaults = argspec.args[:len(argspec.args) - num_default_args]
    args_with_defaults = argspec.args[-1 * num_default_args:]

    details.append('Arguments: %s' % (
        ', '.join(
            args_without_defaults + [
                '%s=%r' % (arg, default)
                for arg, default in zip(args_with_defaults, default_args)
            ])
    ))

    return '\n'.join(details)

def _get_list(env):
    def inner(key):
        return env.get(key, [])
    return inner

class Task(object):
    """
    Abstract base class for objects wishing to be picked up as Fabric tasks.

    Instances of subclasses will be treated as valid tasks when present in
    fabfiles loaded by the :doc:`fab </usage/fab>` tool.

    For details on how to implement and use `~fabric.tasks.Task` subclasses,
    please see the usage documentation on :ref:`new-style tasks
    <new-style-tasks>`.

    .. versionadded:: 1.1
    """
    name = 'undefined'
    use_task_objects = True
    aliases = None
    is_default = False

    # TODO: make it so that this wraps other decorators as expected
    def __init__(self, alias=None, aliases=None, default=False, name=None,
        *args, **kwargs):
        if alias is not None:
            self.aliases = [alias, ]
        if aliases is not None:
            self.aliases = aliases
        if name is not None:
            self.name = name
        self.is_default = default

    def __details__(self):
        return get_task_details(self.run)

    def run(self):
        raise NotImplementedError

    def get_hosts(self, arg_hosts, arg_roles, arg_exclude_hosts, env=None):
        """
        Return the host list the given task should be using.

        See :ref:`host-lists` for detailed documentation on how host lists are
        set.
        """
        env = env or {'hosts': [], 'roles': [], 'exclude_hosts': []}
        roledefs = env.get('roledefs', {})
        # Command line per-task takes precedence over anything else.
        if arg_hosts or arg_roles:
            return merge(arg_hosts, arg_roles, arg_exclude_hosts, roledefs)
        # Decorator-specific hosts/roles go next
        func_hosts = getattr(self, 'hosts', [])
        func_roles = getattr(self, 'roles', [])
        if func_hosts or func_roles:
            return merge(func_hosts, func_roles, arg_exclude_hosts, roledefs)
        # Finally, the env is checked (which might contain globally set lists
        # from the CLI or from module-level code). This will be the empty list
        # if these have not been set -- which is fine, this method should
        # return an empty list if no hosts have been set anywhere.
        env_vars = map(_get_list(env), "hosts roles exclude_hosts".split())
        env_vars.append(roledefs)
        return merge(*env_vars)

    def get_pool_size(self, hosts, default):
        # Default parallel pool size (calculate per-task in case variables
        # change)
        default_pool_size = default or len(hosts)
        # Allow per-task override
        # Also cast to int in case somebody gave a string
        from_task = getattr(self, 'pool_size', None)
        pool_size = int(from_task or default_pool_size)
        # But ensure it's never larger than the number of hosts
        pool_size = min((pool_size, len(hosts)))
        # Inform user of final pool size for this task
        if state.output.debug:
            print("Parallel tasks now using pool size of %d" % pool_size)
        return pool_size

class WrappedCallableTask(Task):
    """
    Wraps a given callable transparently, while marking it as a valid Task.

    Generally used via `~fabric.decorators.task` and not directly.

    .. versionadded:: 1.1

    .. seealso:: `~fabric.docs.unwrap_tasks`, `~fabric.decorators.task`
    """
    def __init__(self, callable, *args, **kwargs):
        super(WrappedCallableTask, self).__init__(*args, **kwargs)
        self.wrapped = callable
        # Don't use getattr() here -- we want to avoid touching self.name
        # entirely so the superclass' value remains default.
        if hasattr(callable, '__name__'):
            if self.name == 'undefined':
                self.__name__ = self.name = callable.__name__
            else:
                self.__name__ = self.name
        if hasattr(callable, '__doc__'):
            self.__doc__ = callable.__doc__
        if hasattr(callable, '__module__'):
            self.__module__ = callable.__module__

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)

    def __getattr__(self, k):
        return getattr(self.wrapped, k)

    def __details__(self):
        return get_task_details(self.wrapped)

def requires_parallel(task):
    """
    Returns True if given ``task`` should be run in parallel mode.

    Specifically:

    * It's been explicitly marked with ``@parallel``, or:
    * It's *not* been explicitly marked with ``@serial`` *and* the global
      parallel option (``env.parallel``) is set to ``True``.
    """
    return (
        (state.env.parallel and not getattr(task, 'serial', False))
        or getattr(task, 'parallel', False)
    )

def _parallel_tasks(commands_to_run):
    return any(map(
        lambda x: requires_parallel(crawl(x[0], state.commands)),
        commands_to_run
    ))

def _execute(task, host, my_env, args, kwargs, jobs, queue, multiprocessing):
    """
    Primary single-host work body of execute()
    """
    # Log to stdout
    if state.output.running and not hasattr(task, 'return_value'):
        print("[%s] Executing task '%s'" % (host, my_env['command']))
    # Create per-run env with connection settings
    local_env = to_dict(host)
    local_env.update(my_env)
    # Set a few more env flags for parallelism
    if queue is not None:
        local_env.update({'parallel': True, 'linewise': True})
    # Handle parallel execution
    if queue is not None: # Since queue is only set for parallel
        name = local_env['host_string']
        # Wrap in another callable that:
        # * expands the env it's given to ensure parallel, linewise, etc are
        #   all set correctly and explicitly. Such changes are naturally
        #   insulted from the parent process.
        # * nukes the connection cache to prevent shared-access problems
        # * knows how to send the tasks' return value back over a Queue
        # * captures exceptions raised by the task
        def inner(args, kwargs, queue, name, env):
            state.env.update(env)
            def submit(result):
                queue.put({'name': name, 'result': result})
            try:
                key = normalize_to_string(state.env.host_string)
                state.connections.pop(key, "")
                submit(task.run(*args, **kwargs))
            except BaseException, e: # We really do want to capture everything
                # SystemExit implies use of abort(), which prints its own
                # traceback, host info etc -- so we don't want to double up
                # on that. For everything else, though, we need to make
                # clear what host encountered the exception that will
                # print.
                if e.__class__ is not SystemExit:
                    sys.stderr.write("!!! Parallel execution exception under host %r:\n" % name)
                    submit(e)
                # Here, anything -- unexpected exceptions, or abort()
                # driven SystemExits -- will bubble up and terminate the
                # child process.
                raise

        # Stuff into Process wrapper
        kwarg_dict = {
            'args': args,
            'kwargs': kwargs,
            'queue': queue,
            'name': name,
            'env': local_env,
        }
        p = multiprocessing.Process(target=inner, kwargs=kwarg_dict)
        # Name/id is host string
        p.name = name
        # Add to queue
        jobs.append(p)
    # Handle serial execution
    else:
        with settings(**local_env):
            return task.run(*args, **kwargs)

def _is_task(task):
    return isinstance(task, Task)

def execute(task, *args, **kwargs):
    """
    Execute ``task`` (callable or name), honoring host/role decorators, etc.

    ``task`` may be an actual callable object, or it may be a registered task
    name, which is used to look up a callable just as if the name had been
    given on the command line (including :ref:`namespaced tasks <namespaces>`,
    e.g. ``"deploy.migrate"``.

    The task will then be executed once per host in its host list, which is
    (again) assembled in the same manner as CLI-specified tasks: drawing from
    :option:`-H`, :ref:`env.hosts <hosts>`, the `~fabric.decorators.hosts` or
    `~fabric.decorators.roles` decorators, and so forth.

    ``host``, ``hosts``, ``role``, ``roles`` and ``exclude_hosts`` kwargs will
    be stripped out of the final call, and used to set the task's host list, as
    if they had been specified on the command line like e.g. ``fab
    taskname:host=hostname``.

    Any other arguments or keyword arguments will be passed verbatim into
    ``task`` (the function itself -- not the ``@task`` decorator wrapping your
    function!) when it is called, so ``execute(mytask, 'arg1',
    kwarg1='value')`` will (once per host) invoke ``mytask('arg1',
    kwarg1='value')``.

    :returns:
        a dictionary mapping host strings to the given task's return value for
        that host's execution run. For example, ``execute(foo, hosts=['a',
        'b'])`` might return ``{'a': None, 'b': 'bar'}`` if ``foo`` returned
        nothing on host `a` but returned ``'bar'`` on host `b`.

        In situations where a task execution fails for a given host but overall
        progress does not abort (such as when :ref:`env.skip_bad_hosts
        <skip-bad-hosts>` is True) the return value for that host will be the
        error object or message.

    .. seealso::
        :ref:`The execute usage docs <execute>`, for an expanded explanation
        and some examples.

    .. versionadded:: 1.3
    .. versionchanged:: 1.4
        Added the return value mapping; previously this function had no defined
        return value.
    """
    my_env = {'clean_revert': True}
    results = {}
    # Obtain task
    is_callable = callable(task)
    if not (is_callable or _is_task(task)):
        # Assume string, set env.command to it
        my_env['command'] = task
        task = crawl(task, state.commands)
        if task is None:
            abort("%r is not callable or a valid task name" % (task,))
    # Set env.command if we were given a real function or callable task obj
    else:
        dunder_name = getattr(task, '__name__', None)
        my_env['command'] = getattr(task, 'name', dunder_name)
    # Normalize to Task instance if we ended up with a regular callable
    if not _is_task(task):
        task = WrappedCallableTask(task)
    # Filter out hosts/roles kwargs
    new_kwargs, hosts, roles, exclude_hosts = parse_kwargs(kwargs)
    # Set up host list
    my_env['all_hosts'] = task.get_hosts(hosts, roles, exclude_hosts, state.env)

    parallel = requires_parallel(task)
    if parallel:
        # Import multiprocessing if needed, erroring out usefully
        # if it can't.
        try:
            import multiprocessing
        except ImportError:
            import traceback
            tb = traceback.format_exc()
            abort(tb + """
    At least one task needs to be run in parallel, but the
    multiprocessing module cannot be imported (see above
    traceback.) Please make sure the module is installed
    or that the above ImportError is fixed.""")
    else:
        multiprocessing = None

    # Get pool size for this task
    pool_size = task.get_pool_size(my_env['all_hosts'], state.env.pool_size)
    # Set up job queue in case parallel is needed
    queue = multiprocessing.Queue() if parallel else None
    jobs = JobQueue(pool_size, queue)
    if state.output.debug:
        jobs._debug = True

    # Call on host list
    if my_env['all_hosts']:
        # Attempt to cycle on hosts, skipping if needed
        for host in my_env['all_hosts']:
            try:
                results[host] = _execute(
                    task, host, my_env, args, new_kwargs, jobs, queue,
                    multiprocessing
                )
            except NetworkError, e:
                results[host] = e
                # Backwards compat test re: whether to use an exception or
                # abort
                if not state.env.use_exceptions_for['network']:
                    func = warn if state.env.skip_bad_hosts else abort
                    error(e.message, func=func, exception=e.wrapped)
                else:
                    raise

            # If requested, clear out connections here and not just at the end.
            if state.env.eagerly_disconnect:
                disconnect_all()

        # If running in parallel, block until job queue is emptied
        if jobs:
            err = "One or more hosts failed while executing task '%s'" % (
                my_env['command']
            )
            jobs.close()
            # Abort if any children did not exit cleanly (fail-fast).
            # This prevents Fabric from continuing on to any other tasks.
            # Otherwise, pull in results from the child run.
            ran_jobs = jobs.run()
            for name, d in ran_jobs.iteritems():
                if d['exit_code'] != 0:
                    if isinstance(d['results'], BaseException):
                        error(err, exception=d['results'])
                    else:
                        error(err)
                results[name] = d['results']

    # Or just run once for local-only
    else:
        with settings(**my_env):
            results['<local-only>'] = task.run(*args, **new_kwargs)
    # Return what we can from the inner task executions

    return results
<commit_msg>Monkey-patch inspect.getargspec for Python 2.5<commit_after>from __future__ import with_statement

from functools import wraps
import inspect
import sys

from fabric import state
from fabric.utils import abort, warn, error
from fabric.network import to_dict, normalize_to_string, disconnect_all
from fabric.context_managers import settings
from fabric.job_queue import JobQueue
from fabric.task_utils import crawl, merge, parse_kwargs
from fabric.exceptions import NetworkError

if sys.version_info[:2] == (2, 5):
    # Python 2.5 inspect.getargspec returns a tuple
    # instead of ArgSpec namedtuple.
    class ArgSpec(object):
        def __init__(self, args, varargs, keywords, defaults):
            self.args = args
            self.varargs = varargs
            self.keywords = keywords
            self.defaults = defaults
            self._tuple = (args, varargs, keywords, defaults)

        def __getitem__(self, idx):
            return self._tuple[idx]

    def patched_get_argspec(func):
        return ArgSpec(*inspect._getargspec(func))

    inspect._getargspec = inspect.getargspec
    inspect.getargspec = patched_get_argspec

def get_task_details(task):
    details = [task.__doc__ if task.__doc__ else 'No docstring provided']
    argspec = inspect.getargspec(task)

    default_args = [] if not argspec.defaults else argspec.defaults
    num_default_args = len(default_args)
    args_without_defaults = argspec.args[:len(argspec.args) - num_default_args]
    args_with_defaults = argspec.args[-1 * num_default_args:]

    details.append('Arguments: %s' % (
        ', '.join(
            args_without_defaults + [
                '%s=%r' % (arg, default)
                for arg, default in zip(args_with_defaults, default_args)
            ])
    ))

    return '\n'.join(details)

def _get_list(env):
    def inner(key):
        return env.get(key, [])
    return inner

class Task(object):
    """
    Abstract base class for objects wishing to be picked up as Fabric tasks.

    Instances of subclasses will be treated as valid tasks when present in
    fabfiles loaded by the :doc:`fab </usage/fab>` tool.

    For details on how to implement and use `~fabric.tasks.Task` subclasses,
    please see the usage documentation on :ref:`new-style tasks
    <new-style-tasks>`.

    .. versionadded:: 1.1
    """
    name = 'undefined'
    use_task_objects = True
    aliases = None
    is_default = False

    # TODO: make it so that this wraps other decorators as expected
    def __init__(self, alias=None, aliases=None, default=False, name=None,
        *args, **kwargs):
        if alias is not None:
            self.aliases = [alias, ]
        if aliases is not None:
            self.aliases = aliases
        if name is not None:
            self.name = name
        self.is_default = default

    def __details__(self):
        return get_task_details(self.run)

    def run(self):
        raise NotImplementedError

    def get_hosts(self, arg_hosts, arg_roles, arg_exclude_hosts, env=None):
        """
        Return the host list the given task should be using.

        See :ref:`host-lists` for detailed documentation on how host lists are
        set.
        """
        env = env or {'hosts': [], 'roles': [], 'exclude_hosts': []}
        roledefs = env.get('roledefs', {})
        # Command line per-task takes precedence over anything else.
        if arg_hosts or arg_roles:
            return merge(arg_hosts, arg_roles, arg_exclude_hosts, roledefs)
        # Decorator-specific hosts/roles go next
        func_hosts = getattr(self, 'hosts', [])
        func_roles = getattr(self, 'roles', [])
        if func_hosts or func_roles:
            return merge(func_hosts, func_roles, arg_exclude_hosts, roledefs)
        # Finally, the env is checked (which might contain globally set lists
        # from the CLI or from module-level code). This will be the empty list
        # if these have not been set -- which is fine, this method should
        # return an empty list if no hosts have been set anywhere.
        env_vars = map(_get_list(env), "hosts roles exclude_hosts".split())
        env_vars.append(roledefs)
        return merge(*env_vars)

    def get_pool_size(self, hosts, default):
        # Default parallel pool size (calculate per-task in case variables
        # change)
        default_pool_size = default or len(hosts)
        # Allow per-task override
        # Also cast to int in case somebody gave a string
        from_task = getattr(self, 'pool_size', None)
        pool_size = int(from_task or default_pool_size)
        # But ensure it's never larger than the number of hosts
        pool_size = min((pool_size, len(hosts)))
        # Inform user of final pool size for this task
        if state.output.debug:
            print("Parallel tasks now using pool size of %d" % pool_size)
        return pool_size

class WrappedCallableTask(Task):
    """
    Wraps a given callable transparently, while marking it as a valid Task.

    Generally used via `~fabric.decorators.task` and not directly.

    .. versionadded:: 1.1

    .. seealso:: `~fabric.docs.unwrap_tasks`, `~fabric.decorators.task`
    """
    def __init__(self, callable, *args, **kwargs):
        super(WrappedCallableTask, self).__init__(*args, **kwargs)
        self.wrapped = callable
        # Don't use getattr() here -- we want to avoid touching self.name
        # entirely so the superclass' value remains default.
        if hasattr(callable, '__name__'):
            if self.name == 'undefined':
                self.__name__ = self.name = callable.__name__
            else:
                self.__name__ = self.name
        if hasattr(callable, '__doc__'):
            self.__doc__ = callable.__doc__
        if hasattr(callable, '__module__'):
            self.__module__ = callable.__module__

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)

    def __getattr__(self, k):
        return getattr(self.wrapped, k)

    def __details__(self):
        return get_task_details(self.wrapped)

def requires_parallel(task):
    """
    Returns True if given ``task`` should be run in parallel mode.

    Specifically:

    * It's been explicitly marked with ``@parallel``, or:
    * It's *not* been explicitly marked with ``@serial`` *and* the global
      parallel option (``env.parallel``) is set to ``True``.
    """
    return (
        (state.env.parallel and not getattr(task, 'serial', False))
        or getattr(task, 'parallel', False)
    )

def _parallel_tasks(commands_to_run):
    return any(map(
        lambda x: requires_parallel(crawl(x[0], state.commands)),
        commands_to_run
    ))

def _execute(task, host, my_env, args, kwargs, jobs, queue, multiprocessing):
    """
    Primary single-host work body of execute()
    """
    # Log to stdout
    if state.output.running and not hasattr(task, 'return_value'):
        print("[%s] Executing task '%s'" % (host, my_env['command']))
    # Create per-run env with connection settings
    local_env = to_dict(host)
    local_env.update(my_env)
    # Set a few more env flags for parallelism
    if queue is not None:
        local_env.update({'parallel': True, 'linewise': True})
    # Handle parallel execution
    if queue is not None: # Since queue is only set for parallel
        name = local_env['host_string']
        # Wrap in another callable that:
        # * expands the env it's given to ensure parallel, linewise, etc are
        #   all set correctly and explicitly. Such changes are naturally
        #   insulted from the parent process.
        # * nukes the connection cache to prevent shared-access problems
        # * knows how to send the tasks' return value back over a Queue
        # * captures exceptions raised by the task
        def inner(args, kwargs, queue, name, env):
            state.env.update(env)
            def submit(result):
                queue.put({'name': name, 'result': result})
            try:
                key = normalize_to_string(state.env.host_string)
                state.connections.pop(key, "")
                submit(task.run(*args, **kwargs))
            except BaseException, e: # We really do want to capture everything
                # SystemExit implies use of abort(), which prints its own
                # traceback, host info etc -- so we don't want to double up
                # on that. For everything else, though, we need to make
                # clear what host encountered the exception that will
                # print.
                if e.__class__ is not SystemExit:
                    sys.stderr.write("!!! Parallel execution exception under host %r:\n" % name)
                    submit(e)
                # Here, anything -- unexpected exceptions, or abort()
                # driven SystemExits -- will bubble up and terminate the
                # child process.
                raise

        # Stuff into Process wrapper
        kwarg_dict = {
            'args': args,
            'kwargs': kwargs,
            'queue': queue,
            'name': name,
            'env': local_env,
        }
        p = multiprocessing.Process(target=inner, kwargs=kwarg_dict)
        # Name/id is host string
        p.name = name
        # Add to queue
        jobs.append(p)
    # Handle serial execution
    else:
        with settings(**local_env):
            return task.run(*args, **kwargs)

def _is_task(task):
    return isinstance(task, Task)

def execute(task, *args, **kwargs):
    """
    Execute ``task`` (callable or name), honoring host/role decorators, etc.

    ``task`` may be an actual callable object, or it may be a registered task
    name, which is used to look up a callable just as if the name had been
    given on the command line (including :ref:`namespaced tasks <namespaces>`,
    e.g. ``"deploy.migrate"``.

    The task will then be executed once per host in its host list, which is
    (again) assembled in the same manner as CLI-specified tasks: drawing from
    :option:`-H`, :ref:`env.hosts <hosts>`, the `~fabric.decorators.hosts` or
    `~fabric.decorators.roles` decorators, and so forth.

    ``host``, ``hosts``, ``role``, ``roles`` and ``exclude_hosts`` kwargs will
    be stripped out of the final call, and used to set the task's host list, as
    if they had been specified on the command line like e.g. ``fab
    taskname:host=hostname``.

    Any other arguments or keyword arguments will be passed verbatim into
    ``task`` (the function itself -- not the ``@task`` decorator wrapping your
    function!) when it is called, so ``execute(mytask, 'arg1',
    kwarg1='value')`` will (once per host) invoke ``mytask('arg1',
    kwarg1='value')``.

    :returns:
        a dictionary mapping host strings to the given task's return value for
        that host's execution run. For example, ``execute(foo, hosts=['a',
        'b'])`` might return ``{'a': None, 'b': 'bar'}`` if ``foo`` returned
        nothing on host `a` but returned ``'bar'`` on host `b`.

        In situations where a task execution fails for a given host but overall
        progress does not abort (such as when :ref:`env.skip_bad_hosts
        <skip-bad-hosts>` is True) the return value for that host will be the
        error object or message.

    .. seealso::
        :ref:`The execute usage docs <execute>`, for an expanded explanation
        and some examples.

    .. versionadded:: 1.3
    .. versionchanged:: 1.4
        Added the return value mapping; previously this function had no defined
        return value.
    """
    my_env = {'clean_revert': True}
    results = {}
    # Obtain task
    is_callable = callable(task)
    if not (is_callable or _is_task(task)):
        # Assume string, set env.command to it
        my_env['command'] = task
        task = crawl(task, state.commands)
        if task is None:
            abort("%r is not callable or a valid task name" % (task,))
    # Set env.command if we were given a real function or callable task obj
    else:
        dunder_name = getattr(task, '__name__', None)
        my_env['command'] = getattr(task, 'name', dunder_name)
    # Normalize to Task instance if we ended up with a regular callable
    if not _is_task(task):
        task = WrappedCallableTask(task)
    # Filter out hosts/roles kwargs
    new_kwargs, hosts, roles, exclude_hosts = parse_kwargs(kwargs)
    # Set up host list
    my_env['all_hosts'] = task.get_hosts(hosts, roles, exclude_hosts, state.env)

    parallel = requires_parallel(task)
    if parallel:
        # Import multiprocessing if needed, erroring out usefully
        # if it can't.
        try:
            import multiprocessing
        except ImportError:
            import traceback
            tb = traceback.format_exc()
            abort(tb + """
    At least one task needs to be run in parallel, but the
    multiprocessing module cannot be imported (see above
    traceback.) Please make sure the module is installed
    or that the above ImportError is fixed.""")
    else:
        multiprocessing = None

    # Get pool size for this task
    pool_size = task.get_pool_size(my_env['all_hosts'], state.env.pool_size)
    # Set up job queue in case parallel is needed
    queue = multiprocessing.Queue() if parallel else None
    jobs = JobQueue(pool_size, queue)
    if state.output.debug:
        jobs._debug = True

    # Call on host list
    if my_env['all_hosts']:
        # Attempt to cycle on hosts, skipping if needed
        for host in my_env['all_hosts']:
            try:
                results[host] = _execute(
                    task, host, my_env, args, new_kwargs, jobs, queue,
                    multiprocessing
                )
            except NetworkError, e:
                results[host] = e
                # Backwards compat test re: whether to use an exception or
                # abort
                if not state.env.use_exceptions_for['network']:
                    func = warn if state.env.skip_bad_hosts else abort
                    error(e.message, func=func, exception=e.wrapped)
                else:
                    raise

            # If requested, clear out connections here and not just at the end.
            if state.env.eagerly_disconnect:
                disconnect_all()

        # If running in parallel, block until job queue is emptied
        if jobs:
            err = "One or more hosts failed while executing task '%s'" % (
                my_env['command']
            )
            jobs.close()
            # Abort if any children did not exit cleanly (fail-fast).
            # This prevents Fabric from continuing on to any other tasks.
            # Otherwise, pull in results from the child run.
            ran_jobs = jobs.run()
            for name, d in ran_jobs.iteritems():
                if d['exit_code'] != 0:
                    if isinstance(d['results'], BaseException):
                        error(err, exception=d['results'])
                    else:
                        error(err)
                results[name] = d['results']

    # Or just run once for local-only
    else:
        with settings(**my_env):
            results['<local-only>'] = task.run(*args, **new_kwargs)
    # Return what we can from the inner task executions

    return results
<|endoftext|>


<commit_before>"""
SmartOS packages
===============

This module provides tools to manage SmartOS packages.

"""
from __future__ import with_statement

from fabric.api import *
from fabtools.files import is_file

MANAGER = 'pkgin'

def update_index(force=False):
    """
    Update pkgin package definitions.
    """
    manager = MANAGER
    if force:
        with quiet():
            sudo("%(manager)s cl" % locals())
        sudo("%(manager)s -f up" % locals())
    else:
        sudo("%(manager)s up" % locals())

def upgrade(full=False):
    """
    Upgrade all packages.
    """
    manager = MANAGER
    cmds = {'pkgin': {False: 'ug', True: 'fug'}}
    cmd = cmds[manager][full]
    sudo("%(manager)s -y %(cmd)s" % locals())

def is_installed(pkg_name):
    """
    Check if a package is installed.
    """
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("pkgin list | grep -qio %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, update=False, options=None):
    """
    Install one or more packages.

    If *update* is ``True``, the package definitions will be updated
    first, using :py:func:`~fabtools.pkg.update_index`.

    Extra *options* may be passed to ``pkgin`` if necessary.

    Example::

        import fabtools

        # Update index, then install a single package
        fabtools.pkg.install('redis', update=True)

        # Install multiple packages
        fabtools.pkg.install([
            'unzip',
            'top'
        ])

    """
    manager = MANAGER
    if update:
        update_index()
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    sudo('%(manager)s %(options)s in %(packages)s' % locals())

def uninstall(packages, orphan=False, options=None):
    """
    Remove one or more packages.

    If *orphan* is ``True``, orphan dependencies will be
    removed from the system.

    Extra *options* may be passed to ``pkgin`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if orphan:
        sudo('%(manager)s -y ar' % manager)
    sudo('%(manager)s %(options)s %(command)s %(packages)s' % locals())

def smartos_build():
    """
    Get the build of SmartOS. Useful to determine provider for example.

    Example::

        from fabtools.pkg import smartos_build

        if smartos_build().startswith('joyent'):
            print('SmartOS Joyent')

    """
    with settings(hide('running', 'stdout')):
        return run('uname -v')

def smartos_image():
    """
    Get the SmartOS image. Useful to determine the image/dataset for example.
    Returns 'NA' if it can't be determined.

    Example::

        from fabtools.pkg import smartos_image

        if smartos_image().startswith('percona'):
            sudo('mysql -uroot -psecretpassword -e 'show databases;'')

    """
    with settings(hide('running', 'stdout')):
        if is_file('/etc/product'):
            return run('cat /etc/product | head -n 2 | tail -n 1 | awk \'{ print $2 " " $3 }\'')
        else:
            return 'NA'
<commit_msg>move keyword arg in pkg.install<commit_after>"""
SmartOS packages
===============

This module provides tools to manage SmartOS packages.

"""
from __future__ import with_statement

from fabric.api import *
from fabtools.files import is_file

MANAGER = 'pkgin'

def update_index(force=False):
    """
    Update pkgin package definitions.
    """
    manager = MANAGER
    if force:
        with quiet():
            sudo("%(manager)s cl" % locals())
        sudo("%(manager)s -f up" % locals())
    else:
        sudo("%(manager)s up" % locals())

def upgrade(full=False):
    """
    Upgrade all packages.
    """
    manager = MANAGER
    cmds = {'pkgin': {False: 'ug', True: 'fug'}}
    cmd = cmds[manager][full]
    sudo("%(manager)s -y %(cmd)s" % locals())

def is_installed(pkg_name):
    """
    Check if a package is installed.
    """
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("pkgin list | grep -qio %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, options=None, update=False):
    """
    Install one or more packages.

    If *update* is ``True``, the package definitions will be updated
    first, using :py:func:`~fabtools.pkg.update_index`.

    Extra *options* may be passed to ``pkgin`` if necessary.

    Example::

        import fabtools

        # Update index, then install a single package
        fabtools.pkg.install('redis', update=True)

        # Install multiple packages
        fabtools.pkg.install([
            'unzip',
            'top'
        ])

    """
    manager = MANAGER
    if update:
        update_index()
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    sudo('%(manager)s %(options)s in %(packages)s' % locals())

def uninstall(packages, orphan=False, options=None):
    """
    Remove one or more packages.

    If *orphan* is ``True``, orphan dependencies will be
    removed from the system.

    Extra *options* may be passed to ``pkgin`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options.append("-y")
    options = " ".join(options)
    if orphan:
        sudo('%(manager)s -y ar' % manager)
    sudo('%(manager)s %(options)s %(command)s %(packages)s' % locals())

def smartos_build():
    """
    Get the build of SmartOS. Useful to determine provider for example.

    Example::

        from fabtools.pkg import smartos_build

        if smartos_build().startswith('joyent'):
            print('SmartOS Joyent')

    """
    with settings(hide('running', 'stdout')):
        return run('uname -v')

def smartos_image():
    """
    Get the SmartOS image. Useful to determine the image/dataset for example.
    Returns 'NA' if it can't be determined.

    Example::

        from fabtools.pkg import smartos_image

        if smartos_image().startswith('percona'):
            sudo('mysql -uroot -psecretpassword -e 'show databases;'')

    """
    with settings(hide('running', 'stdout')):
        if is_file('/etc/product'):
            return run('cat /etc/product | head -n 2 | tail -n 1 | awk \'{ print $2 " " $3 }\'')
        else:
            return 'NA'
<|endoftext|>


<commit_before>import logging

from flask import Flask
from flask_alembic import Alembic
from flask_sqlalchemy import SQLAlchemy
from raven.contrib.flask import Sentry

from zeus.utils.celery import Celery
from zeus.utils.redis import Redis

import os

ROOT = os.path.join(os.path.dirname(__file__), os.pardir)

alembic = Alembic()
celery = Celery()
db = SQLAlchemy()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def create_app(_read_config=True, **config):
    app = Flask(
        __name__,
        static_folder=os.path.join(ROOT, 'static'),
        template_folder=os.path.join(ROOT, 'templates'),
    )

    # support for kubernetes
    # https://kubernetes.io/docs/concepts/services-networking/service/
    if os.environ.get('GET_HOSTS_FROM') == 'env':
        REDIS_URL = 'redis://{}:{}/0'.format(
            os.environ['REDIS_MASTER_SERVICE_HOST'],
            os.environ['REDIS_MASTER_SERVICE_PORT'],
        )
        # Cloud SQL
        # https://cloud.google.com/sql/docs/postgres/connect-container-engine
        SQLALCHEMY_URI = 'postgresql://{}:{}@127.0.0.1:5432/zeus'.format(
            os.environ['DB_USER'],
            os.environ['DB_PASSWORD'],
        )
    else:
        REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost/0')
        SQLALCHEMY_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql:///zeus')

    if os.environ.get('SERVER_NAME'):
        app.config['SERVER_NAME'] = os.environ['SERVER_NAME']

    app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

    # if we're not running in debug mode, we require SSL
    app.config['SESSION_COOKIE_SECURE'] = not app.debug
    app.config['PREFERRED_URL_SCHEME'] = not app.debug

    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_URI
    app.config['SQLALCHEMY_POOL_SIZE'] = 60
    app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    app.config['REDIS_URL'] = REDIS_URL

    app.config['SENTRY_DSN'] = os.environ.get('SENTRY_DSN') or None
    app.config['SENTRY_INCLUDE_PATHS'] = [
        'changes',
    ]

    app.config['GITHUB_CLIENT_ID'] = os.environ.get('GITHUB_CLIENT_ID') or None
    app.config['GITHUB_CLIENT_SECRET'] = os.environ.get('GITHUB_CLIENT_SECRET') or None

    app.config['CELERY_ACCEPT_CONTENT'] = ['zeus_json', 'json']
    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = 'direct'
    app.config['CELERY_DEFAULT_ROUTING_KEY'] = 'default'
    app.config['CELERY_DISABLE_RATE_LIMITS'] = True
    app.config['CELERY_EVENT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_IGNORE_RESULT'] = True
    app.config['CELERY_IMPORTS'] = ('zeus.tasks', )
    app.config['CELERY_RESULT_BACKEND'] = None
    app.config['CELERY_RESULT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_SEND_EVENTS'] = False
    app.config['CELERY_TASK_RESULT_EXPIRES'] = 1
    app.config['CELERY_TASK_SERIALIZER'] = 'zeus_json'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000

    app.config['REPO_ROOT'] = os.environ.get('REPO_ROOT', '/usr/local/cache/zeus-repos')

    # app.config['DEFAULT_FILE_STORAGE'] = ''

    if _read_config:
        if os.environ.get('ZEUS_CONF'):
            # ZEUS_CONF=/etc/zeus.conf.py
            app.config.from_envvar('ZEUS_CONF')
        else:
            # Look for ~/.zeus/zeus.conf.py
            path = os.path.normpath(os.path.expanduser('~/.zeus/zeus.config.py'))
            app.config.from_pyfile(path, silent=True)

    app.config.update(config)

    from zeus.testutils.client import ZeusTestClient
    app.test_client_class = ZeusTestClient

    # init sentry first
    sentry.init_app(app)

    configure_db(app)

    redis.init_app(app)

    celery.init_app(app, sentry)

    configure_api(app)
    configure_web(app)

    from . import models  # NOQA

    return app

def configure_db(app):
    from sqlalchemy import event
    from sqlalchemy.orm import mapper
    from sqlalchemy.inspection import inspect

    alembic.init_app(app)
    db.init_app(app)

    @event.listens_for(mapper, "init")
    def instant_defaults_listener(target, args, kwargs):
        for key, column in inspect(type(target)).columns.items():
            if column.default is not None:
                if callable(column.default.arg):
                    setattr(target, key, column.default.arg(target))
                else:
                    setattr(target, key, column.default.arg)

    event.listen(mapper, 'init', instant_defaults_listener)

def configure_api(app):
    from zeus import api

    app.register_blueprint(api.app, url_prefix='/api')

def configure_web(app):
    from zeus import web

    app.register_blueprint(web.app, url_prefix='')
<commit_msg>fix bad preferred url scheme<commit_after>import logging

from flask import Flask
from flask_alembic import Alembic
from flask_sqlalchemy import SQLAlchemy
from raven.contrib.flask import Sentry

from zeus.utils.celery import Celery
from zeus.utils.redis import Redis

import os

ROOT = os.path.join(os.path.dirname(__file__), os.pardir)

alembic = Alembic()
celery = Celery()
db = SQLAlchemy()
redis = Redis()
sentry = Sentry(logging=True, level=logging.WARN)

def create_app(_read_config=True, **config):
    app = Flask(
        __name__,
        static_folder=os.path.join(ROOT, 'static'),
        template_folder=os.path.join(ROOT, 'templates'),
    )

    # support for kubernetes
    # https://kubernetes.io/docs/concepts/services-networking/service/
    if os.environ.get('GET_HOSTS_FROM') == 'env':
        REDIS_URL = 'redis://{}:{}/0'.format(
            os.environ['REDIS_MASTER_SERVICE_HOST'],
            os.environ['REDIS_MASTER_SERVICE_PORT'],
        )
        # Cloud SQL
        # https://cloud.google.com/sql/docs/postgres/connect-container-engine
        SQLALCHEMY_URI = 'postgresql://{}:{}@127.0.0.1:5432/zeus'.format(
            os.environ['DB_USER'],
            os.environ['DB_PASSWORD'],
        )
    else:
        REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost/0')
        SQLALCHEMY_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql:///zeus')

    if os.environ.get('SERVER_NAME'):
        app.config['SERVER_NAME'] = os.environ['SERVER_NAME']

    app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY')

    # if we're not running in debug mode, we require SSL
    app.config['SESSION_COOKIE_SECURE'] = not app.debug
    app.config['PREFERRED_URL_SCHEME'] = ('https' if not not app.debug else 'http')

    app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
    app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_URI
    app.config['SQLALCHEMY_POOL_SIZE'] = 60
    app.config['SQLALCHEMY_MAX_OVERFLOW'] = 20
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

    app.config['REDIS_URL'] = REDIS_URL

    app.config['SENTRY_DSN'] = os.environ.get('SENTRY_DSN') or None
    app.config['SENTRY_INCLUDE_PATHS'] = [
        'changes',
    ]

    app.config['GITHUB_CLIENT_ID'] = os.environ.get('GITHUB_CLIENT_ID') or None
    app.config['GITHUB_CLIENT_SECRET'] = os.environ.get('GITHUB_CLIENT_SECRET') or None

    app.config['CELERY_ACCEPT_CONTENT'] = ['zeus_json', 'json']
    app.config['CELERY_ACKS_LATE'] = True
    app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
    app.config['CELERY_DEFAULT_QUEUE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE'] = 'default'
    app.config['CELERY_DEFAULT_EXCHANGE_TYPE'] = 'direct'
    app.config['CELERY_DEFAULT_ROUTING_KEY'] = 'default'
    app.config['CELERY_DISABLE_RATE_LIMITS'] = True
    app.config['CELERY_EVENT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_IGNORE_RESULT'] = True
    app.config['CELERY_IMPORTS'] = ('zeus.tasks', )
    app.config['CELERY_RESULT_BACKEND'] = None
    app.config['CELERY_RESULT_SERIALIZER'] = 'zeus_json'
    app.config['CELERY_SEND_EVENTS'] = False
    app.config['CELERY_TASK_RESULT_EXPIRES'] = 1
    app.config['CELERY_TASK_SERIALIZER'] = 'zeus_json'
    app.config['CELERYD_PREFETCH_MULTIPLIER'] = 1
    app.config['CELERYD_MAX_TASKS_PER_CHILD'] = 10000

    app.config['REPO_ROOT'] = os.environ.get('REPO_ROOT', '/usr/local/cache/zeus-repos')

    # app.config['DEFAULT_FILE_STORAGE'] = ''

    if _read_config:
        if os.environ.get('ZEUS_CONF'):
            # ZEUS_CONF=/etc/zeus.conf.py
            app.config.from_envvar('ZEUS_CONF')
        else:
            # Look for ~/.zeus/zeus.conf.py
            path = os.path.normpath(os.path.expanduser('~/.zeus/zeus.config.py'))
            app.config.from_pyfile(path, silent=True)

    app.config.update(config)

    from zeus.testutils.client import ZeusTestClient
    app.test_client_class = ZeusTestClient

    # init sentry first
    sentry.init_app(app)

    configure_db(app)

    redis.init_app(app)

    celery.init_app(app, sentry)

    configure_api(app)
    configure_web(app)

    from . import models  # NOQA

    return app

def configure_db(app):
    from sqlalchemy import event
    from sqlalchemy.orm import mapper
    from sqlalchemy.inspection import inspect

    alembic.init_app(app)
    db.init_app(app)

    @event.listens_for(mapper, "init")
    def instant_defaults_listener(target, args, kwargs):
        for key, column in inspect(type(target)).columns.items():
            if column.default is not None:
                if callable(column.default.arg):
                    setattr(target, key, column.default.arg(target))
                else:
                    setattr(target, key, column.default.arg)

    event.listen(mapper, 'init', instant_defaults_listener)

def configure_api(app):
    from zeus import api

    app.register_blueprint(api.app, url_prefix='/api')

def configure_web(app):
    from zeus import web

    app.register_blueprint(web.app, url_prefix='')
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-

from __future__ import unicode_literals
import re
import json
from time import time
from random import random
import warnings
import logging
from .models import *

try:
    from urllib.parse import urlencode
    basestring = (str, bytes)
except ImportError:
    from urllib import urlencode
    basestring = basestring

# Python 2's `input` executes the input, whereas `raw_input` just returns the input
try:
    input = raw_input
except NameError:
    pass

# Log settings
log = logging.getLogger("client")
log.setLevel(logging.DEBUG)
# Creates the console handler
handler = logging.StreamHandler()
log.addHandler(handler)

#: Default list of user agents
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/601.1.10 (KHTML, like Gecko) Version/8.0.5 Safari/601.1.10",
    "Mozilla/5.0 (Windows NT 6.3; WOW64; ; NCT50_AAP285C84A1328) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
]

LIKES = {
    'large': EmojiSize.LARGE,
    'medium': EmojiSize.MEDIUM,
    'small': EmojiSize.SMALL,
    'l': EmojiSize.LARGE,
    'm': EmojiSize.MEDIUM,
    's': EmojiSize.SMALL
}

MessageReactionFix = {
    '😍': ('0001f60d', '%F0%9F%98%8D'),
    '😆': ('0001f606', '%F0%9F%98%86'),
    '😮': ('0001f62e', '%F0%9F%98%AE'),
    '😢': ('0001f622', '%F0%9F%98%A2'),
    '😠': ('0001f620', '%F0%9F%98%A0'),
    '👍': ('0001f44d', '%F0%9F%91%8D'),
    '👎': ('0001f44e', '%F0%9F%91%8E')
}

GENDERS = {
    # For standard requests
    0: 'unknown',
    1: 'female_singular',
    2: 'male_singular',
    3: 'female_singular_guess',
    4: 'male_singular_guess',
    5: 'mixed',
    6: 'neuter_singular',
    7: 'unknown_singular',
    8: 'female_plural',
    9: 'male_plural',
    10: 'neuter_plural',
    11: 'unknown_plural',

    # For graphql requests
    'UNKNOWN': 'unknown',
    'FEMALE': 'female_singular',
    'MALE': 'male_singular',
    #'': 'female_singular_guess',
    #'': 'male_singular_guess',
    #'': 'mixed',
    'NEUTER': 'neuter_singular',
    #'': 'unknown_singular',
    #'': 'female_plural',
    #'': 'male_plural',
    #'': 'neuter_plural',
    #'': 'unknown_plural',
}

class ReqUrl(object):
    """A class containing all urls used by `fbchat`"""
    SEARCH = "https://www.facebook.com/ajax/typeahead/search.php"
    LOGIN = "https://m.facebook.com/login.php?login_attempt=1"
    SEND = "https://www.facebook.com/messaging/send/"
    UNREAD_THREADS = "https://www.facebook.com/ajax/mercury/unread_threads.php"
    UNSEEN_THREADS = "https://www.facebook.com/mercury/unseen_thread_ids/"
    THREADS = "https://www.facebook.com/ajax/mercury/threadlist_info.php"
    MOVE_THREAD = "https://www.facebook.com/ajax/mercury/move_thread.php"
    ARCHIVED_STATUS = "https://www.facebook.com/ajax/mercury/change_archived_status.php?dpr=1"
    PINNED_STATUS = "https://www.facebook.com/ajax/mercury/change_pinned_status.php?dpr=1"
    MESSAGES = "https://www.facebook.com/ajax/mercury/thread_info.php"
    READ_STATUS = "https://www.facebook.com/ajax/mercury/change_read_status.php"
    DELIVERED = "https://www.facebook.com/ajax/mercury/delivery_receipts.php"
    MARK_SEEN = "https://www.facebook.com/ajax/mercury/mark_seen.php"
    BASE = "https://www.facebook.com"
    MOBILE = "https://m.facebook.com/"
    STICKY = "https://0-edge-chat.facebook.com/pull"
    PING = "https://0-edge-chat.facebook.com/active_ping"
    UPLOAD = "https://upload.facebook.com/ajax/mercury/upload.php"
    INFO = "https://www.facebook.com/chat/user_info/"
    CONNECT = "https://www.facebook.com/ajax/add_friend/action.php?dpr=1"
    REMOVE_USER = "https://www.facebook.com/chat/remove_participants/"
    LOGOUT = "https://www.facebook.com/logout.php"
    ALL_USERS = "https://www.facebook.com/chat/user_info_all"
    SAVE_DEVICE = "https://m.facebook.com/login/save-device/cancel/"
    CHECKPOINT = "https://m.facebook.com/login/checkpoint/"
    THREAD_COLOR = "https://www.facebook.com/messaging/save_thread_color/?source=thread_settings&dpr=1"
    THREAD_NICKNAME = "https://www.facebook.com/messaging/save_thread_nickname/?source=thread_settings&dpr=1"
    THREAD_EMOJI = "https://www.facebook.com/messaging/save_thread_emoji/?source=thread_settings&dpr=1"
    THREAD_IMAGE = "https://www.facebook.com/messaging/set_thread_image/?dpr=1"
    THREAD_NAME = "https://www.facebook.com/messaging/set_thread_name/?dpr=1"
    MESSAGE_REACTION = "https://www.facebook.com/webgraphql/mutation"
    TYPING = "https://www.facebook.com/ajax/messaging/typ.php"
    GRAPHQL = "https://www.facebook.com/api/graphqlbatch/"
    ATTACHMENT_PHOTO = "https://www.facebook.com/mercury/attachments/photo/"
    EVENT_REMINDER = "https://www.facebook.com/ajax/eventreminder/create"
    MODERN_SETTINGS_MENU = "https://www.facebook.com/bluebar/modern_settings_menu/"
    REMOVE_FRIEND = "https://m.facebook.com/a/removefriend.php"
    BLOCK_USER = "https://www.facebook.com/messaging/block_messages/?dpr=1"
    UNBLOCK_USER = "https://www.facebook.com/messaging/unblock_messages/?dpr=1"
    SAVE_ADMINS = "https://www.facebook.com/messaging/save_admins/?dpr=1"
    APPROVAL_MODE = "https://www.facebook.com/messaging/set_approval_mode/?dpr=1"
    CREATE_GROUP = "https://m.facebook.com/messages/send/?icm=1"
    DELETE_THREAD = "https://www.facebook.com/ajax/mercury/delete_thread.php?dpr=1"
    MUTE_THREAD = "https://www.facebook.com/ajax/mercury/change_mute_thread.php?dpr=1"
    MUTE_REACTIONS = "https://www.facebook.com/ajax/mercury/change_reactions_mute_thread/?dpr=1"
    MUTE_MENTIONS = "https://www.facebook.com/ajax/mercury/change_mentions_mute_thread/?dpr=1"

    pull_channel = 0

    def change_pull_channel(self, channel=None):
        if channel is None:
            self.pull_channel = (self.pull_channel + 1) % 5 # Pull channel will be 0-4
        else:
            self.pull_channel = channel
        self.STICKY = "https://{}-edge-chat.facebook.com/pull".format(self.pull_channel)
        self.PING = "https://{}-edge-chat.facebook.com/active_ping".format(self.pull_channel)

facebookEncoding = 'UTF-8'

def now():
    return int(time()*1000)

def strip_to_json(text):
    try:
        return text[text.index('{'):]
    except ValueError:
        raise FBchatException('No JSON object found: {!r}'.format(text))

def get_decoded_r(r):
    return get_decoded(r._content)

def get_decoded(content):
    return content.decode(facebookEncoding)

def parse_json(content):
    return json.loads(content)

def get_json(r):
    return json.loads(strip_to_json(get_decoded_r(r)))

def digitToChar(digit):
    if digit < 10:
        return str(digit)
    return chr(ord('a') + digit - 10)

def str_base(number, base):
    if number < 0:
        return '-' + str_base(-number, base)
    (d, m) = divmod(number, base)
    if d > 0:
        return str_base(d, base) + digitToChar(m)
    return digitToChar(m)

def generateMessageID(client_id=None):
    k = now()
    l = int(random() * 4294967295)
    return "<{}:{}-{}@mail.projektitan.com>".format(k, l, client_id)

def getSignatureID():
    return hex(int(random() * 2147483648))

def generateOfflineThreadingID():
    ret = now()
    value = int(random() * 4294967295)
    string = ("0000000000000000000000" + format(value, 'b'))[-22:]
    msgs = format(ret, 'b') + string
    return str(int(msgs, 2))

def check_json(j):
    if j.get('error') is None:
        return
    if 'errorDescription' in j:
        # 'errorDescription' is in the users own language!
        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error'], j['errorDescription']), fb_error_code=j['error'], fb_error_message=j['errorDescription'])
    elif 'debug_info' in j['error'] and 'code' in j['error']:
        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info'])), fb_error_code=j['error']['code'], fb_error_message=j['error']['debug_info'])
    else:
        raise FBchatFacebookError('Error {} when sending request'.format(j['error']), fb_error_code=j['error'])

def check_request(r, as_json=True):
    if not r.ok:
        raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_status_code=r.status_code)

    content = get_decoded_r(r)

    if content is None or len(content) == 0:
        raise FBchatFacebookError('Error when sending request: Got empty response')

    if as_json:
        content = strip_to_json(content)
        try:
            j = json.loads(content)
        except ValueError:
            raise FBchatFacebookError('Error while parsing JSON: {!r}'.format(content))
        check_json(j)
        log.debug(j)
        return j
    else:
        return content

def get_jsmods_require(j, index):
    if j.get('jsmods') and j['jsmods'].get('require'):
        try:
            return j['jsmods']['require'][0][index][0]
        except (KeyError, IndexError) as e:
            log.warning('Error when getting jsmods_require: {}. Facebook might have changed protocol'.format(j))
    return None

def get_emojisize_from_tags(tags):
    if tags is None:
        return None
    tmp = [tag for tag in tags if tag.startswith('hot_emoji_size:')]
    if len(tmp) > 0:
        try:
            return LIKES[tmp[0].split(':')[1]]
        except (KeyError, IndexError):
            log.exception('Could not determine emoji size from {} - {}'.format(tags, tmp))
    return None
<commit_msg>Added deleteMesseges request url<commit_after># -*- coding: UTF-8 -*-

from __future__ import unicode_literals
import re
import json
from time import time
from random import random
import warnings
import logging
from .models import *

try:
    from urllib.parse import urlencode
    basestring = (str, bytes)
except ImportError:
    from urllib import urlencode
    basestring = basestring

# Python 2's `input` executes the input, whereas `raw_input` just returns the input
try:
    input = raw_input
except NameError:
    pass

# Log settings
log = logging.getLogger("client")
log.setLevel(logging.DEBUG)
# Creates the console handler
handler = logging.StreamHandler()
log.addHandler(handler)

#: Default list of user agents
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/601.1.10 (KHTML, like Gecko) Version/8.0.5 Safari/601.1.10",
    "Mozilla/5.0 (Windows NT 6.3; WOW64; ; NCT50_AAP285C84A1328) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"
]

LIKES = {
    'large': EmojiSize.LARGE,
    'medium': EmojiSize.MEDIUM,
    'small': EmojiSize.SMALL,
    'l': EmojiSize.LARGE,
    'm': EmojiSize.MEDIUM,
    's': EmojiSize.SMALL
}

MessageReactionFix = {
    '😍': ('0001f60d', '%F0%9F%98%8D'),
    '😆': ('0001f606', '%F0%9F%98%86'),
    '😮': ('0001f62e', '%F0%9F%98%AE'),
    '😢': ('0001f622', '%F0%9F%98%A2'),
    '😠': ('0001f620', '%F0%9F%98%A0'),
    '👍': ('0001f44d', '%F0%9F%91%8D'),
    '👎': ('0001f44e', '%F0%9F%91%8E')
}

GENDERS = {
    # For standard requests
    0: 'unknown',
    1: 'female_singular',
    2: 'male_singular',
    3: 'female_singular_guess',
    4: 'male_singular_guess',
    5: 'mixed',
    6: 'neuter_singular',
    7: 'unknown_singular',
    8: 'female_plural',
    9: 'male_plural',
    10: 'neuter_plural',
    11: 'unknown_plural',

    # For graphql requests
    'UNKNOWN': 'unknown',
    'FEMALE': 'female_singular',
    'MALE': 'male_singular',
    #'': 'female_singular_guess',
    #'': 'male_singular_guess',
    #'': 'mixed',
    'NEUTER': 'neuter_singular',
    #'': 'unknown_singular',
    #'': 'female_plural',
    #'': 'male_plural',
    #'': 'neuter_plural',
    #'': 'unknown_plural',
}

class ReqUrl(object):
    """A class containing all urls used by `fbchat`"""
    SEARCH = "https://www.facebook.com/ajax/typeahead/search.php"
    LOGIN = "https://m.facebook.com/login.php?login_attempt=1"
    SEND = "https://www.facebook.com/messaging/send/"
    UNREAD_THREADS = "https://www.facebook.com/ajax/mercury/unread_threads.php"
    UNSEEN_THREADS = "https://www.facebook.com/mercury/unseen_thread_ids/"
    THREADS = "https://www.facebook.com/ajax/mercury/threadlist_info.php"
    MOVE_THREAD = "https://www.facebook.com/ajax/mercury/move_thread.php"
    ARCHIVED_STATUS = "https://www.facebook.com/ajax/mercury/change_archived_status.php?dpr=1"
    PINNED_STATUS = "https://www.facebook.com/ajax/mercury/change_pinned_status.php?dpr=1"
    MESSAGES = "https://www.facebook.com/ajax/mercury/thread_info.php"
    READ_STATUS = "https://www.facebook.com/ajax/mercury/change_read_status.php"
    DELIVERED = "https://www.facebook.com/ajax/mercury/delivery_receipts.php"
    MARK_SEEN = "https://www.facebook.com/ajax/mercury/mark_seen.php"
    BASE = "https://www.facebook.com"
    MOBILE = "https://m.facebook.com/"
    STICKY = "https://0-edge-chat.facebook.com/pull"
    PING = "https://0-edge-chat.facebook.com/active_ping"
    UPLOAD = "https://upload.facebook.com/ajax/mercury/upload.php"
    INFO = "https://www.facebook.com/chat/user_info/"
    CONNECT = "https://www.facebook.com/ajax/add_friend/action.php?dpr=1"
    REMOVE_USER = "https://www.facebook.com/chat/remove_participants/"
    LOGOUT = "https://www.facebook.com/logout.php"
    ALL_USERS = "https://www.facebook.com/chat/user_info_all"
    SAVE_DEVICE = "https://m.facebook.com/login/save-device/cancel/"
    CHECKPOINT = "https://m.facebook.com/login/checkpoint/"
    THREAD_COLOR = "https://www.facebook.com/messaging/save_thread_color/?source=thread_settings&dpr=1"
    THREAD_NICKNAME = "https://www.facebook.com/messaging/save_thread_nickname/?source=thread_settings&dpr=1"
    THREAD_EMOJI = "https://www.facebook.com/messaging/save_thread_emoji/?source=thread_settings&dpr=1"
    THREAD_IMAGE = "https://www.facebook.com/messaging/set_thread_image/?dpr=1"
    THREAD_NAME = "https://www.facebook.com/messaging/set_thread_name/?dpr=1"
    MESSAGE_REACTION = "https://www.facebook.com/webgraphql/mutation"
    TYPING = "https://www.facebook.com/ajax/messaging/typ.php"
    GRAPHQL = "https://www.facebook.com/api/graphqlbatch/"
    ATTACHMENT_PHOTO = "https://www.facebook.com/mercury/attachments/photo/"
    EVENT_REMINDER = "https://www.facebook.com/ajax/eventreminder/create"
    MODERN_SETTINGS_MENU = "https://www.facebook.com/bluebar/modern_settings_menu/"
    REMOVE_FRIEND = "https://m.facebook.com/a/removefriend.php"
    BLOCK_USER = "https://www.facebook.com/messaging/block_messages/?dpr=1"
    UNBLOCK_USER = "https://www.facebook.com/messaging/unblock_messages/?dpr=1"
    SAVE_ADMINS = "https://www.facebook.com/messaging/save_admins/?dpr=1"
    APPROVAL_MODE = "https://www.facebook.com/messaging/set_approval_mode/?dpr=1"
    CREATE_GROUP = "https://m.facebook.com/messages/send/?icm=1"
    DELETE_THREAD = "https://www.facebook.com/ajax/mercury/delete_thread.php?dpr=1"
    DELETE_MESSAGES = "https://www.facebook.com/ajax/mercury/delete_messages.php?dpr=1"
    MUTE_THREAD = "https://www.facebook.com/ajax/mercury/change_mute_thread.php?dpr=1"
    MUTE_REACTIONS = "https://www.facebook.com/ajax/mercury/change_reactions_mute_thread/?dpr=1"
    MUTE_MENTIONS = "https://www.facebook.com/ajax/mercury/change_mentions_mute_thread/?dpr=1"

    pull_channel = 0

    def change_pull_channel(self, channel=None):
        if channel is None:
            self.pull_channel = (self.pull_channel + 1) % 5 # Pull channel will be 0-4
        else:
            self.pull_channel = channel
        self.STICKY = "https://{}-edge-chat.facebook.com/pull".format(self.pull_channel)
        self.PING = "https://{}-edge-chat.facebook.com/active_ping".format(self.pull_channel)

facebookEncoding = 'UTF-8'

def now():
    return int(time()*1000)

def strip_to_json(text):
    try:
        return text[text.index('{'):]
    except ValueError:
        raise FBchatException('No JSON object found: {!r}'.format(text))

def get_decoded_r(r):
    return get_decoded(r._content)

def get_decoded(content):
    return content.decode(facebookEncoding)

def parse_json(content):
    return json.loads(content)

def get_json(r):
    return json.loads(strip_to_json(get_decoded_r(r)))

def digitToChar(digit):
    if digit < 10:
        return str(digit)
    return chr(ord('a') + digit - 10)

def str_base(number, base):
    if number < 0:
        return '-' + str_base(-number, base)
    (d, m) = divmod(number, base)
    if d > 0:
        return str_base(d, base) + digitToChar(m)
    return digitToChar(m)

def generateMessageID(client_id=None):
    k = now()
    l = int(random() * 4294967295)
    return "<{}:{}-{}@mail.projektitan.com>".format(k, l, client_id)

def getSignatureID():
    return hex(int(random() * 2147483648))

def generateOfflineThreadingID():
    ret = now()
    value = int(random() * 4294967295)
    string = ("0000000000000000000000" + format(value, 'b'))[-22:]
    msgs = format(ret, 'b') + string
    return str(int(msgs, 2))

def check_json(j):
    if j.get('error') is None:
        return
    if 'errorDescription' in j:
        # 'errorDescription' is in the users own language!
        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error'], j['errorDescription']), fb_error_code=j['error'], fb_error_message=j['errorDescription'])
    elif 'debug_info' in j['error'] and 'code' in j['error']:
        raise FBchatFacebookError('Error #{} when sending request: {}'.format(j['error']['code'], repr(j['error']['debug_info'])), fb_error_code=j['error']['code'], fb_error_message=j['error']['debug_info'])
    else:
        raise FBchatFacebookError('Error {} when sending request'.format(j['error']), fb_error_code=j['error'])

def check_request(r, as_json=True):
    if not r.ok:
        raise FBchatFacebookError('Error when sending request: Got {} response'.format(r.status_code), request_status_code=r.status_code)

    content = get_decoded_r(r)

    if content is None or len(content) == 0:
        raise FBchatFacebookError('Error when sending request: Got empty response')

    if as_json:
        content = strip_to_json(content)
        try:
            j = json.loads(content)
        except ValueError:
            raise FBchatFacebookError('Error while parsing JSON: {!r}'.format(content))
        check_json(j)
        log.debug(j)
        return j
    else:
        return content

def get_jsmods_require(j, index):
    if j.get('jsmods') and j['jsmods'].get('require'):
        try:
            return j['jsmods']['require'][0][index][0]
        except (KeyError, IndexError) as e:
            log.warning('Error when getting jsmods_require: {}. Facebook might have changed protocol'.format(j))
    return None

def get_emojisize_from_tags(tags):
    if tags is None:
        return None
    tmp = [tag for tag in tags if tag.startswith('hot_emoji_size:')]
    if len(tmp) > 0:
        try:
            return LIKES[tmp[0].split(':')[1]]
        except (KeyError, IndexError):
            log.exception('Could not determine emoji size from {} - {}'.format(tags, tmp))
    return None
<|endoftext|>