<commit_before>#   -*- coding: utf-8 -*-
#
#   This file is part of PyBuilder
#
#   Copyright 2011-2015 PyBuilder Team
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
    The PyBuilder vcs module.
    Provides version control system utilities.
"""

from pybuilder.utils import execute_command_and_capture_output
from pybuilder.errors import PyBuilderException

class VCSRevision(object):
    """
    An object representing the VCS revision of the current working directory.
    """

    @property
    def count(self):
        """
        Returns the current revision number as a string.
        """
        if self.is_a_git_repo():
            return self.get_git_revision_count()

        if self.is_a_svn_repo():
            return self.get_svn_revision_count()

        raise PyBuilderException(
            "Cannot determine VCS revision: project is neither a git nor a svn repo.")

    def get_git_revision_count(self):
        # NOTE: git rev-list HEAD --count does not work on RHEL6, hence we count ourselves.
        exit_code, stdout, stderr = execute_command_and_capture_output(
            "git", "rev-list", "HEAD")
        if exit_code != 0:
            raise PyBuilderException("Cannot determine git revision: git rev-list HEAD failed:\n{0}".
                                     format(stderr))
        return str(len(stdout.splitlines()))

    def get_svn_revision_count(self):
        exit_code, stdout, stderr = execute_command_and_capture_output(
            "svnversion")
        if exit_code != 0 or "Unversioned directory" in stdout or "Uncommitted" in stdout:
            raise PyBuilderException("Cannot determine svn revision: svnversion failed or unversioned directory:\n{0}".
                                     format(stderr))
        return stdout.strip().replace("M", "").replace("S", "").replace("P", "").split(":")[0]

    def is_a_git_repo(self):
        exit_code, _, __ = execute_command_and_capture_output("git", "status")
        if exit_code == 0:
            return True
        return False

    def is_a_svn_repo(self):
        exit_code, stdout, stderr = execute_command_and_capture_output("svn", "status")
        if "not a working copy" in stderr or exit_code != 0:
            return False

        return True
<commit_msg>implement get_git_hash<commit_after>#   -*- coding: utf-8 -*-
#
#   This file is part of PyBuilder
#
#   Copyright 2011-2015 PyBuilder Team
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
    The PyBuilder vcs module.
    Provides version control system utilities.
"""

from pybuilder.utils import execute_command_and_capture_output
from pybuilder.errors import PyBuilderException

class VCSRevision(object):
    """
    An object representing the VCS revision of the current working directory.
    """

    @property
    def count(self):
        """
        Returns the current revision number as a string.
        """
        if self.is_a_git_repo():
            return self.get_git_revision_count()

        if self.is_a_svn_repo():
            return self.get_svn_revision_count()

        raise PyBuilderException(
            "Cannot determine VCS revision: project is neither a git nor a svn repo.")

    def get_git_hash(self, abbreviate=7):
        if self.is_a_git_repo():
            exit_code, stdout, stderr = execute_command_and_capture_output(
                "git", "rev-parse", "--short={0}".format(abbreviate), "HEAD")
            if exit_code != 0:
                raise PyBuilderException("Cannot determine git hash: git rev-parse HEAD failed:\n{0}".
                                         format(stderr))
            else:
                return stdout.strip()
        else:
            raise PyBuilderException("Cannot determine git hash: project is not a git repo.")

    def get_git_revision_count(self):
        # NOTE: git rev-list HEAD --count does not work on RHEL6, hence we count ourselves.
        exit_code, stdout, stderr = execute_command_and_capture_output(
            "git", "rev-list", "HEAD")
        if exit_code != 0:
            raise PyBuilderException("Cannot determine git revision: git rev-list HEAD failed:\n{0}".
                                     format(stderr))
        return str(len(stdout.splitlines()))

    def get_svn_revision_count(self):
        exit_code, stdout, stderr = execute_command_and_capture_output(
            "svnversion")
        if exit_code != 0 or "Unversioned directory" in stdout or "Uncommitted" in stdout:
            raise PyBuilderException("Cannot determine svn revision: svnversion failed or unversioned directory:\n{0}".
                                     format(stderr))
        return stdout.strip().replace("M", "").replace("S", "").replace("P", "").split(":")[0]

    def is_a_git_repo(self):
        exit_code, _, __ = execute_command_and_capture_output("git", "status")
        if exit_code == 0:
            return True
        return False

    def is_a_svn_repo(self):
        exit_code, stdout, stderr = execute_command_and_capture_output("svn", "status")
        if "not a working copy" in stderr or exit_code != 0:
            return False

        return True
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#jiandan-meizitu,并保存至本地
import requests
import os
import re

a = 2423
print('开始爬取煎蛋网美女图片')
allpicurl=[]
def getHTMLText(url):
	try:
		headers = { "Accept":"text/html,application/xhtml+xml,application/xml;",
            "Accept-Encoding":"gzip",
            "Accept-Language":"zh-CN,zh;q=0.8",
            "Referer":"http://www.example.com/",
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36"
            }
		r = requests.get(url, timeout=30,headers=headers)
		r.raise_for_status()
		return r.text
	except:
		return ""

def foo(lst,a):
    return[a+str(i) for i in lst]

def LoadPic(picurl,path):
	try:
		if not os.path.exists(root):
			os.mkdir(root)
		if not os.path.exists(path):
			r = requests.get(picurl)
			with open(path,'wb') as f:
				f.write(r.content)
				f.close()
				print("文件保存成功")
		else:
				print("文件已保存")
	except:
		print("爬取失败")

def main():
	infoList = []
	rex = r'href="//w[xw][1-4].sinaimg.cn/large.*?.[jpg,png,gif]"'
	plt = re.findall(rex,getHTMLText(url))
	for i in range(len(plt)):
		picurl = eval(plt[i].split('=')[1])
		infoList.append(picurl)
	picfile = foo(infoList,'http:')
	for i in picfile:
		path = root+url.split('page-')[1]+i.split('/')[-1]
		LoadPic(i,path)

while a >= 2420:
    print(a)
    b=str(a)
    root = "./pic/"+b+"/"
    url = 'http://jandan.net/ooxx/page-'+str(a)
    main()
    a=a-1

<commit_msg>第二次修改<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#jiandan-meizitu,并保存至本地
import requests
import os
import re

a = 2426
print('开始爬取煎蛋网美女图片')
allpicurl=[]
def getHTMLText(url):
	try:
		headers = { "Accept":"text/html,application/xhtml+xml,application/xml;",
            "Accept-Encoding":"gzip",
            "Accept-Language":"zh-CN,zh;q=0.8",
            "Referer":"http://www.example.com/",
            "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36"
            }
		r = requests.get(url, timeout=30,headers=headers)
		r.raise_for_status()
		return r.text
	except:
		return ""

def foo(lst,a):
    return[a+str(i) for i in lst]

def LoadPic(picurl,path):
	try:
		if not os.path.exists(root):
			os.mkdir(root)
		if not os.path.exists(path):
			r = requests.get(picurl)
			with open(path,'wb') as f:
				f.write(r.content)
				f.close()
				print("文件保存成功")
		else:
				print("文件已保存")
	except:
		print("爬取失败")

def main():
	infoList = []
	rex = r'href="//w[xw][1-4].sinaimg.cn/large.*?.[jpg,png,gif]"'
	plt = re.findall(rex,getHTMLText(url))
	for i in range(len(plt)):
		picurl = eval(plt[i].split('=')[1])
		infoList.append(picurl)
	picfile = foo(infoList,'http:')
	for i in picfile:
		path = root+url.split('page-')[1]+i.split('/')[-1]
		LoadPic(i,path)

while a >= 2420:
    print(a)
    b=str(a)
    root = "./pic/"+b+"/"
    url = 'http://jandan.net/ooxx/page-'+str(a)
    main()
    a=a-1

<|endoftext|>


<commit_before>from __future__ import absolute_import
from kafka.common import OffsetRequest
from kafka.util import kafka_bytestring

from corehq.apps.change_feed.connection import get_kafka_client
from corehq.apps.change_feed.exceptions import UnavailableKafkaOffset
from .document_types import CASE, FORM, DOMAIN, META, APP

# this is redundant but helps avoid import warnings until nothing references these
CASE = CASE
FORM = FORM
DOMAIN = DOMAIN
META = META
APP = APP

# new models
CASE_SQL = 'case-sql'
FORM_SQL = 'form-sql'
SMS = 'sms'
LEDGER = 'ledger'
COMMCARE_USER = 'commcare-user'
GROUP = 'group'
WEB_USER = 'web-user'
LOCATION = 'location'

CASE_TOPICS = (CASE, CASE_SQL)
FORM_TOPICS = (FORM, FORM_SQL)
USER_TOPICS = (COMMCARE_USER, WEB_USER)
ALL = (
    CASE,
    CASE_SQL,
    COMMCARE_USER,
    DOMAIN,
    FORM,
    FORM_SQL,
    GROUP,
    LEDGER,
    META,
    SMS,
    WEB_USER,
    APP,
    LOCATION,
)

def get_topic(document_type_object):
    return document_type_object.primary_type

def get_topic_offset(topic):
    """
    :returns: The kafka offset dict for the topic."""
    return get_multi_topic_offset([topic])

def get_all_offsets():
    """
    :returns: A dict of offsets keyed by topic and parition"""
    return get_multi_topic_offset(ALL)

def get_multi_topic_offset(topics):
    """
    :returns: A dict of offsets keyed by topic and partition"""
    return _get_topic_offsets(topics, latest=True)

def get_multi_topic_first_available_offsets(topics):
    """
    :returns: A dict of offsets keyed by topic and partition"""
    return _get_topic_offsets(topics, latest=False)

def _get_topic_offsets(topics, latest):
    """
    :param topics: list of topics
    :param latest: True to fetch latest offsets, False to fetch earliest available
    :return: dict: { (topic, partition): offset, ... }
    """

    # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetRequest
    # https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/
    assert set(topics) <= set(ALL)
    client = get_kafka_client()
    partition_meta = client.topic_partitions

    # only return the offset of the latest message in the partition
    num_offsets = 1
    time_value = -1 if latest else -2

    offsets = {}
    offset_requests = []
    for topic in topics:
        partitions = list(partition_meta.get(topic, {}))
        for partition in partitions:
            offsets[(topic, partition)] = None
            offset_requests.append(OffsetRequest(topic, partition, time_value, num_offsets))

    responses = client.send_offset_request(offset_requests)
    for r in responses:
        offsets[(r.topic, r.partition)] = r.offsets[0]

    return offsets

def validate_offsets(expected_offsets):
    """
    Takes in a dictionary of offsets (topics to checkpoint numbers) and ensures they are all available
    in the current kafka feed
    """
    if expected_offsets:
        topics = {kafka_bytestring(x[0]) for x in expected_offsets.keys()}
        available_offsets = get_multi_topic_first_available_offsets(topics)
        for topic_partition, offset in expected_offsets.items():
            topic, partition = topic_partition
            if topic_partition not in available_offsets:
                raise UnavailableKafkaOffset("Invalid partition '{}' for topic '{}'".format(partition, topic))

            if expected_offsets[topic_partition] < available_offsets[topic_partition]:
                message = (
                    'First available topic offset for {}:{} is {} but needed {}.'
                ).format(topic, partition, available_offsets[topic_partition], expected_offsets[topic_partition])
                raise UnavailableKafkaOffset(message)
<commit_msg>Proactively convert to bytestring for kafka.<commit_after>from __future__ import absolute_import
from kafka.common import OffsetRequest
from kafka.util import kafka_bytestring

from corehq.apps.change_feed.connection import get_kafka_client
from corehq.apps.change_feed.exceptions import UnavailableKafkaOffset
from .document_types import CASE, FORM, DOMAIN, META, APP

# this is redundant but helps avoid import warnings until nothing references these
CASE = CASE
FORM = FORM
DOMAIN = DOMAIN
META = META
APP = APP

# new models
CASE_SQL = 'case-sql'
FORM_SQL = 'form-sql'
SMS = 'sms'
LEDGER = 'ledger'
COMMCARE_USER = 'commcare-user'
GROUP = 'group'
WEB_USER = 'web-user'
LOCATION = 'location'

CASE_TOPICS = (CASE, CASE_SQL)
FORM_TOPICS = (FORM, FORM_SQL)
USER_TOPICS = (COMMCARE_USER, WEB_USER)
ALL = (
    CASE,
    CASE_SQL,
    COMMCARE_USER,
    DOMAIN,
    FORM,
    FORM_SQL,
    GROUP,
    LEDGER,
    META,
    SMS,
    WEB_USER,
    APP,
    LOCATION,
)

def get_topic(document_type_object):
    return document_type_object.primary_type

def get_topic_offset(topic):
    """
    :returns: The kafka offset dict for the topic."""
    return get_multi_topic_offset([topic])

def get_all_offsets():
    """
    :returns: A dict of offsets keyed by topic and parition"""
    return get_multi_topic_offset(ALL)

def get_multi_topic_offset(topics):
    """
    :returns: A dict of offsets keyed by topic and partition"""
    return _get_topic_offsets(topics, latest=True)

def get_multi_topic_first_available_offsets(topics):
    """
    :returns: A dict of offsets keyed by topic and partition"""
    return _get_topic_offsets(topics, latest=False)

def _get_topic_offsets(topics, latest):
    """
    :param topics: list of topics
    :param latest: True to fetch latest offsets, False to fetch earliest available
    :return: dict: { (topic, partition): offset, ... }
    """

    # https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetRequest
    # https://cfchou.github.io/blog/2015/04/23/a-closer-look-at-kafka-offsetrequest/
    assert set(topics) <= set(ALL)
    client = get_kafka_client()
    partition_meta = client.topic_partitions

    # only return the offset of the latest message in the partition
    num_offsets = 1
    time_value = -1 if latest else -2

    offsets = {}
    offset_requests = []
    for topic in topics:
        partitions = list(partition_meta.get(topic, {}))
        for partition in partitions:
            offsets[(kafka_bytestring(topic), partition)] = None
            offset_requests.append(OffsetRequest(kafka_bytestring(topic), partition, time_value, num_offsets))

    responses = client.send_offset_request(offset_requests)
    for r in responses:
        offsets[(kafka_bytestring(r.topic), r.partition)] = r.offsets[0]

    return offsets

def validate_offsets(expected_offsets):
    """
    Takes in a dictionary of offsets (topics to checkpoint numbers) and ensures they are all available
    in the current kafka feed
    """
    if expected_offsets:
        topics = {kafka_bytestring(x[0]) for x in expected_offsets.keys()}
        available_offsets = get_multi_topic_first_available_offsets(topics)
        for topic_partition, offset in expected_offsets.items():
            topic, partition = topic_partition
            if topic_partition not in available_offsets:
                raise UnavailableKafkaOffset("Invalid partition '{}' for topic '{}'".format(partition, topic))

            if expected_offsets[topic_partition] < available_offsets[topic_partition]:
                message = (
                    'First available topic offset for {}:{} is {} but needed {}.'
                ).format(topic, partition, available_offsets[topic_partition], expected_offsets[topic_partition])
                raise UnavailableKafkaOffset(message)
<|endoftext|>


<commit_before># HologramCloud.py - Hologram Python SDK Cloud interface
#
# Author: Hologram <support@hologram.io>
#
# Copyright 2016 - Hologram (Konekt, Inc.)
#
#
# LICENSE: Distributed under the terms of the MIT License
#

import binascii
import json
import sys
from Hologram.CustomCloud import CustomCloud
from HologramAuth import TOTPAuthentication, SIMOTPAuthentication
from Hologram.Authentication import CSRPSKAuthentication
from Exceptions.HologramError import HologramError

DEFAULT_SEND_MESSAGE_TIMEOUT = 5
HOLOGRAM_HOST_SEND = 'cloudsocket.hologram.io'
HOLOGRAM_PORT_SEND = 9999
HOLOGRAM_HOST_RECEIVE = '0.0.0.0'
HOLOGRAM_PORT_RECEIVE = 4010
MAX_SMS_LENGTH = 160

# Hologram error codes
ERR_OK = 0
ERR_CONNCLOSED = 1 # Connection was closed before a terminating character
                   # but message might be fine
ERR_MSGINVALID = 2 # Couldn't parse the message
ERR_AUTHINVALID = 3 # Auth section of message was invalid
ERR_PAYLOADINVALID = 4 # Payload type was invalid
ERR_PROTINVALID = 5 # Protocol type was invalid
ERR_INTERNAL = 6 # An internal error occurred
ERR_METADATA = 7 # Metadata was formatted incorrectly
ERR_TOPICINVALID = 8 # Topic was formatted incorrectly
ERR_UNKNOWN = -1 # Unknown error

class HologramCloud(CustomCloud):

    _authentication_handlers = {
        'csrpsk' : CSRPSKAuthentication.CSRPSKAuthentication,
        'totp' : TOTPAuthentication.TOTPAuthentication,
        'sim-otp' : SIMOTPAuthentication.SIMOTPAuthentication,
    }

    _errorCodeDescription = {

        ERR_OK: 'Message sent successfully',
        ERR_CONNCLOSED: 'Connection was closed so we couldn\'t read the whole message',
        ERR_MSGINVALID: 'Failed to parse the message',
        ERR_AUTHINVALID: 'Auth section of the message was invalid',
        ERR_PAYLOADINVALID: 'Payload type was invalid',
        ERR_PROTINVALID: 'Protocol type was invalid',
        ERR_INTERNAL: 'Internal error in Hologram Cloud',
        ERR_METADATA: 'Metadata was formatted incorrectly',
        ERR_TOPICINVALID: 'Topic was formatted incorrectly',
        ERR_UNKNOWN: 'Unknown error'
    }

    def __init__(self, credentials, enable_inbound=False, network='',
                 authentication_type='totp'):
        super().__init__(credentials,
                         send_host=HOLOGRAM_HOST_SEND,
                         send_port=HOLOGRAM_PORT_SEND,
                         receive_host=HOLOGRAM_HOST_RECEIVE,
                         receive_port=HOLOGRAM_PORT_RECEIVE,
                         enable_inbound=enable_inbound,
                         network=network)

        self.setAuthenticationType(credentials, authentication_type=authentication_type)

        if self.authenticationType == 'totp':
            self.__populate_totp_credentials()

    # EFFECTS: Authentication Configuration
    def setAuthenticationType(self, credentials, authentication_type='csrpsk'):

        if authentication_type not in HologramCloud._authentication_handlers:
            raise HologramError('Invalid authentication type: %s' % authentication_type)

        self.authenticationType = authentication_type

        self.authentication = HologramCloud._authentication_handlers[self.authenticationType](credentials)

    # EFFECTS: Sends the message to the cloud.
    def sendMessage(self, message, topics=None, timeout=DEFAULT_SEND_MESSAGE_TIMEOUT):

        if not self.is_ready_to_send():
            self.addPayloadToBuffer(message)
            return ''

        # Set the appropriate credentials required for sim otp authentication.
        if self.authenticationType == 'sim-otp':
            self.__populate_sim_otp_credentials()

        modem_type = None
        modem_id = None
        if self.network is not None:
            modem_id = self.network.modem_id
            modem_type = str(self.network.modem)

        output = self.authentication.buildPayloadString(message,
                                                        topics=topics,
                                                        modem_type=modem_type,
                                                        modem_id=modem_id,
                                                        version=self.version)

        result = super().sendMessage(output, timeout)
        return self.__parse_result(result)

    def __parse_result(self, result):
        resultList = None
        if self.authenticationType == 'csrpsk':
            resultList = self.__parse_hologram_json_result(result)
        else:
            resultList = self.__parse_hologram_compact_result(result)

        return resultList[0]

    def __populate_totp_credentials(self):
        try:
            self.authentication.credentials['device_id'] = self.network.iccid
            self.authentication.credentials['private_key'] = self.network.imsi
        except Exception as e:
            self.logger.error('Unable to fetch device id or private key')

    def __populate_sim_otp_credentials(self):
        nonce = self.request_hex_nonce()
        command = self.authentication.generate_sim_otp_command(imsi=self.network.imsi,
                                                               iccid=self.network.iccid,
                                                               nonce=nonce)
        modem_response = self.network.get_sim_otp_response(command)
        self.authentication.generate_sim_otp_token(modem_response)

    def sendSMS(self, destination_number, message):

        self.__enforce_authentication_type_supported_for_sms()
        self.__enforce_valid_destination_number(destination_number)
        self.__enforce_max_sms_length(message)

        output = self.authentication.buildSMSPayloadString(destination_number,
                                                           message)

        self.logger.debug('Destination number: %s', destination_number)
        self.logger.debug('SMS: %s', message)

        result = super().sendMessage(output)

        resultList = self.__parse_hologram_compact_result(result)
        return resultList[0]

    # REQUIRES: Called only when sim otp authentication is required.
    # EFFECTS: Request for a hex nonce.
    def request_hex_nonce(self):

        self.open_send_socket()

        # build nonce request payload string
        nonce_request = self.authentication.buildNonceRequestPayloadString()

        self.logger.debug("Sending nonce request with body of length %d", len(nonce_request))
        self.logger.debug('Send: %s', nonce_request)

        nonce = super().sendMessage(message=nonce_request, timeout=10, close_socket=False)
        self.logger.debug('Nonce request sent.')

        resultbuf_hex = binascii.b2a_hex(nonce)

        if resultbuf_hex is None:
            raise HologramError('Internal nonce error')

        return resultbuf_hex

    def enableSMS(self):
        return self.network.enableSMS()

    def disableSMS(self):
        return self.network.disableSMS()

    def popReceivedSMS(self):
        return self.network.popReceivedSMS()

    # EFFECTS: Parses the hologram send response.
    def __parse_hologram_json_result(self, result):
        try:
            resultList = json.loads(result)
            resultList[0] = int(resultList[0])
        except ValueError:
            self.logger.error('Server replied with invalid JSON [%s]', result)
            resultList = [ERR_UNKNOWN]
        return resultList

    def __parse_hologram_compact_result(self, result):

        # convert the returned response to formatted list.
        if result is None:
            return [ERR_UNKNOWN]

        resultList = []
        for x in result:
            resultList.append(int(x))

        if len(resultList) == 0:
            resultList = [ERR_UNKNOWN]

        return resultList

    def __enforce_max_sms_length(self, message):
        if len(message) > MAX_SMS_LENGTH:
            raise HologramError('SMS cannot be more than %d characters long' % MAX_SMS_LENGTH)

    def __enforce_valid_destination_number(self, destination_number):
        if not destination_number.startswith('+'):
            raise HologramError('SMS destination number must start with a \'+\' sign')

    def __enforce_authentication_type_supported_for_sms(self):
        if self.authenticationType != 'csrpsk':
            raise HologramError('%s does not support SDK SMS features' % self.authenticationType)

    # REQUIRES: A result code (int).
    # EFFECTS: Returns a translated string based on the given hologram result code.
    def getResultString(self, result_code):
        if result_code not in self._errorCodeDescription:
            return 'Unknown response code'
        return self._errorCodeDescription[result_code]

    def resultWasSuccess(self, result_code):
        return result_code in (ERR_OK, ERR_CONNCLOSED)
<commit_msg>properly convert bytestring ascii value<commit_after># HologramCloud.py - Hologram Python SDK Cloud interface
#
# Author: Hologram <support@hologram.io>
#
# Copyright 2016 - Hologram (Konekt, Inc.)
#
#
# LICENSE: Distributed under the terms of the MIT License
#

import binascii
import json
import sys
from Hologram.CustomCloud import CustomCloud
from HologramAuth import TOTPAuthentication, SIMOTPAuthentication
from Hologram.Authentication import CSRPSKAuthentication
from Exceptions.HologramError import HologramError

DEFAULT_SEND_MESSAGE_TIMEOUT = 5
HOLOGRAM_HOST_SEND = 'cloudsocket.hologram.io'
HOLOGRAM_PORT_SEND = 9999
HOLOGRAM_HOST_RECEIVE = '0.0.0.0'
HOLOGRAM_PORT_RECEIVE = 4010
MAX_SMS_LENGTH = 160

# Hologram error codes
ERR_OK = 0
ERR_CONNCLOSED = 1 # Connection was closed before a terminating character
                   # but message might be fine
ERR_MSGINVALID = 2 # Couldn't parse the message
ERR_AUTHINVALID = 3 # Auth section of message was invalid
ERR_PAYLOADINVALID = 4 # Payload type was invalid
ERR_PROTINVALID = 5 # Protocol type was invalid
ERR_INTERNAL = 6 # An internal error occurred
ERR_METADATA = 7 # Metadata was formatted incorrectly
ERR_TOPICINVALID = 8 # Topic was formatted incorrectly
ERR_UNKNOWN = -1 # Unknown error

class HologramCloud(CustomCloud):

    _authentication_handlers = {
        'csrpsk' : CSRPSKAuthentication.CSRPSKAuthentication,
        'totp' : TOTPAuthentication.TOTPAuthentication,
        'sim-otp' : SIMOTPAuthentication.SIMOTPAuthentication,
    }

    _errorCodeDescription = {

        ERR_OK: 'Message sent successfully',
        ERR_CONNCLOSED: 'Connection was closed so we couldn\'t read the whole message',
        ERR_MSGINVALID: 'Failed to parse the message',
        ERR_AUTHINVALID: 'Auth section of the message was invalid',
        ERR_PAYLOADINVALID: 'Payload type was invalid',
        ERR_PROTINVALID: 'Protocol type was invalid',
        ERR_INTERNAL: 'Internal error in Hologram Cloud',
        ERR_METADATA: 'Metadata was formatted incorrectly',
        ERR_TOPICINVALID: 'Topic was formatted incorrectly',
        ERR_UNKNOWN: 'Unknown error'
    }

    def __init__(self, credentials, enable_inbound=False, network='',
                 authentication_type='totp'):
        super().__init__(credentials,
                         send_host=HOLOGRAM_HOST_SEND,
                         send_port=HOLOGRAM_PORT_SEND,
                         receive_host=HOLOGRAM_HOST_RECEIVE,
                         receive_port=HOLOGRAM_PORT_RECEIVE,
                         enable_inbound=enable_inbound,
                         network=network)

        self.setAuthenticationType(credentials, authentication_type=authentication_type)

        if self.authenticationType == 'totp':
            self.__populate_totp_credentials()

    # EFFECTS: Authentication Configuration
    def setAuthenticationType(self, credentials, authentication_type='csrpsk'):

        if authentication_type not in HologramCloud._authentication_handlers:
            raise HologramError('Invalid authentication type: %s' % authentication_type)

        self.authenticationType = authentication_type

        self.authentication = HologramCloud._authentication_handlers[self.authenticationType](credentials)

    # EFFECTS: Sends the message to the cloud.
    def sendMessage(self, message, topics=None, timeout=DEFAULT_SEND_MESSAGE_TIMEOUT):

        if not self.is_ready_to_send():
            self.addPayloadToBuffer(message)
            return ''

        # Set the appropriate credentials required for sim otp authentication.
        if self.authenticationType == 'sim-otp':
            self.__populate_sim_otp_credentials()

        modem_type = None
        modem_id = None
        if self.network is not None:
            modem_id = self.network.modem_id
            modem_type = str(self.network.modem)

        output = self.authentication.buildPayloadString(message,
                                                        topics=topics,
                                                        modem_type=modem_type,
                                                        modem_id=modem_id,
                                                        version=self.version)

        result = super().sendMessage(output, timeout)
        return self.__parse_result(result)

    def __parse_result(self, result):
        resultList = None
        if self.authenticationType == 'csrpsk':
            resultList = self.__parse_hologram_json_result(result)
        else:
            resultList = self.__parse_hologram_compact_result(result)

        return resultList[0]

    def __populate_totp_credentials(self):
        try:
            self.authentication.credentials['device_id'] = self.network.iccid
            self.authentication.credentials['private_key'] = self.network.imsi
        except Exception as e:
            self.logger.error('Unable to fetch device id or private key')

    def __populate_sim_otp_credentials(self):
        nonce = self.request_hex_nonce()
        command = self.authentication.generate_sim_otp_command(imsi=self.network.imsi,
                                                               iccid=self.network.iccid,
                                                               nonce=nonce)
        modem_response = self.network.get_sim_otp_response(command)
        self.authentication.generate_sim_otp_token(modem_response)

    def sendSMS(self, destination_number, message):

        self.__enforce_authentication_type_supported_for_sms()
        self.__enforce_valid_destination_number(destination_number)
        self.__enforce_max_sms_length(message)

        output = self.authentication.buildSMSPayloadString(destination_number,
                                                           message)

        self.logger.debug('Destination number: %s', destination_number)
        self.logger.debug('SMS: %s', message)

        result = super().sendMessage(output)

        resultList = self.__parse_hologram_compact_result(result)
        return resultList[0]

    # REQUIRES: Called only when sim otp authentication is required.
    # EFFECTS: Request for a hex nonce.
    def request_hex_nonce(self):

        self.open_send_socket()

        # build nonce request payload string
        nonce_request = self.authentication.buildNonceRequestPayloadString()

        self.logger.debug("Sending nonce request with body of length %d", len(nonce_request))
        self.logger.debug('Send: %s', nonce_request)

        nonce = super().sendMessage(message=nonce_request, timeout=10, close_socket=False)
        self.logger.debug('Nonce request sent.')

        resultbuf_hex = binascii.b2a_hex(nonce)

        if resultbuf_hex is None:
            raise HologramError('Internal nonce error')

        return resultbuf_hex

    def enableSMS(self):
        return self.network.enableSMS()

    def disableSMS(self):
        return self.network.disableSMS()

    def popReceivedSMS(self):
        return self.network.popReceivedSMS()

    # EFFECTS: Parses the hologram send response.
    def __parse_hologram_json_result(self, result):
        try:
            resultList = json.loads(result)
            if isinstance(resultList, bytes):
                resultList[0] = int(chr(resultList[0]))
            else:
                resultList[0] = int(resultList[0])
        except ValueError:
            self.logger.error('Server replied with invalid JSON [%s]', result)
            resultList = [ERR_UNKNOWN]
        return resultList

    def __parse_hologram_compact_result(self, result):

        # convert the returned response to formatted list.
        if result is None:
            return [ERR_UNKNOWN]

        resultList = []
        for x in result:
            resultList.append(int(x))

        if len(resultList) == 0:
            resultList = [ERR_UNKNOWN]

        return resultList

    def __enforce_max_sms_length(self, message):
        if len(message) > MAX_SMS_LENGTH:
            raise HologramError('SMS cannot be more than %d characters long' % MAX_SMS_LENGTH)

    def __enforce_valid_destination_number(self, destination_number):
        if not destination_number.startswith('+'):
            raise HologramError('SMS destination number must start with a \'+\' sign')

    def __enforce_authentication_type_supported_for_sms(self):
        if self.authenticationType != 'csrpsk':
            raise HologramError('%s does not support SDK SMS features' % self.authenticationType)

    # REQUIRES: A result code (int).
    # EFFECTS: Returns a translated string based on the given hologram result code.
    def getResultString(self, result_code):
        if result_code not in self._errorCodeDescription:
            return 'Unknown response code'
        return self._errorCodeDescription[result_code]

    def resultWasSuccess(self, result_code):
        return result_code in (ERR_OK, ERR_CONNCLOSED)
<|endoftext|>


<commit_before>import builtins
from   math import floor, log10
import re
import shutil
import sys

import numpy as np
import pandas as pd
import pln.ctr
from   pln.terminal import ansi

from   . import *

#-------------------------------------------------------------------------------

# FIXME: We need a proper cascading configuration system.

# FIXME: Make hierarchical.
# FIXME: Don't support None; it's silly.
DEFAULT_CFG = {
    "bottom.line"                   : "-",
    "bottom.separator.between"      : " ",
    "bottom.separator.end"          : "",
    "bottom.separator.index"        : "  ",
    "bottom.separator.start"        : "",
    "bottom.show"                   : False,

    "float.inf"                     : "\u221e",
    "float.max_precision"           : 8,
    "float.min_precision"           : 1,
    "float.nan"                     : "NaN",

    "formatters"                    : {},

    "header.elide.position"         : 0.7,
    "header.prefix"                 : "",
    "header.separator.between"      : " ",
    "header.separator.end"          : "",
    "header.separator.index"        : " ",
    "header.separator.start"        : "",
    "header.show"                   : True,
    "header.style.prefix"           : "",
    "header.style.suffix"           : "",
    "header.suffix"                 : "",

    "separator.between"             : " ",
    "separator.end"                 : "",
    "separator.index"               : " | ",
    "separator.start"               : "",

    "row_ellipsis.separator.end"    : "",
    "row_ellipsis.separator.start"  : "",
    "row_ellipsis.pad"              : " ",
    "row_ellipsis.position"         : 0.85,
    "row_ellipsis.format"           : "\u2026 skipping {skipped} rows \u2026",

    "str.min_size"                  :  1,
    "str.max_size"                  : 32,

    "top.line"                      : "-",
    "top.separator.between"         : " ",
    "top.separator.end"             : "",
    "top.separator.index"           : "  ",
    "top.separator.start"           : "",
    "top.show"                      : False,

    "underline.line"                : "=",
    "underline.separator.between"   : " ",
    "underline.separator.end"       : "",
    "underline.separator.index"     : " ",
    "underline.separator.start"     : "",
    "underline.show"                : True,
}

UNICODE_BOX_CFG = {
    "bottom.line"                   : "\u2500",
    "bottom.separator.between"      : "\u2500\u2534\u2500",
    "bottom.separator.end"          : "\u2500\u2518",
    "bottom.separator.index"        : "\u2500\u2534\u2500",
    "bottom.separator.start"        : "\u2514\u2500",
    "bottom.show"                   : True,
    "header.separator.between"      : " \u2502 ",
    "header.separator.end"          : " \u2502",
    "header.separator.index"        : " \u2551 ",
    "header.separator.start"        : "\u2502 ",
    "row_ellipsis.separator.end"    : "\u2561",
    "row_ellipsis.separator.start"  : "\u255e",
    "row_ellipsis.pad"              : "\u2550",
    "row_ellipsis.format"           : " (skipped {skipped} rows) ",
    "separator.between"             : " \u2502 ",
    "separator.end"                 : " \u2502",
    "separator.index"               : " \u2551 ",
    "separator.start"               : "\u2502 ",
    "top.line"                      : "\u2500",
    "top.separator.between"         : "\u2500\u252c\u2500",
    "top.separator.end"             : "\u2500\u2510",
    "top.separator.index"           : "\u2500\u252c\u2500",
    "top.separator.start"           : "\u250c\u2500",
    "top.show"                      : True,
    "underline.line"                : "\u2500",
    "underline.separator.between"   : "\u2500\u253c\u2500",
    "underline.separator.end"       : "\u2500\u2524",
    "underline.separator.index"     : " \u253c ",
    "underline.separator.start"     : "\u251c\u2500",
}

COLOR_CFG = {
    "float.inf"                     :
      ansi.style(color=ansi.YELLOW, light=False)("\u221e"),
    "float.nan"                     :
      ansi.style(color=ansi.RED, light=False)("NaN"),
    "header.style.prefix"           : ansi.sgr(underline=True, bold=True),
    "header.style.suffix"           : ansi.RESET,
    "row_ellipsis.format"           :
      ansi.style(color=ansi.BLACK, light=True)
      ("\u2026 skipping {skipped} rows \u2026"),
    "underline.show"                : False,
}

def _add_array_to_table(table, arr, fmt):
    dtype = arr.dtype
    if dtype == np.dtype("int8"):
        table.add_int8(arr, fmt)
    elif dtype == np.dtype("int16"):
        table.add_int16(arr, fmt)
    elif dtype == np.dtype("int32"):
        table.add_int32(arr, fmt)
    elif dtype == np.dtype("int64"):
        table.add_int64(arr, fmt)
    elif dtype == np.dtype("float32"):
        table.add_float32(arr, fmt)
    elif dtype == np.dtype("float64"):
        table.add_float64(arr, fmt)
    elif dtype == np.dtype("bool"):
        table.add_bool(arr, fmt)
    elif dtype == np.dtype("object"):
        table.add_str_object(arr, fmt)
    else:
        raise TypeError("unsupported dtype: {}".format(dtype))

def _get_num_digits(value):
    """
    Returns the number of digits required to represent a value.
    """
    return 1 if value == 0 else max(int(floor(log10(abs(value)) + 1)), 1)

def _choose_formatter_bool(values, cfg):
    # Not much to do.
    return Bool()

def _choose_formatter_float(values, cfg):
    # FIXME: This could be done with a single pass more efficiently.

    inf         = cfg["float.inf"]
    nan         = cfg["float.nan"]

    # Remove NaN and infinite values.
    is_nan      = np.isnan(values)
    has_nan     = is_nan.any()
    is_inf      = np.isinf(values)
    has_inf     = is_inf.any()
    has_neg_inf = (values[is_inf] < 0).any()
    vals        = values[~(is_nan | is_inf)]

    special_width = max(
        string_length(nan) if has_nan else 0,
        string_length(inf) if has_inf else 0)

    if len(vals) == 0:
        # No values left.
        return Float(max(1, special_width))

    # Determine the scale.
    min_val = vals.min()
    max_val = vals.max()
    sign    = "-" if has_neg_inf or min_val < 0 else " "
    size    = _get_num_digits(max(abs(min_val), abs(max_val)))

    # Try progressively higher precision until rounding won't leave any
    # residuals larger the maximum pecision.
    precision_min   = cfg["float.min_precision"]
    precision_min   = max(precision_min, special_width - size - 1)
    precision_max   = cfg["float.max_precision"]
    tolerance       = (10 ** -precision_max) / 2
    for precision in range(precision_min, precision_max + 1):
        if (abs(np.round(vals, precision) - vals) < tolerance).all():
            break

    return Number(size, precision, sign=sign, nan=nan, inf=inf)

def _choose_formatter_int(values, cfg):
    if len(values) == 0:
        return Number(1, None, sign=" ")

    min_val = values.min()
    max_val = values.max()
    size    = _get_num_digits(max(abs(min_val), abs(max_val)))
    sign    = " " if min_val >= 0 else "-"
    return Number(size, sign=sign)

def _choose_formatter_str(values, cfg):
    size = np.vectorize(len)(values).max()
    size = max(size, cfg["str.min_size"])
    size = min(size, cfg["str.max_size"])
    return String(size)

def _get_default_formatter(arr, cfg):
    dtype = arr.dtype
    if dtype.kind == "b":
        return _choose_formatter_bool(arr, cfg=cfg)
    elif dtype.kind == "f":
        return _choose_formatter_float(arr, cfg=cfg)
    elif dtype.kind == "i":
        return _choose_formatter_int(arr, cfg=cfg)
    elif dtype.kind == "O":
        return _choose_formatter_str(arr, cfg=cfg)
    else:
        raise TypeError("no default formatter for {}".format(dtype))

def _get_formatter(formatters, name, arr, cfg):
    fmt = None
    for regex, value in formatters.items():
        if isinstance(regex, str) and re.match(regex, name) is not None:
            fmt = value
    if fmt is None:
        fmt = formatters.get(arr.dtype, None)
    if fmt is None:
        fmt = _get_default_formatter(arr, cfg=cfg)
    assert fmt is not None
    return fmt

def _get_header_justification(fmt):
    """
    Returns true if the format is right-justified.
    """
    if isinstance(fmt, Bool):
        return fmt.pad_left
    elif isinstance(fmt, Number):
        return True
    elif isinstance(fmt, String):
        return fmt.pad_left
    else:
        raise TypeError("unrecognized formatter: {!r}".format(fmt))

def _table_for_dataframe(df, names, cfg={}):
    table = Table()

    formatters  = cfg["formatters"]
    begin_sep   = cfg["separator.start"]
    sep         = cfg["separator.between"]
    end_sep     = cfg["separator.end"]

    if begin_sep:
        table.add_string(begin_sep)

    fmts = []
    arrs = []
    for i, name in enumerate(names):
        arr = df[name].values
        arrs.append(arr)

        if arr.dtype.kind == "O":
            # Objects are stringified.  Do this now.
            arr = np.vectorize(str, [object])(arr)

        fmt = _get_formatter(formatters, name, arr, cfg)
        # FIXME: Add accessor to table.
        fmts.append(fmt)
        _add_array_to_table(table, arr, fmt)
        if i == len(names) - 1:
            if end_sep:
                table.add_string(end_sep)
        else:
            if sep:
                table.add_string(sep)

    return table, fmts, arrs

# FIXME: By screen (repeating header?)
# FIXME: Index, with underlining.
# FIXME: Special sep after index.

def _print_header(names, fmts, cfg):
    if cfg["header.show"]:
        pfx         = cfg["header.prefix"]
        start       = cfg["header.separator.start"]
        sep         = cfg["header.separator.between"]
        end         = cfg["header.separator.end"]
        style_pfx   = cfg["header.style.prefix"]
        style_sfx   = cfg["header.style.suffix"]
        sfx         = cfg["header.suffix"]
        position    = cfg["header.elide.position"]

        assert string_length(style_pfx) == 0
        assert string_length(style_sfx) == 0

        def format_name(name, fmt):
            name = pfx + name + sfx
            left = _get_header_justification(fmt)
            name = palide(name, fmt.width, position=position, left=left)
            return style_pfx + name + style_sfx

        header = sep.join( format_name(n, f) for n, f in zip(names, fmts) )
        builtins.print(start + header + end)

# FIXME: Support subconfigs.
def _print_line(names, fmts, prefix, cfg):
    if cfg[prefix + ".show"]:
        begin_sep   = cfg[prefix + ".separator.start"]
        sep         = cfg[prefix + ".separator.between"]
        end_sep     = cfg[prefix + ".separator.end"]
        line        = cfg[prefix + ".line"]

        # FIXME: Relax this.
        if len(line) != 1:
            raise ValueError("{}.line must be one character".format(prefix))

        builtins.print(
              begin_sep
            + sep.join( line * f.width for f in fmts )
            + end_sep
        )

# FIXME: Do what when it's too wide???

def _print_dataframe(df, cfg):
    # Slog through configuration.
    formatters  = cfg["formatters"]
    max_rows    = cfg.get("max_rows", "terminal")  # FIXME

    extra_rows  = sum([
        1,
        cfg["top.show"],
        cfg["header.show"],
        cfg["underline.show"],
        cfg["bottom.show"],
    ])

    if max_rows == "terminal":
        # FIXME
        max_rows = shutil.get_terminal_size().lines - 1
    names = cfg.get("names", pln.ctr.ALL)

    names = pln.ctr.select_ordered(df.columns, names)
    # FIXME: Get formats from table columns.
    table, fmts, arrs = _table_for_dataframe(df, names, cfg)

    _print_line(names, fmts, "top", cfg)
    _print_header(names, fmts, cfg)
    _print_line(names, fmts, "underline", cfg)

    num_rows = len(table)
    if num_rows <= max_rows - extra_rows:
        for i in range(len(table)):
            builtins.print(table(i))
    else:
        num_rows_top        = int(cfg["row_ellipsis.position"] * max_rows)
        num_rows_bottom     = max_rows - extra_rows - num_rows_top
        num_rows_skipped    = num_rows - num_rows_top - num_rows_bottom

        # Print rows from the top.
        for i in range(num_rows_top):
            builtins.print(table(i))

        # Print the row ellipsis.
        ell = cfg["row_ellipsis.format"].format(
            bottom  =num_rows_bottom,
            rows    =num_rows,
            skipped =num_rows_skipped,
            top     =num_rows_top,
        )
        ell_start   = cfg["row_ellipsis.separator.start"]
        ell_end     = cfg["row_ellipsis.separator.end"]
        ell_pad     = cfg["row_ellipsis.pad"]
        ell_width   = (
            table.width - string_length(ell_start) - string_length(ell_end))
        ell         = center(ell, ell_width, ell_pad)
        builtins.print(ell_start + ell + ell_end)

        # Print rows from the bottom.
        for i in range(num_rows - num_rows_bottom, num_rows):
            builtins.print(table(i))

    _print_line(names, fmts, "bottom", cfg)

def print(df, *, cfg=DEFAULT_CFG):
    _print_dataframe(df, cfg)

#-------------------------------------------------------------------------------

def main():
    from   argparse import ArgumentParser
    from   pln.io import load_pickle

    parser = ArgumentParser()
    parser.add_argument(
        "filename", metavar="FILENAME",
        help="read from FILENAME")
    args = parser.parse_args()

    # FIXME
    cfg = DEFAULT_CFG.copy()
    # cfg.update(UNICODE_BOX_CFG)
    cfg.update(COLOR_CFG)

    # FIXME: Support "-".
    df = load_pickle(args.filename)
    print(df, cfg=cfg)

if __name__ == "__main__":
    main()

<commit_msg>Use pln.cfg for configuration.<commit_after>import builtins
from   math import floor, log10
import re
import shutil
import sys

import numpy as np
import pandas as pd
from   pln.cfg import Group, Var, Cfg
import pln.ctr
from   pln.terminal import ansi

from   . import *

#-------------------------------------------------------------------------------
# Configuration

SEPARATOR_CONFIGURATION = Group(
    between                 = " ",
    end                     = "",
    start                   = "",
)

CONFIGURATION = Group(
    bottom = Group(
        line                        = "-",
        separator                   = SEPARATOR_CONFIGURATION,
        show                        = False,
    ),
    columns = Group(
        names                       = Var(default=pln.ctr.ALL),
    ),
    data = Group(
        max_rows                    = Var(default="terminal"),
    ),
    float = Group(
        inf                         = "\u221e",
        max_precision               = 8,
        min_precision               = 1,
        nan                         = "NaN",
    ),
    formatters                      = {},
    header = Group(
        elide = Group(
            position                = 0.7,
        ),
        prefix                      = "",
        separator                   = SEPARATOR_CONFIGURATION,
        show                        = True,
        style = Group(
            prefix                  = "",
            suffix                  = "",
        ),
        suffix                      = "",
    ),
    row = Group(
        separator                   = SEPARATOR_CONFIGURATION,
    ),
    row_ellipsis = Group(
        separator                   = SEPARATOR_CONFIGURATION,
        pad                         = " ",
        position                    = 0.85,
        format                      = "\u2026 skipping {skipped} rows \u2026",
    ),
    str = Group(
        min_size                    =  1,
        max_size                    = 32,
    ),
    top = Group(
        line                        = "-",
        separator                   = SEPARATOR_CONFIGURATION,
        show                        = False,
    ),
    underline = Group(
        line                        = "=",
        separator                   = SEPARATOR_CONFIGURATION,
        show                        = True,
    ),
)

DEFAULT_CFG = Cfg(CONFIGURATION)

UNICODE_BOX_CFG = Cfg(CONFIGURATION)(
    bottom = dict(
        line                        = "\u2500",
        separator = dict(
            between                 = "\u2500\u2534\u2500",
            end                     = "\u2500\u2518",
            start                   = "\u2514\u2500",
        ),
        show                        = True,
    ),
    header = dict(
        separator = dict(
            between             = " \u2502 ",
            end                 = " \u2502",
            start               = "\u2502 ",
        ),
    ),
    row_ellipsis = dict(
        separator = dict(
            end                     = "\u2561",
            start                   = "\u255e",
        ),
        pad                         = "\u2550",
        format                      = " skipped {skipped} rows ",
    ),
    row = dict(
        separator = dict(
            between                 = " \u2502 ",
            end                     = " \u2502",
            start                   = "\u2502 ",
        ),
    ),
    top = dict(
        line                        = "\u2500",
        separator = dict(
            between                 = "\u2500\u252c\u2500",
            end                     = "\u2500\u2510",
            start                   = "\u250c\u2500",
        ),
        show                        = True,
    ),
    underline = dict(
        line                        = "\u2500",
        separator = dict(
            between                 = "\u2500\u253c\u2500",
            end                     = "\u2500\u2524",
            start                   = "\u251c\u2500",
        ),
    ),
)

def _add_color(cfg):
    """
    ANSI-colorizes a configuration.
    """
    # Color Inf and Nan, for visibility.
    cfg.float.inf = ansi.style(color=ansi.YELLOW, light=False)(cfg.float.inf)
    cfg.float.nan = ansi.style(color=ansi.RED   , light=False)(cfg.float.nan)

    # Use underlining instead of drawing an underline.
    cfg.header.style.prefix = ansi.sgr(underline=True, bold=True)
    cfg.header.style.suffix = ansi.RESET
    cfg.underline.show = False

    # Color the ellipsis row.
    cfg.row_ellipsis.format = ansi.style(color=ansi.BLACK, light=True)(
        cfg.row_ellipsis.format)

#-------------------------------------------------------------------------------

def _add_array_to_table(table, arr, fmt):
    dtype = arr.dtype
    if dtype == np.dtype("int8"):
        table.add_int8(arr, fmt)
    elif dtype == np.dtype("int16"):
        table.add_int16(arr, fmt)
    elif dtype == np.dtype("int32"):
        table.add_int32(arr, fmt)
    elif dtype == np.dtype("int64"):
        table.add_int64(arr, fmt)
    elif dtype == np.dtype("float32"):
        table.add_float32(arr, fmt)
    elif dtype == np.dtype("float64"):
        table.add_float64(arr, fmt)
    elif dtype == np.dtype("bool"):
        table.add_bool(arr, fmt)
    elif dtype == np.dtype("object"):
        table.add_str_object(arr, fmt)
    else:
        raise TypeError("unsupported dtype: {}".format(dtype))

def _get_num_digits(value):
    """
    Returns the number of digits required to represent a value.
    """
    return 1 if value == 0 else max(int(floor(log10(abs(value)) + 1)), 1)

def _choose_formatter_bool(values, cfg):
    # Not much to do.
    return Bool()

def _choose_formatter_float(values, cfg):
    # FIXME: This could be done with a single pass more efficiently.

    inf         = cfg.float.inf
    nan         = cfg.float.nan

    # Remove NaN and infinite values.
    is_nan      = np.isnan(values)
    has_nan     = is_nan.any()
    is_inf      = np.isinf(values)
    has_inf     = is_inf.any()
    has_neg_inf = (values[is_inf] < 0).any()
    vals        = values[~(is_nan | is_inf)]

    special_width = max(
        string_length(nan) if has_nan else 0,
        string_length(inf) if has_inf else 0)

    if len(vals) == 0:
        # No values left.
        return Float(max(1, special_width))

    # Determine the scale.
    min_val = vals.min()
    max_val = vals.max()
    sign    = "-" if has_neg_inf or min_val < 0 else " "
    size    = _get_num_digits(max(abs(min_val), abs(max_val)))

    # Try progressively higher precision until rounding won't leave any
    # residuals larger the maximum pecision.
    precision_min   = cfg.float.min_precision
    precision_min   = max(precision_min, special_width - size - 1)
    precision_max   = cfg.float.max_precision
    tolerance       = (10 ** -precision_max) / 2
    for precision in range(precision_min, precision_max + 1):
        if (abs(np.round(vals, precision) - vals) < tolerance).all():
            break

    return Number(size, precision, sign=sign, nan=nan, inf=inf)

def _choose_formatter_int(values, cfg):
    if len(values) == 0:
        return Number(1, None, sign=" ")

    min_val = values.min()
    max_val = values.max()
    size    = _get_num_digits(max(abs(min_val), abs(max_val)))
    sign    = " " if min_val >= 0 else "-"
    return Number(size, sign=sign)

def _choose_formatter_str(values, cfg):
    size = np.vectorize(len)(values).max()
    size = max(size, cfg.str.min_size)
    size = min(size, cfg.str.max_size)
    return String(size)

def _get_default_formatter(arr, cfg):
    dtype = arr.dtype
    if dtype.kind == "b":
        return _choose_formatter_bool(arr, cfg=cfg)
    elif dtype.kind == "f":
        return _choose_formatter_float(arr, cfg=cfg)
    elif dtype.kind == "i":
        return _choose_formatter_int(arr, cfg=cfg)
    elif dtype.kind == "O":
        return _choose_formatter_str(arr, cfg=cfg)
    else:
        raise TypeError("no default formatter for {}".format(dtype))

def _get_formatter(formatters, name, arr, cfg):
    fmt = None
    for regex, value in formatters.items():
        if isinstance(regex, str) and re.match(regex, name) is not None:
            fmt = value
    if fmt is None:
        fmt = formatters.get(arr.dtype, None)
    if fmt is None:
        fmt = _get_default_formatter(arr, cfg=cfg)
    assert fmt is not None
    return fmt

def _get_header_justification(fmt):
    """
    Returns true if the format is right-justified.
    """
    if isinstance(fmt, Bool):
        return fmt.pad_left
    elif isinstance(fmt, Number):
        return True
    elif isinstance(fmt, String):
        return fmt.pad_left
    else:
        raise TypeError("unrecognized formatter: {!r}".format(fmt))

def _table_for_dataframe(df, names, cfg={}):
    table = Table()

    formatters  = cfg.formatters
    begin_sep   = cfg.row.separator.start
    sep         = cfg.row.separator.between
    end_sep     = cfg.row.separator.end

    if begin_sep:
        table.add_string(begin_sep)

    fmts = []
    arrs = []
    for i, name in enumerate(names):
        arr = df[name].values
        arrs.append(arr)

        if arr.dtype.kind == "O":
            # Objects are stringified.  Do this now.
            arr = np.vectorize(str, [object])(arr)

        fmt = _get_formatter(formatters, name, arr, cfg)
        # FIXME: Add accessor to table.
        fmts.append(fmt)
        _add_array_to_table(table, arr, fmt)
        if i == len(names) - 1:
            if end_sep:
                table.add_string(end_sep)
        else:
            if sep:
                table.add_string(sep)

    return table, fmts, arrs

# FIXME: By screen (repeating header?)
# FIXME: Index, with underlining.
# FIXME: Special sep after index.

def _print_header(names, fmts, cfg):
    if cfg.header.show:
        pfx         = cfg.header.prefix
        start       = cfg.header.separator.start
        sep         = cfg.header.separator.between
        end         = cfg.header.separator.end
        style_pfx   = cfg.header.style.prefix
        style_sfx   = cfg.header.style.suffix
        sfx         = cfg.header.suffix
        position    = cfg.header.elide.position

        assert string_length(style_pfx) == 0
        assert string_length(style_sfx) == 0

        def format_name(name, fmt):
            name = pfx + name + sfx
            left = _get_header_justification(fmt)
            name = palide(name, fmt.width, position=position, left=left)
            return style_pfx + name + style_sfx

        header = sep.join( format_name(n, f) for n, f in zip(names, fmts) )
        builtins.print(start + header + end)

# FIXME: Support subconfigs.
def _print_line(names, fmts, cfg):
    if cfg.show:
        begin_sep   = cfg.separator.start
        sep         = cfg.separator.between
        end_sep     = cfg.separator.end
        line        = cfg.line

        # FIXME: Relax this.
        if len(line) != 1:
            raise ValueError("line must be one character")

        builtins.print(
              begin_sep
            + sep.join( line * f.width for f in fmts )
            + end_sep
        )

# FIXME: Do what when it's too wide???

def _print_dataframe(df, cfg):
    # Slog through configuration.
    formatters  = cfg.formatters

    extra_rows  = sum([
        1,
        cfg.top.show,
        cfg.header.show,
        cfg.underline.show,
        cfg.bottom.show,
    ])

    max_rows = cfg.data.max_rows
    if max_rows == "terminal":
        # FIXME
        max_rows = shutil.get_terminal_size().lines - 1

    names = cfg.columns.names
    names = pln.ctr.select_ordered(df.columns, names)
    # FIXME: Get formats from table columns.
    table, fmts, arrs = _table_for_dataframe(df, names, cfg)

    _print_line(names, fmts, cfg.top)
    _print_header(names, fmts, cfg)
    _print_line(names, fmts, cfg.underline)

    num_rows = len(table)
    if num_rows <= max_rows - extra_rows:
        for i in range(len(table)):
            builtins.print(table(i))
    else:
        num_rows_top        = int(cfg.row_ellipsis.position * max_rows)
        num_rows_bottom     = max_rows - extra_rows - num_rows_top
        num_rows_skipped    = num_rows - num_rows_top - num_rows_bottom

        # Print rows from the top.
        for i in range(num_rows_top):
            builtins.print(table(i))

        # Print the row ellipsis.
        ell = cfg.row_ellipsis.format.format(
            bottom  =num_rows_bottom,
            rows    =num_rows,
            skipped =num_rows_skipped,
            top     =num_rows_top,
        )
        ell_start   = cfg.row_ellipsis.separator.start
        ell_end     = cfg.row_ellipsis.separator.end
        ell_pad     = cfg.row_ellipsis.pad
        ell_width   = (
            table.width - string_length(ell_start) - string_length(ell_end))
        ell         = center(ell, ell_width, ell_pad)
        builtins.print(ell_start + ell + ell_end)

        # Print rows from the bottom.
        for i in range(num_rows - num_rows_bottom, num_rows):
            builtins.print(table(i))

    _print_line(names, fmts, cfg.bottom)

def print(df, *, cfg=DEFAULT_CFG):
    _print_dataframe(df, cfg)

#-------------------------------------------------------------------------------

def main():
    from   argparse import ArgumentParser
    from   pln.io import load_pickle

    parser = ArgumentParser()
    parser.add_argument(
        "filename", metavar="FILENAME",
        help="read from FILENAME")
    args = parser.parse_args()

    # FIXME
    cfg = DEFAULT_CFG
    # cfg = UNICODE_BOX_CFG
    _add_color(cfg)
    # cfg.update(COLOR_CFG)
    builtins.print(cfg)

    # FIXME: Support "-".
    df = load_pickle(args.filename)
    print(df, cfg=cfg)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Based on previous work by
# Charles Menguy (see: http://stackoverflow.com/questions/10217067/implementing-a-full-python-unix-style-daemon-process)
# and Sander Marechal (see: http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/)

# Adapted by M.Hendrix [2015]

# daemon15.py measures the size of selected logfiles.
# These are all counters, therefore no averaging is needed.

import os, sys, time, math, commands, syslog
from libdaemon import Daemon

DEBUG = False
IS_SYSTEMD = os.path.isfile('/bin/journalctl')

class MyDaemon(Daemon):
	def run(self):
		sampleptr = 0
		samples = 1
		datapoints = 3
		if IS_SYSTEMD:
			sampleTime = 180
		else:
			sampleTime = 60
		cycleTime = samples * sampleTime
		# sync to whole minute
		waitTime = (cycleTime + sampleTime) - (time.time() % cycleTime)
		if DEBUG:
			print "NOT waiting {0} s.".format(waitTime)
		else:
			time.sleep(waitTime)
		while True:
			startTime = time.time()

			result = do_work().split(',')
			data = map(int, result)

			sampleptr = sampleptr + 1
			if (sampleptr == samples):
				do_report(data)
				sampleptr = 0

			waitTime = sampleTime - (time.time() - startTime) - (startTime%sampleTime)
			if (waitTime > 0):
				if DEBUG:print "Waiting {0} s".format(waitTime)
				time.sleep(waitTime)

def do_work():
	# 3 datapoints gathered here
	kernlog=messlog=syslog=0

	if IS_SYSTEMD:
		#
		kernlog = commands.getoutput("sudo journalctl --since=12:00:00 --no-pager |grep -i 'fail' |wc -l").split()[0]
		messlog = commands.getoutput("sudo journalctl --since=12:00:00 --no-pager |grep -i 'warn\|error' |wc -l").split()[0]
		syslog  = commands.getoutput("sudo journalctl --since=12:00:00 --no-pager |wc -l").split()[0]
	else:
		kernlog = commands.getoutput("wc -l /var/log/kern.log").split()[0]
		messlog = commands.getoutput("wc -l /var/log/messages").split()[0]
		syslog  = commands.getoutput("wc -l /var/log/syslog").split()[0]

	return '{0}, {1}, {2}'.format(kernlog, messlog, syslog)

def do_report(result):
	# Get the time and date in human-readable form and UN*X-epoch...
	outDate = commands.getoutput("date '+%F %H:%M:%S, %s'")

	result = ', '.join(map(str, result))
	flock = '/tmp/raspdiagd/15.lock'
	lock(flock)
	f = file('/tmp/raspdiagd/15-cnt-loglines.csv', 'a')
	f.write('{0}, {1}\n'.format(outDate, result) )
	f.close()
	unlock(flock)
	return

def lock(fname):
	open(fname, 'a').close()

def unlock(fname):
	if os.path.isfile(fname):
		os.remove(fname)

if __name__ == "__main__":
	daemon = MyDaemon('/tmp/raspdiagd/15.pid')
	if len(sys.argv) == 2:
		if 'start' == sys.argv[1]:
			daemon.start()
		elif 'stop' == sys.argv[1]:
			daemon.stop()
		elif 'restart' == sys.argv[1]:
			daemon.restart()
		elif 'foreground' == sys.argv[1]:
			# assist with debugging.
			print "Debug-mode started. Use <Ctrl>+C to stop."
			DEBUG = True
			if DEBUG:
				logtext = "Daemon logging is ON"
				syslog.syslog(syslog.LOG_DEBUG, logtext)
			daemon.run()
		else:
			print "Unknown command"
			sys.exit(2)
		sys.exit(0)
	else:
		print "usage: %s start|stop|restart|foreground" % sys.argv[0]
		sys.exit(2)
<commit_msg>count logging since midnight<commit_after>#!/usr/bin/env python

# Based on previous work by
# Charles Menguy (see: http://stackoverflow.com/questions/10217067/implementing-a-full-python-unix-style-daemon-process)
# and Sander Marechal (see: http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/)

# Adapted by M.Hendrix [2015]

# daemon15.py measures the size of selected logfiles.
# These are all counters, therefore no averaging is needed.

import os, sys, time, math, commands, syslog
from libdaemon import Daemon

DEBUG = False
IS_SYSTEMD = os.path.isfile('/bin/journalctl')

class MyDaemon(Daemon):
	def run(self):
		sampleptr = 0
		samples = 1
		datapoints = 3
		if IS_SYSTEMD:
			sampleTime = 180
		else:
			sampleTime = 60
		cycleTime = samples * sampleTime
		# sync to whole minute
		waitTime = (cycleTime + sampleTime) - (time.time() % cycleTime)
		if DEBUG:
			print "NOT waiting {0} s.".format(waitTime)
		else:
			time.sleep(waitTime)
		while True:
			startTime = time.time()

			result = do_work().split(',')
			data = map(int, result)

			sampleptr = sampleptr + 1
			if (sampleptr == samples):
				do_report(data)
				sampleptr = 0

			waitTime = sampleTime - (time.time() - startTime) - (startTime%sampleTime)
			if (waitTime > 0):
				if DEBUG:print "Waiting {0} s".format(waitTime)
				time.sleep(waitTime)

def do_work():
	# 3 datapoints gathered here
	kernlog=messlog=syslog=0

	if IS_SYSTEMD:
		#
		kernlog = commands.getoutput("journalctl --since=00:00:00 --no-pager |grep -i 'fail' |wc -l").split()[0]
		messlog = commands.getoutput("journalctl --since=00:00:00 --no-pager |grep -i 'warn\|error' |wc -l").split()[0]
		syslog  = commands.getoutput("journalctl --since=00:00:00 --no-pager |wc -l").split()[0]
	else:
		kernlog = commands.getoutput("wc -l /var/log/kern.log").split()[0]
		messlog = commands.getoutput("wc -l /var/log/messages").split()[0]
		syslog  = commands.getoutput("wc -l /var/log/syslog").split()[0]

	return '{0}, {1}, {2}'.format(kernlog, messlog, syslog)

def do_report(result):
	# Get the time and date in human-readable form and UN*X-epoch...
	outDate = commands.getoutput("date '+%F %H:%M:%S, %s'")

	result = ', '.join(map(str, result))
	flock = '/tmp/raspdiagd/15.lock'
	lock(flock)
	f = file('/tmp/raspdiagd/15-cnt-loglines.csv', 'a')
	f.write('{0}, {1}\n'.format(outDate, result) )
	f.close()
	unlock(flock)
	return

def lock(fname):
	open(fname, 'a').close()

def unlock(fname):
	if os.path.isfile(fname):
		os.remove(fname)

if __name__ == "__main__":
	daemon = MyDaemon('/tmp/raspdiagd/15.pid')
	if len(sys.argv) == 2:
		if 'start' == sys.argv[1]:
			daemon.start()
		elif 'stop' == sys.argv[1]:
			daemon.stop()
		elif 'restart' == sys.argv[1]:
			daemon.restart()
		elif 'foreground' == sys.argv[1]:
			# assist with debugging.
			print "Debug-mode started. Use <Ctrl>+C to stop."
			DEBUG = True
			if DEBUG:
				logtext = "Daemon logging is ON"
				syslog.syslog(syslog.LOG_DEBUG, logtext)
			daemon.run()
		else:
			print "Unknown command"
			sys.exit(2)
		sys.exit(0)
	else:
		print "usage: %s start|stop|restart|foreground" % sys.argv[0]
		sys.exit(2)
<|endoftext|>


<commit_before># ©  2008-2020 Deltatech
#              Dorin Hongu <dhongu(@)gmail(.)com
# See README.rst file on addons root folder for license details

from odoo import api, fields, models

class ResPartner(models.Model):
    _inherit = "res.partner"

    mean_transp = fields.Char(string="Mean transport")

class AccountInvoice(models.Model):
    _inherit = "account.move"

    delegate_id = fields.Many2one(
        "res.partner",
        string="Delegate",
        readonly=True,
        states={"draft": [("readonly", False)]},
        domain=[("is_company", "=", False)],
    )

    mean_transp = fields.Char(
        string="Mean transport",
        readonly=True,
        states={"draft": [("readonly", False)]},
    )

    @api.model
    def default_get(self, fields_list):
        defaults = super(AccountInvoice, self).default_get(fields_list)
        if "delegate_id" not in defaults:
            if "default_delegate_id" in self.env.context:
                defaults["default_delegate_id"] = defaults["default_delegate_id"]
        return defaults

    @api.onchange("delegate_id")
    def on_change_delegate_id(self):
        if self.delegate_id:
            self.mean_transp = self.delegate_id.mean_transp

    # anularea facturilor cu zero

    def action_invoice_cancel(self):
        for invoice in self:
            if invoice.amount_total == 0.0 and invoice.state == "paid":
                invoice.state = "open"
                # invoice.write({'state':'open'})

        return super(AccountInvoice, self).action_invoice_cancel()

#
# class account_invoice_line(models.Model):
#     _inherit = "account.move.line"
#
#     sequence = fields.Integer(default=1)
#
#     price_unit_without_taxes = fields.Float(string='Unit Price without taxes', store=True, readonly=True,
#                                             compute='_compute_price')
#
#     price_taxes = fields.Float(string='Taxes', digits='Account', store=True, readonly=True,
#                                compute='_compute_price')
#
#     price_normal_taxes = fields.Float(tring='Normal Taxes', digits='Account', store=True,
#                                       readonly=True, compute='_compute_price')
#
#     """
#     # campurile standard
#     price_unit = fields.Float(string='Unit Price', required=True, digits=dp.get_precision('Product Price'))
#     price_subtotal = fields.Monetary(string='Amount',
#         store=True, readonly=True, compute='_compute_price', help="Total amount without taxes")
#     price_total = fields.Monetary(string='Amount',
#         store=True, readonly=True, compute='_compute_price', help="Total amount with taxes")
#     price_subtotal_signed = fields.Monetary(string='Amount Signed', currency_field='company_currency_id',
#         store=True, readonly=True, compute='_compute_price',
#         help="Total amount in the currency of the company, negative for credit note.")
#     """
#
#     @api.depends('price_unit', 'discount', 'invoice_line_tax_ids', 'quantity',
#                  'product_id', 'invoice_id.partner_id', 'invoice_id.currency_id', 'invoice_id.company_id',
#                  'invoice_id.invoice_date', 'invoice_id.date')
#     def _compute_price(self):
#
#         super(account_invoice_line, self)._compute_price()
#         # if self.price_subtotal:
#         #      self.price_unit_without_taxes = self.price_subtotal / self.quantity
#         #      self.price_taxes = (self.price_total - self.price_subtotal ) / self.quantity
#
#         # Versiunea mai complexa
#         price = self.price_unit * (1 - (self.discount or 0.0) / 100.0)
#
#         currency = self.invoice_id and self.invoice_id.currency_id or None
#
#         taxes = False
#         if self.invoice_line_tax_ids:
#             taxes = self.invoice_line_tax_ids.compute_all(price, currency, self.quantity, product=self.product_id,
#                                                           partner=self.invoice_id.partner_id)
#
#         # de ce se seteaza cele doua valori ? nu au fost setate in
#         if taxes:
#             self.price_subtotal = taxes['total_excluded'] if taxes else self.quantity * price
#             self.price_taxes = taxes['total_included'] - self.price_subtotal
#
#         taxes_unit = self.invoice_line_tax_ids.compute_all(price, currency=currency,
#                                                            quantity=1, product=self.product_id,
#                                                            partner=self.invoice_id.partner_id)
#
#         self.price_unit_without_taxes = taxes_unit['total_excluded']
#         # Compute normal taxes in case of Customer Invoices to have the value
#         # in Inverse Taxation
#         if self.invoice_id.type in ['out_invoice', 'out_refund']:
#             taxes_ids = self.product_id.taxes_id.filtered(lambda r: r.company_id == self.invoice_id.company_id)
#             normal_taxes = taxes_ids.compute_all(price, currency=currency,
#                                                  quantity=self.quantity, product=self.product_id,
#                                                  partner=self.invoice_id.partner_id)
#             self.price_normal_taxes = normal_taxes['total_included'] - normal_taxes['total_excluded']
#         # aplicare rotunjiri . asta nu trebuie facuta in functie de config
#         if self.invoice_id:
#             self.price_subtotal = self.invoice_id.currency_id.round(self.price_subtotal)
#             self.price_taxes = self.invoice_id.currency_id.round(self.price_taxes)
#             self.price_unit_without_taxes = self.invoice_id.currency_id.round(self.price_unit_without_taxes)
#             self.price_normal_taxes = self.invoice_id.currency_id.round(self.price_normal_taxes)
<commit_msg>adaugare metoda set_origin_with_picking<commit_after># ©  2008-2020 Deltatech
#              Dorin Hongu <dhongu(@)gmail(.)com
# See README.rst file on addons root folder for license details

from odoo import api, fields, models

class ResPartner(models.Model):
    _inherit = "res.partner"

    mean_transp = fields.Char(string="Mean transport")

class AccountInvoice(models.Model):
    _inherit = "account.move"

    delegate_id = fields.Many2one(
        "res.partner",
        string="Delegate",
        readonly=True,
        states={"draft": [("readonly", False)]},
        domain=[("is_company", "=", False)],
    )

    mean_transp = fields.Char(
        string="Mean transport",
        readonly=True,
        states={"draft": [("readonly", False)]},
    )

    @api.model
    def default_get(self, fields_list):
        defaults = super(AccountInvoice, self).default_get(fields_list)
        if "delegate_id" not in defaults:
            if "default_delegate_id" in self.env.context:
                defaults["default_delegate_id"] = defaults["default_delegate_id"]
        return defaults

    @api.onchange("delegate_id")
    def on_change_delegate_id(self):
        if self.delegate_id:
            self.mean_transp = self.delegate_id.mean_transp

    # anularea facturilor cu zero

    def action_invoice_cancel(self):
        for invoice in self:
            if invoice.amount_total == 0.0 and invoice.state == "paid":
                invoice.state = "open"
                # invoice.write({'state':'open'})

        return super(AccountInvoice, self).action_invoice_cancel()

    def set_origin_with_picking(self):
        for invoice in self:
            pickings = self.env["stock.picking"]
            for line in invoice.invoice_line_ids:
                for sale_line in line.sale_line_ids:
                    for move in sale_line.move_ids:
                        if move.picking_id.state == "done":
                            pickings |= move.picking_id
                if line.purchase_line_id:
                    for move in line.purchase_line_id.move_ids:
                        if move.picking_id.state == "done":
                            pickings |= move.picking_id
            origin = ""
            for picking in pickings:
                origin += " " + picking.name

            if origin:
                invoice.write({"origin": origin})

#
# class account_invoice_line(models.Model):
#     _inherit = "account.move.line"
#
#     sequence = fields.Integer(default=1)
#
#     price_unit_without_taxes = fields.Float(string='Unit Price without taxes', store=True, readonly=True,
#                                             compute='_compute_price')
#
#     price_taxes = fields.Float(string='Taxes', digits='Account', store=True, readonly=True,
#                                compute='_compute_price')
#
#     price_normal_taxes = fields.Float(tring='Normal Taxes', digits='Account', store=True,
#                                       readonly=True, compute='_compute_price')
#
#     """
#     # campurile standard
#     price_unit = fields.Float(string='Unit Price', required=True, digits=dp.get_precision('Product Price'))
#     price_subtotal = fields.Monetary(string='Amount',
#         store=True, readonly=True, compute='_compute_price', help="Total amount without taxes")
#     price_total = fields.Monetary(string='Amount',
#         store=True, readonly=True, compute='_compute_price', help="Total amount with taxes")
#     price_subtotal_signed = fields.Monetary(string='Amount Signed', currency_field='company_currency_id',
#         store=True, readonly=True, compute='_compute_price',
#         help="Total amount in the currency of the company, negative for credit note.")
#     """
#
#     @api.depends('price_unit', 'discount', 'invoice_line_tax_ids', 'quantity',
#                  'product_id', 'invoice_id.partner_id', 'invoice_id.currency_id', 'invoice_id.company_id',
#                  'invoice_id.invoice_date', 'invoice_id.date')
#     def _compute_price(self):
#
#         super(account_invoice_line, self)._compute_price()
#         # if self.price_subtotal:
#         #      self.price_unit_without_taxes = self.price_subtotal / self.quantity
#         #      self.price_taxes = (self.price_total - self.price_subtotal ) / self.quantity
#
#         # Versiunea mai complexa
#         price = self.price_unit * (1 - (self.discount or 0.0) / 100.0)
#
#         currency = self.invoice_id and self.invoice_id.currency_id or None
#
#         taxes = False
#         if self.invoice_line_tax_ids:
#             taxes = self.invoice_line_tax_ids.compute_all(price, currency, self.quantity, product=self.product_id,
#                                                           partner=self.invoice_id.partner_id)
#
#         # de ce se seteaza cele doua valori ? nu au fost setate in
#         if taxes:
#             self.price_subtotal = taxes['total_excluded'] if taxes else self.quantity * price
#             self.price_taxes = taxes['total_included'] - self.price_subtotal
#
#         taxes_unit = self.invoice_line_tax_ids.compute_all(price, currency=currency,
#                                                            quantity=1, product=self.product_id,
#                                                            partner=self.invoice_id.partner_id)
#
#         self.price_unit_without_taxes = taxes_unit['total_excluded']
#         # Compute normal taxes in case of Customer Invoices to have the value
#         # in Inverse Taxation
#         if self.invoice_id.type in ['out_invoice', 'out_refund']:
#             taxes_ids = self.product_id.taxes_id.filtered(lambda r: r.company_id == self.invoice_id.company_id)
#             normal_taxes = taxes_ids.compute_all(price, currency=currency,
#                                                  quantity=self.quantity, product=self.product_id,
#                                                  partner=self.invoice_id.partner_id)
#             self.price_normal_taxes = normal_taxes['total_included'] - normal_taxes['total_excluded']
#         # aplicare rotunjiri . asta nu trebuie facuta in functie de config
#         if self.invoice_id:
#             self.price_subtotal = self.invoice_id.currency_id.round(self.price_subtotal)
#             self.price_taxes = self.invoice_id.currency_id.round(self.price_taxes)
#             self.price_unit_without_taxes = self.invoice_id.currency_id.round(self.price_unit_without_taxes)
#             self.price_normal_taxes = self.invoice_id.currency_id.round(self.price_normal_taxes)
<|endoftext|>


<commit_before>from logging import getLogger

from ..macros.iterable import find_item

class AuthorizationGuard():

    def __init__(self, configuration, safe):
        self.configuration = configuration
        self.safe = safe

    def check(self, request, permission_id, automation_definition=None):
        if not automation_definition:
            automation_definition = self.configuration
        group_definitions = automation_definition.group_definitions
        if not group_definitions:
            return True
        token = get_token(request)
        if not token:
            return False
        try:
            payload = self.safe.get(token)
        except KeyError:
            return False
        value_by_name = dict(payload, ip_address=request.remote_addr)
        try:
            group_definition = find_group_definition(
                group_definitions, value_by_name)
        except StopIteration:
            return False
        for permission_definition in group_definition.permissions:
            if permission_definition.id != permission_id:
                continue
            action = permission_definition.action
            if action == 'match':
                return define_is_match(payload)
            else:
                return True
        return False

    def put(self, payload, time_in_seconds):
        return self.safe.put(payload, time_in_seconds)

def get_token(request):
    params = request.params
    headers = request.headers
    cookies = request.cookies
    if '_token' in params:
        token = params['_token']
    elif 'Authorization' in headers:
        try:
            token = headers['Authorization'].split(maxsplit=1)[1]
        except IndexError:
            token = ''
    elif 'crosscompute' in cookies:
        token = cookies['crosscompute']
    else:
        token = ''
    return token

def define_is_match(payload):

    def is_match(batch):
        automation_definition = batch.automation_definition
        variable_definitions = automation_definition.get_variable_definitions(
            'input')
        for name, value in payload.items():
            try:
                variable_definition = find_item(
                    variable_definitions, 'id', name)
            except StopIteration:
                continue
            data = batch.get_data(variable_definition)
            if not has_match(value, data.get('value')):
                return False
        return True

    return is_match

def find_group_definition(group_definitions, value_by_name):
    for group_definition in group_definitions:
        is_match = True
        for name, value in group_definition.configuration.items():
            try:
                v = value_by_name[name]
            except KeyError:
                L.error('"%s" is not defined in token payload', name)
                is_match = False
                continue
            if not has_match(value, v):
                is_match = False
        if is_match:
            return group_definition
    raise StopIteration

def has_match(value1, value2):
    if not isinstance(value1, list):
        value1 = [value1]
    if not isinstance(value2, list):
        value2 = [value2]
    for v1 in value1:
        for v2 in value2:
            if v1 == v2:
                return True
    return False

L = getLogger(__name__)
<commit_msg>Fix https://github.com/crosscompute/crosscompute/issues/59<commit_after>from logging import getLogger

from ..macros.iterable import find_item

class AuthorizationGuard():

    def __init__(self, configuration, safe):
        self.configuration = configuration
        self.safe = safe

    def check(self, request, permission_id, automation_definition=None):
        if not automation_definition:
            automation_definition = self.configuration
        group_definitions = automation_definition.group_definitions
        if not group_definitions:
            return True
        token = get_token(request)
        if not token:
            return False
        try:
            payload = self.safe.get(token)
        except KeyError:
            return False
        value_by_name = dict(payload, ip_address=request.remote_addr)
        try:
            group_definition = find_group_definition(
                group_definitions, value_by_name)
        except StopIteration:
            return False
        for permission_definition in group_definition.permissions:
            if permission_definition.id != permission_id:
                continue
            action = permission_definition.action
            if action == 'match':
                return define_is_match(payload)
            else:
                return True
        return False

    def put(self, payload, time_in_seconds):
        return self.safe.put(payload, time_in_seconds)

def get_token(request):
    params = request.params
    headers = request.headers
    cookies = request.cookies
    if '_token' in params:
        token = params['_token']
        request.response.set_cookie(
            'crosscompute', value=token, secure=True, httponly=True)
    elif 'Authorization' in headers:
        try:
            token = headers['Authorization'].split(maxsplit=1)[1]
        except IndexError:
            token = ''
    elif 'crosscompute' in cookies:
        token = cookies['crosscompute']
    else:
        token = ''
    return token

def define_is_match(payload):

    def is_match(batch):
        automation_definition = batch.automation_definition
        variable_definitions = automation_definition.get_variable_definitions(
            'input')
        for name, value in payload.items():
            try:
                variable_definition = find_item(
                    variable_definitions, 'id', name)
            except StopIteration:
                continue
            data = batch.get_data(variable_definition)
            if not has_match(value, data.get('value')):
                return False
        return True

    return is_match

def find_group_definition(group_definitions, value_by_name):
    for group_definition in group_definitions:
        is_match = True
        for name, value in group_definition.configuration.items():
            try:
                v = value_by_name[name]
            except KeyError:
                L.error('"%s" is not defined in token payload', name)
                is_match = False
                continue
            if not has_match(value, v):
                is_match = False
        if is_match:
            return group_definition
    raise StopIteration

def has_match(value1, value2):
    if not isinstance(value1, list):
        value1 = [value1]
    if not isinstance(value2, list):
        value2 = [value2]
    for v1 in value1:
        for v2 in value2:
            if v1 == v2:
                return True
    return False

L = getLogger(__name__)
<|endoftext|>


<commit_before>from . helpers import (
    roundtrip_LZ4FrameCompressor,
    roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor,
    decompress_truncated,
)
import os
import pytest

test_data=[
    b'',
    (128 * (32 * os.urandom(32))),
    (256 * (32 * os.urandom(32))),
    (512 * (32 * os.urandom(32))),
    (1024 * (32 * os.urandom(32))),
]

@pytest.fixture(
    params=test_data,
    ids=[
        'data' + str(i) for i in range(len(test_data))
    ]
)
def data(request):
    return request.param

@pytest.fixture(
    params=[
        (True),
        (False)
    ]
)
def reset(request):
    return request.param

@pytest.fixture(
    params=[
        (1),
        (8)
    ]
)
def chunks(request):
    return request.param

def test_roundtrip_LZ4FrameCompressor(data, chunks, block_size, reset, block_checksum, content_checksum):
    roundtrip_LZ4FrameCompressor(
        data,
        chunks=chunks,
        block_size=block_size,
        reset=reset,
        block_checksum=block_checksum,
        content_checksum=content_checksum,
    )

def test_roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor(
        data, chunks, block_size, reset, block_checksum, content_checksum):
    roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor(
        data,
        chunks=chunks,
        block_size=block_size,
        reset=reset,
        block_checksum=block_checksum,
        content_checksum=content_checksum,
    )

def test_decompress_truncated(data):
    decompress_truncated(data)

# class TestLZ4FrameModern(unittest.TestCase):

#     def test_checksum_failure(self):
#         input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
#         compressed = lz4frame.compress(input_data, content_checksum=lz4frame.CONTENTCHECKSUM_ENABLED)
#         with self.assertRaisesRegexp(RuntimeError, r'^LZ4F_decompress failed with code: ERROR_contentChecksum_invalid'):
#             last = struct.unpack('B', compressed[-1:])[0]
#             lz4frame.decompress(compressed[:-1] + struct.pack('B', last ^ 0x42))
#         # NB: blockChecksumFlag is not supported by lz4 at the moment, so some
#         # random 1-bit modifications of input may actually trigger valid output
#         # without errors. And content checksum remains the same!

#     def test_decompress_trailer(self):
#         input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
#         compressed = lz4frame.compress(input_data)
#         with self.assertRaisesRegexp(ValueError, r'^Extra data: 64 trailing bytes'):
#             lz4frame.decompress(compressed + b'A'*64)
#         # This API does not support frame concatenation!
#         with self.assertRaisesRegexp(ValueError, r'^Extra data: \d+ trailing bytes'):
#             lz4frame.decompress(compressed + compressed)

#     def test_LZ4FrameCompressor_fails(self):
#         input_data = b"2099023098234882923049823094823094898239230982349081231290381209380981203981209381238901283098908123109238098123"
#         with self.assertRaisesRegexp(RuntimeError, r'compress called after flush'):
#             with lz4frame.LZ4FrameCompressor() as compressor:
#                 compressed = compressor.compress_begin()
#                 compressed += compressor.compress(input_data)
#                 compressed += compressor.flush()
#                 compressed = compressor.compress(input_data)

# if sys.version_info < (2, 7):
#     # Poor-man unittest.TestCase.skip for Python 2.6
#     del TestLZ4FrameModern

# if __name__ == '__main__':
#     unittest.main()
<commit_msg>Add tests for block and content checksum failures<commit_after>from . helpers import (
    roundtrip_LZ4FrameCompressor,
    roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor,
    decompress_truncated,
)
import os
import pytest

test_data=[
    b'',
    (128 * (32 * os.urandom(32))),
    (256 * (32 * os.urandom(32))),
    (512 * (32 * os.urandom(32))),
    (1024 * (32 * os.urandom(32))),
]

@pytest.fixture(
    params=test_data,
    ids=[
        'data' + str(i) for i in range(len(test_data))
    ]
)
def data(request):
    return request.param

@pytest.fixture(
    params=[
        (True),
        (False)
    ]
)
def reset(request):
    return request.param

@pytest.fixture(
    params=[
        (1),
        (8)
    ]
)
def chunks(request):
    return request.param

def test_roundtrip_LZ4FrameCompressor(data, chunks, block_size, reset, block_checksum, content_checksum):
    roundtrip_LZ4FrameCompressor(
        data,
        chunks=chunks,
        block_size=block_size,
        reset=reset,
        block_checksum=block_checksum,
        content_checksum=content_checksum,
    )

def test_roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor(
        data, chunks, block_size, reset, block_checksum, content_checksum):
    roundtrip_LZ4FrameCompressor_LZ4FrameDecompressor(
        data,
        chunks=chunks,
        block_size=block_size,
        reset=reset,
        block_checksum=block_checksum,
        content_checksum=content_checksum,
    )

def test_decompress_truncated(data):
    decompress_truncated(data)

import struct
import lz4.frame as lz4frame

def test_content_checksum_failure(data):
    compressed = lz4frame.compress(data, content_checksum=True)
    message = r'^LZ4F_decompress failed with code: ERROR_contentChecksum_invalid$'
    with pytest.raises(RuntimeError, message=message):
        last = struct.unpack('B', compressed[-1:])[0]
        lz4frame.decompress(compressed[:-1] + struct.pack('B', last ^ 0x42))

def test_block_checksum_failure(data):
    compressed = lz4frame.compress(
        data,
        content_checksum=True,
        block_checksum=True,
    )
    message = r'^LZ4F_decompress failed with code: ERROR_blockChecksum_invalid$'
    if len(compressed) > 64:
        with pytest.raises(RuntimeError, message=message):
            lz4frame.decompress(
                compressed[0:31] +
                struct.pack('B', compressed[32] ^ 0x42) +
                compressed[33:]
            )
<|endoftext|>