<commit_before># stdlib
from typing import Any
from typing import Dict
from typing import Optional
from typing import Union

# third party
from nacl.encoding import HexEncoder
from nacl.signing import SigningKey

# syft relative
from ...core.common.message import EventualSyftMessageWithoutReply
from ...core.common.message import ImmediateSyftMessageWithReply
from ...core.common.message import ImmediateSyftMessageWithoutReply
from ...core.common.message import SignedImmediateSyftMessageWithReply
from ...core.common.message import SignedImmediateSyftMessageWithoutReply
from ...core.common.message import SyftMessage
from ...core.io.address import Address
from ...core.io.connection import ClientConnection
from ...core.io.location.specific import SpecificLocation
from ...core.io.route import SoloRoute
from ...core.node.common.client import Client
from ...core.node.device.client import DeviceClient
from ...core.node.domain.client import DomainClient
from ...core.node.network.client import NetworkClient
from ...core.node.vm.client import VirtualMachineClient
from ..messages.setup_messages import CreateInitialSetUpMessage
from ..messages.setup_messages import GetSetUpMessage
from .request_api.association_api import AssociationRequestAPI
from .request_api.group_api import GroupRequestAPI
from .request_api.role_api import RoleRequestAPI
from .request_api.user_api import UserRequestAPI
from .request_api.worker_api import WorkerRequestAPI

def connect(
    url: str,
    conn_type: ClientConnection,
    client_type: Client,
    credentials: Dict = {},
) -> Any:
    class GridClient(client_type):  # type: ignore
        def __init__(
            self,
            credentials: Dict,
            url: str,
            conn_type: ClientConnection,
            client_type: Client,
        ) -> None:
            # Use Server metadata
            # to build client route
            conn = conn_type(url=url)  # type: ignore

            if credentials:
                metadata, user_key = conn.login(credentials=credentials)
                user_key = SigningKey(user_key.encode("utf-8"), encoder=HexEncoder)
            else:
                metadata = conn._get_metadata()
                user_key = SigningKey.generate()

            (
                spec_location,
                name,
                client_id,
            ) = client_type.deserialize_client_metadata_from_node(metadata=metadata)

            # Create a new Solo Route using the selected connection type
            route = SoloRoute(destination=spec_location, connection=conn)

            location_args = self.__route_client_location(
                client_type=client_type, location=spec_location
            )

            self.proxy_address: Optional[Address] = None

            # Create a new client using the selected client type
            super().__init__(
                network=location_args[NetworkClient],
                domain=location_args[DomainClient],
                device=location_args[DeviceClient],
                vm=location_args[VirtualMachineClient],
                name=name,
                routes=[route],
                signing_key=user_key,
            )

            self.groups = GroupRequestAPI(send=self.__perform_grid_request)
            self.users = UserRequestAPI(send=self.__perform_grid_request)
            self.roles = RoleRequestAPI(send=self.__perform_grid_request)
            self.workers = WorkerRequestAPI(send=self.__perform_grid_request)
            self.association_requests = AssociationRequestAPI(
                send=self.__perform_grid_request
            )

        def proxy(self, vm_address: Address) -> None:
            self.proxy_address = vm_address

        def unproxy(self) -> None:
            self.proxy_address = None

        def initial_setup(self, **kwargs: Any) -> Any:
            return self.__perform_grid_request(
                grid_msg=CreateInitialSetUpMessage, content=kwargs
            )

        def get_setup(self, **kwargs: Any) -> Any:
            return self.__perform_grid_request(grid_msg=GetSetUpMessage, content=kwargs)

        def send_immediate_msg_with_reply(
            self,
            msg: Union[
                SignedImmediateSyftMessageWithReply, ImmediateSyftMessageWithReply
            ],
            route_index: int = 0,
        ) -> SyftMessage:

            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_immediate_msg_with_reply(
                msg=msg, route_index=route_index
            )

        def send_immediate_msg_without_reply(
            self,
            msg: Union[
                SignedImmediateSyftMessageWithoutReply, ImmediateSyftMessageWithoutReply
            ],
            route_index: int = 0,
        ) -> None:
            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_immediate_msg_without_reply(
                msg=msg, route_index=route_index
            )

        def send_eventual_msg_without_reply(
            self,
            msg: EventualSyftMessageWithoutReply,
            route_index: int = 0,
        ) -> None:
            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_eventual_msg_without_reply(
                msg=msg, route_index=route_index
            )

        def __route_client_location(
            self, client_type: Any, location: SpecificLocation
        ) -> Dict[Any, Any]:
            locations: Dict[Any, Optional[SpecificLocation]] = {
                NetworkClient: None,
                DomainClient: None,
                DeviceClient: None,
                VirtualMachineClient: None,
            }
            locations[client_type] = location
            return locations

        def __perform_grid_request(
            self, grid_msg: Any, content: Dict[Any, Any] = {}
        ) -> Dict[Any, Any]:
            signed_msg = self.__build_msg(grid_msg=grid_msg, content=content)
            response = self.send_immediate_msg_with_reply(msg=signed_msg)
            return self.__process_response(response=response)

        def __build_msg(self, grid_msg: Any, content: Dict[Any, Any]) -> Any:
            args = {
                "address": self.address,
                "content": content,
                "reply_to": self.address,
            }
            return grid_msg(**args).sign(signing_key=self.signing_key)

        def __process_response(self, response: SyftMessage) -> Dict[Any, Any]:
            if response.status_code == 200:  # type: ignore
                return response.content  # type: ignore
            else:
                raise Exception(response.content["error"])  # type: ignore

    return GridClient(
        credentials=credentials, url=url, conn_type=conn_type, client_type=client_type
    )
<commit_msg>Grid Client Small changes - Remove proxy/unproxy methods - ADD load method - Add user_key optional parameter<commit_after># stdlib
from typing import Any
from typing import Dict
from typing import Optional
from typing import Type
from typing import Union

# third party
from nacl.encoding import HexEncoder
from nacl.signing import SigningKey

# syft relative
from ...core.common.message import EventualSyftMessageWithoutReply
from ...core.common.message import ImmediateSyftMessageWithReply
from ...core.common.message import ImmediateSyftMessageWithoutReply
from ...core.common.message import SignedImmediateSyftMessageWithReply
from ...core.common.message import SignedImmediateSyftMessageWithoutReply
from ...core.common.message import SyftMessage
from ...core.io.address import Address
from ...core.io.connection import ClientConnection
from ...core.io.location.specific import SpecificLocation
from ...core.io.route import SoloRoute
from ...core.node.common.client import Client
from ...core.node.device.client import DeviceClient
from ...core.node.domain.client import DomainClient
from ...core.node.network.client import NetworkClient
from ...core.node.vm.client import VirtualMachineClient
from ...core.pointer.pointer import Pointer
from ..messages.setup_messages import CreateInitialSetUpMessage
from ..messages.setup_messages import GetSetUpMessage
from ..messages.transfer_messages import LoadObjectMessage
from .request_api.association_api import AssociationRequestAPI
from .request_api.group_api import GroupRequestAPI
from .request_api.role_api import RoleRequestAPI
from .request_api.user_api import UserRequestAPI
from .request_api.worker_api import WorkerRequestAPI

def connect(
    url: str,
    conn_type: ClientConnection,
    client_type: Client,
    credentials: Dict = {},
    user_key: Optional[SigningKey] = None,
) -> Any:
    class GridClient(client_type):  # type: ignore
        def __init__(
            self,
            credentials: Dict,
            url: str,
            conn_type: ClientConnection,
            client_type: Client,
        ) -> None:
            # Use Server metadata
            # to build client route
            conn = conn_type(url=url)  # type: ignore

            if credentials:
                metadata, _user_key = conn.login(credentials=credentials)
                _user_key = SigningKey(_user_key.encode("utf-8"), encoder=HexEncoder)
            else:
                metadata = conn._get_metadata()
                if not user_key:
                    _user_key = SigningKey.generate()
                else:
                    _user_key = user_key

            (
                spec_location,
                name,
                client_id,
            ) = client_type.deserialize_client_metadata_from_node(metadata=metadata)

            # Create a new Solo Route using the selected connection type
            route = SoloRoute(destination=spec_location, connection=conn)

            location_args = self.__route_client_location(
                client_type=client_type, location=spec_location
            )

            self.proxy_address: Optional[Address] = None

            # Create a new client using the selected client type
            super().__init__(
                network=location_args[NetworkClient],
                domain=location_args[DomainClient],
                device=location_args[DeviceClient],
                vm=location_args[VirtualMachineClient],
                name=name,
                routes=[route],
                signing_key=_user_key,
            )

            self.groups = GroupRequestAPI(send=self.__perform_grid_request)
            self.users = UserRequestAPI(send=self.__perform_grid_request)
            self.roles = RoleRequestAPI(send=self.__perform_grid_request)
            self.workers = WorkerRequestAPI(
                send=self.__perform_grid_request, client=self
            )
            self.association_requests = AssociationRequestAPI(
                send=self.__perform_grid_request
            )

        def load(
            self, obj_ptr: Type[Pointer], address: Address, searchable: bool = False
        ) -> None:
            content = {
                "address": address.serialize().SerializeToString().decode("ISO-8859-1"),  # type: ignore
                "uid": str(obj_ptr.id_at_location.value),
                "searchable": searchable,
            }
            self.__perform_grid_request(grid_msg=LoadObjectMessage, content=content)

        def initial_setup(self, **kwargs: Any) -> Any:
            return self.__perform_grid_request(
                grid_msg=CreateInitialSetUpMessage, content=kwargs
            )

        def get_setup(self, **kwargs: Any) -> Any:
            return self.__perform_grid_request(grid_msg=GetSetUpMessage, content=kwargs)

        def send_immediate_msg_with_reply(
            self,
            msg: Union[
                SignedImmediateSyftMessageWithReply, ImmediateSyftMessageWithReply
            ],
            route_index: int = 0,
        ) -> SyftMessage:
            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_immediate_msg_with_reply(
                msg=msg, route_index=route_index
            )

        def send_immediate_msg_without_reply(
            self,
            msg: Union[
                SignedImmediateSyftMessageWithoutReply, ImmediateSyftMessageWithoutReply
            ],
            route_index: int = 0,
        ) -> None:
            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_immediate_msg_without_reply(
                msg=msg, route_index=route_index
            )

        def send_eventual_msg_without_reply(
            self,
            msg: EventualSyftMessageWithoutReply,
            route_index: int = 0,
        ) -> None:
            if self.proxy_address:
                msg.address = self.proxy_address

            return super(GridClient, self).send_eventual_msg_without_reply(
                msg=msg, route_index=route_index
            )

        def __route_client_location(
            self, client_type: Any, location: SpecificLocation
        ) -> Dict[Any, Any]:
            locations: Dict[Any, Optional[SpecificLocation]] = {
                NetworkClient: None,
                DomainClient: None,
                DeviceClient: None,
                VirtualMachineClient: None,
            }
            locations[client_type] = location
            return locations

        def __perform_grid_request(
            self, grid_msg: Any, content: Dict[Any, Any] = {}
        ) -> Dict[Any, Any]:
            signed_msg = self.__build_msg(grid_msg=grid_msg, content=content)
            response = self.send_immediate_msg_with_reply(msg=signed_msg)
            return self.__process_response(response=response)

        def __build_msg(self, grid_msg: Any, content: Dict[Any, Any]) -> Any:
            args = {
                "address": self.address,
                "content": content,
                "reply_to": self.address,
            }
            return grid_msg(**args).sign(signing_key=self.signing_key)

        def __process_response(self, response: SyftMessage) -> Dict[Any, Any]:
            if response.status_code == 200:  # type: ignore
                return response.content  # type: ignore
            else:
                raise Exception(response.content["error"])  # type: ignore

    return GridClient(
        credentials=credentials, url=url, conn_type=conn_type, client_type=client_type
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################
from openerp import models, fields, api

class CrmClaim(models.Model):
    _inherit = 'crm.claim'

    @api.multi
    def _links_get(self):
        link_obj = self.env['res.request.link']
        return [(r.object, r.name) for r in link_obj.search([])]

    @api.one
    @api.depends('ref2')
    def _generate_ref_model_name2(self):
        model_obj = self.env['ir.model']
        self.ref_model_name2 = False
        if self.ref2:
            cond = [('model', '=', str(self.ref2._model))]
            model = model_obj.search(cond)
            self.ref_model_name2 = model.name

    @api.one
    @api.depends('ref2')
    def _generate_ref_name2(self):
        self.ref_name2 = False
        if self.ref2:
            self.ref_name2 = self.ref2.name

    @api.one
    @api.depends('ref3')
    def _generate_ref_model_name3(self):
        model_obj = self.env['ir.model']
        self.ref_model_name3 = False
        if self.ref3:
            cond = [('model', '=', str(self.ref3._model))]
            model = model_obj.search(cond)
            self.ref_model_name3 = model.name

    @api.one
    @api.depends('ref3')
    def _generate_ref_name3(self):
        self.ref_name3 = False
        if self.ref3:
            self.ref_name3 = self.ref3.name

    ref2 = fields.Reference(string='Reference 2', selection=_links_get)
    ref_model_name2 = fields.Char(
        string='Ref. Model 2', compute='_generate_ref_model_name2', store=True)
    ref_name2 = fields.Char(
        string='Ref. Name 2', compute='_generate_ref_name2', store=True)
    ref3 = fields.Reference(string='Reference 3', selection=_links_get)
    ref_model_name3 = fields.Char(
        string='Ref. Model 3', compute='_generate_ref_model_name3', store=True)
    ref_name3 = fields.Char(
        string='Ref. Name 3', compute='_generate_ref_name3', store=True)
<commit_msg>[FIX] crm_claim_extra_ref: Improve programming.<commit_after># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################
from openerp import models, fields, api

class CrmClaim(models.Model):
    _inherit = 'crm.claim'

    @api.multi
    def _links_get(self):
        link_obj = self.env['res.request.link']
        return [(r.object, r.name) for r in link_obj.search([])]

    @api.one
    @api.depends('ref2')
    def _generate_ref_model_name2(self):
        model_obj = self.env['ir.model']
        self.ref_model_name2 = False
        if self.ref2:
            cond = [('model', '=', str(self.ref2._model))]
            model = model_obj.search(cond)
            self.ref_model_name2 = model.name

    @api.one
    @api.depends('ref2')
    def _generate_ref_name2(self):
        self.ref_name2 = False
        if self.ref2:
            self.ref_name2 = self.ref2.name_get()

    @api.one
    @api.depends('ref3')
    def _generate_ref_model_name3(self):
        model_obj = self.env['ir.model']
        self.ref_model_name3 = False
        if self.ref3:
            cond = [('model', '=', str(self.ref3._model))]
            model = model_obj.search(cond)
            self.ref_model_name3 = model.name

    @api.one
    @api.depends('ref3')
    def _generate_ref_name3(self):
        self.ref_name3 = False
        if self.ref3:
            self.ref_name3 = self.ref3.name_get()

    ref2 = fields.Reference(string='Reference 2', selection=_links_get)
    ref_model_name2 = fields.Char(
        string='Ref. Model 2', compute='_generate_ref_model_name2', store=True)
    ref_name2 = fields.Char(
        string='Ref. Name 2', compute='_generate_ref_name2', store=True)
    ref3 = fields.Reference(string='Reference 3', selection=_links_get)
    ref_model_name3 = fields.Char(
        string='Ref. Model 3', compute='_generate_ref_model_name3', store=True)
    ref_name3 = fields.Char(
        string='Ref. Name 3', compute='_generate_ref_name3', store=True)
<|endoftext|>


<commit_before>import re
import time
import random
from math import floor
from cwbot.managers.BaseChatManager import BaseChatManager
from cwbot.util.textProcessing import stringToList
from cwbot.common.exceptions import FatalError
from kol.request.UseItemRequest import UseItemRequest
from cwbot.util.tryRequest import tryRequest
from kol.Error import Error

class WalkieTalkieRepeater(BaseChatManager):
    """ A manager that relays between /clan and /talkie on a randomly-
        selected frequency. """
    capabilities = ['inventory']

    def _chatApplies(self, x, _checkNum):
        """ Override of _chatApplies. Here, only chats from a single channel
        (or optionally, a PM) are accepted. """
        return ((x['type'] in ['normal', 'listen', 'emote'] and
                 x['channel'].lower() in ['clan', 'talkie'])
                or x['type'] == 'private')

    def __init__(self, parent, name, iData, config):
        """ Initialize the manager """
        self._ready = False
        self._otherBots = None
        self._myFreq = None
        self._numPlayersForFreqChange = None
        self._freqChangeTimeout = None
        self._format = None
        super(WalkieTalkieRepeater, self).__init__(parent, name, iData, config)
        self._invMan.refreshInventory()
        self._lastOutsiderCheck = 0
        self._changeFreqRequests = {}
        inventory = self._invMan.inventory()
        numTalkies = inventory.get(6846, 0)
        if numTalkies == 0:
            numUnusedTalkies = inventory.get(6845, 0)
            if numUnusedTalkies > 0:
                try:
                    r = UseItemRequest(self._s, 6845)
                    d = tryRequest(r)
                    numTalkies = 1
                except Error:
                    raise FatalError("Could not use unused walkie talkie.")
            else:
                raise FatalError("Cannot use WalkieTalkieRepeater with "
                                 "no walkie talkie!")
        replies = self.sendChatMessage("/kenneth", None,
                                   waitForReply=True, raw=True)
        for reply in replies:
            txt = reply['text']
            freq = re.search(r"The frequency is (\d+.\d), Mr. Rather", txt)
            if freq:
                self._myFreq = freq.group(1)
                self._log.debug("My walkie talkie frequency is {}"
                                .format(freq.group(1)))
        if self._myFreq is None:
            raise RuntimeError("Could not determine walkie talkie frequency.")
        self._ready = True

    def _configure(self, config):
        BaseChatManager._configure(self, config)
        try:
            otherBots = stringToList(config.setdefault('other_bots', ""))
            self._otherBots = map(int, otherBots)
        except ValueError:
            raise FatalError("Error configuring WalkieTalkieRepeater: "
                             "other_bots must be list of integers (userIds)")
        try:
            self._numPlayersForFreqChange = int(config.setdefault(
                                                 'num_players_to_change', 3))
            self._freqChangeTimeout = int(config.setdefault('change_timeout',
                                                            10))
        except ValueError:
            raise FatalError("Error configuring WalkieTalkieRepeater: "
                             "num_players_to_change and change_timeout must "
                             "be integral")
        self._format = config.setdefault('format',
                                         "%username% (%hash%%userid%): %text%")

    def defaultChannel(self):
        return "clan"

    def _showCommandSummary(self, msg, availableCommands, availableAdmin):
        """ Override of _showCommandSummary """
        if self.director.clanMemberInfo(msg['userId']):
            return ["!newfrequency to request walkie-talkie change; "
                    "!frequency to get the current walkie-talkie channel"]
        else:
            return []

    def parseChat(self, msg, checkNum):
        """ Say text in opposite channel """
        if not self._chatApplies(msg, checkNum):
            return []

        clanMember = self.director.clanMemberInfo(msg['userId'])
        text = msg['text']
        if msg['type'] != 'private':
            inChannel = msg.get('channel', self.defaultChannel).lower()
            outChannel = {'clan': "talkie", 'talkie': "clan"}[inChannel]
            userName = msg.get('userName', "???")
            userId = msg['userId']
            if userId not in self._otherBots:
                newText = self._format
                print map(ord, self._format)
                newText = (newText.replace("%username%", userName)
                                  .replace("%userid%", str(userId))
                                  .replace("%text%", text)
                                  .replace("%hash%", "#"))
            else:
                newText = text
            self.sendChatMessage(newText, outChannel, False, True)
            if inChannel == "talkie" and not clanMember:
                self._handleOutsider(msg['userId'], msg['userName'])

        if clanMember:
            if re.search(r"^!frequency($|\s)", text):
                return [self._handleFrequencyRequest(msg)]
            elif re.search(r"^!newfrequency($|\s)", text):
                self._handleNewFrequencyRequest(msg)
            elif re.search(r"^!help\s*$", text):
                return self._showCommandSummary(msg, None, None)
        return []

    def _handleFrequencyRequest(self, msg):
        if msg.get('channel', "").lower() == "talkie":
            return "You're already here, Kenneth."
        return ("Use /kenneth {} to listen to the clan chat relay."
                .format(self._myFreq))

    def _handleNewFrequencyRequest(self, msg):
        uid = msg['userId']
        self._changeFreqRequests[uid] = time.time()
        numRequests = len([v for v in self._changeFreqRequests.values()
                          if v >= time.time() - self._freqChangeTimeout * 60])

        if numRequests >= self._numPlayersForFreqChange:
            self._changeFreqRequests = {}
            self._changeFrequencies()
        else:
            self._dualChat("{}/{} clan members have requested a frequency "
                           "change."
                           .format(numRequests, self._numPlayersForFreqChange))

    def _changeFrequencies(self):
        newFreq = random.randint(1, 99999)
        self._myFreq = "{}.{}".format(int(floor(newFreq / 10)), newFreq % 10)
        self._dualChat("Changing walkie-talkie frequency to {}."
                       .format(self._myFreq),
                       "Changing walkie-talkie frequency; send '!frequency' "
                       "in a PM to get the new frequency.", waitForReply=True)
        self.sendChatMessage("/kenneth {}".format(self._myFreq),
                             waitForReply=True, raw=True)
        self._lastOutsiderCheck = 0

    def _handleOutsider(self, uid, userName):
        self._dualChat("Non-clan member {} (#{}) detected in /talkie."
                       .format(userName, uid))
        self._changeFrequencies()

    def _dualChat(self, txtClan, txtTalkie=None, **kwargs):
        if txtTalkie is None:
            txtTalkie = txtClan
        self.sendChatMessage(txtClan, channel="clan", **kwargs)
        self.sendChatMessage(txtTalkie, channel="talkie", **kwargs)

    def _heartbeat(self):
        if not self._ready:
            return
        timeSinceLastOutsiderCheck = time.time() - self._lastOutsiderCheck
        if timeSinceLastOutsiderCheck >= 5 * 60:
            self._lastOutsiderCheck = time.time()
            whoVals = self.sendChatMessage("/who talkie", "DEFAULT",
                                           waitForReply=True, raw=True)
            allUsers = whoVals[0]['users']
            self._log.debug("Users in /talkie: {}"
                            .format(', '.join(user['userName']
                                            for user in allUsers)))
            for user in allUsers:
                if not self.director.clanMemberInfo(user['userId']):
                    self._handleOutsider(user['userId'], user['userName'])
                    break
<commit_msg>* Removed debug print statement<commit_after>import re
import time
import random
from math import floor
from cwbot.managers.BaseChatManager import BaseChatManager
from cwbot.util.textProcessing import stringToList
from cwbot.common.exceptions import FatalError
from kol.request.UseItemRequest import UseItemRequest
from cwbot.util.tryRequest import tryRequest
from kol.Error import Error

class WalkieTalkieRepeater(BaseChatManager):
    """ A manager that relays between /clan and /talkie on a randomly-
        selected frequency. """
    capabilities = ['inventory']

    def _chatApplies(self, x, _checkNum):
        """ Override of _chatApplies. Here, only chats from a single channel
        (or optionally, a PM) are accepted. """
        return ((x['type'] in ['normal', 'listen', 'emote'] and
                 x['channel'].lower() in ['clan', 'talkie'])
                or x['type'] == 'private')

    def __init__(self, parent, name, iData, config):
        """ Initialize the manager """
        self._ready = False
        self._otherBots = None
        self._myFreq = None
        self._numPlayersForFreqChange = None
        self._freqChangeTimeout = None
        self._format = None
        super(WalkieTalkieRepeater, self).__init__(parent, name, iData, config)
        self._invMan.refreshInventory()
        self._lastOutsiderCheck = 0
        self._changeFreqRequests = {}
        inventory = self._invMan.inventory()
        numTalkies = inventory.get(6846, 0)
        if numTalkies == 0:
            numUnusedTalkies = inventory.get(6845, 0)
            if numUnusedTalkies > 0:
                try:
                    r = UseItemRequest(self._s, 6845)
                    d = tryRequest(r)
                    numTalkies = 1
                except Error:
                    raise FatalError("Could not use unused walkie talkie.")
            else:
                raise FatalError("Cannot use WalkieTalkieRepeater with "
                                 "no walkie talkie!")
        replies = self.sendChatMessage("/kenneth", None,
                                   waitForReply=True, raw=True)
        for reply in replies:
            txt = reply['text']
            freq = re.search(r"The frequency is (\d+.\d), Mr. Rather", txt)
            if freq:
                self._myFreq = freq.group(1)
                self._log.debug("My walkie talkie frequency is {}"
                                .format(freq.group(1)))
        if self._myFreq is None:
            raise RuntimeError("Could not determine walkie talkie frequency.")
        self._ready = True

    def _configure(self, config):
        BaseChatManager._configure(self, config)
        try:
            otherBots = stringToList(config.setdefault('other_bots', ""))
            self._otherBots = map(int, otherBots)
        except ValueError:
            raise FatalError("Error configuring WalkieTalkieRepeater: "
                             "other_bots must be list of integers (userIds)")
        try:
            self._numPlayersForFreqChange = int(config.setdefault(
                                                 'num_players_to_change', 3))
            self._freqChangeTimeout = int(config.setdefault('change_timeout',
                                                            10))
        except ValueError:
            raise FatalError("Error configuring WalkieTalkieRepeater: "
                             "num_players_to_change and change_timeout must "
                             "be integral")
        self._format = config.setdefault('format',
                                         "%username% (%hash%%userid%): %text%")

    def defaultChannel(self):
        return "clan"

    def _showCommandSummary(self, msg, availableCommands, availableAdmin):
        """ Override of _showCommandSummary """
        if self.director.clanMemberInfo(msg['userId']):
            return ["!newfrequency to request walkie-talkie change; "
                    "!frequency to get the current walkie-talkie channel"]
        else:
            return []

    def parseChat(self, msg, checkNum):
        """ Say text in opposite channel """
        if not self._chatApplies(msg, checkNum):
            return []

        clanMember = self.director.clanMemberInfo(msg['userId'])
        text = msg['text']
        if msg['type'] != 'private':
            inChannel = msg.get('channel', self.defaultChannel).lower()
            outChannel = {'clan': "talkie", 'talkie': "clan"}[inChannel]
            userName = msg.get('userName', "???")
            userId = msg['userId']
            if userId not in self._otherBots:
                newText = self._format
                newText = (newText.replace("%username%", userName)
                                  .replace("%userid%", str(userId))
                                  .replace("%text%", text)
                                  .replace("%hash%", "#"))
            else:
                newText = text
            self.sendChatMessage(newText, outChannel, False, True)
            if inChannel == "talkie" and not clanMember:
                self._handleOutsider(msg['userId'], msg['userName'])

        if clanMember:
            if re.search(r"^!frequency($|\s)", text):
                return [self._handleFrequencyRequest(msg)]
            elif re.search(r"^!newfrequency($|\s)", text):
                self._handleNewFrequencyRequest(msg)
            elif re.search(r"^!help\s*$", text):
                return self._showCommandSummary(msg, None, None)
        return []

    def _handleFrequencyRequest(self, msg):
        if msg.get('channel', "").lower() == "talkie":
            return "You're already here, Kenneth."
        return ("Use /kenneth {} to listen to the clan chat relay."
                .format(self._myFreq))

    def _handleNewFrequencyRequest(self, msg):
        uid = msg['userId']
        self._changeFreqRequests[uid] = time.time()
        numRequests = len([v for v in self._changeFreqRequests.values()
                          if v >= time.time() - self._freqChangeTimeout * 60])

        if numRequests >= self._numPlayersForFreqChange:
            self._changeFreqRequests = {}
            self._changeFrequencies()
        else:
            self._dualChat("{}/{} clan members have requested a frequency "
                           "change."
                           .format(numRequests, self._numPlayersForFreqChange))

    def _changeFrequencies(self):
        newFreq = random.randint(1, 99999)
        self._myFreq = "{}.{}".format(int(floor(newFreq / 10)), newFreq % 10)
        self._dualChat("Changing walkie-talkie frequency to {}."
                       .format(self._myFreq),
                       "Changing walkie-talkie frequency; send '!frequency' "
                       "in a PM to get the new frequency.", waitForReply=True)
        self.sendChatMessage("/kenneth {}".format(self._myFreq),
                             waitForReply=True, raw=True)
        self._lastOutsiderCheck = 0

    def _handleOutsider(self, uid, userName):
        self._dualChat("Non-clan member {} (#{}) detected in /talkie."
                       .format(userName, uid))
        self._changeFrequencies()

    def _dualChat(self, txtClan, txtTalkie=None, **kwargs):
        if txtTalkie is None:
            txtTalkie = txtClan
        self.sendChatMessage(txtClan, channel="clan", **kwargs)
        self.sendChatMessage(txtTalkie, channel="talkie", **kwargs)

    def _heartbeat(self):
        if not self._ready:
            return
        timeSinceLastOutsiderCheck = time.time() - self._lastOutsiderCheck
        if timeSinceLastOutsiderCheck >= 5 * 60:
            self._lastOutsiderCheck = time.time()
            whoVals = self.sendChatMessage("/who talkie", "DEFAULT",
                                           waitForReply=True, raw=True)
            allUsers = whoVals[0]['users']
            self._log.debug("Users in /talkie: {}"
                            .format(', '.join(user['userName']
                                            for user in allUsers)))
            for user in allUsers:
                if not self.director.clanMemberInfo(user['userId']):
                    self._handleOutsider(user['userId'], user['userName'])
                    break
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import warnings
import unittest
import time

from cwharaj.database_factory import DatabaseFactory, CollectionTypes
from cwharaj.parser.utils.section_item import SectionItem
from cwharaj.utils.crawl_utils import CrawlUtils
from cwharaj.items import CacheItem, HistoryItem, Ad, WebsiteTypes, City, Member
from datetime import datetime
from cwharaj import settings

class MysqlDBTest(unittest.TestCase):
    def setUp(self):
        database_factory = DatabaseFactory(settings.SQL_HOST, settings.SQL_PORT,
                                           settings.SQL_USER, settings.SQL_PASSWD,
                                           settings.SQL_DB, settings.SQL_COLLECTION_NAME)

        self._cache_db = database_factory.get_database(CollectionTypes.cache)
        self._history_db = database_factory.get_database(CollectionTypes.history)
        self._item_db = database_factory.get_database(CollectionTypes.item)

        self._cache_url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
        self._cache_from = WebsiteTypes.opensooq.value

    # def test_insert_cache_row(self):
    #     _guid = "1234321"
    #     _id = CrawlUtils.url_parse_id_from_page_url(self._cache_url, 3)
    #
    #     self._cache_db.open_spider()
    #     item = CacheItem(
    #         url=self._cache_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         ID=_id,
    #         url_from=self._cache_from
    #     )
    #     self._cache_db.insert_for_cache(item)

    # def test_oldest_cache(self):
    #     self._cache_db.open_spider()
    #
    #     # row = self._cache_db.find_oldest_for_cache()
    #
    #     self._cache_db.delete_row(self._cache_url, self._cache_from)

    # def test_insert_history_row(self):
    #     _url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
    #     _guid = "1234321"
    #     # _id = CrawlUtils.url_parse_id_from_page_url(_url, 3)
    #     _id = "123"
    #
    #     self._history_db.open_spider()
    #     item = HistoryItem(
    #         url=_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         ID=_id
    #     )
    #     self._history_db.update_for_history(_id, item)

    # def test_insert_item_row(self):
    #     _url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
    #     _guid = "1234321"
    #     # _id = CrawlUtils.url_parse_id_from_page_url(_url, 3)
    #     _id = "123"
    #
    #     # _section = ['section']
    #     _section = []
    #
    #     self._item_db.open_spider()
    #     item = Ad(
    #         url=_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         updated_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #
    #         ID=_id,
    #         city='city',
    #         time='time',
    #         title='title',
    #         pictures=['pic1', 'pic2'],
    #         subject='subject',
    #         contact='contact',
    #         number='number',
    #
    #         # cache form where, such as opensooq,mstaml.(WebsiteTypes variable)
    #         url_from='opensooq',
    #
    #         address='address',
    #         memberName='member name',
    #         description='description',
    #         section=_section,
    #     )
    #     self._item_db.insert_for_item(item)

    # def test_get_year_id(self):
    #     _years_row = {
    #         "id": 58,
    #         "text": '2014'
    #     }
    #     _years_id = self._item_db.get_year_id(_years_row['text'])
    #     expect = _years_row['id']
    #     self.assertEqual(expect, _years_id)
    #
    # def test_insert_exist_cities(self):
    #     _cities_row = {
    #         "id": 12,
    #         "text": "الجوف"
    #     }
    #     _city_id = self._item_db.save_city(City.get_default(_cities_row['text']))
    #     expect = _cities_row['id']
    #     self.assertEqual(expect, _city_id)
    #
    # def test_insert_new_cities(self):
    #     _cities_row = {
    #         "id": 26,
    #         "text": "دبي",
    #         "sql": "INSERT INTO cities(text) VALUES ('دبي')"
    #     }
    #     _city_id = self._item_db.save_city(City.get_default(_cities_row['text']))
    #     expect = _cities_row['id']
    #     self.assertEqual(expect, _city_id)

    # def test_insert_exist_members(self):
    #     _memberName = "ابراهيم لطفي"
    #     expect = 51
    #     _member_id = self._item_db.save_member(Member.get_default(_memberName))
    #     self.assertEqual(expect, _member_id)

    # def test_insert_new_members(self):
    #     _memberName = "djzhang"
    #     expect = 60
    #     member = Member.get_default(_memberName)
    #
    #     sql = " INSERT INTO " + "members" + " (username, password, groupnumber, email, timeregister, member_code, documentingmobile, Documentingemail, phone, sendtime, active, now, Lastactivity, subscribe_1, subscribe_2, subscribe_3, The_pay_commission) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(
    #         member['username'], member['password'], member['groupnumber'], member['email'], member['timeregister'],
    #         member['member_code'], member['documentingmobile'], member['Documentingemail'], member['phone'],
    #         member['sendtime'], member['active'], member['now'], member['Lastactivity'], member['subscribe_1'],
    #         member['subscribe_2'], member['subscribe_3'], member['The_pay_commission']
    #     )
    #
    #     _member_id = self._item_db.save_member(member)
    #     self.assertEqual(expect, _member_id)

    # def test_insert_exist_ads(self):
    #     expect = 21
    #     item = Ad()
    #     item['ads_title'] = 'جمس سوبربان  دبل  موديل 2012 اللون فضي للبيع'
    #     _ads_id = self._item_db.save_ad(item)
    #     self.assertEqual(expect, _ads_id)

    def test_insert_new_ads(self):
        expect = 27
        section_item = SectionItem.get_default()
        item = Ad.get_default(
            section_item=section_item,
            _ads_title="اسكاليد موديل 2016 بسعر جي",
            _city_id='44',
            _ads_contact="123454321",
            _ads_body="test mysql db",
            _image_link="https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1ba3366cd.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bab49a3b.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bb0b6ca6.jpeg",
            _His_announcement_id=60,
            _type_ads_or=1, _close_ads=0
        )

        sql = " INSERT INTO ads (ads_title, ads_city, ads_tags_R, ads_tags_F, ads_tags_FF, ads_contact, ads_body, image_link, type_ads_other_final, un_model, status, fixing, Time_added, His_announcement, type_ads_or, close_ads, Last_updated_Ad, closecomment, fixed_home, fixed_tub, fixed_sec, fixed_sec2, fixed_sec3, timer_mazad) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(
            item['ads_title'], item['ads_city'], item['ads_tags_R'], item['ads_tags_F'], item['ads_tags_FF'],
            item['ads_contact'], item['ads_body'], item['image_link'], item['type_ads_other_final'],
            item['un_model'], item['status'], item['fixing'], item['Time_added'], item['His_announcement'],
            item['type_ads_or'], item['close_ads'], item['Last_updated_Ad'], item['closecomment'],
            item['fixed_home'], item['fixed_tub'], item['fixed_sec'], item['fixed_sec2'], item['fixed_sec3'],
            item['timer_mazad']
        )

        _ads_id = self._item_db.save_ad(item)
        self.assertEqual(expect, _ads_id)
<commit_msg>charset="utf8mb4",<commit_after># -*- coding: utf-8 -*-

import warnings
import unittest
import time

from cwharaj.database_factory import DatabaseFactory, CollectionTypes
from cwharaj.parser.utils.section_item import SectionItem
from cwharaj.utils.crawl_utils import CrawlUtils
from cwharaj.items import CacheItem, HistoryItem, Ad, WebsiteTypes, City, Member
from datetime import datetime
from cwharaj import settings

class MysqlDBTest(unittest.TestCase):
    def setUp(self):
        database_factory = DatabaseFactory(settings.SQL_HOST, settings.SQL_PORT,
                                           settings.SQL_USER, settings.SQL_PASSWD,
                                           settings.SQL_DB, settings.SQL_COLLECTION_NAME)

        self._cache_db = database_factory.get_database(CollectionTypes.cache)
        self._history_db = database_factory.get_database(CollectionTypes.history)
        self._item_db = database_factory.get_database(CollectionTypes.item)

        self._cache_url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
        self._cache_from = WebsiteTypes.opensooq.value

    # def test_insert_cache_row(self):
    #     _guid = "1234321"
    #     _id = CrawlUtils.url_parse_id_from_page_url(self._cache_url, 3)
    #
    #     self._cache_db.open_spider()
    #     item = CacheItem(
    #         url=self._cache_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         ID=_id,
    #         url_from=self._cache_from
    #     )
    #     self._cache_db.insert_for_cache(item)

    # def test_oldest_cache(self):
    #     self._cache_db.open_spider()
    #
    #     # row = self._cache_db.find_oldest_for_cache()
    #
    #     self._cache_db.delete_row(self._cache_url, self._cache_from)

    # def test_insert_history_row(self):
    #     _url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
    #     _guid = "1234321"
    #     # _id = CrawlUtils.url_parse_id_from_page_url(_url, 3)
    #     _id = "123"
    #
    #     self._history_db.open_spider()
    #     item = HistoryItem(
    #         url=_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         ID=_id
    #     )
    #     self._history_db.update_for_history(_id, item)

    # def test_insert_item_row(self):
    #     _url = "https://sa.opensooq.com/ar/search/30002057/استراحة-سديم-للايجار-اليومي-والشهري-والسنوي-حي-الأمانة-شمال-الرياض"
    #     _guid = "1234321"
    #     # _id = CrawlUtils.url_parse_id_from_page_url(_url, 3)
    #     _id = "123"
    #
    #     # _section = ['section']
    #     _section = []
    #
    #     self._item_db.open_spider()
    #     item = Ad(
    #         url=_url,
    #         guid=_guid,
    #         created_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #         updated_at=datetime.utcnow().replace(microsecond=0).isoformat(' '),
    #
    #         ID=_id,
    #         city='city',
    #         time='time',
    #         title='title',
    #         pictures=['pic1', 'pic2'],
    #         subject='subject',
    #         contact='contact',
    #         number='number',
    #
    #         # cache form where, such as opensooq,mstaml.(WebsiteTypes variable)
    #         url_from='opensooq',
    #
    #         address='address',
    #         memberName='member name',
    #         description='description',
    #         section=_section,
    #     )
    #     self._item_db.insert_for_item(item)

    # def test_get_year_id(self):
    #     _years_row = {
    #         "id": 58,
    #         "text": '2014'
    #     }
    #     _years_id = self._item_db.get_year_id(_years_row['text'])
    #     expect = _years_row['id']
    #     self.assertEqual(expect, _years_id)
    #
    # def test_insert_exist_cities(self):
    #     _cities_row = {
    #         "id": 12,
    #         "text": "الجوف"
    #     }
    #     _city_id = self._item_db.save_city(City.get_default(_cities_row['text']))
    #     expect = _cities_row['id']
    #     self.assertEqual(expect, _city_id)
    #
    # def test_insert_new_cities(self):
    #     _cities_row = {
    #         "id": 26,
    #         "text": "دبي",
    #         "sql": "INSERT INTO cities(text) VALUES ('دبي')"
    #     }
    #     _city_id = self._item_db.save_city(City.get_default(_cities_row['text']))
    #     expect = _cities_row['id']
    #     self.assertEqual(expect, _city_id)

    # def test_insert_exist_members(self):
    #     _memberName = "ابراهيم لطفي"
    #     expect = 51
    #     _member_id = self._item_db.save_member(Member.get_default(_memberName))
    #     self.assertEqual(expect, _member_id)

    # def test_insert_new_members(self):
    #     _memberName = "djzhang"
    #     expect = 60
    #     member = Member.get_default(_memberName)
    #
    #     sql = " INSERT INTO " + "members" + " (username, password, groupnumber, email, timeregister, member_code, documentingmobile, Documentingemail, phone, sendtime, active, now, Lastactivity, subscribe_1, subscribe_2, subscribe_3, The_pay_commission) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(
    #         member['username'], member['password'], member['groupnumber'], member['email'], member['timeregister'],
    #         member['member_code'], member['documentingmobile'], member['Documentingemail'], member['phone'],
    #         member['sendtime'], member['active'], member['now'], member['Lastactivity'], member['subscribe_1'],
    #         member['subscribe_2'], member['subscribe_3'], member['The_pay_commission']
    #     )
    #
    #     _member_id = self._item_db.save_member(member)
    #     self.assertEqual(expect, _member_id)

    # def test_insert_exist_ads(self):
    #     expect = 21
    #     item = Ad()
    #     item['ads_title'] = 'جمس سوبربان  دبل  موديل 2012 اللون فضي للبيع'
    #     _ads_id = self._item_db.save_ad(item)
    #     self.assertEqual(expect, _ads_id)

    def test_insert_new_ads(self):
        expect = 26
        section_item = SectionItem.get_default()
        item = Ad.get_default(
            section_item=section_item,
            _ads_title="اسكاليد موديل 2016 بسعر جي",
            _city_id='44',
            _ads_contact="123454321",
            _ads_body="test mysql db",
            _image_link="https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1ba3366cd.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bab49a3b.jpeg,https://img1cdn.haraj.com.sa/userfiles30/2015-07-18/55aa1bb0b6ca6.jpeg",
            _His_announcement_id=60,
            _type_ads_or=1, _close_ads=0
        )

        sql = " INSERT INTO ads (ads_title, ads_city, ads_tags_R, ads_tags_F, ads_tags_FF, ads_contact, ads_body, image_link, type_ads_other_final, un_model, status, fixing, Time_added, His_announcement, type_ads_or, close_ads, Last_updated_Ad, closecomment, fixed_home, fixed_tub, fixed_sec, fixed_sec2, fixed_sec3, timer_mazad) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(
            item['ads_title'], item['ads_city'], item['ads_tags_R'], item['ads_tags_F'], item['ads_tags_FF'],
            item['ads_contact'], item['ads_body'], item['image_link'], item['type_ads_other_final'],
            item['un_model'], item['status'], item['fixing'], item['Time_added'], item['His_announcement'],
            item['type_ads_or'], item['close_ads'], item['Last_updated_Ad'], item['closecomment'],
            item['fixed_home'], item['fixed_tub'], item['fixed_sec'], item['fixed_sec2'], item['fixed_sec3'],
            item['timer_mazad']
        )

        _ads_id = self._item_db.save_ad(item)
        self.assertEqual(expect, _ads_id)
<|endoftext|>


<commit_before>import numpy as np
import pandas as pd
import pyther as pt

# properties thermodynamics

Solid_Density = "Solid Density", "[kmol/m^3]", "A+B*T+C*T^2+D*T^3+E*T^4", 0
Liquid_Density = "Liquid Density", "[kmol/m^3]", "A/B^(1+(1-T/C)^D)", 1
Vapour_Pressure = "Vapour Pressure", "[Pa]", "exp(A+B/T+C*ln(T)+D*T^E)", 2
Heat_of_Vaporization = "Heat of Vaporization", "[J/kmol]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 3
Solid_Heat_Capacity = "Solid Heat Capacity", "[J/(kmol*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 4
Liquid_Heat_Capacity = "Liquid Heat Capacity", "[J/(kmol*K)]", "A^2/(1-Tr)+B-2*A*C*(1-Tr)-A*D*(1-Tr)^2-C^2*(1-Tr)^3/3-C*D*(1-Tr)^4/2-D^2*(1-Tr)^5/5", 5
Ideal_Gas_Heat_Capacity = "Ideal Gas Heat Capacity" "[J/(kmol*K)]", "A+B*(C/T/sinh(C/T))^2+D*(E/T/cosh(E/T))^2", 6
Second_Virial_Coefficient = "Second	Virial	Coefficient", "[m^3/kmol]", "A+B/T+C/T^3+D/T^8+E/T^9", 7
Liquid_Viscosity = "Liquid	Viscosity", "[kg/(m*s)]", "exp(A+B/T+C*ln(T)+D*T^E)", 8
Vapour_Viscosity = "Vapour	Viscosity", "[kg/(m*s)]", "A*T^B/(1+C/T+D/T^2)", 9
Liquid_Thermal_Conductivity = "Liquid Thermal Conductivity", "[J/(m*s*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 10
Vapour_Thermal_Conductivity = "Vapour Thermal Conductivity", "[J/(m*s*K)]", "A*T^B/(1+C/T+D/T^2)", 11
Surface_Tension = "Surface Tension", "[kg/s^2]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 12

#----------------------------------------------------------------------------------------------------------------

dppr_file = "PureFull_mod_properties.xls"

#print(dppr_file)

properties_data = pt.Data_parse()

def read_dppr(dppr_file):
        #self.dppr_data = pd.read_excel(dppr_file).head().set_index('Name').ix[:, 1:12]

        #dppr_data = pd.read_excel(dppr_file).set_index("Name").ix[:, 1:5]
        dppr_data = pd.read_excel(dppr_file).ix[:, 0:8]

        # component_names = dppr_data.index.get_values()
        return dppr_data

data = read_dppr(dppr_file)

#print(data)

g = 30
components_labels = [x for x in range(0, 13*g, 13)]
data_name = data.ix[components_labels, 0].get_values()

def property_cal(component, property_thermodynamics, temperature = None):

	"""
	properties thermodynamics

	Solid_Density = "Solid Density", "[kmol/m^3]", "A+B*T+C*T^2+D*T^3+E*T^4", 0
	Liquid_Density = "Liquid Density", "[kmol/m^3]", "A/B^(1+(1-T/C)^D)", 1
	Vapour_Pressure = "Vapour Pressure", "[Pa]", "exp(A+B/T+C*ln(T)+D*T^E)", 2
	Heat_of_Vaporization = "Heat of Vaporization", "[J/kmol]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 3
	Solid_Heat_Capacity = "Solid Heat Capacity", "[J/(kmol*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 4
	Liquid_Heat_Capacity = "Liquid Heat Capacity", "[J/(kmol*K)]", "A^2/(1-Tr)+B-2*A*C*(1-Tr)-A*D*(1-Tr)^2-C^2*(1-Tr)^3/3-C*D*(1-Tr)^4/2-D^2*(1-Tr)^5/5", 5
	Ideal_Gas_Heat_Capacity = "Ideal Gas Heat Capacity" "[J/(kmol*K)]", "A+B*(C/T/sinh(C/T))^2+D*(E/T/cosh(E/T))^2", 6
	Second_Virial_Coefficient = "Second	Virial	Coefficient", "[m^3/kmol]", "A+B/T+C/T^3+D/T^8+E/T^9", 7
	Liquid_Viscosity = "Liquid	Viscosity", "[kg/(m*s)]", "exp(A+B/T+C*ln(T)+D*T^E)", 8
	Vapour_Viscosity = "Vapour	Viscosity", "[kg/(m*s)]", "A*T^B/(1+C/T+D/T^2)", 9
	Liquid_Thermal_Conductivity = "Liquid Thermal Conductivity", "[J/(m*s*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 10
	Vapour_Thermal_Conductivity = "Vapour Thermal Conductivity", "[J/(m*s*K)]", "A*T^B/(1+C/T+D/T^2)", 11
	Surface_Tension = "Surface Tension", "[kg/s^2]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 12
	"""

	select_constans = [x + property_thermodynamics[3] for x in range(0, 13*g, 13)]
	values_constans = data.ix[select_constans, 1:8].get_values()
	table_constans = pd.DataFrame(data=values_constans,index=data_name,
						 columns=["A", "B", "C", "D", "E", "T Min [K]", "T Max [K]"])

	print(table_constans)
	component_constans = table_constans.loc[component]

	A, B, C, D, E, Min, Max = component_constans

	if temperature == None:
		Temp_vector = np.array([Temp_vector for Temp_vector in np.arange(Min, Max)])
	elif (Min < np.array(temperature) < Max):
		Temp_vector = np.array(temperature)
	else:
		return print("temperature is not valid")

	print(component_constans)

	if property_thermodynamics == Solid_Density:

		solid_Density = A + B * Temp_vector + C * Temp_vector ** 2 + D * Temp_vector ** 3 + E * Temp_vector **4
		#print(solid_Density)
		return solid_Density

	elif property_thermodynamics == Liquid_Density:

		liquid_Density = A / B ** (1 + (1 - Temp_vector / C) ** D)
		#print("liquid_Density = {0}".format(liquid_Density))
		return liquid_Density

	elif property_thermodynamics == Vapour_Pressure:

		vapour_Pressure = np.exp(A + B/Temp_vector + C * np.log(Temp_vector)+D*Temp_vector **E)
		#print(vapour_Pressure)
		return vapour_Pressure

	elif property_thermodynamics == Heat_of_Vaporization:

		heat_of_Vaporization = A*(1-Tr) ** (B+C*Tr+D*Tr**2)
		#print(heat_of_Vaporization)
		return heat_of_Vaporization

	elif property_thermodynamics == Solid_Heat_Capacity:
		solid_Heat_Capacity = A + B * Temp_vector + C * Temp_vector ** 2 + D * Temp_vector ** 3 + E * Temp_vector ** 4
		#print(solid_Heat_Capacity)
		return solid_Heat_Capacity
	elif property_thermodynamics == Liquid_Heat_Capacity:
		liquid_Heat_Capacity = A ** 2 / (1-Tr) + B-2*A*C*(1-Tr)-A*D*(1-Tr)**2-C**2*(1-Tr)**3/3-C*D*(1-Tr)**4/2-D**2*(1-Tr)**5/5
		#print(liquid_Heat_Capacity)
		return(liquid_Heat_Capacity)
	elif property_thermodynamics == Ideal_Gas_Heat_Capacity:
		ideal_Gas_Heat_Capacity = A+B*(C/Temp_vector/np.sinh(C/Temp_vector))**2+D*(E/Temp_vector/np.cosh(E/Temp_vector))**2
		#print(ideal_Gas_Heat_Capacity)
		return ideal_Gas_Heat_Capacity
	elif property_thermodynamics == Second_Virial_Coefficient:
		second_Virial_Coefficient = A+B/Temp_vector+C/Temp_vector**3+D/Temp_vector**8+E/Temp_vector**9
		#print(second_Virial_Coefficient)
		return second_Virial_Coefficient
	elif property_thermodynamics == Liquid_Viscosity:
		liquid_Viscosity = np.exp(A + B / Temp_vector + C * np.log(Temp_vector) + D * Temp_vector ** E)
		#print(liquid_Viscosity)
		return liquid_Viscosity
	elif property_thermodynamics == Vapour_Viscosity:
		vapour_Viscosity = A*Temp_vector**B/(1+C/Temp_vector+D/Temp_vector**2)
		#print(vapour_Viscosity)
		return vapour_Viscosity
	elif property_thermodynamics == Liquid_Thermal_Conductivity:
		liquid_Thermal_Conductivity = A+B*Temp_vector+C*Temp_vector**2+D*Temp_vector**3+E*Temp_vector**4
		#print(liquid_Thermal_Conductivity)
		return liquid_Thermal_Conductivity
	elif property_thermodynamics == Vapour_Thermal_Conductivity:
		vapour_Thermal_Conductivity = A*Temp_vector**B/(1+C/Temp_vector+D/Temp_vector**2)
		#print(vapour_Thermal_Conductivity)
		return vapour_Thermal_Conductivity
	elif property_thermodynamics == Surface_Tension:
		surface_Tension = A*(1-Tr)**(B+C*Tr+D*Tr*2)
		#print(surface_Tension)
		return surface_Tension

component = 'METHANE'
#component = "ETHANE"
#component = "3-METHYLHEPTANE"
#component = "n-PENTACOSANE"
#component = "ISOBUTANE"
#component = "n-TETRADECANE"

#components = ["METHANE", "n-TETRACOSANE"]

#property_thermodynamics = property_cal(component, Vapour_Pressure, [180.4, 181.4, 185.3])
property_thermodynamics = property_cal(component, Vapour_Pressure, 180.4)
#property_thermodynamics = property_cal(component, Vapour_Pressure)
print(property_thermodynamics)

Min = 30
Max = 90
t = np.array([40, 34, 56, 98.4, 25, 456])

Temperature_valid = [i if Min < np.array(i) < Max else "{0} K is a temperature not valid".format(i) for i in t]

print(Temperature_valid)

# Tareas

# estado de referencias del estado triple para las propiedades del gas ideal

# graficar la correlación de la densidad de liquido para comparar y sobreponer
# con la parte de la campana calculada con la ecuación de estado para el líquido

# revisar las ecuaciones para utilizar el enfoque de fugacidad de solidó para determinar
# las demás propiedades termodinámicas

# para los graficos de velocidad de sonido tener en cuenta "por ejemplo" las isotermas
# para el caso de una sola fase

# criterio de parar el calculo es que sea hasta el 5% de la presión de vapor de la
# isoterma a la cual se está calculando. En caso de las isotermas super criticas,
# se toma como parametro el 5% de la presión critica

# incluir el calculo del corrimiento de volumen

# dato experiental de líquido saturado para determinar un corrimiento C
# para adicionar dicho corrimiento

# escribir un borrador del artículo

# introduccion#
# arquitectura diagrama
# calculo termodinámicas
# referencias del libro de mollerup, conceptos de algoritmos por ejemmplo la cmapana
# - subsecciópn calculo de la cmapana de saturación,, parametros de inicio
# sección para calcular los otros diagramas, descripción general del calculo
# resultados o ejemplos
# -sub sección pantallazos del notebook para mostrar capacidades de ussuarios
# conclusiones

<commit_msg>solid pure<commit_after>import numpy as np
import pandas as pd
import pyther as pt

# properties thermodynamics

Solid_Density = "Solid Density", "[kmol/m^3]", "A+B*T+C*T^2+D*T^3+E*T^4", 0
Liquid_Density = "Liquid Density", "[kmol/m^3]", "A/B^(1+(1-T/C)^D)", 1
Vapour_Pressure = "Vapour Pressure", "[Pa]", "exp(A+B/T+C*ln(T)+D*T^E)", 2
Heat_of_Vaporization = "Heat of Vaporization", "[J/kmol]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 3
Solid_Heat_Capacity = "Solid Heat Capacity", "[J/(kmol*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 4
Liquid_Heat_Capacity = "Liquid Heat Capacity", "[J/(kmol*K)]", "A^2/(1-Tr)+B-2*A*C*(1-Tr)-A*D*(1-Tr)^2-C^2*(1-Tr)^3/3-C*D*(1-Tr)^4/2-D^2*(1-Tr)^5/5", 5
Ideal_Gas_Heat_Capacity = "Ideal Gas Heat Capacity" "[J/(kmol*K)]", "A+B*(C/T/sinh(C/T))^2+D*(E/T/cosh(E/T))^2", 6
Second_Virial_Coefficient = "Second	Virial	Coefficient", "[m^3/kmol]", "A+B/T+C/T^3+D/T^8+E/T^9", 7
Liquid_Viscosity = "Liquid	Viscosity", "[kg/(m*s)]", "exp(A+B/T+C*ln(T)+D*T^E)", 8
Vapour_Viscosity = "Vapour	Viscosity", "[kg/(m*s)]", "A*T^B/(1+C/T+D/T^2)", 9
Liquid_Thermal_Conductivity = "Liquid Thermal Conductivity", "[J/(m*s*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 10
Vapour_Thermal_Conductivity = "Vapour Thermal Conductivity", "[J/(m*s*K)]", "A*T^B/(1+C/T+D/T^2)", 11
Surface_Tension = "Surface Tension", "[kg/s^2]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 12

#----------------------------------------------------------------------------------------------------------------

dppr_file = "PureFull_mod_properties.xls"

#print(dppr_file)

properties_data = pt.Data_parse()

def read_dppr(dppr_file):
        #self.dppr_data = pd.read_excel(dppr_file).head().set_index('Name').ix[:, 1:12]

        #dppr_data = pd.read_excel(dppr_file).set_index("Name").ix[:, 1:5]
        dppr_data = pd.read_excel(dppr_file).ix[:, 0:8]

        # component_names = dppr_data.index.get_values()
        return dppr_data

data = read_dppr(dppr_file)

#print(data)

g = 30
components_labels = [x for x in range(0, 13*g, 13)]
data_name = data.ix[components_labels, 0].get_values()

def property_cal(component, property_thermodynamics, temperature = None):

	"""
	properties thermodynamics

	Solid_Density = "Solid Density", "[kmol/m^3]", "A+B*T+C*T^2+D*T^3+E*T^4", 0
	Liquid_Density = "Liquid Density", "[kmol/m^3]", "A/B^(1+(1-T/C)^D)", 1
	Vapour_Pressure = "Vapour Pressure", "[Pa]", "exp(A+B/T+C*ln(T)+D*T^E)", 2
	Heat_of_Vaporization = "Heat of Vaporization", "[J/kmol]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 3
	Solid_Heat_Capacity = "Solid Heat Capacity", "[J/(kmol*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 4
	Liquid_Heat_Capacity = "Liquid Heat Capacity", "[J/(kmol*K)]", "A^2/(1-Tr)+B-2*A*C*(1-Tr)-A*D*(1-Tr)^2-C^2*(1-Tr)^3/3-C*D*(1-Tr)^4/2-D^2*(1-Tr)^5/5", 5
	Ideal_Gas_Heat_Capacity = "Ideal Gas Heat Capacity" "[J/(kmol*K)]", "A+B*(C/T/sinh(C/T))^2+D*(E/T/cosh(E/T))^2", 6
	Second_Virial_Coefficient = "Second	Virial	Coefficient", "[m^3/kmol]", "A+B/T+C/T^3+D/T^8+E/T^9", 7
	Liquid_Viscosity = "Liquid	Viscosity", "[kg/(m*s)]", "exp(A+B/T+C*ln(T)+D*T^E)", 8
	Vapour_Viscosity = "Vapour	Viscosity", "[kg/(m*s)]", "A*T^B/(1+C/T+D/T^2)", 9
	Liquid_Thermal_Conductivity = "Liquid Thermal Conductivity", "[J/(m*s*K)]", "A+B*T+C*T^2+D*T^3+E*T^4", 10
	Vapour_Thermal_Conductivity = "Vapour Thermal Conductivity", "[J/(m*s*K)]", "A*T^B/(1+C/T+D/T^2)", 11
	Surface_Tension = "Surface Tension", "[kg/s^2]", "A*(1-Tr)^(B+C*Tr+D*Tr^2)", 12
	"""

	select_constans = [x + property_thermodynamics[3] for x in range(0, 13*g, 13)]
	values_constans = data.ix[select_constans, 1:8].get_values()
	table_constans = pd.DataFrame(data=values_constans,index=data_name,
						 columns=["A", "B", "C", "D", "E", "T Min [K]", "T Max [K]"])

	print(table_constans)
	component_constans = table_constans.loc[component]

	A, B, C, D, E, Min, Max = component_constans

	if temperature == None:
		Temp_vector = np.array([Temp_vector for Temp_vector in np.arange(Min, Max)])
	elif (Min < np.array(temperature) < Max):
		Temp_vector = np.array(temperature)
	else:
		return print("temperature is not valid")

	#Temperature_enter = [i if Min < np.array(i) < Max else "{0} K is a temperature not valid".format(i) for i in t]
	#Temperature_invalid = [i for i in Temperature_enter if type(i) == str]
	#Temperature_valid = [i for i in Temperature_enter if type(i) != str]

	#print(Temperature_enter)
	#print(Temperature_invalid)
	#print(Temperature_valid)

	print(component_constans)

	if property_thermodynamics == Solid_Density:

		solid_Density = A + B * Temp_vector + C * Temp_vector ** 2 + D * Temp_vector ** 3 + E * Temp_vector **4
		#print(solid_Density)
		return solid_Density

	elif property_thermodynamics == Liquid_Density:

		liquid_Density = A / B ** (1 + (1 - Temp_vector / C) ** D)
		#print("liquid_Density = {0}".format(liquid_Density))
		return liquid_Density

	elif property_thermodynamics == Vapour_Pressure:

		vapour_Pressure = np.exp(A + B/Temp_vector + C * np.log(Temp_vector)+D*Temp_vector **E)
		#print(vapour_Pressure)
		return vapour_Pressure

	elif property_thermodynamics == Heat_of_Vaporization:

		heat_of_Vaporization = A*(1-Tr) ** (B+C*Tr+D*Tr**2)
		#print(heat_of_Vaporization)
		return heat_of_Vaporization

	elif property_thermodynamics == Solid_Heat_Capacity:
		solid_Heat_Capacity = A + B * Temp_vector + C * Temp_vector ** 2 + D * Temp_vector ** 3 + E * Temp_vector ** 4
		#print(solid_Heat_Capacity)
		return solid_Heat_Capacity
	elif property_thermodynamics == Liquid_Heat_Capacity:
		liquid_Heat_Capacity = A ** 2 / (1-Tr) + B-2*A*C*(1-Tr)-A*D*(1-Tr)**2-C**2*(1-Tr)**3/3-C*D*(1-Tr)**4/2-D**2*(1-Tr)**5/5
		#print(liquid_Heat_Capacity)
		return(liquid_Heat_Capacity)
	elif property_thermodynamics == Ideal_Gas_Heat_Capacity:
		ideal_Gas_Heat_Capacity = A+B*(C/Temp_vector/np.sinh(C/Temp_vector))**2+D*(E/Temp_vector/np.cosh(E/Temp_vector))**2
		#print(ideal_Gas_Heat_Capacity)
		return ideal_Gas_Heat_Capacity
	elif property_thermodynamics == Second_Virial_Coefficient:
		second_Virial_Coefficient = A+B/Temp_vector+C/Temp_vector**3+D/Temp_vector**8+E/Temp_vector**9
		#print(second_Virial_Coefficient)
		return second_Virial_Coefficient
	elif property_thermodynamics == Liquid_Viscosity:
		liquid_Viscosity = np.exp(A + B / Temp_vector + C * np.log(Temp_vector) + D * Temp_vector ** E)
		#print(liquid_Viscosity)
		return liquid_Viscosity
	elif property_thermodynamics == Vapour_Viscosity:
		vapour_Viscosity = A*Temp_vector**B/(1+C/Temp_vector+D/Temp_vector**2)
		#print(vapour_Viscosity)
		return vapour_Viscosity
	elif property_thermodynamics == Liquid_Thermal_Conductivity:
		liquid_Thermal_Conductivity = A+B*Temp_vector+C*Temp_vector**2+D*Temp_vector**3+E*Temp_vector**4
		#print(liquid_Thermal_Conductivity)
		return liquid_Thermal_Conductivity
	elif property_thermodynamics == Vapour_Thermal_Conductivity:
		vapour_Thermal_Conductivity = A*Temp_vector**B/(1+C/Temp_vector+D/Temp_vector**2)
		#print(vapour_Thermal_Conductivity)
		return vapour_Thermal_Conductivity
	elif property_thermodynamics == Surface_Tension:
		surface_Tension = A*(1-Tr)**(B+C*Tr+D*Tr*2)
		#print(surface_Tension)
		return surface_Tension

component = 'METHANE'
#component = "ETHANE"
#component = "3-METHYLHEPTANE"
#component = "n-PENTACOSANE"
#component = "ISOBUTANE"
#component = "n-TETRADECANE"

#components = ["METHANE", "n-TETRACOSANE"]

#property_thermodynamics = property_cal(component, Vapour_Pressure, [180.4, 181.4, 185.3])
property_thermodynamics = property_cal(component, Vapour_Pressure, 180.4)
#property_thermodynamics = property_cal(component, Vapour_Pressure)
print(property_thermodynamics)

Min = 30
Max = 90
t = np.array([40, 98.4, 34, 56, 25, 456])

t = np.array([40])

Temperature_enter = [i if Min < np.array(i) < Max else "{0} K is a temperature not valid".format(i) for i in t]
Temperature_invalid = [i for i in Temperature_enter if type(i) == str]
Temperature_valid = [i for i in Temperature_enter if type(i) != str]

print(Temperature_enter)
print(Temperature_invalid)
print(Temperature_valid)

# Tareas

# estado de referencias del estado triple para las propiedades del gas ideal

# graficar la correlación de la densidad de liquido para comparar y sobreponer
# con la parte de la campana calculada con la ecuación de estado para el líquido

# revisar las ecuaciones para utilizar el enfoque de fugacidad de solidó para determinar
# las demás propiedades termodinámicas

# para los graficos de velocidad de sonido tener en cuenta "por ejemplo" las isotermas
# para el caso de una sola fase

# criterio de parar el calculo es que sea hasta el 5% de la presión de vapor de la
# isoterma a la cual se está calculando. En caso de las isotermas super criticas,
# se toma como parametro el 5% de la presión critica

# incluir el calculo del corrimiento de volumen

# dato experiental de líquido saturado para determinar un corrimiento C
# para adicionar dicho corrimiento

# escribir un borrador del artículo

# introduccion#
# arquitectura diagrama
# calculo termodinámicas
# referencias del libro de mollerup, conceptos de algoritmos por ejemmplo la cmapana
# - subsecciópn calculo de la cmapana de saturación,, parametros de inicio
# sección para calcular los otros diagramas, descripción general del calculo
# resultados o ejemplos
# -sub sección pantallazos del notebook para mostrar capacidades de ussuarios
# conclusiones

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import glob
import grp
import json
import logging
import os
import pwd
import shutil
import sys

# TODO(rhallisey): add docstring.
logging.basicConfig()
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)

class ExitingException(Exception):
    def __init__(self, message, exit_code=1):
        super(ExitingException, self).__init__(message)
        self.exit_code = exit_code

class ImmutableConfig(ExitingException):
    pass

class InvalidConfig(ExitingException):
    pass

class MissingRequiredSource(ExitingException):
    pass

class UserNotFound(ExitingException):
    pass

class ConfigFileBadState(ExitingException):
    pass

class ConfigFile(object):

    def __init__(self, source, dest, owner=None, perm=None, optional=False,
                 preserve_properties=False, merge=False):
        self.source = source
        self.dest = dest
        self.owner = owner
        self.perm = perm
        self.optional = optional
        self.merge = merge
        self.preserve_properties = preserve_properties

    def __str__(self):
        return '<ConfigFile source:"{}" dest:"{}">'.format(self.source,
                                                           self.dest)

    def _copy_file(self, source, dest):
        self._delete_path(dest)
        # dest endswith / means copy the <source> to <dest> folder
        LOG.info('Copying %s to %s', source, dest)
        if os.path.islink(source):
            link_target = os.readlink(source)
            os.symlink(link_target, dest)
        else:
            shutil.copy(source, dest)
            self._set_properties(source, dest)

    def _merge_directories(self, source, dest):
        if os.path.isdir(source):
            if os.path.lexists(dest) and not os.path.isdir(dest):
                self._delete_path(dest)
            if not os.path.isdir(dest):
                LOG.info('Creating directory %s', dest)
                os.makedirs(dest)
            self._set_properties(source, dest)

            dir_content = os.listdir(source)
            for to_copy in dir_content:
                self._merge_directories(os.path.join(source, to_copy),
                                        os.path.join(dest, to_copy))
        else:
            self._copy_file(source, dest)

    def _delete_path(self, path):
        if not os.path.lexists(path):
            return
        LOG.info('Deleting %s', path)
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)

    def _create_parent_dirs(self, path):
        parent_path = os.path.dirname(path)
        if not os.path.exists(parent_path):
            os.makedirs(parent_path)

    def _set_properties(self, source, dest):
        if self.preserve_properties:
            self._set_properties_from_file(source, dest)
        else:
            self._set_properties_from_conf(dest)

    def _set_properties_from_file(self, source, dest):
        shutil.copystat(source, dest)
        stat = os.stat(source)
        os.chown(dest, stat.st_uid, stat.st_gid)

    def _set_properties_from_conf(self, path):
        config = {'permissions':
                  [{'owner': self.owner, 'path': path, 'perm': self.perm}]}
        handle_permissions(config)

    def copy(self):

        sources = glob.glob(self.source)

        if not self.optional and not sources:
            raise MissingRequiredSource('%s file is not found' % self.source)
        # skip when there is no sources and optional
        elif self.optional and not sources:
            return

        for source in sources:
            dest = self.dest
            # dest endswith / means copy the <source> into <dest> folder,
            # otherwise means copy the source to dest
            if dest.endswith(os.sep):
                dest = os.path.join(dest, os.path.basename(source))
            if not self.merge:
                self._delete_path(dest)
            self._create_parent_dirs(dest)
            self._merge_directories(source, dest)

    def _cmp_file(self, source, dest):
        # check exsit
        if (os.path.exists(source) and
                not self.optional and
                not os.path.exists(dest)):
            return False
        # check content
        with open(source) as f1, open(dest) as f2:
            if f1.read() != f2.read():
                LOG.error('The content of source file(%s) and'
                          ' dest file(%s) are not equal.', source, dest)
                return False
        # check perm
        file_stat = os.stat(dest)
        actual_perm = oct(file_stat.st_mode)[-4:]
        if self.perm != actual_perm:
            LOG.error('Dest file does not have expected perm: %s, actual: %s',
                      self.perm, actual_perm)
            return False
        # check owner
        desired_user, desired_group = user_group(self.owner)
        actual_user = pwd.getpwuid(file_stat.st_uid)
        if actual_user.pw_name != desired_user:
            LOG.error('Dest file does not have expected user: %s,'
                      ' actual: %s ', desired_user, actual_user.pw_name)
            return False
        actual_group = grp.getgrgid(file_stat.st_gid)
        if actual_group.gr_name != desired_group:
            LOG.error('Dest file does not have expected group: %s,'
                      ' actual: %s ', desired_group, actual_group.gr_name)
            return False
        return True

    def _cmp_dir(self, source, dest):
        for root, dirs, files in os.walk(source):
            for dir_ in dirs:
                full_path = os.path.join(root, dir_)
                dest_full_path = os.path.join(dest, os.path.relpath(source,
                                                                    full_path))
                dir_stat = os.stat(dest_full_path)
                actual_perm = oct(dir_stat.st_mode)[-4:]
                if self.perm != actual_perm:
                    LOG.error('Dest dir does not have expected perm: %s,'
                              ' actual %s', self.perm, actual_perm)
                    return False
            for file_ in files:
                full_path = os.path.join(root, file_)
                dest_full_path = os.path.join(dest, os.path.relpath(source,
                                                                    full_path))
                if not self._cmp_file(full_path, dest_full_path):
                    return False
        return True

    def check(self):
        bad_state_files = []
        sources = glob.glob(self.source)

        if not sources and not self.optional:
            raise MissingRequiredSource('%s file is not found' % self.source)
        elif self.optional and not sources:
            return

        for source in sources:
            dest = self.dest
            # dest endswith / means copy the <source> into <dest> folder,
            # otherwise means copy the source to dest
            if dest.endswith(os.sep):
                dest = os.path.join(dest, os.path.basename(source))
            if os.path.isdir(source) and not self._cmp_dir(source, dest):
                bad_state_files.append(source)
            elif not self._cmp_file(source, dest):
                bad_state_files.append(source)
        if len(bad_state_files) != 0:
            msg = 'Following files are in bad state: %s' % bad_state_files
            raise ConfigFileBadState(msg)

def validate_config(config):
    required_keys = {'source', 'dest'}

    if 'command' not in config:
        raise InvalidConfig('Config is missing required "command" key')

    # Validate config sections
    for data in config.get('config_files', list()):
        # Verify required keys exist.
        if not data.viewkeys() >= required_keys:
            message = 'Config is missing required keys: %s' % required_keys
            raise InvalidConfig(message)
        if ('owner' not in data or 'perm' not in data) \
                and not data.get('preserve_properties', False):
            raise InvalidConfig(
                'Config needs preserve_properties or owner and perm')

def validate_source(data):
    source = data.get('source')

    # Only check existence if no wildcard found
    if '*' not in source:
        if not os.path.exists(source):
            if data.get('optional'):
                LOG.info("%s does not exist, but is not required", source)
                return False
            else:
                raise MissingRequiredSource(
                    "The source to copy does not exist: %s" % source)

    return True

def load_config():
    def load_from_env():
        config_raw = os.environ.get("KOLLA_CONFIG")
        if config_raw is None:
            return None

        # Attempt to read config
        try:
            return json.loads(config_raw)
        except ValueError:
            raise InvalidConfig('Invalid json for Kolla config')

    def load_from_file():
        config_file = os.environ.get("KOLLA_CONFIG_FILE")
        if not config_file:
            config_file = '/var/lib/kolla/config_files/config.json'
        LOG.info("Loading config file at %s", config_file)

        # Attempt to read config file
        with open(config_file) as f:
            try:
                return json.load(f)
            except ValueError:
                raise InvalidConfig(
                    "Invalid json file found at %s" % config_file)
            except IOError as e:
                raise InvalidConfig(
                    "Could not read file %s: %r" % (config_file, e))

    config = load_from_env()
    if config is None:
        config = load_from_file()

    LOG.info('Validating config file')
    validate_config(config)
    return config

def copy_config(config):
    if 'config_files' in config:
        LOG.info('Copying service configuration files')
        for data in config['config_files']:
            config_file = ConfigFile(**data)
            config_file.copy()
    else:
        LOG.debug('No files to copy found in config')

    LOG.info('Writing out command to execute')
    LOG.debug("Command is: %s", config['command'])
    # The value from the 'command' key will be written to '/run_command'
    with open('/run_command', 'w+') as f:
        f.write(config['command'])

def user_group(owner):
    if ':' in owner:
        user, group = owner.split(':', 1)
        if not group:
            group = user
    else:
        user, group = owner, owner
    return user, group

def handle_permissions(config):
    for permission in config.get('permissions', list()):
        path = permission.get('path')
        owner = permission.get('owner')
        recurse = permission.get('recurse', False)
        perm = permission.get('perm')

        desired_user, desired_group = user_group(owner)
        uid = pwd.getpwnam(desired_user).pw_uid
        gid = grp.getgrnam(desired_group).gr_gid

        def set_perms(path, uid, gid, perm):
            LOG.info('Setting permission for %s', path)
            if not os.path.exists(path):
                LOG.warning('%s does not exist', path)
                return

            try:
                os.chown(path, uid, gid)
            except OSError:
                LOG.exception('Failed to change ownership of %s to %s:%s',
                              path, uid, gid)

            if perm:
                # NOTE(Jeffrey4l): py3 need '0oXXX' format for octal literals,
                # and py2 support such format too.
                if len(perm) == 4 and perm[1] != 'o':
                    perm = ''.join([perm[:1], 'o', perm[1:]])
                perm = int(perm, base=0)

                try:
                    os.chmod(path, perm)
                except OSError:
                    LOG.exception('Failed to set permission of %s to %s',
                                  path, perm)

        for dest in glob.glob(path):
            set_perms(dest, uid, gid, perm)
            if recurse and os.path.isdir(dest):
                for root, dirs, files in os.walk(dest):
                    for dir_ in dirs:
                        set_perms(os.path.join(root, dir_), uid, gid, perm)
                    for file_ in files:
                        set_perms(os.path.join(root, file_), uid, gid, perm)

def execute_config_strategy(config):
    config_strategy = os.environ.get("KOLLA_CONFIG_STRATEGY")
    LOG.info("Kolla config strategy set to: %s", config_strategy)
    if config_strategy == "COPY_ALWAYS":
        copy_config(config)
        handle_permissions(config)
    elif config_strategy == "COPY_ONCE":
        if os.path.exists('/configured'):
            raise ImmutableConfig(
                "The config strategy prevents copying new configs",
                exit_code=0)
        else:
            copy_config(config)
            handle_permissions(config)
            os.mknod('/configured')
    else:
        raise InvalidConfig('KOLLA_CONFIG_STRATEGY is not set properly')

def execute_config_check(config):
    for data in config['config_files']:
        config_file = ConfigFile(**data)
        config_file.check()

def main():
    try:
        parser = argparse.ArgumentParser()
        parser.add_argument('--check',
                            action='store_true',
                            required=False,
                            help='Check whether the configs changed')
        args = parser.parse_args()
        config = load_config()

        if args.check:
            execute_config_check(config)
        else:
            execute_config_strategy(config)
    except ExitingException as e:
        LOG.error("%s: %s", e.__class__.__name__, e)
        return e.exit_code
    except Exception:
        LOG.exception('Unexpected error:')
        return 2
    return 0

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Only copy symlinks when merge and preserve_properties are set<commit_after>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import glob
import grp
import json
import logging
import os
import pwd
import shutil
import sys

# TODO(rhallisey): add docstring.
logging.basicConfig()
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)

class ExitingException(Exception):
    def __init__(self, message, exit_code=1):
        super(ExitingException, self).__init__(message)
        self.exit_code = exit_code

class ImmutableConfig(ExitingException):
    pass

class InvalidConfig(ExitingException):
    pass

class MissingRequiredSource(ExitingException):
    pass

class UserNotFound(ExitingException):
    pass

class ConfigFileBadState(ExitingException):
    pass

class ConfigFile(object):

    def __init__(self, source, dest, owner=None, perm=None, optional=False,
                 preserve_properties=False, merge=False):
        self.source = source
        self.dest = dest
        self.owner = owner
        self.perm = perm
        self.optional = optional
        self.merge = merge
        self.preserve_properties = preserve_properties

    def __str__(self):
        return '<ConfigFile source:"{}" dest:"{}">'.format(self.source,
                                                           self.dest)

    def _copy_file(self, source, dest):
        self._delete_path(dest)
        # dest endswith / means copy the <source> to <dest> folder
        LOG.info('Copying %s to %s', source, dest)
        if self.merge and self.preserve_properties and os.path.islink(source):
            link_target = os.readlink(source)
            os.symlink(link_target, dest)
        else:
            shutil.copy(source, dest)
            self._set_properties(source, dest)

    def _merge_directories(self, source, dest):
        if os.path.isdir(source):
            if os.path.lexists(dest) and not os.path.isdir(dest):
                self._delete_path(dest)
            if not os.path.isdir(dest):
                LOG.info('Creating directory %s', dest)
                os.makedirs(dest)
            self._set_properties(source, dest)

            dir_content = os.listdir(source)
            for to_copy in dir_content:
                self._merge_directories(os.path.join(source, to_copy),
                                        os.path.join(dest, to_copy))
        else:
            self._copy_file(source, dest)

    def _delete_path(self, path):
        if not os.path.lexists(path):
            return
        LOG.info('Deleting %s', path)
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.remove(path)

    def _create_parent_dirs(self, path):
        parent_path = os.path.dirname(path)
        if not os.path.exists(parent_path):
            os.makedirs(parent_path)

    def _set_properties(self, source, dest):
        if self.preserve_properties:
            self._set_properties_from_file(source, dest)
        else:
            self._set_properties_from_conf(dest)

    def _set_properties_from_file(self, source, dest):
        shutil.copystat(source, dest)
        stat = os.stat(source)
        os.chown(dest, stat.st_uid, stat.st_gid)

    def _set_properties_from_conf(self, path):
        config = {'permissions':
                  [{'owner': self.owner, 'path': path, 'perm': self.perm}]}
        handle_permissions(config)

    def copy(self):

        sources = glob.glob(self.source)

        if not self.optional and not sources:
            raise MissingRequiredSource('%s file is not found' % self.source)
        # skip when there is no sources and optional
        elif self.optional and not sources:
            return

        for source in sources:
            dest = self.dest
            # dest endswith / means copy the <source> into <dest> folder,
            # otherwise means copy the source to dest
            if dest.endswith(os.sep):
                dest = os.path.join(dest, os.path.basename(source))
            if not self.merge:
                self._delete_path(dest)
            self._create_parent_dirs(dest)
            self._merge_directories(source, dest)

    def _cmp_file(self, source, dest):
        # check exsit
        if (os.path.exists(source) and
                not self.optional and
                not os.path.exists(dest)):
            return False
        # check content
        with open(source) as f1, open(dest) as f2:
            if f1.read() != f2.read():
                LOG.error('The content of source file(%s) and'
                          ' dest file(%s) are not equal.', source, dest)
                return False
        # check perm
        file_stat = os.stat(dest)
        actual_perm = oct(file_stat.st_mode)[-4:]
        if self.perm != actual_perm:
            LOG.error('Dest file does not have expected perm: %s, actual: %s',
                      self.perm, actual_perm)
            return False
        # check owner
        desired_user, desired_group = user_group(self.owner)
        actual_user = pwd.getpwuid(file_stat.st_uid)
        if actual_user.pw_name != desired_user:
            LOG.error('Dest file does not have expected user: %s,'
                      ' actual: %s ', desired_user, actual_user.pw_name)
            return False
        actual_group = grp.getgrgid(file_stat.st_gid)
        if actual_group.gr_name != desired_group:
            LOG.error('Dest file does not have expected group: %s,'
                      ' actual: %s ', desired_group, actual_group.gr_name)
            return False
        return True

    def _cmp_dir(self, source, dest):
        for root, dirs, files in os.walk(source):
            for dir_ in dirs:
                full_path = os.path.join(root, dir_)
                dest_full_path = os.path.join(dest, os.path.relpath(source,
                                                                    full_path))
                dir_stat = os.stat(dest_full_path)
                actual_perm = oct(dir_stat.st_mode)[-4:]
                if self.perm != actual_perm:
                    LOG.error('Dest dir does not have expected perm: %s,'
                              ' actual %s', self.perm, actual_perm)
                    return False
            for file_ in files:
                full_path = os.path.join(root, file_)
                dest_full_path = os.path.join(dest, os.path.relpath(source,
                                                                    full_path))
                if not self._cmp_file(full_path, dest_full_path):
                    return False
        return True

    def check(self):
        bad_state_files = []
        sources = glob.glob(self.source)

        if not sources and not self.optional:
            raise MissingRequiredSource('%s file is not found' % self.source)
        elif self.optional and not sources:
            return

        for source in sources:
            dest = self.dest
            # dest endswith / means copy the <source> into <dest> folder,
            # otherwise means copy the source to dest
            if dest.endswith(os.sep):
                dest = os.path.join(dest, os.path.basename(source))
            if os.path.isdir(source) and not self._cmp_dir(source, dest):
                bad_state_files.append(source)
            elif not self._cmp_file(source, dest):
                bad_state_files.append(source)
        if len(bad_state_files) != 0:
            msg = 'Following files are in bad state: %s' % bad_state_files
            raise ConfigFileBadState(msg)

def validate_config(config):
    required_keys = {'source', 'dest'}

    if 'command' not in config:
        raise InvalidConfig('Config is missing required "command" key')

    # Validate config sections
    for data in config.get('config_files', list()):
        # Verify required keys exist.
        if not data.viewkeys() >= required_keys:
            message = 'Config is missing required keys: %s' % required_keys
            raise InvalidConfig(message)
        if ('owner' not in data or 'perm' not in data) \
                and not data.get('preserve_properties', False):
            raise InvalidConfig(
                'Config needs preserve_properties or owner and perm')

def validate_source(data):
    source = data.get('source')

    # Only check existence if no wildcard found
    if '*' not in source:
        if not os.path.exists(source):
            if data.get('optional'):
                LOG.info("%s does not exist, but is not required", source)
                return False
            else:
                raise MissingRequiredSource(
                    "The source to copy does not exist: %s" % source)

    return True

def load_config():
    def load_from_env():
        config_raw = os.environ.get("KOLLA_CONFIG")
        if config_raw is None:
            return None

        # Attempt to read config
        try:
            return json.loads(config_raw)
        except ValueError:
            raise InvalidConfig('Invalid json for Kolla config')

    def load_from_file():
        config_file = os.environ.get("KOLLA_CONFIG_FILE")
        if not config_file:
            config_file = '/var/lib/kolla/config_files/config.json'
        LOG.info("Loading config file at %s", config_file)

        # Attempt to read config file
        with open(config_file) as f:
            try:
                return json.load(f)
            except ValueError:
                raise InvalidConfig(
                    "Invalid json file found at %s" % config_file)
            except IOError as e:
                raise InvalidConfig(
                    "Could not read file %s: %r" % (config_file, e))

    config = load_from_env()
    if config is None:
        config = load_from_file()

    LOG.info('Validating config file')
    validate_config(config)
    return config

def copy_config(config):
    if 'config_files' in config:
        LOG.info('Copying service configuration files')
        for data in config['config_files']:
            config_file = ConfigFile(**data)
            config_file.copy()
    else:
        LOG.debug('No files to copy found in config')

    LOG.info('Writing out command to execute')
    LOG.debug("Command is: %s", config['command'])
    # The value from the 'command' key will be written to '/run_command'
    with open('/run_command', 'w+') as f:
        f.write(config['command'])

def user_group(owner):
    if ':' in owner:
        user, group = owner.split(':', 1)
        if not group:
            group = user
    else:
        user, group = owner, owner
    return user, group

def handle_permissions(config):
    for permission in config.get('permissions', list()):
        path = permission.get('path')
        owner = permission.get('owner')
        recurse = permission.get('recurse', False)
        perm = permission.get('perm')

        desired_user, desired_group = user_group(owner)
        uid = pwd.getpwnam(desired_user).pw_uid
        gid = grp.getgrnam(desired_group).gr_gid

        def set_perms(path, uid, gid, perm):
            LOG.info('Setting permission for %s', path)
            if not os.path.exists(path):
                LOG.warning('%s does not exist', path)
                return

            try:
                os.chown(path, uid, gid)
            except OSError:
                LOG.exception('Failed to change ownership of %s to %s:%s',
                              path, uid, gid)

            if perm:
                # NOTE(Jeffrey4l): py3 need '0oXXX' format for octal literals,
                # and py2 support such format too.
                if len(perm) == 4 and perm[1] != 'o':
                    perm = ''.join([perm[:1], 'o', perm[1:]])
                perm = int(perm, base=0)

                try:
                    os.chmod(path, perm)
                except OSError:
                    LOG.exception('Failed to set permission of %s to %s',
                                  path, perm)

        for dest in glob.glob(path):
            set_perms(dest, uid, gid, perm)
            if recurse and os.path.isdir(dest):
                for root, dirs, files in os.walk(dest):
                    for dir_ in dirs:
                        set_perms(os.path.join(root, dir_), uid, gid, perm)
                    for file_ in files:
                        set_perms(os.path.join(root, file_), uid, gid, perm)

def execute_config_strategy(config):
    config_strategy = os.environ.get("KOLLA_CONFIG_STRATEGY")
    LOG.info("Kolla config strategy set to: %s", config_strategy)
    if config_strategy == "COPY_ALWAYS":
        copy_config(config)
        handle_permissions(config)
    elif config_strategy == "COPY_ONCE":
        if os.path.exists('/configured'):
            raise ImmutableConfig(
                "The config strategy prevents copying new configs",
                exit_code=0)
        else:
            copy_config(config)
            handle_permissions(config)
            os.mknod('/configured')
    else:
        raise InvalidConfig('KOLLA_CONFIG_STRATEGY is not set properly')

def execute_config_check(config):
    for data in config['config_files']:
        config_file = ConfigFile(**data)
        config_file.check()

def main():
    try:
        parser = argparse.ArgumentParser()
        parser.add_argument('--check',
                            action='store_true',
                            required=False,
                            help='Check whether the configs changed')
        args = parser.parse_args()
        config = load_config()

        if args.check:
            execute_config_check(config)
        else:
            execute_config_strategy(config)
    except ExitingException as e:
        LOG.error("%s: %s", e.__class__.__name__, e)
        return e.exit_code
    except Exception:
        LOG.exception('Unexpected error:')
        return 2
    return 0

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>