<commit_before># -*- coding: utf-8 -*-

"""
Provide classes and functions to interact with Bluetooth LE devices.
"""

import struct
import socket
import contextlib
import binascii
import datetime
import logging
import pathlib

from typing import Tuple, Optional, Set, Any, Type

from hci_constants import ScanType, AddressType, FilterPolicy, HCI_MAX_EVENT_SIZE, \
    PacketType, Event, LeEvent, AdType, CompanyId, DiscoveryType, ALL_16BIT_SERVICES
from bluez_ffi import hci_get_route, hci_le_set_scan_parameters, hci_le_set_scan_enable

class Advertisement(object):
    def analyse(self) -> None:
        if self.type is AdType.ManufacturerSpecificData:
            vendor, data = struct.unpack("<H{}s".format(len(self.data) - 2), self.data)
            self.vendor = CompanyId(vendor)
        elif self.type in (AdType.CompleteListOf16BitServiceClassUUIDs, AdType.IncompleteListOf16BitServiceClassUUIDs):
            service_count = len(self.data) // 2
            self.services = set(struct.unpack("<{}H".format(service_count), self.data))

    def __init__(self,
                 ad_type: AdType,
                 ad_data: bytes,
                 vendor: Optional[CompanyId] = None,
                 svcs: Set[int] = set()) -> None:
        self.type = ad_type
        self.data = ad_data
        self.vendor = vendor
        self.services = svcs

    def __hash__(self) -> int:
        return hash(self.type) + hash(self.data)

    def __str__(self) -> str:
        if self.type is AdType.ManufacturerSpecificData and self.vendor is not None:
            return "[{}:{}:{}]".format(self.vendor.name, self.type.name, binascii.hexlify(self.data).decode("ascii"))
        elif self.type in (AdType.CompleteListOf16BitServiceClassUUIDs, AdType.IncompleteListOf16BitServiceClassUUIDs) and len(self.services) > 0:
            return "[{}:{}]".format(self.type.name, [ALL_16BIT_SERVICES.get(s, s) for s in self.services])
        else:
            return "[{}:{}]".format(self.type.name, binascii.hexlify(self.data).decode("ascii"))

    def __repr__(self) -> str:
        return str(self)

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, Advertisement):
            return self.type is other.type and self.data == other.data
        else:
            return NotImplemented

class AdvertisementPacket(object):
    def __init__(self,
                 dtype: DiscoveryType,
                 atype: AddressType,
                 addr: bytes,
                 rssi: int,
                 ads: Tuple[Advertisement, ...]
                 ) -> None:
        self.discovery_type = dtype
        self.address_type = atype
        self.address = addr
        self.rssi = rssi,
        self.advertisements = ads

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, AdvertisementPacket):
            return self.discovery_type is other.discovery_type \
                and self.address_type is other.address_type \
                and self.address == other.address \
                and self.advertisements == other.advertisements
        else:
            return NotImplemented

    @classmethod
    def create(cls, packet: bytes, offset: int = 0) -> "AdvertisementPacket":
        report_header = struct.unpack_from(">BB7sB", packet, offset)
        discovery_type = DiscoveryType(report_header[0])
        address_type = AddressType(report_header[1])
        address = report_header[2]
        ad_data_length = report_header[3]

        report_body = struct.unpack_from(">{}sb".format(ad_data_length), packet, offset + 10)
        report_data = report_body[0]
        rssi = report_body[1]

        advertisements = list()

        i = 0
        while i < len(report_data):
            ad_length = report_data[i]
            ad_type = AdType(report_data[i + 1])
            advertisements.append(
                Advertisement(ad_type, report_data[i + 2:i + 1 + ad_length])
            )
            i += ad_length + 1

        return cls(
            discovery_type,
            address_type,
            address,
            rssi,
            tuple(advertisements)
        )

class Device(object):
    def union(self, other: "Device") -> bool:
        if self.vendor is other.vendor:
            new_addrs = self.addresses | other.addresses
            new_svcs = self.services | other.services
            new_ads = self.advertisements | other.advertisements

            if new_addrs == self.addresses and new_svcs == self.services and new_ads == self.advertisements:
                return False
            else:
                self.addresses = new_addrs
                self.services = new_svcs
                self.advertisements = new_ads
                return True
        else:
            raise ValueError("The two devices do not have the same vendor.")

    def __init__(self,
                 dtype: DiscoveryType,
                 atype: AddressType,
                 addrs: Set[bytes],
                 vendor: Optional[CompanyId] = None,
                 svcs: set = set(),
                 ads: Set[Advertisement] = set()) -> None:
        self.discovery_type = dtype
        self.address_type = atype
        self.addresses = addrs
        self.vendor = vendor
        self.services = svcs
        self.advertisements = ads

    def __str__(self) -> str:
        addr_str = {binascii.hexlify(e).decode("ascii") for e in self.addresses}

        return "<Device: {}>".format(addr_str)

    def __repr__(self):
        if self.vendor is None:
            vendor_str = "Unknown"
        else:
            vendor_str = self.vendor.name

        addr_str = {binascii.hexlify(e).decode("ascii") for e in self.addresses}

        return "<{}: {}, {}>".format(vendor_str, addr_str, self.advertisements)

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, Device):
            shared_addrs = len(self.addresses & other.addresses) > 0
            same_vendor = self.vendor is other.vendor
            same_services = len(self.services) > 0 and self.services >= other.services
            same_advertisements = len(self.advertisements) > 0 and self.advertisements >= other.advertisements
            return shared_addrs or (same_vendor and (same_services or same_advertisements))
        else:
            return NotImplemented

class HciParser(object):
    """
    HciParser implements a context manager that can scan for BTLE devices,
    and maintain a database of the recognised devices.
    """
    def run(self) -> None:
        """
        Run the main discovery loop.
        """
        if self._sock is not None:
            iterations: int = 0
            while True:
                packet = self._sock.recv(HCI_MAX_EVENT_SIZE)
                advert = self._parse_packet(packet)
                if advert is not None:
                    device = self._analyse_advert(advert)
                    self._register_device(device)

                # if iterations % 50 == 0:
                #     self._backup_registry()

                iterations += 1
        else:
            raise ValueError("You must use HciParser as context manager before calling HciParser.run().")

    def _find_device(self, device: Device) -> Optional[int]:
        for idx, rd in enumerate(self.registry):
            if device == rd:
                return idx
        else:
            return None

    def _register_device(self, device: Device) -> None:
        if len(self.registry) > 0:
            idx = self._find_device(device)
            if idx is None:
                self._log.info("New device: {}".format(device))
                self._log.debug("Full info: {!r}".format(device))
                self.registry.append(device)
            elif self.registry[idx].union(device):
                self._log.info("Device update: {}".format(self.registry[idx]))
                self._log.debug("Full info: {!r}".format(self.registry[idx]))
        else:
            self._log.info("New device: {}".format(device))
            self._log.debug("Full info: {!r}".format(device))
            self.registry.append(device)

    def _backup_registry(self) -> None:
        if self.backup_path is not None:
            pass

    def _parse_packet(self, packet: bytes) -> Optional[AdvertisementPacket]:
        """
        Parse the next packet in the socket.
        """
        self._log.debug("Raw: {}".format(binascii.hexlify(packet).decode("ascii")))
        if packet[0] == PacketType.Event and packet[1] == Event.Le and packet[3] == LeEvent.LeAdvertisingReport:
            return AdvertisementPacket.create(packet, 4)

        return None

    def _analyse_advert(self, advert: AdvertisementPacket) -> Device:
        """
        Analyse the content of the specified advertisement packet.
        """
        device = Device(
            advert.discovery_type,
            advert.address_type,
            {advert.address}
        )
        for ad_block in advert.advertisements:
            ad_block.analyse()
            device.vendor = ad_block.vendor
            device.services |= ad_block.services
            device.advertisements.add(ad_block)

        return device

    def __init__(self, bkp_path: Optional[pathlib.Path] = None) -> None:
        self.backup_path: Optional[pathlib.Path] = bkp_path
        self.registry: list = list()
        self._sock: Optional[socket.socket] = None
        self._log: logging.Logger = logging.getLogger(__name__)

    def __enter__(self) -> "HciParser":
        required_constants = ("AF_BLUETOOTH", "BTPROTO_HCI", "SOL_HCI", "HCI_FILTER")
        if not all(hasattr(socket, a) for a in required_constants):
            raise RuntimeError("The current system does not support Unix Bluetooth sockets.")

        dev_id = hci_get_route()
        self._sock = socket.socket(
            socket.AF_BLUETOOTH,
            socket.SOCK_RAW,
            socket.BTPROTO_HCI
        )
        self._sock.bind((dev_id,))

        hci_le_set_scan_parameters(
            self._sock,
            ScanType.PassiveScan,
            0x0010,
            0x0010,
            AddressType.PublicDeviceAddress,
            FilterPolicy.UndirectedAdsOnly,
            2000
        )

        hci_filter = struct.pack(
            "<IQH",
            0x00000010,
            0x4000000000000000,
            0x0000
        )
        self._sock.setsockopt(socket.SOL_HCI, socket.HCI_FILTER, hci_filter)

        hci_le_set_scan_enable(self._sock, True, False, 2000)

        return self

    def __exit__(self,
                 exc_type: Optional[Type[Exception]],
                 exc_val: Optional[Exception],
                 trcbak: Any) -> bool:
        if self._sock is not None:
            hci_le_set_scan_enable(self._sock, False, False, 2000)
            self._sock.close()
            self._sock = None
        return False
<commit_msg>[+] Added periodic registry backup to a Pickle file.<commit_after># -*- coding: utf-8 -*-

"""
Provide classes and functions to interact with Bluetooth LE devices.
"""

import struct
import socket
import binascii
import logging
import pathlib
import pickle

from typing import Tuple, Optional, Set, Any, Type

from hci_constants import ScanType, AddressType, FilterPolicy, HCI_MAX_EVENT_SIZE, \
    PacketType, Event, LeEvent, AdType, CompanyId, DiscoveryType, ALL_16BIT_SERVICES
from bluez_ffi import hci_get_route, hci_le_set_scan_parameters, hci_le_set_scan_enable

class Advertisement(object):
    def analyse(self) -> None:
        if self.type is AdType.ManufacturerSpecificData:
            vendor, data = struct.unpack("<H{}s".format(len(self.data) - 2), self.data)
            self.vendor = CompanyId(vendor)
        elif self.type in (AdType.CompleteListOf16BitServiceClassUUIDs, AdType.IncompleteListOf16BitServiceClassUUIDs):
            service_count = len(self.data) // 2
            self.services = set(struct.unpack("<{}H".format(service_count), self.data))

    def __init__(self,
                 ad_type: AdType,
                 ad_data: bytes,
                 vendor: Optional[CompanyId] = None,
                 svcs: Set[int] = set()) -> None:
        self.type = ad_type
        self.data = ad_data
        self.vendor = vendor
        self.services = svcs

    def __hash__(self) -> int:
        return hash(self.type) + hash(self.data)

    def __str__(self) -> str:
        if self.type is AdType.ManufacturerSpecificData and self.vendor is not None:
            return "[{}:{}:{}]".format(self.vendor.name, self.type.name, binascii.hexlify(self.data).decode("ascii"))
        elif self.type in (AdType.CompleteListOf16BitServiceClassUUIDs, AdType.IncompleteListOf16BitServiceClassUUIDs) and len(self.services) > 0:
            return "[{}:{}]".format(self.type.name, [ALL_16BIT_SERVICES.get(s, s) for s in self.services])
        else:
            return "[{}:{}]".format(self.type.name, binascii.hexlify(self.data).decode("ascii"))

    def __repr__(self) -> str:
        return str(self)

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, Advertisement):
            return self.type is other.type and self.data == other.data
        else:
            return NotImplemented

class AdvertisementPacket(object):
    def __init__(self,
                 dtype: DiscoveryType,
                 atype: AddressType,
                 addr: bytes,
                 rssi: int,
                 ads: Tuple[Advertisement, ...]
                 ) -> None:
        self.discovery_type = dtype
        self.address_type = atype
        self.address = addr
        self.rssi = rssi,
        self.advertisements = ads

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, AdvertisementPacket):
            return self.discovery_type is other.discovery_type \
                and self.address_type is other.address_type \
                and self.address == other.address \
                and self.advertisements == other.advertisements
        else:
            return NotImplemented

    @classmethod
    def create(cls, packet: bytes, offset: int = 0) -> "AdvertisementPacket":
        report_header = struct.unpack_from(">BB7sB", packet, offset)
        discovery_type = DiscoveryType(report_header[0])
        address_type = AddressType(report_header[1])
        address = report_header[2]
        ad_data_length = report_header[3]

        report_body = struct.unpack_from(">{}sb".format(ad_data_length), packet, offset + 10)
        report_data = report_body[0]
        rssi = report_body[1]

        advertisements = list()

        i = 0
        while i < len(report_data):
            ad_length = report_data[i]
            ad_type = AdType(report_data[i + 1])
            advertisements.append(
                Advertisement(ad_type, report_data[i + 2:i + 1 + ad_length])
            )
            i += ad_length + 1

        return cls(
            discovery_type,
            address_type,
            address,
            rssi,
            tuple(advertisements)
        )

class Device(object):
    def union(self, other: "Device") -> bool:
        if self.vendor is other.vendor:
            new_addrs = self.addresses | other.addresses
            new_svcs = self.services | other.services
            new_ads = self.advertisements | other.advertisements

            if new_addrs == self.addresses and new_svcs == self.services and new_ads == self.advertisements:
                return False
            else:
                self.addresses = new_addrs
                self.services = new_svcs
                self.advertisements = new_ads
                return True
        else:
            raise ValueError("The two devices do not have the same vendor.")

    def __init__(self,
                 dtype: DiscoveryType,
                 atype: AddressType,
                 addrs: Set[bytes],
                 vendor: Optional[CompanyId] = None,
                 svcs: set = set(),
                 ads: Set[Advertisement] = set()) -> None:
        self.discovery_type = dtype
        self.address_type = atype
        self.addresses = addrs
        self.vendor = vendor
        self.services = svcs
        self.advertisements = ads

    def __str__(self) -> str:
        addr_str = {binascii.hexlify(e).decode("ascii") for e in self.addresses}

        return "<Device: {}>".format(addr_str)

    def __repr__(self):
        if self.vendor is None:
            vendor_str = "Unknown"
        else:
            vendor_str = self.vendor.name

        addr_str = {binascii.hexlify(e).decode("ascii") for e in self.addresses}

        return "<{}: {}, {}>".format(vendor_str, addr_str, self.advertisements)

    def __eq__(self, other: Any) -> bool:
        if isinstance(other, Device):
            shared_addrs = len(self.addresses & other.addresses) > 0
            same_vendor = self.vendor is other.vendor
            same_services = len(self.services) > 0 and self.services >= other.services
            same_advertisements = len(self.advertisements) > 0 and self.advertisements >= other.advertisements
            return shared_addrs or (same_vendor and (same_services or same_advertisements))
        else:
            return NotImplemented

class HciParser(object):
    """
    HciParser implements a context manager that can scan for BTLE devices,
    and maintain a database of the recognised devices.
    """
    def run(self) -> None:
        """
        Run the main discovery loop.
        """
        if self._sock is not None:
            iterations: int = 0
            while True:
                packet = self._sock.recv(HCI_MAX_EVENT_SIZE)
                advert = self._parse_packet(packet)
                if advert is not None:
                    device = self._analyse_advert(advert)
                    self._register_device(device)

                if iterations % self.backup_interval == 0:
                    self._backup_registry()

                iterations += 1
        else:
            raise ValueError("You must use HciParser as context manager before calling HciParser.run().")

    def _find_device(self, device: Device) -> Optional[int]:
        for idx, rd in enumerate(self.registry):
            if device == rd:
                return idx
        else:
            return None

    def _register_device(self, device: Device) -> None:
        if len(self.registry) > 0:
            idx = self._find_device(device)
            if idx is None:
                self._log.info("New device: {}".format(device))
                self._log.debug("Full info: {!r}".format(device))
                self.registry.append(device)
            elif self.registry[idx].union(device):
                self._log.info("Device update: {}".format(self.registry[idx]))
                self._log.debug("Full info: {!r}".format(self.registry[idx]))
        else:
            self._log.info("New device: {}".format(device))
            self._log.debug("Full info: {!r}".format(device))
            self.registry.append(device)

    def _backup_registry(self) -> None:
        if self.backup_path is not None:
            with self.backup_path.open("wb") as f:
                pickle.dump(self.registry, f, pickle.HIGHEST_PROTOCOL)

    def _parse_packet(self, packet: bytes) -> Optional[AdvertisementPacket]:
        """
        Parse the next packet in the socket.
        """
        if packet[0] == PacketType.Event and packet[1] == Event.Le and packet[3] == LeEvent.LeAdvertisingReport:
            return AdvertisementPacket.create(packet, 4)

        return None

    def _analyse_advert(self, advert: AdvertisementPacket) -> Device:
        """
        Analyse the content of the specified advertisement packet.
        """
        device = Device(
            advert.discovery_type,
            advert.address_type,
            {advert.address}
        )
        for ad_block in advert.advertisements:
            ad_block.analyse()
            device.vendor = ad_block.vendor
            device.services |= ad_block.services
            device.advertisements.add(ad_block)

        return device

    def __init__(self, bkp_path: Optional[pathlib.Path] = None, backup_interval: int = 50) -> None:
        self.backup_path: Optional[pathlib.Path] = bkp_path
        self.backup_interval: int = backup_interval
        self.registry: list = list()
        self._sock: Optional[socket.socket] = None
        self._log: logging.Logger = logging.getLogger(__name__)

    def __enter__(self) -> "HciParser":
        required_constants = ("AF_BLUETOOTH", "BTPROTO_HCI", "SOL_HCI", "HCI_FILTER")
        if not all(hasattr(socket, a) for a in required_constants):
            raise RuntimeError("The current system does not support Unix Bluetooth sockets.")

        dev_id = hci_get_route()
        self._sock = socket.socket(
            socket.AF_BLUETOOTH,
            socket.SOCK_RAW,
            socket.BTPROTO_HCI
        )
        self._sock.bind((dev_id,))

        hci_le_set_scan_parameters(
            self._sock,
            ScanType.PassiveScan,
            0x0010,
            0x0010,
            AddressType.PublicDeviceAddress,
            FilterPolicy.UndirectedAdsOnly,
            2000
        )

        hci_filter = struct.pack(
            "<IQH",
            0x00000010,
            0x4000000000000000,
            0x0000
        )
        self._sock.setsockopt(socket.SOL_HCI, socket.HCI_FILTER, hci_filter)

        hci_le_set_scan_enable(self._sock, True, False, 2000)

        return self

    def __exit__(self,
                 exc_type: Optional[Type[Exception]],
                 exc_val: Optional[Exception],
                 trcbak: Any) -> bool:
        if self._sock is not None:
            hci_le_set_scan_enable(self._sock, False, False, 2000)
            self._sock.close()
            self._sock = None
        return False
<|endoftext|>


<commit_before># Copyright 2018 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Functions to load skill data such as intents and regular expressions."""

import collections
import csv
import re
from os import walk
from os.path import splitext, join

from mycroft.util.format import expand_options

def read_vocab_file(path):
    """ Read voc file.

        This reads a .voc file, stripping out empty lines comments and expand
        parentheses. It retruns each line as a list of all expanded
        alternatives.

        Arguments:
            path (str): path to vocab file.

        Returns:
            List of Lists of strings.
    """
    vocab = []
    with open(path, 'r', encoding='utf8') as voc_file:
        for line in voc_file.readlines():
            if line.startswith('#') or line.strip() == '':
                continue
            vocab.append(expand_options(line.lower()))
    return vocab

def load_regex_from_file(path, skill_id):
    """Load regex from file
    The regex is sent to the intent handler using the message bus

    Args:
        path:       path to vocabulary file (*.voc)
        skill_id:   skill_id to the regex is tied to
    """
    regexes = []
    if path.endswith('.rx'):
        with open(path, 'r', encoding='utf8') as reg_file:
            for line in reg_file.readlines():
                if line.startswith("#"):
                    continue
                regex = munge_regex(line.strip(), skill_id)
                # Raise error if regex can't be compiled
                re.compile(regex)
                regexes.append(regex)

    return regexes

def load_vocabulary(basedir, skill_id):
    """Load vocabulary from all files in the specified directory.

    Arguments:
        basedir (str): path of directory to load from (will recurse)
        skill_id: skill the data belongs to
    Returns:
        dict with intent_type as keys and list of list of lists as value.
    """
    vocabs = {}
    for path, _, files in walk(basedir):
        for f in files:
            if f.endswith(".voc"):
                vocab_type = to_alnum(skill_id) + splitext(f)[0]
                vocs = read_vocab_file(join(path, f))
                if vocs:
                    vocabs[vocab_type] = vocs
    return vocabs

def load_regex(basedir, skill_id):
    """Load regex from all files in the specified directory.

    Args:
        basedir (str): path of directory to load from
        bus (messagebus emitter): messagebus instance used to send the vocab to
                                  the intent service
        skill_id (str): skill identifier
    """
    regexes = []
    for path, _, files in walk(basedir):
        for f in files:
            if f.endswith(".rx"):
                regexes += load_regex_from_file(join(path, f), skill_id)
    return regexes

def to_alnum(skill_id):
    """Convert a skill id to only alphanumeric characters

     Non alpha-numeric characters are converted to "_"

    Args:
        skill_id (str): identifier to be converted
    Returns:
        (str) String of letters
    """
    return ''.join(c if c.isalnum() else '_' for c in str(skill_id))

def munge_regex(regex, skill_id):
    """Insert skill id as letters into match groups.

    Args:
        regex (str): regex string
        skill_id (str): skill identifier
    Returns:
        (str) munged regex
    """
    base = '(?P<' + to_alnum(skill_id)
    return base.join(regex.split('(?P<'))

def munge_intent_parser(intent_parser, name, skill_id):
    """Rename intent keywords to make them skill exclusive
    This gives the intent parser an exclusive name in the
    format <skill_id>:<name>.  The keywords are given unique
    names in the format <Skill id as letters><Intent name>.

    The function will not munge instances that's already been
    munged

    Args:
        intent_parser: (IntentParser) object to update
        name: (str) Skill name
        skill_id: (int) skill identifier
    """
    # Munge parser name
    if not name.startswith(str(skill_id) + ':'):
        intent_parser.name = str(skill_id) + ':' + name
    else:
        intent_parser.name = name

    # Munge keywords
    skill_id = to_alnum(skill_id)
    # Munge required keyword
    reqs = []
    for i in intent_parser.requires:
        if not i[0].startswith(skill_id):
            kw = (skill_id + i[0], skill_id + i[0])
            reqs.append(kw)
        else:
            reqs.append(i)
    intent_parser.requires = reqs

    # Munge optional keywords
    opts = []
    for i in intent_parser.optional:
        if not i[0].startswith(skill_id):
            kw = (skill_id + i[0], skill_id + i[0])
            opts.append(kw)
        else:
            opts.append(i)
    intent_parser.optional = opts

    # Munge at_least_one keywords
    at_least_one = []
    for i in intent_parser.at_least_one:
        element = [skill_id + e.replace(skill_id, '') for e in i]
        at_least_one.append(tuple(element))
    intent_parser.at_least_one = at_least_one

def read_value_file(filename, delim):
    """Read value file.

    The value file is a simple csv structure with a key and value.

    Arguments:
        filename (str): file to read
        delim (str): csv delimiter

    Returns:
        OrderedDict with results.
    """
    result = collections.OrderedDict()

    if filename:
        with open(filename) as f:
            reader = csv.reader(f, delimiter=delim)
            for row in reader:
                # skip blank or comment lines
                if not row or row[0].startswith("#"):
                    continue
                if len(row) != 2:
                    continue

                result[row[0]] = row[1]
    return result

def read_translated_file(filename, data):
    """Read a file inserting data.

    Arguments:
        filename (str): file to read
        data (dict): dictionary with data to insert into file

    Returns:
        list of lines.
    """
    if filename:
        with open(filename) as f:
            text = f.read().replace('{{', '{').replace('}}', '}')
            return text.format(**data or {}).rstrip('\n').split('\n')
    else:
        return None
<commit_msg>Add log messages that provide insight into the disambiguation behavior<commit_after># Copyright 2018 Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Functions to load skill data such as intents and regular expressions."""

import collections
import csv
import re
from os import walk
from os.path import splitext, join

from mycroft.util.format import expand_options
from mycroft.util.log import LOG

def read_vocab_file(path):
    """ Read voc file.

        This reads a .voc file, stripping out empty lines comments and expand
        parentheses. It retruns each line as a list of all expanded
        alternatives.

        Arguments:
            path (str): path to vocab file.

        Returns:
            List of Lists of strings.
    """
    vocab = []
    with open(path, 'r', encoding='utf8') as voc_file:
        for line in voc_file.readlines():
            if line.startswith('#') or line.strip() == '':
                continue
            vocab.append(expand_options(line.lower()))
    return vocab

def load_regex_from_file(path, skill_id):
    """Load regex from file
    The regex is sent to the intent handler using the message bus

    Args:
        path:       path to vocabulary file (*.voc)
        skill_id:   skill_id to the regex is tied to
    """
    regexes = []
    if path.endswith('.rx'):
        with open(path, 'r', encoding='utf8') as reg_file:
            for line in reg_file.readlines():
                if line.startswith("#"):
                    continue
                LOG.debug('regex pre-munge: ' + line.strip())
                regex = munge_regex(line.strip(), skill_id)
                LOG.debug('regex post-munge: ' + regex)
                # Raise error if regex can't be compiled
                re.compile(regex)
                regexes.append(regex)

    return regexes

def load_vocabulary(basedir, skill_id):
    """Load vocabulary from all files in the specified directory.

    Arguments:
        basedir (str): path of directory to load from (will recurse)
        skill_id: skill the data belongs to
    Returns:
        dict with intent_type as keys and list of list of lists as value.
    """
    vocabs = {}
    for path, _, files in walk(basedir):
        for f in files:
            if f.endswith(".voc"):
                vocab_type = to_alnum(skill_id) + splitext(f)[0]
                vocs = read_vocab_file(join(path, f))
                if vocs:
                    vocabs[vocab_type] = vocs
    return vocabs

def load_regex(basedir, skill_id):
    """Load regex from all files in the specified directory.

    Args:
        basedir (str): path of directory to load from
        bus (messagebus emitter): messagebus instance used to send the vocab to
                                  the intent service
        skill_id (str): skill identifier
    """
    regexes = []
    for path, _, files in walk(basedir):
        for f in files:
            if f.endswith(".rx"):
                regexes += load_regex_from_file(join(path, f), skill_id)
    return regexes

def to_alnum(skill_id):
    """Convert a skill id to only alphanumeric characters

     Non alpha-numeric characters are converted to "_"

    Args:
        skill_id (str): identifier to be converted
    Returns:
        (str) String of letters
    """
    return ''.join(c if c.isalnum() else '_' for c in str(skill_id))

def munge_regex(regex, skill_id):
    """Insert skill id as letters into match groups.

    Args:
        regex (str): regex string
        skill_id (str): skill identifier
    Returns:
        (str) munged regex
    """
    base = '(?P<' + to_alnum(skill_id)
    return base.join(regex.split('(?P<'))

def munge_intent_parser(intent_parser, name, skill_id):
    """Rename intent keywords to make them skill exclusive
    This gives the intent parser an exclusive name in the
    format <skill_id>:<name>.  The keywords are given unique
    names in the format <Skill id as letters><Intent name>.

    The function will not munge instances that's already been
    munged

    Args:
        intent_parser: (IntentParser) object to update
        name: (str) Skill name
        skill_id: (int) skill identifier
    """
    # Munge parser name
    if not name.startswith(str(skill_id) + ':'):
        intent_parser.name = str(skill_id) + ':' + name
    else:
        intent_parser.name = name

    # Munge keywords
    skill_id = to_alnum(skill_id)
    # Munge required keyword
    reqs = []
    for i in intent_parser.requires:
        if not i[0].startswith(skill_id):
            kw = (skill_id + i[0], skill_id + i[0])
            reqs.append(kw)
        else:
            reqs.append(i)
    intent_parser.requires = reqs

    # Munge optional keywords
    opts = []
    for i in intent_parser.optional:
        if not i[0].startswith(skill_id):
            kw = (skill_id + i[0], skill_id + i[0])
            opts.append(kw)
        else:
            opts.append(i)
    intent_parser.optional = opts

    # Munge at_least_one keywords
    at_least_one = []
    for i in intent_parser.at_least_one:
        element = [skill_id + e.replace(skill_id, '') for e in i]
        at_least_one.append(tuple(element))
    intent_parser.at_least_one = at_least_one

def read_value_file(filename, delim):
    """Read value file.

    The value file is a simple csv structure with a key and value.

    Arguments:
        filename (str): file to read
        delim (str): csv delimiter

    Returns:
        OrderedDict with results.
    """
    result = collections.OrderedDict()

    if filename:
        with open(filename) as f:
            reader = csv.reader(f, delimiter=delim)
            for row in reader:
                # skip blank or comment lines
                if not row or row[0].startswith("#"):
                    continue
                if len(row) != 2:
                    continue

                result[row[0]] = row[1]
    return result

def read_translated_file(filename, data):
    """Read a file inserting data.

    Arguments:
        filename (str): file to read
        data (dict): dictionary with data to insert into file

    Returns:
        list of lines.
    """
    if filename:
        with open(filename) as f:
            text = f.read().replace('{{', '{').replace('}}', '}')
            return text.format(**data or {}).rstrip('\n').split('\n')
    else:
        return None
<|endoftext|>


<commit_before>"""
A list of the languages that Pycco supports, mapping the file extension to
the name of the Pygments lexer and the symbol that indicates a comment. To
add another language to Pycco's repertoire, add it here.
"""

__all__ = ("supported_languages",)

HASH = "#"
SLASH_STAR = "/*"
STAR_SLASH = "*/"
SLASH_SLASH = "//"
DASH_DASH = "--"
TRIPLE_QUOTE = '"""'

def lang(name, comment_symbol, multistart=None, multiend=None):
    """
    Generate a language entry dictionary, given a name and comment symbol and
    optional start/end strings for multiline comments.
    """
    result = {
        "name": name,
        "comment_symbol": comment_symbol
    }
    if multistart is not None and multiend is not None:
        result.update(multistart=multistart, multiend=multiend)
    return result

c_lang = lang("c", SLASH_SLASH, SLASH_STAR, STAR_SLASH)

supported_languages = {
    ".coffee": lang("coffee-script", HASH, "###", "###"),

    ".pl": lang("perl", HASH),

    ".sql": lang("sql", DASH_DASH, SLASH_STAR, STAR_SLASH),

    ".sh": lang("bash", HASH),

    ".c": c_lang,

    ".h": c_lang,

    ".cl": c_lang,

    ".cpp": lang("cpp", SLASH_SLASH),

    ".js": lang("javascript", SLASH_SLASH, SLASH_STAR, STAR_SLASH),

    ".rb": lang("ruby", HASH, "=begin", "=end"),

    ".py": lang("python", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE),

    ".pyx": lang("cython", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE),

    ".scm": lang("scheme", ";;", "#|", "|#"),

    ".lua": lang("lua", DASH_DASH, "--[[", "--]]"),

    ".erl": lang("erlang", "%%"),

    ".tcl": lang("tcl", HASH),

    ".hs": lang("haskell", DASH_DASH, "{-", "-}"),
}
<commit_msg>Add R, Julia, Matlab, and Stata to supported languages<commit_after>"""
A list of the languages that Pycco supports, mapping the file extension to
the name of the Pygments lexer and the symbol that indicates a comment. To
add another language to Pycco's repertoire, add it here.
"""

__all__ = ("supported_languages",)

HASH = "#"
SLASH_STAR = "/*"
STAR_SLASH = "*/"
SLASH_SLASH = "//"
DASH_DASH = "--"
TRIPLE_QUOTE = '"""'

def lang(name, comment_symbol, multistart=None, multiend=None):
    """
    Generate a language entry dictionary, given a name and comment symbol and
    optional start/end strings for multiline comments.
    """
    result = {
        "name": name,
        "comment_symbol": comment_symbol
    }
    if multistart is not None and multiend is not None:
        result.update(multistart=multistart, multiend=multiend)
    return result

c_lang = lang("c", SLASH_SLASH, SLASH_STAR, STAR_SLASH)

supported_languages = {
    ".coffee": lang("coffee-script", HASH, "###", "###"),

    ".pl": lang("perl", HASH),

    ".sql": lang("sql", DASH_DASH, SLASH_STAR, STAR_SLASH),

    ".sh": lang("bash", HASH),

    ".c": c_lang,

    ".h": c_lang,

    ".cl": c_lang,

    ".cpp": lang("cpp", SLASH_SLASH),

    ".js": lang("javascript", SLASH_SLASH, SLASH_STAR, STAR_SLASH),

    ".rb": lang("ruby", HASH, "=begin", "=end"),

    ".py": lang("python", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE),

    ".pyx": lang("cython", HASH, TRIPLE_QUOTE, TRIPLE_QUOTE),

    ".scm": lang("scheme", ";;", "#|", "|#"),

    ".lua": lang("lua", DASH_DASH, "--[[", "--]]"),

    ".erl": lang("erlang", "%%"),

    ".tcl": lang("tcl", HASH),

    ".hs": lang("haskell", DASH_DASH, "{-", "-}"),

    ".r": lang("r", HASH),

    ".jl": lang("julia", HASH, "#=", "=#"),

    ".m": lang("matlab", "%", "%{", "%}"),

    ".do": lang("stata", SLASH_SLASH, SLASH_STAR, STAR_SLASH)

}
<|endoftext|>


<commit_before>import os
import sys
import fnmatch
import subprocess
import datetime
import argparse
import logging
import re

def isLeapYear(year):
	'''Returns TRUE if the given year (int) is leap and FALSE otherwise'''
	leapyear = False
	if year % 4 != 0:
		leapyear = False
	elif year % 100 != 0:
		leapyear = True
	elif year % 400 == 0:
		leapyear = True
	else:
		leapyear = False
	return leapyear

def listFiles(path, pattern):
	res = list()
	for path, subdirs, files in os.walk(path):
		for name in files:
			filepath = os.path.join(path, name)
			if(re.match(pattern, filepath, flags=0)):
				res.append(filepath)
	return res

def testGribModis2SciDB():
	'''Test if GRibeiro's modis2scidb is installed'''
	res = True
	try:
		subprocess.check_output(["modis2scidb", "--h"])
	except:
		res = False
	return res

def date2grid(dateFileName, period, startyear):
	'''Return an time index (timid) from the input date (MODIS DOY) and time period (e.g 8 days). '''
	res = -1
	if period > 0:
		dateYYYYDOY = dateFileName[1:] #Remove the A precedding the date
		year = int(dateYYYYDOY[0:4])
		doy = int(dateYYYYDOY[4:7])
		ppy = int(365 / period) + 1 # Periods per year
		if(period > 0 and (doy - 1) % period == 0):
			idd = (doy - 1) / period
			idy = (year - startyear) * ppy
			res = idy + idd
		else:
			logging.error("date2grid: Invalid date")
	elif period == -319980101: # Monthly - given as YYYYMMDD i.e 19980101, 19980201, 19980301
		dateYYYYMMDD = dateFileName
		year = int(dateYYYYMMDD[0:4])
		mm = int(dateYYYYMMDD[4:6])
		#dd = int(dateYYYYMMDD[6:8])
		idy = (year - startyear) * 12
		idd = mm - 1
		res = idy + idd
	return res

#********************************************************
# MAIN
#********************************************************
def main(argv):
	t0 = datetime.datetime.now()
	parser = argparse.ArgumentParser(description = "Exports MODIS-HDFs to binary files for uploading to SCIDB using GRibeiro's tool")
	parser.add_argument("hdfFile", help = "Path to the HDF")
	parser.add_argument("loadFolder", help = "Folder from where the binary files are uploaded to SCIDB")
	parser.add_argument("product", help = "Product. e.g MOD09Q1")
	parser.add_argument("--log", help = "Log level. Default = WARNING", default = 'WARNING')
	#Get paramters
	args = parser.parse_args()
	hdfFile = os.path.join(args.hdfFile, '')
	loadFolder = os.path.join(args.loadFolder, '')
	product = args.product
	log = args.log
	####################################################
	# CONFIG
	####################################################
	prodList = ['MOD09Q1', 'MOD13Q1', 'TRMM3B43']
	prodTemporalResolution = {
		'MOD09Q1':	8,
		'MOD13Q1':	16,
		'TRMM3B43':	-319980101
	}
	prodStartYear = {
		'MOD09Q1':	2000,
		'MOD13Q1':	2000,
		'TRMM3B43':	1998
	}
	prodBands = {
		'MOD09Q1':	'0,1,2',
		'MOD13Q1':	'0,1,2,3,4,5,6,7,8,9,10,11',
		'TRMM3B43':	'0,1,2'
	}
	numeric_loglevel = getattr(logging, log.upper(), None)
	if not isinstance(numeric_loglevel, int):
		raise ValueError('Invalid log level: %s' % log)
	logging.basicConfig(filename = 'log_hdf2sdbin.log', level = numeric_loglevel, format = '%(asctime)s %(levelname)s: %(message)s')
	logging.info("log_hdf2sdbin.py: " + str(args))
	####################################################
	# VALIDATION
	####################################################
	if product in prodList == False:
		logging.exception("Unknown product!")
		raise Exception("Unknown product!")
	if testGribModis2SciDB() == False:
		logging.exception("GRibeiro's mod2scidb not found")
		raise Exception("GRibeiro's mod2scidb not found")
	####################################################
	#
	####################################################
	cmd = ""
	try:
		period = prodTemporalResolution[product]
		startyear = prodStartYear[product]
		bands = prodBands[product]
		filename = os.path.basename(hdfFile)
		time_id = date2grid(filename.split(".")[1], period, startyear)
		arg0 = "modis2scidb"
		arg1 = " --f " + hdfFile
		arg2 = " --o " + loadFolder + os.path.splitext(filename)[0] + ".sdbbin"
		arg3 = " --b " + bands
		arg4 = " --t " + str(time_id)
		cmd = arg0 + arg1 + arg2 + arg3 + arg4
		subprocess.check_call(str(cmd), shell = True)
	except subprocess.CalledProcessError as e:
		logging.exception("CalledProcessError: " + cmd + "\n" + str(e.message))
	except ValueError as e:
		logging.exception("ValueError: " + cmd + "\n" + str(e.message))
	except OSError as e:
		logging.exception("OSError: " + cmd + "\n" + str(e.message))
	except:
		e = sys.exc_info()[0]
		logging.exception("Unknown exception: " + cmd + "\n" + str(e.message))

	t1 = datetime.datetime.now()
	tt = t1 - t0
	logging.info("Finished in " + str(tt))

if __name__ == "__main__":
   main(sys.argv[1:])

<commit_msg>Fixed bug on file path building<commit_after>import os
import sys
import fnmatch
import subprocess
import datetime
import argparse
import logging
import re

def isLeapYear(year):
	'''Returns TRUE if the given year (int) is leap and FALSE otherwise'''
	leapyear = False
	if year % 4 != 0:
		leapyear = False
	elif year % 100 != 0:
		leapyear = True
	elif year % 400 == 0:
		leapyear = True
	else:
		leapyear = False
	return leapyear

def listFiles(path, pattern):
	res = list()
	for path, subdirs, files in os.walk(path):
		for name in files:
			filepath = os.path.join(path, name)
			if(re.match(pattern, filepath, flags=0)):
				res.append(filepath)
	return res

def testGribModis2SciDB():
	'''Test if GRibeiro's modis2scidb is installed'''
	res = True
	try:
		subprocess.check_output(["modis2scidb", "--h"])
	except:
		res = False
	return res

def date2grid(dateFileName, period, startyear):
	'''Return an time index (timid) from the input date (MODIS DOY) and time period (e.g 8 days). '''
	res = -1
	if period > 0:
		dateYYYYDOY = dateFileName[1:] #Remove the A precedding the date
		year = int(dateYYYYDOY[0:4])
		doy = int(dateYYYYDOY[4:7])
		ppy = int(365 / period) + 1 # Periods per year
		if(period > 0 and (doy - 1) % period == 0):
			idd = (doy - 1) / period
			idy = (year - startyear) * ppy
			res = idy + idd
		else:
			logging.error("date2grid: Invalid date")
	elif period == -319980101: # Monthly - given as YYYYMMDD i.e 19980101, 19980201, 19980301
		dateYYYYMMDD = dateFileName
		year = int(dateYYYYMMDD[0:4])
		mm = int(dateYYYYMMDD[4:6])
		#dd = int(dateYYYYMMDD[6:8])
		idy = (year - startyear) * 12
		idd = mm - 1
		res = idy + idd
	return res

#********************************************************
# MAIN
#********************************************************
def main(argv):
	t0 = datetime.datetime.now()
	parser = argparse.ArgumentParser(description = "Exports MODIS-HDFs to binary files for uploading to SCIDB using GRibeiro's tool")
	parser.add_argument("hdfFile", help = "Path to the HDF")
	parser.add_argument("loadFolder", help = "Folder from where the binary files are uploaded to SCIDB")
	parser.add_argument("product", help = "Product. e.g MOD09Q1")
	parser.add_argument("--log", help = "Log level. Default = WARNING", default = 'WARNING')
	#Get paramters
	args = parser.parse_args()
	hdfFile = args.hdfFile
	loadFolder = os.path.join(args.loadFolder, '')
	product = args.product
	log = args.log
	####################################################
	# CONFIG
	####################################################
	prodList = ['MOD09Q1', 'MOD13Q1', 'TRMM3B43']
	prodTemporalResolution = {
		'MOD09Q1':	8,
		'MOD13Q1':	16,
		'TRMM3B43':	-319980101
	}
	prodStartYear = {
		'MOD09Q1':	2000,
		'MOD13Q1':	2000,
		'TRMM3B43':	1998
	}
	prodBands = {
		'MOD09Q1':	'0,1,2',
		'MOD13Q1':	'0,1,2,3,4,5,6,7,8,9,10,11',
		'TRMM3B43':	'0,1,2'
	}
	numeric_loglevel = getattr(logging, log.upper(), None)
	if not isinstance(numeric_loglevel, int):
		raise ValueError('Invalid log level: %s' % log)
	logging.basicConfig(filename = 'log_hdf2sdbin.log', level = numeric_loglevel, format = '%(asctime)s %(levelname)s: %(message)s')
	logging.info("log_hdf2sdbin.py: " + str(args))
	####################################################
	# VALIDATION
	####################################################
	if product in prodList == False:
		logging.exception("Unknown product!")
		raise Exception("Unknown product!")
	if testGribModis2SciDB() == False:
		logging.exception("GRibeiro's mod2scidb not found")
		raise Exception("GRibeiro's mod2scidb not found")
	####################################################
	#
	####################################################
	cmd = ""
	try:
		period = prodTemporalResolution[product]
		startyear = prodStartYear[product]
		bands = prodBands[product]
		filename = os.path.basename(hdfFile)
		time_id = date2grid(filename.split(".")[1], period, startyear)
		arg0 = "modis2scidb"
		arg1 = " --f " + hdfFile
		arg2 = " --o " + loadFolder + os.path.splitext(filename)[0] + ".sdbbin"
		arg3 = " --b " + bands
		arg4 = " --t " + str(time_id)
		cmd = arg0 + arg1 + arg2 + arg3 + arg4
		subprocess.check_call(str(cmd), shell = True)
	except subprocess.CalledProcessError as e:
		logging.exception("CalledProcessError: " + cmd + "\n" + str(e.message))
	except ValueError as e:
		logging.exception("ValueError: " + cmd + "\n" + str(e.message))
	except OSError as e:
		logging.exception("OSError: " + cmd + "\n" + str(e.message))
	except:
		e = sys.exc_info()[0]
		logging.exception("Unknown exception: " + cmd + "\n" + str(e.message))

	t1 = datetime.datetime.now()
	tt = t1 - t0
	logging.info("Finished in " + str(tt))

if __name__ == "__main__":
   main(sys.argv[1:])

<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from os.path import join as join_path, exists as path_exists, abspath

class WebsiteTheme(Document):
	def validate(self):
		self.validate_if_customizable()
		self.render_theme()
		self.validate_theme()

	def on_update(self):
		if (not self.custom
			and frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test)):

			self.export_doc()

		self.clear_cache_if_current_theme()

	def is_standard_and_not_valid_user(self):
		return (not self.custom
			and not frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test or frappe.flags.in_migrate))

	def on_trash(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("You are not allowed to delete a standard Website Theme"),
				frappe.PermissionError)

	def validate_if_customizable(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("Please Duplicate this Website Theme to customize."))

	def render_theme(self):
		self.theme_scss = frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', self.as_dict())

	def validate_theme(self):
		'''Generate theme css if theme_scss has changed'''
		doc_before_save = self.get_doc_before_save()
		if doc_before_save is None or get_scss(self) != get_scss(doc_before_save):
			self.generate_bootstrap_theme()

	def export_doc(self):
		"""Export to standard folder `[module]/website_theme/[name]/[name].json`."""
		from frappe.modules.export_file import export_to_files
		export_to_files(record_list=[['Website Theme', self.name]], create_init=True)

	def clear_cache_if_current_theme(self):
		if frappe.flags.in_install == 'frappe': return
		website_settings = frappe.get_doc("Website Settings", "Website Settings")
		if getattr(website_settings, "website_theme", None) == self.name:
			website_settings.clear_cache()

	def generate_bootstrap_theme(self):
		from subprocess import Popen, PIPE

		# create theme file in site public files folder
		folder_path = abspath(frappe.utils.get_files_path('website_theme', is_private=False))
		# create folder if not exist
		frappe.create_folder(folder_path)

		if not self.custom:
			self.delete_old_theme_files(folder_path)

		# add a random suffix
		suffix = frappe.generate_hash('Website Theme', 8) if self.custom else 'style'
		file_name = frappe.scrub(self.name) + '_' + suffix + '.css'
		output_path = join_path(folder_path, file_name)

		self.theme_scss = content = get_scss(self)
		content = content.replace('\n', '\\n')
		command = ['node', 'generate_bootstrap_theme.js', output_path, content]

		process = Popen(command, cwd=frappe.get_app_path('frappe', '..'), stdout=PIPE, stderr=PIPE)

		stderr = process.communicate()[1]

		if stderr:
			stderr = frappe.safe_decode(stderr)
			stderr = stderr.replace('\n', '
')
			frappe.throw('
{stderr}
'.format(stderr=stderr))
		else:
			self.theme_url = '/files/website_theme/' + file_name

		frappe.msgprint(_('Compiled Successfully'), alert=True)

	def delete_old_theme_files(self, folder_path):
		import os
		for fname in os.listdir(folder_path):
			if fname.startswith(frappe.scrub(self.name) + '_') and fname.endswith('.css'):
				os.remove(os.path.join(folder_path, fname))

	def generate_theme_if_not_exist(self):
		bench_path = frappe.utils.get_bench_path()
		if self.theme_url:
			theme_path = join_path(bench_path, 'sites', self.theme_url[1:])
			if not path_exists(theme_path):
				self.generate_bootstrap_theme()
		else:
			self.generate_bootstrap_theme()

	def set_as_default(self):
		website_settings = frappe.get_doc('Website Settings')
		website_settings.website_theme = self.name
		website_settings.ignore_validate = True
		website_settings.save()

def add_website_theme(context):
	context.theme = frappe._dict()

	if not context.disable_website_theme:
		website_theme = get_active_theme()
		context.theme = website_theme or frappe._dict()

def get_active_theme():
	website_theme = frappe.db.get_value("Website Settings", "Website Settings", "website_theme")
	if website_theme:
		try:
			return frappe.get_doc("Website Theme", website_theme)
		except frappe.DoesNotExistError:
			pass

def get_scss(doc):
	def trim_list(list_of_strings):
		return [s.strip() for s in list_of_strings]

	opts = doc.as_dict()
	opts['website_theme_scss'] = trim_list(frappe.get_hooks('website_theme_scss', []))
	opts['imports_to_ignore'] = trim_list((opts.get('imports_to_ignore') or '').split(','))
	return frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', opts)
<commit_msg>fix: always regenerate the website theme<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document
from os.path import join as join_path, exists as path_exists, abspath

class WebsiteTheme(Document):
	def validate(self):
		self.validate_if_customizable()
		self.render_theme()
		self.generate_bootstrap_theme()

	def on_update(self):
		if (not self.custom
			and frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test)):

			self.export_doc()

		self.clear_cache_if_current_theme()

	def is_standard_and_not_valid_user(self):
		return (not self.custom
			and not frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test or frappe.flags.in_migrate))

	def on_trash(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("You are not allowed to delete a standard Website Theme"),
				frappe.PermissionError)

	def validate_if_customizable(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("Please Duplicate this Website Theme to customize."))

	def render_theme(self):
		self.theme_scss = frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', self.as_dict())

	def export_doc(self):
		"""Export to standard folder `[module]/website_theme/[name]/[name].json`."""
		from frappe.modules.export_file import export_to_files
		export_to_files(record_list=[['Website Theme', self.name]], create_init=True)

	def clear_cache_if_current_theme(self):
		if frappe.flags.in_install == 'frappe': return
		website_settings = frappe.get_doc("Website Settings", "Website Settings")
		if getattr(website_settings, "website_theme", None) == self.name:
			website_settings.clear_cache()

	def generate_bootstrap_theme(self):
		from subprocess import Popen, PIPE

		# create theme file in site public files folder
		folder_path = abspath(frappe.utils.get_files_path('website_theme', is_private=False))
		# create folder if not exist
		frappe.create_folder(folder_path)

		if not self.custom:
			self.delete_old_theme_files(folder_path)

		# add a random suffix
		suffix = frappe.generate_hash('Website Theme', 8) if self.custom else 'style'
		file_name = frappe.scrub(self.name) + '_' + suffix + '.css'
		output_path = join_path(folder_path, file_name)

		self.theme_scss = content = get_scss(self)
		content = content.replace('\n', '\\n')
		command = ['node', 'generate_bootstrap_theme.js', output_path, content]

		process = Popen(command, cwd=frappe.get_app_path('frappe', '..'), stdout=PIPE, stderr=PIPE)

		stderr = process.communicate()[1]

		if stderr:
			stderr = frappe.safe_decode(stderr)
			stderr = stderr.replace('\n', '
')
			frappe.throw('
{stderr}
'.format(stderr=stderr))
		else:
			self.theme_url = '/files/website_theme/' + file_name

		frappe.msgprint(_('Compiled Successfully'), alert=True)

	def delete_old_theme_files(self, folder_path):
		import os
		for fname in os.listdir(folder_path):
			if fname.startswith(frappe.scrub(self.name) + '_') and fname.endswith('.css'):
				os.remove(os.path.join(folder_path, fname))

	def generate_theme_if_not_exist(self):
		bench_path = frappe.utils.get_bench_path()
		if self.theme_url:
			theme_path = join_path(bench_path, 'sites', self.theme_url[1:])
			if not path_exists(theme_path):
				self.generate_bootstrap_theme()
		else:
			self.generate_bootstrap_theme()

	def set_as_default(self):
		website_settings = frappe.get_doc('Website Settings')
		website_settings.website_theme = self.name
		website_settings.ignore_validate = True
		website_settings.save()

def add_website_theme(context):
	context.theme = frappe._dict()

	if not context.disable_website_theme:
		website_theme = get_active_theme()
		context.theme = website_theme or frappe._dict()

def get_active_theme():
	website_theme = frappe.db.get_value("Website Settings", "Website Settings", "website_theme")
	if website_theme:
		try:
			return frappe.get_doc("Website Theme", website_theme)
		except frappe.DoesNotExistError:
			pass

def get_scss(doc):
	def trim_list(list_of_strings):
		return [s.strip() for s in list_of_strings]

	opts = doc.as_dict()
	opts['website_theme_scss'] = trim_list(frappe.get_hooks('website_theme_scss', []))
	opts['imports_to_ignore'] = trim_list((opts.get('imports_to_ignore') or '').split(','))
	return frappe.render_template('frappe/website/doctype/website_theme/website_theme_template.scss', opts)
<|endoftext|>


<commit_before>from browser import console, window

def wrap(func):
    # Transforms a function f into another function that prints a
    # traceback in case of exception
    def f(*args, **kw):
        try:
            return func(*args, **kw)
        except Exception as exc:
            msg = ''
            try:
                if exc.args:
                    msg = '{0.info}\n{0.__name__}: {0.args[0]}'.format(exc)
                else:
                    msg = str(exc)
                import sys
                sys.stderr.write(msg)
            except Exception as exc2:
                console.log("Error printing exception traceback", exc2, func,
                    args, kw)
    return f

clear_interval = window.clearInterval

clear_timeout = window.clearTimeout

def set_interval(func, interval):
    return window.setInterval(wrap(func), interval)

def set_timeout(func, interval, *args):
    return int(window.setTimeout(wrap(func), interval, *args))

def request_animation_frame(func):
    return int(window.requestAnimationFrame(func))

def cancel_animation_frame(int_id):
    window.cancelAnimationFrame(int_id)

def set_loop_timeout(x):
    # set a variable used to stop loops that last more than x seconds
    assert isinstance(x, int)
    __BRYTHON__.loop_timeout = x<commit_msg>Fix bug in error reporting in browser/timer.py<commit_after>from browser import console, window

def wrap(func):
    # Transforms a function f into another function that prints a
    # traceback in case of exception
    def f(*args, **kw):
        try:
            return func(*args, **kw)
        except Exception as exc:
            msg = ''
            try:
                if exc.args:
                    msg = '{0.info}\n{0.__class__.__name__}: {0.args[0]}'.format(exc)
                else:
                    msg = str(exc)
                import sys
                sys.stderr.write(msg)
            except Exception as exc2:
                console.log("Error printing exception traceback", exc2, func,
                    args, kw)
    return f

clear_interval = window.clearInterval

clear_timeout = window.clearTimeout

def set_interval(func, interval):
    return window.setInterval(wrap(func), interval)

def set_timeout(func, interval, *args):
    return int(window.setTimeout(wrap(func), interval, *args))

def request_animation_frame(func):
    return int(window.requestAnimationFrame(func))

def cancel_animation_frame(int_id):
    window.cancelAnimationFrame(int_id)

def set_loop_timeout(x):
    # set a variable used to stop loops that last more than x seconds
    assert isinstance(x, int)
    __BRYTHON__.loop_timeout = x<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

"""
Python distutils setup script for pychecker.

This code was originally contributed by Nicolas Chauvat, and has been rewritten
to generalize it so it works properly with --prefix and other distutils
options.

This install script needs to customize two distutils behaviors: it needs to
install the documentation to the configured package install directory, and it
needs to create a pychecker (or pychecker.bat) script containing the correct
path to the installed checker.py module and the Python interpreter.

Nicolas' original attempt at this worked fine in the normal case (i.e. a root
user install).  However, because it assumed that the package directory was
always within sysconfig.get_python_lib(), it broke when users wanted to specify
a prefix or a home directory install to, etc.

After some research, I've decided that the best way to make this work is to
customize (override) some of the distutils action classes.  This way, we get
access to the distutils configuration and can "do the right thing" when options
are specified.  It turns out that I needed to customize the build_scripts and
install_data sections.  There are more notes below by each customized action.

@author: Kenneth J. Pronovici <pronovic@ieee.org>, after Nicolas Chauvat.
"""

###################
# Imported modules
###################

import sys
import os
from distutils import core
from distutils.util import execute
from distutils.command.install_data import install_data
from distutils.command.build_scripts import build_scripts

###############################
# Overridden distutils actions
###############################

class my_install_data(install_data):
   """
   Customized install_data distutils action.

   This customized action forces all data files to all be installed relative to
   the normal library install directory rather than in the standard location.
   This directory is usually something like /usr/share/python2.3/site-packages.

   This action does not obey any --install-data flag that the user specifies.
   Distutils apparently does not provide a way to tell whether the install data
   directory on the Distribution class is "standard" or has already been
   overrwritten.  This means that we don't have a way to say "change this only
   if it hasn't been changed already".  All we can do is override the location
   all of the time.

   Note: If you want your files to go in the "pychecker" package directory,
   make sure that you specify "pychecker" as the prefix in the setup target.
   """
   def finalize_options(self):
      self.set_undefined_options('install', ('install_lib', 'install_dir'))
      install_data.finalize_options(self) # invoke "standard" action

class my_build_scripts(build_scripts):
   """
   Customized build_scripts distutils action.

   This action looks through the configured scripts list, and if "pychecker" is
   in the list, replaces that entry with the real name of the script to be
   created within the build directory (including the .bat extension if needed).
   Then, it builds the script (either Windows or shell style, depending on
   platform) using the proper path to the checker.py file and Python
   interpreter.  This is done through the execute() method, so that the action
   obeys the --dry-run flag, etc.

   This action completely ignores any scripts other than "pychecker" which are
   listed in the setup configuration, and it only does anything if "pychecker"
   is listed in the first place.  This way, new scripts with constant contents
   (if any) can be added to the setup configuration without writing any new
   code.
   """
   def run(self):
      if self.scripts is not None and "pychecker" in self.scripts:
         if sys.platform == "win32":
            script_path = os.path.join(self.build_dir, "pychecker.bat")
         else:
            script_path = os.path.join(self.build_dir, "pychecker")
         self.scripts.remove("pychecker")
         self.scripts.append(script_path)
         self.mkpath(self.build_dir)
         install_lib = self.distribution.get_command_obj("install").install_lib
         package_path = os.path.join(install_lib, "pychecker")
         self.execute(func=create_pychecker, args=[script_path, package_path], msg="Building %s" % script_path)
      build_scripts.run(self) # invoke "standard" action

def create_pychecker(script_path, package_path):
   """
   Creates the pychecker script at the indicated path.

   The pychecker script will be created to point to checker.py in the package
   directory, using the Python executable specified in sys.executable.  If the
   platform is Windows, a batch-style script will be created.  Otherwise, a
   Bourne-shell script will be created.  Note that we don't worry about what
   permissions mode the created file will have because the distutils install
   process takes care of that for us.

   @param script_path: Path to the script to be created
   @param package_path: Path to the package that checker.py can be found within

   @raise Exception: If script cannot be created on disk.
   """
   try:
      checker_path = os.path.join(package_path, "checker.py")
      if sys.platform == "win32":
         script_str = "%s %s %%1 %%2 %%3 %%4 %%5 %%6 %%7 %%8 %%9\n" % (sys.executable, checker_path)
      else:
         script_str = '#! /bin/sh\n\n%s %s "$@"\n' % (sys.executable, checker_path)
      open(script_path, "w").write(script_str)
   except Exception, e:
      print "ERROR: Unable to create %s: %s" % (script_path, e)
      raise e

######################
# Setup configuration
######################

CUSTOMIZED_ACTIONS = { 'build_scripts'  : my_build_scripts,
                       'install_data'   : my_install_data,
                     }

DATA_FILES = [ 'COPYRIGHT', 'README', 'VERSION', 'CHANGELOG',
                'KNOWN_BUGS', 'MAINTAINERS', 'TODO',
             ]

LONG_DESCRIPTION = """
PyChecker is a tool for finding bugs in python source code.  It finds problems
that are typically caught by a compiler for less dynamic languages, like C and
C++. Because of the dynamic nature of Python, some warnings may be incorrect;
however, spurious warnings should be fairly infrequent.
"""

kw =  { 'name'             : "PyChecker",
        'version'          : "0.8.15",
        'license'          : "BSD-like",
        'description'      : "Python source code checking tool",
        'author'           : "Neal Norwitz, MetaSlash, Inc.",
        'author_email'     : "nnorwitz@gmail.com",
        'url'              : "http://pychecker.sourceforge.net/",
        'packages'         : [ 'pychecker', ],
        'scripts'          : [ "pychecker" ],   # note: will be replaced by customized action
        'data_files'       : [ ( "pychecker", DATA_FILES, ) ],
        'long_description' : LONG_DESCRIPTION,
        'cmdclass'         : CUSTOMIZED_ACTIONS,
      }

if hasattr(core, 'setup_keywords') and 'classifiers' in core.setup_keywords:
   kw['classifiers'] =  [ 'Development Status :: 4 - Beta',
                          'Environment :: Console',
                          'Intended Audience :: Developers',
                          'License :: OSI Approved :: BSD License',
                          'Operating System :: OS Independent',
                          'Programming Language :: Python',
                          'Topic :: Software Development :: Debuggers',
                          'Topic :: Software Development :: Quality Assurance',
                          'Topic :: Software Development :: Testing',
                        ]

if __name__ == '__main__' :
   core.setup(**kw)

<commit_msg>Fix setup.py behavior when invoking just build rule (bug #1279223)<commit_after>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-

"""
Python distutils setup script for pychecker.

This code was originally contributed by Nicolas Chauvat, and has been rewritten
to generalize it so it works properly with --prefix and other distutils
options.

This install script needs to customize two distutils behaviors: it needs to
install the documentation to the configured package install directory, and it
needs to create a pychecker (or pychecker.bat) script containing the correct
path to the installed checker.py module and the Python interpreter.

Nicolas' original attempt at this worked fine in the normal case (i.e. a root
user install).  However, because it assumed that the package directory was
always within sysconfig.get_python_lib(), it broke when users wanted to specify
a prefix or a home directory install to, etc.

After some research, I've decided that the best way to make this work is to
customize (override) some of the distutils action classes.  This way, we get
access to the distutils configuration and can "do the right thing" when options
are specified.  It turns out that I needed to customize the build_scripts,
install_scripts and install_data sections.  There are more notes below by each
customized action.

@author: Kenneth J. Pronovici <pronovic@ieee.org>, after Nicolas Chauvat.
"""

###################
# Imported modules
###################

import sys
import os
from distutils import core
from distutils.util import execute
from distutils.command.install_data import install_data
from distutils.command.install_scripts import install_scripts
from distutils.command.build_scripts import build_scripts

###############################
# Overridden distutils actions
###############################

class my_install_data(install_data):
   """
   Customized install_data distutils action.

   This customized action forces all data files to all be installed relative to
   the normal library install directory rather than in the standard location.
   This directory is usually something like /usr/share/python2.3/site-packages.

   This action does not obey any --install-data flag that the user specifies.
   Distutils apparently does not provide a way to tell whether the install data
   directory on the Distribution class is "standard" or has already been
   overrwritten.  This means that we don't have a way to say "change this only
   if it hasn't been changed already".  All we can do is override the location
   all of the time.

   Note: If you want your files to go in the "pychecker" package directory,
   make sure that you specify "pychecker" as the prefix in the setup target.
   """
   def finalize_options(self):
      self.set_undefined_options('install', ('install_lib', 'install_dir'))
      install_data.finalize_options(self) # invoke "standard" action

class my_install_scripts(install_scripts):
   """
   Customized install_scripts distutils action.

   This customized action fills in the pychecker script (either Windows or
   shell style, depending on platform) using the proper path to the checker.py
   file and Python interpreter.  This all is done through the execute() method,
   so that the action obeys the --dry-run flag, etc.

   The pychecker script must be built here, in the install action, because the
   proper path to checker.py cannot be known before this point.
   """
   def run(self):
      install_lib = self.distribution.get_command_obj("install").install_lib
      scripts = self.distribution.get_command_obj("build_scripts").scripts
      script_path = get_script_path(self.build_dir)
      if scripts is not None and script_path in scripts:
         package_path = os.path.join(install_lib, "pychecker")
         self.execute(func=create_script, args=[script_path, package_path], msg="filling in script %s" % script_path)
      install_scripts.run(self) # invoke "standard" action

class my_build_scripts(build_scripts):
   """
   Customized build_scripts distutils action.

   This action looks through the configured scripts list, and if "pychecker" is
   in the list, replaces that entry with the real name of the script to be
   created within the build directory (including the .bat extension if needed).
   Then, it creates an empty script file at that path to make distutils happy.
   This is all done through the execute() method, so that the action obeys the
   --dry-run flag, etc.

   It might be surprising that we don't fill in the pychecker script here.
   This is because in order to fill in the script, we need to know the
   installed location of checker.py.  This information isn't available until
   the install action is executed.

   This action completely ignores any scripts other than "pychecker" which are
   listed in the setup configuration, and it only does anything if "pychecker"
   is listed in the first place.  This way, new scripts with constant contents
   (if any) can be added to the setup configuration without writing any new
   code.  It also helps OS package maintainers (like for the Debian package)
   because they can still avoid installing scripts just by commenting out the
   scripts line in configuration, just like usual.
   """
   def run(self):
      if self.scripts is not None and "pychecker" in self.scripts:
         script_path = get_script_path(self.build_dir)
         self.scripts.remove("pychecker")
         self.scripts.append(script_path)
         self.mkpath(self.build_dir)
         self.execute(func=open, args=[script_path, "w"], msg="creating empty script %s" % script_path)
      build_scripts.run(self) # invoke "standard" action

def get_script_path(build_dir):
   """
   Returns the platform-specific path to the pychecker script within the build dir.
   """
   if sys.platform == "win32":
      return os.path.join(build_dir, "pychecker.bat")
   else:
      return os.path.join(build_dir, "pychecker")

def create_script(script_path, package_path):
   """
   Creates the pychecker script at the indicated path.

   The pychecker script will be created to point to checker.py in the package
   directory, using the Python executable specified in sys.executable.  If the
   platform is Windows, a batch-style script will be created.  Otherwise, a
   Bourne-shell script will be created.  Note that we don't worry about what
   permissions mode the created file will have because the distutils install
   process takes care of that for us.

   @param script_path: Path to the script to be created
   @param package_path: Path to the package that checker.py can be found within

   @raise Exception: If script cannot be created on disk.
   """
   try:
      checker_path = os.path.join(package_path, "checker.py")
      if sys.platform == "win32":
         script_str = "%s %s %%1 %%2 %%3 %%4 %%5 %%6 %%7 %%8 %%9\n" % (sys.executable, checker_path)
      else:
         script_str = '#! /bin/sh\n\n%s %s "$@"\n' % (sys.executable, checker_path)
      open(script_path, "w").write(script_str)
   except Exception, e:
      print "ERROR: Unable to create %s: %s" % (script_path, e)
      raise e

######################
# Setup configuration
######################

CUSTOMIZED_ACTIONS = { 'build_scripts'  : my_build_scripts,
                       'install_scripts': my_install_scripts,
                       'install_data'   : my_install_data,
                     }

DATA_FILES = [ 'COPYRIGHT', 'README', 'VERSION', 'CHANGELOG',
                'KNOWN_BUGS', 'MAINTAINERS', 'TODO',
             ]

LONG_DESCRIPTION = """
PyChecker is a tool for finding bugs in python source code.  It finds problems
that are typically caught by a compiler for less dynamic languages, like C and
C++. Because of the dynamic nature of Python, some warnings may be incorrect;
however, spurious warnings should be fairly infrequent.
"""

kw =  { 'name'             : "PyChecker",
        'version'          : "0.8.15",
        'license'          : "BSD-like",
        'description'      : "Python source code checking tool",
        'author'           : "Neal Norwitz, MetaSlash, Inc.",
        'author_email'     : "nnorwitz@gmail.com",
        'url'              : "http://pychecker.sourceforge.net/",
        'packages'         : [ 'pychecker', ],
        'scripts'          : [ "pychecker" ],   # note: will be replaced by customized action
        'data_files'       : [ ( "pychecker", DATA_FILES, ) ],
        'long_description' : LONG_DESCRIPTION,
        'cmdclass'         : CUSTOMIZED_ACTIONS,
      }

if hasattr(core, 'setup_keywords') and 'classifiers' in core.setup_keywords:
   kw['classifiers'] =  [ 'Development Status :: 4 - Beta',
                          'Environment :: Console',
                          'Intended Audience :: Developers',
                          'License :: OSI Approved :: BSD License',
                          'Operating System :: OS Independent',
                          'Programming Language :: Python',
                          'Topic :: Software Development :: Debuggers',
                          'Topic :: Software Development :: Quality Assurance',
                          'Topic :: Software Development :: Testing',
                        ]

if __name__ == '__main__' :
   core.setup(**kw)

<|endoftext|>


<commit_before>from .darkobject import DarkObject
import collections

fmt = collections.OrderedDict()
fmt['itemid' ] = '{:>8}'
fmt['name'   ] = '{:>24}'
fmt['sell01' ] = '{:>6}'
fmt['buy01'  ] = '{:>6}'
fmt['price01'] = '{:>16}'
fmt['stock01'] = '{:>7}'
fmt['sell12' ] = '{:>6}'
fmt['buy12'  ] = '{:>6}'
fmt['price12'] = '{:>16}'
fmt['stock12'] = '{:>7}'

def titles():
    return ', '.join([fmt[k].format(k) for k in fmt.keys()]) + '\n'

def values(*objs):
    return ', '.join([fmt[fmt.keys()[i]].format(objs[i]) for i in range(len(objs))]) + '\n'

class Item(DarkObject):
    """
    Item properties.

    :param itemid: unique item id
    :param name: item name

    :param sell01: sell single
    :param buy01: buy single
    :param price01: price (>= 1) for single
    :param stock01: restock count (>= 0) for single

    :param sell12: sell stack
    :param buy12: buy stack
    :param price12: price (>= 1) for stack
    :param stock12: restock count (>= 0) for stack
    """

    keys = ['itemid', 'name',
            'sell01', 'buy01', 'price01', 'stock01',
            'sell12', 'buy12', 'price12', 'stock12']

    @property
    def values(self):
        return [self.itemid, self.name,
                self.sell01, self.buy01, self.price01, self.stock01,
                self.sell12, self.buy12, self.price12, self.stock12]

    def __init__(self, itemid, name=None,
            sell01=None, buy01=None, price01=None, stock01=None,
            sell12=None, buy12=None, price12=None, stock12=None):
        super(Item, self).__init__()

        self._itemid   = int(itemid)
        self._name     = None

        self._sell01   = None
        self._sell12   = None

        self._buy01    = None
        self._buy12    = None

        self._price01  = None
        self._price12  = None

        self._stock01  = None
        self._stock12  = None

        self.name = name

        self.sell01 = sell01
        self.buy01 = buy01
        self.price01 = price01
        self.stock01 = stock01

        self.sell12 = sell12
        self.buy12 = buy12
        self.price12 = price12
        self.stock12 = stock12

        if not self._itemid >= 0:
            raise ValueError('itemid must be positive: %d' % self._itemid)

    def _init_notify(self):
        pass

    def __str__(self):
        return 'Item({self.itemid})'.format(self=self)

    @property
    def itemid(self):
        return self._itemid

    @property
    def name(self):
        return self._name
    @name.setter
    def name(self, value):
        if value is None:
            value = '?'
        self._name = str(value)

    @property
    def sell01(self):
        return self._sell01
    @sell01.setter
    def sell01(self, value):
        if value is None:
            value = True
        self._sell01 = bool(value)

    @property
    def buy01(self):
        return self._buy01
    @buy01.setter
    def buy01(self, value):
        if value is None:
            value = True
        self._buy01 = bool(value)

    @property
    def price01(self):
        return self._price01
    @price01.setter
    def price01(self, value):
        if value is None:
            value = 1
        self._price01 = int(value)
        if self._price01 < 1:
            raise ValueError('price01 must be positive definite: %d' % self._price01)

    @property
    def stock01(self):
        return self._stock01
    @stock01.setter
    def stock01(self, value):
        if value is None:
            value = 0
        self._stock01 = int(value)
        if self._stock01 < 0:
            raise ValueError('stock01 must be positive: %d' % self._stock01)

    @property
    def sell12(self):
        return self._sell12
    @sell12.setter
    def sell12(self, value):
        if value is None:
            value = True
        self._sell12 = bool(value)

    @property
    def buy12(self):
        return self._buy12
    @buy12.setter
    def buy12(self, value):
        if value is None:
            value = True
        self._buy12 = bool(value)

    @property
    def price12(self):
        return self._price12
    @price12.setter
    def price12(self, value):
        if value is None:
            value = 1
        self._price12 = int(value)
        if self._price12 < 1:
            raise ValueError('price12 must be positive definite: %d' % self._price12)

    @property
    def stock12(self):
        return self._stock12
    @stock12.setter
    def stock12(self, value):
        if value is None:
            value = 0
        self._stock12 = int(value)
        if self._stock12 < 0:
            raise ValueError('stock12 must be positive: %d' % self._stock12)

if __name__ == '__main__':
    pass
<commit_msg>Update __str__ output.<commit_after>from .darkobject import DarkObject
import collections

fmt = collections.OrderedDict()
fmt['itemid' ] = '{:>8}'
fmt['name'   ] = '{:>24}'
fmt['sell01' ] = '{:>6}'
fmt['buy01'  ] = '{:>6}'
fmt['price01'] = '{:>16}'
fmt['stock01'] = '{:>7}'
fmt['sell12' ] = '{:>6}'
fmt['buy12'  ] = '{:>6}'
fmt['price12'] = '{:>16}'
fmt['stock12'] = '{:>7}'

def titles():
    return ', '.join([fmt[k].format(k) for k in fmt.keys()]) + '\n'

def values(*objs):
    return ', '.join([fmt[fmt.keys()[i]].format(objs[i]) for i in range(len(objs))]) + '\n'

_template = \
"""
[Item]
    addr        = {addr}
    itemid      = {self.itemid}
    name        = {self.name}
    sell01      = {self.sell01}
    buy01       = {self.buy01}
    price01     = {self.price01}
    stock01     = {self.stock01}
    sell12      = {self.sell12}
    buy12       = {self.buy12}
    price12     = {self.price12}
    stock12     = {self.stock12}
"""[:-1]

class Item(DarkObject):
    """
    Item properties.

    :param itemid: unique item id
    :param name: item name

    :param sell01: sell single
    :param buy01: buy single
    :param price01: price (>= 1) for single
    :param stock01: restock count (>= 0) for single

    :param sell12: sell stack
    :param buy12: buy stack
    :param price12: price (>= 1) for stack
    :param stock12: restock count (>= 0) for stack
    """

    keys = ['itemid', 'name',
            'sell01', 'buy01', 'price01', 'stock01',
            'sell12', 'buy12', 'price12', 'stock12']

    @property
    def values(self):
        return [self.itemid, self.name,
                self.sell01, self.buy01, self.price01, self.stock01,
                self.sell12, self.buy12, self.price12, self.stock12]

    def __init__(self, itemid, name=None,
            sell01=None, buy01=None, price01=None, stock01=None,
            sell12=None, buy12=None, price12=None, stock12=None):
        super(Item, self).__init__()

        self._itemid   = int(itemid)
        self._name     = None

        self._sell01   = None
        self._sell12   = None

        self._buy01    = None
        self._buy12    = None

        self._price01  = None
        self._price12  = None

        self._stock01  = None
        self._stock12  = None

        self.name = name

        self.sell01 = sell01
        self.buy01 = buy01
        self.price01 = price01
        self.stock01 = stock01

        self.sell12 = sell12
        self.buy12 = buy12
        self.price12 = price12
        self.stock12 = stock12

        if not self._itemid >= 0:
            raise ValueError('itemid must be positive: %d' % self._itemid)

    def _init_notify(self):
        pass

    def __str__(self):
        return _template.format(self=self, addr=hex(id(self)))

    @property
    def itemid(self):
        return self._itemid

    @property
    def name(self):
        return self._name
    @name.setter
    def name(self, value):
        if value is None:
            value = '?'
        self._name = str(value)

    @property
    def sell01(self):
        return self._sell01
    @sell01.setter
    def sell01(self, value):
        if value is None:
            value = True
        self._sell01 = bool(value)

    @property
    def buy01(self):
        return self._buy01
    @buy01.setter
    def buy01(self, value):
        if value is None:
            value = True
        self._buy01 = bool(value)

    @property
    def price01(self):
        return self._price01
    @price01.setter
    def price01(self, value):
        if value is None:
            value = 1
        self._price01 = int(value)
        if self._price01 < 1:
            raise ValueError('price01 must be positive definite: %d' % self._price01)

    @property
    def stock01(self):
        return self._stock01
    @stock01.setter
    def stock01(self, value):
        if value is None:
            value = 0
        self._stock01 = int(value)
        if self._stock01 < 0:
            raise ValueError('stock01 must be positive: %d' % self._stock01)

    @property
    def sell12(self):
        return self._sell12
    @sell12.setter
    def sell12(self, value):
        if value is None:
            value = True
        self._sell12 = bool(value)

    @property
    def buy12(self):
        return self._buy12
    @buy12.setter
    def buy12(self, value):
        if value is None:
            value = True
        self._buy12 = bool(value)

    @property
    def price12(self):
        return self._price12
    @price12.setter
    def price12(self, value):
        if value is None:
            value = 1
        self._price12 = int(value)
        if self._price12 < 1:
            raise ValueError('price12 must be positive definite: %d' % self._price12)

    @property
    def stock12(self):
        return self._stock12
    @stock12.setter
    def stock12(self, value):
        if value is None:
            value = 0
        self._stock12 = int(value)
        if self._stock12 < 0:
            raise ValueError('stock12 must be positive: %d' % self._stock12)

if __name__ == '__main__':
    pass
<|endoftext|>


<commit_before>from itertools import izip

from pydigree import Chromosome, Population, Individual
from pydigree.io import smartopen as open

def read_vcf(filename):
    with open(filename) as f:
        pop = Population()

        # Pass 1: build the chromosomes
        last_chromid = None
        for i, line in enumerate(f):
            if line.startswith('#'):
                continue
            chromid, pos, varid, ref, alt, qual, filter_passed, rest = line.strip().split(None, 7)
            if chromid != last_chromid:
                if last_chromid is not None:
                    pop.add_chromosome(chromobj)
                chromobj = Chromosome()

            chromobj.add_genotype(None, None, bp=pos, label=varid)
            last_chromid = chromid
        pop.add_chromosome(chromobj)

        # Pass 2: Add genotypes for individuals
        f.seek(0) # Return to start of file
        last_chrom = None
        chr_idx = -1
        for i, line in enumerate(f):
            if line.startswith('##'):
                # Skip the header lines
                continue

            elif line.startswith('#'):
                ind_ids = line.strip().split()[9:]
                inds = [Individual(pop, ind_id) for ind_id in ind_ids]
                for ind in inds:
                    # Initialize new genotypes with a string datatype
                    ind._init_genotypes(dtype='S')
                continue

            l = line.strip().split()
            chrom = l[0]
            if chrom != last_chrom:
                chr_idx += 1
                pos_idx = 0

            loc = (chr_idx, pos_idx)

            data_format = l[8]
            gtidx = data_format.split(':').index('GT')

            for ind, data in izip(inds, l[9:]):
                data = data.split(':')

                # The genotypes come in the form A|C or A/C depending on phasing.
                # I treat all genotypes as phased (or unphased, I guess) so I'm
                # not doing anything with that.
                gt = data[gtidx][0:3:2]
                ind.set_genotype(loc, gt)

            pos_idx += 1
            last_chrom = chrom

        return pop
<commit_msg>add a vcf record object to clarify how the file is parsed<commit_after>from itertools import izip

from pydigree import Chromosome, Population, Individual
from pydigree.io import smartopen as open

class VCFRecord(object):
    def __init__(self, line):
        chromid, pos, varid, ref, alt, qual, filter_passed, info, format, data = line.strip().split('\t', 9)
        self.chrom = chromid
        self.pos = pos
        self.label = varid
        self.ref = ref
        self.alt = alt
        self.qual = qual
        self.filter_passed = filter_passed == 'PASS'
        self.info = info
        self.format = format
        self.data = data.split('\t')

def read_vcf(filename):
    with open(filename) as f:
        pop = Population()

        # Pass 1: build the chromosomes
        last_chromid = None
        for i, line in enumerate(f):
            if line.startswith('#'):
                continue
            chromid, pos, varid, ref, alt, qual, filter_passed, rest = line.strip().split(None, 7)
            if chromid != last_chromid:
                if last_chromid is not None:
                    pop.add_chromosome(chromobj)
                chromobj = Chromosome()

            chromobj.add_genotype(None, None, bp=pos, label=varid)
            last_chromid = chromid
        pop.add_chromosome(chromobj)

        # Pass 2: Add genotypes for individuals
        f.seek(0) # Return to start of file
        last_chrom = None
        chr_idx = -1
        for i, line in enumerate(f):
            if line.startswith('##'):
                # Skip the header lines
                continue

            elif line.startswith('#'):
                ind_ids = line.strip().split()[9:]
                inds = [Individual(pop, ind_id) for ind_id in ind_ids]
                for ind in inds:
                    # Initialize new genotypes with a string datatype
                    ind._init_genotypes(dtype='S')
                continue

            r = VCFRecord(line)
            if r.chrom != last_chrom:
                chr_idx += 1
                pos_idx = 0

            loc = (chr_idx, pos_idx)

            gtidx = r.format.split(':').index('GT')

            for ind, data in izip(inds, r.data):
                data = data.split(':')

                # The genotypes come in the form A|C or A/C depending on phasing.
                # I treat all genotypes as phased (or unphased, I guess) so I'm
                # not doing anything with that.
                gt = data[gtidx][0:3:2]
                ind.set_genotype(loc, gt)

            pos_idx += 1
            last_chrom = r.chrom

        return pop
<|endoftext|>