<commit_before># Copyright 2015-2016 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import boto3
import logging

from os import path

LOG = logging.getLogger(__name__)
MAX_PACKAGE_SIZE = 50000000

class PackageUploader(object):
    '''TODO: Should we decouple the config from the Object Init'''
    def __init__(self, config, profile_name):
        self._config = config
        self._vpc_config = self._format_vpc_config()
        self._aws_session = boto3.session.Session(region_name=config.region,
                                                  profile_name=profile_name)
        self._lambda_client = self._aws_session.client('lambda')
        self.version = None

    '''
    Calls the AWS methods to upload an existing package and update
    the function configuration

    returns the package version
    '''
    def upload_existing(self, pkg):
        environment = {'Variables': self._config.variables}
        self._validate_package_size(pkg.zip_file)
        with open(pkg.zip_file, "rb") as fil:
            zip_file = fil.read()

        LOG.debug('running update_function_code')
        conf_update_resp = None
        if self._config.s3_bucket:
            self._upload_s3(pkg.zip_file)
            conf_update_resp = self._lambda_client.update_function_code(
                FunctionName=self._config.name,
                S3Bucket=self._config.s3_bucket,
                S3Key=self._config.s3_package_name(),
                Publish=False,
            )
        else:
            conf_update_resp = self._lambda_client.update_function_code(
                FunctionName=self._config.name,
                ZipFile=zip_file,
                Publish=False,
            )
        LOG.debug("AWS update_function_code response: %s"
                  % conf_update_resp)
        LOG.debug('running update_function_configuration')
        response = self._lambda_client.update_function_configuration(
            FunctionName=self._config.name,
            Handler=self._config.handler,
            Role=self._config.role,
            Description=self._config.description,
            Timeout=self._config.timeout,
            MemorySize=self._config.memory,
            VpcConfig=self._vpc_config,
            Environment=environment,
            TracingConfig=self._config.tracing,
        )
        LOG.debug("AWS update_function_configuration response: %s"
                  % response)

        version = response.get('Version')
        # Publish the version after upload and config update if needed
        if self._config.publish:
            resp = self._lambda_client.publish_version(
                    FunctionName=self._config.name,
                    )
            LOG.debug("AWS publish_version response: %s" % resp)
            version = resp.get('Version')

        return version

    '''
    Creates and uploads a new lambda function

    returns the package version
    '''
    def upload_new(self, pkg):
        environment = {'Variables': self._config.variables}
        code = {}
        if self._config.s3_bucket:
            code = {'S3Bucket': self._config.s3_bucket,
                    'S3Key': self._config.s3_package_name()}
            self._upload_s3(pkg.zip_file)
        else:
            self._validate_package_size(pkg.zip_file)
            with open(pkg.zip_file, "rb") as fil:
                zip_file = fil.read()
            code = {'ZipFile': zip_file}

        LOG.debug('running create_function_code')
        response = self._lambda_client.create_function(
            FunctionName=self._config.name,
            Runtime=self._config.runtime,
            Handler=self._config.handler,
            Role=self._config.role,
            Code=code,
            Description=self._config.description,
            Timeout=self._config.timeout,
            MemorySize=self._config.memory,
            Publish=self._config.publish,
            VpcConfig=self._vpc_config,
            Environment=environment,
            TracingConfig=self._config.tracing,
        )
        LOG.debug("AWS create_function response: %s" % response)

        return response.get('Version')

    '''
    Auto determines whether the function exists or not and calls
    the appropriate method (upload_existing or upload_new).
    '''
    def upload(self, pkg):
        existing_function = True
        try:
            get_resp = self._lambda_client.get_function_configuration(
                    FunctionName=self._config.name)
            LOG.debug("AWS get_function_configuration response: %s" % get_resp)
        except:
            existing_function = False
            LOG.debug("function not found creating new function")

        if existing_function:
            self.version = self.upload_existing(pkg)
        else:
            self.version = self.upload_new(pkg)

    '''
    Create/update an alias to point to the package. Raises an
    exception if the package has not been uploaded.
    '''
    def alias(self):
        # if self.version is still None raise exception
        if self.version is None:
            raise Exception('Must upload package before applying alias')

        if self._alias_exists():
            self._update_alias()
        else:
            self._create_alias()

    '''
    Pulls down the current list of aliases and checks to see if
    an alias exists.
    '''
    def _alias_exists(self):
        resp = self._lambda_client.list_aliases(
                FunctionName=self._config.name)

        for alias in resp.get('Aliases'):
            if alias.get('Name') == self._config.alias:
                return True
        return False

    '''Creates alias'''
    def _create_alias(self):
        LOG.debug("Creating new alias %s" % self._config.alias)
        resp = self._lambda_client.create_alias(
                FunctionName=self._config.name,
                Name=self._config.alias,
                FunctionVersion=self.version,
                Description=self._config.alias_description,
                )
        LOG.debug("AWS create_alias response: %s" % resp)

    '''Update alias'''
    def _update_alias(self):
        LOG.debug("Updating alias %s" % self._config.alias)
        resp = self._lambda_client.update_alias(
                FunctionName=self._config.name,
                Name=self._config.alias,
                FunctionVersion=self.version,
                Description=self._config.alias_description,
                )
        LOG.debug("AWS update_alias response: %s" % resp)

    def _validate_package_size(self, pkg):
        '''
        Logs a warning if the package size is over the current max package size
        '''
        if path.getsize(pkg) > MAX_PACKAGE_SIZE:
            LOG.warning("Size of your deployment package is larger than 50MB!")

    def _format_vpc_config(self):
        '''
        Returns {} if the VPC config is set to None by Config,
        returns the formatted config otherwise
        '''
        if self._config.raw['vpc']:
            return {
                'SubnetIds': self._config.raw['vpc']['subnets'],
                'SecurityGroupIds': self._config.raw['vpc']['security_groups']
            }
        else:
            return {
                'SubnetIds': [],
                'SecurityGroupIds': [],
            }

    def _upload_s3(self, zip_file):
        '''
        Uploads the lambda package to s3
        '''
        s3_client = self._aws_session.client('s3')
        transfer = boto3.s3.transfer.S3Transfer(s3_client)
        transfer.upload_file(zip_file, self._config.s3_bucket,
                             self._config.s3_package_name())
<commit_msg>Allow updates to Runtime<commit_after># Copyright 2015-2016 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import boto3
import logging

from os import path

LOG = logging.getLogger(__name__)
MAX_PACKAGE_SIZE = 50000000

class PackageUploader(object):
    '''TODO: Should we decouple the config from the Object Init'''
    def __init__(self, config, profile_name):
        self._config = config
        self._vpc_config = self._format_vpc_config()
        self._aws_session = boto3.session.Session(region_name=config.region,
                                                  profile_name=profile_name)
        self._lambda_client = self._aws_session.client('lambda')
        self.version = None

    '''
    Calls the AWS methods to upload an existing package and update
    the function configuration

    returns the package version
    '''
    def upload_existing(self, pkg):
        environment = {'Variables': self._config.variables}
        self._validate_package_size(pkg.zip_file)
        with open(pkg.zip_file, "rb") as fil:
            zip_file = fil.read()

        LOG.debug('running update_function_code')
        conf_update_resp = None
        if self._config.s3_bucket:
            self._upload_s3(pkg.zip_file)
            conf_update_resp = self._lambda_client.update_function_code(
                FunctionName=self._config.name,
                S3Bucket=self._config.s3_bucket,
                S3Key=self._config.s3_package_name(),
                Publish=False,
            )
        else:
            conf_update_resp = self._lambda_client.update_function_code(
                FunctionName=self._config.name,
                ZipFile=zip_file,
                Publish=False,
            )
        LOG.debug("AWS update_function_code response: %s"
                  % conf_update_resp)
        LOG.debug('running update_function_configuration')
        response = self._lambda_client.update_function_configuration(
            FunctionName=self._config.name,
            Handler=self._config.handler,
            Role=self._config.role,
            Description=self._config.description,
            Timeout=self._config.timeout,
            MemorySize=self._config.memory,
            VpcConfig=self._vpc_config,
            Environment=environment,
            TracingConfig=self._config.tracing,
            Runtime=self._config.runtime,
        )
        LOG.debug("AWS update_function_configuration response: %s"
                  % response)

        version = response.get('Version')
        # Publish the version after upload and config update if needed
        if self._config.publish:
            resp = self._lambda_client.publish_version(
                    FunctionName=self._config.name,
                    )
            LOG.debug("AWS publish_version response: %s" % resp)
            version = resp.get('Version')

        return version

    '''
    Creates and uploads a new lambda function

    returns the package version
    '''
    def upload_new(self, pkg):
        environment = {'Variables': self._config.variables}
        code = {}
        if self._config.s3_bucket:
            code = {'S3Bucket': self._config.s3_bucket,
                    'S3Key': self._config.s3_package_name()}
            self._upload_s3(pkg.zip_file)
        else:
            self._validate_package_size(pkg.zip_file)
            with open(pkg.zip_file, "rb") as fil:
                zip_file = fil.read()
            code = {'ZipFile': zip_file}

        LOG.debug('running create_function_code')
        response = self._lambda_client.create_function(
            FunctionName=self._config.name,
            Runtime=self._config.runtime,
            Handler=self._config.handler,
            Role=self._config.role,
            Code=code,
            Description=self._config.description,
            Timeout=self._config.timeout,
            MemorySize=self._config.memory,
            Publish=self._config.publish,
            VpcConfig=self._vpc_config,
            Environment=environment,
            TracingConfig=self._config.tracing,
        )
        LOG.debug("AWS create_function response: %s" % response)

        return response.get('Version')

    '''
    Auto determines whether the function exists or not and calls
    the appropriate method (upload_existing or upload_new).
    '''
    def upload(self, pkg):
        existing_function = True
        try:
            get_resp = self._lambda_client.get_function_configuration(
                    FunctionName=self._config.name)
            LOG.debug("AWS get_function_configuration response: %s" % get_resp)
        except:
            existing_function = False
            LOG.debug("function not found creating new function")

        if existing_function:
            self.version = self.upload_existing(pkg)
        else:
            self.version = self.upload_new(pkg)

    '''
    Create/update an alias to point to the package. Raises an
    exception if the package has not been uploaded.
    '''
    def alias(self):
        # if self.version is still None raise exception
        if self.version is None:
            raise Exception('Must upload package before applying alias')

        if self._alias_exists():
            self._update_alias()
        else:
            self._create_alias()

    '''
    Pulls down the current list of aliases and checks to see if
    an alias exists.
    '''
    def _alias_exists(self):
        resp = self._lambda_client.list_aliases(
                FunctionName=self._config.name)

        for alias in resp.get('Aliases'):
            if alias.get('Name') == self._config.alias:
                return True
        return False

    '''Creates alias'''
    def _create_alias(self):
        LOG.debug("Creating new alias %s" % self._config.alias)
        resp = self._lambda_client.create_alias(
                FunctionName=self._config.name,
                Name=self._config.alias,
                FunctionVersion=self.version,
                Description=self._config.alias_description,
                )
        LOG.debug("AWS create_alias response: %s" % resp)

    '''Update alias'''
    def _update_alias(self):
        LOG.debug("Updating alias %s" % self._config.alias)
        resp = self._lambda_client.update_alias(
                FunctionName=self._config.name,
                Name=self._config.alias,
                FunctionVersion=self.version,
                Description=self._config.alias_description,
                )
        LOG.debug("AWS update_alias response: %s" % resp)

    def _validate_package_size(self, pkg):
        '''
        Logs a warning if the package size is over the current max package size
        '''
        if path.getsize(pkg) > MAX_PACKAGE_SIZE:
            LOG.warning("Size of your deployment package is larger than 50MB!")

    def _format_vpc_config(self):
        '''
        Returns {} if the VPC config is set to None by Config,
        returns the formatted config otherwise
        '''
        if self._config.raw['vpc']:
            return {
                'SubnetIds': self._config.raw['vpc']['subnets'],
                'SecurityGroupIds': self._config.raw['vpc']['security_groups']
            }
        else:
            return {
                'SubnetIds': [],
                'SecurityGroupIds': [],
            }

    def _upload_s3(self, zip_file):
        '''
        Uploads the lambda package to s3
        '''
        s3_client = self._aws_session.client('s3')
        transfer = boto3.s3.transfer.S3Transfer(s3_client)
        transfer.upload_file(zip_file, self._config.s3_bucket,
                             self._config.s3_package_name())
<|endoftext|>


<commit_before># support Quandl 3.x.x
try:
    import quandl as Quandl
except:
    # if import fails use Quandl 2.x.x
    import Quandl

from chartpy import Chart, Style

# get your own free Quandl API key from https://www.quandl.com/
quandl_api_key = "xxx"

# choose run_example = 0 for everything
# run_example = 1 - plot US GDP with multiple libraries
# run_example = 2 - plot US and UK unemployment demonstrating multiple line types
run_example = 0

if run_example == 1 or run_example == 0:
    df = Quandl.get("FRED/GDP", authtoken=quandl_api_key)

    # we can use the charting tools in several ways
    chart = Chart()
    chart.plot(df = df)

    # set the style of the plot
    style = Style(title="US GDP", source="Quandl/Fred")

    # Chart object is initialised with the dataframe and our chart style
    chart = Chart(df=df, chart_type='line', style=style)

    # we now plot using multiple plotting libraries, with the same dataframe
    chart.plot(engine='matplotlib')
    chart.plot(engine='bokeh')
    chart.plot(engine='plotly')

if run_example == 2 or run_example == 0:
    # download US and Texas unemployment rate
    df = Quandl.get(["FRED/UNRATE", "FRED/TXUR"], authtoken=quandl_api_key, trim_start="2015-12-01")

    # first plot without any parameters (will use defaults) - note how we can it assign the dataframe to either Chart
    # or the plot method
    Chart(df).plot()
    Chart().plot(df)

    # we can also specify the engine within the Style object if we choose
    style = Style(title="US & Texas unemployment rate", chart_type=['bar', 'line'], engine='matplotlib')

    Chart(df, style=style).plot()
<commit_msg>Added bokeh multiple chart type example<commit_after># support Quandl 3.x.x
try:
    import quandl as Quandl
except:
    # if import fails use Quandl 2.x.x
    import Quandl

from chartpy import Chart, Style

# get your own free Quandl API key from https://www.quandl.com/
quandl_api_key = "x"

# choose run_example = 0 for everything
# run_example = 1 - plot US GDP with multiple libraries
# run_example = 2 - plot US and UK unemployment demonstrating multiple line types
run_example = 2

if run_example == 1 or run_example == 0:
    df = Quandl.get("FRED/GDP", authtoken=quandl_api_key)

    # we can use the charting tools in several ways
    chart = Chart()
    chart.plot(df = df)

    # set the style of the plot
    style = Style(title="US GDP", source="Quandl/Fred")

    # Chart object is initialised with the dataframe and our chart style
    chart = Chart(df=df, chart_type='line', style=style)

    # we now plot using multiple plotting libraries, with the same dataframe
    chart.plot(engine='matplotlib')
    chart.plot(engine='bokeh')
    chart.plot(engine='plotly')

if run_example == 2 or run_example == 0:
    # download US and Texas unemployment rate
    df = Quandl.get(["FRED/UNRATE", "FRED/TXUR"], authtoken=quandl_api_key, trim_start="2015-12-01")

    # first plot without any parameters (will use defaults) - note how we can it assign the dataframe to either Chart
    # or the plot method
    Chart(df).plot()
    Chart().plot(df)

    # we can also specify the engine within the Style object if we choose
    style = Style(title="US & Texas unemployment rate", chart_type=['bar', 'line'], engine='matplotlib')

    Chart(df, style=style).plot()

    style.engine = 'bokeh'
    Chart(df, style=style).plot()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import os
from slugify import slugify
from PIL import Image

CATS = [
    "Koníci",
#    "Pejsci a kočičky",
#    "Jiná zvířátka",
#    "Lidičky a jezdci",
#    "Bytový textil",
#    "Potahy na autosedačky"
]

class BaseObject:
    def __init__(self, title):
        self.title = title

    @property
    def slug(self):
        return slugify(self.title)

class Product(BaseObject):
   pass

class Group(BaseObject):
    def __init__(self, title):
        BaseObject.__init__(self, title)
        self.products = []

    def append(self, ptitle):
        self.products.append(Product(ptitle))

used_products = []

result = ""

for id_cat, cat_title in enumerate(CATS):
    cat_file = "source_data{}".format(id_cat)
    cat_slug = slugify(cat_title)
    result += "- id    : {}\n".format(id_cat)
    result += "  title : {}\n".format(cat_title)
    result += "  slug  : {}\n".format(cat_slug)

    if not os.path.exists(cat_file):
        continue

    result += "  groups :\n"

    groups = []
    current_group = None

    data_feed = open(cat_file)
    for line in data_feed.readlines():
        line = line.strip()
        if not line:
            continue

        if line.startswith("::"):
            group_title = line.lstrip("::")
            groups.append(Group(group_title))
            continue

        if not groups:
            continue

        groups[-1].append(line)

    for id_group, group in enumerate(groups):
        id_group += 1
        result += "    - id    : {}\n".format(id_group)
        result += "      title : {}\n".format(group.title)
        result += "      slug  : {}\n".format(group.slug)

        if group.products:
            result += "      products :\n"

        cpath = "../{}/{}/".format(cat_slug, group.slug)
        if not os.path.exists(cpath):
            os.makedirs(cpath)
        f = open(cpath + "index.html", "w")
        f.write("---\n")
        f.write("layout: product_group\n")
        f.write("title: {}\n".format(group.title))
        f.write("cat_slug : {}\n".format(cat_slug))
        f.write("group_slug: {}\n".format(group.slug))

        #
        # Group thumbnail (barevne sady)
        #

        ifile = os.path.join("products", group.slug+".jpg")
        if os.path.exists(ifile):
            itgt = cpath + group.slug + ".jpg"
            itgt_tn = cpath + group.slug + "_tn.jpg"
            f.write("group_image: {}\n".format(group.slug+".jpg"))
            im_orig = Image.open(ifile)

            im = im_orig.resize((640,400), Image.ANTIALIAS)
            im.save(itgt)

            im = im_orig.resize((261,163), Image.ANTIALIAS)
            im.save(itgt_tn)

        #
        # Products
        #

        if group.products:
            f.write("products:\n")

        for id_product, product in enumerate(group.products):
            id_product += 1000*id_group
            result += "        - id   : {}\n".format(id_product)
            result += "          title: {}\n".format(product.title)
            result += "          slug : {}\n".format(product.slug)

            f.write("  - id   : {}\n".format(id_product))
            f.write("    title: {}\n".format(product.title))
            f.write("    slug : {}\n".format(product.slug))

            pfile = os.path.join("products", product.slug+".txt")
            ifile = os.path.join("products", product.slug+".jpg")
            ptext = ""
            pprices = []
            if os.path.exists(pfile):
                for pline in open(pfile).readlines():
                    r = pline.split(":")
                    if len(r) == 2 and r[1].strip().isdigit():
                        pprices.append(r)
                        continue
                    ptext += pline
            else:
                print (pfile, "does not exist")

            ptfile = cpath + product.slug + ".md"
            pt = open(ptfile, "w")
            pt.write("---\n")
            pt.write("layout: product\n")
            pt.write("title: {}\n".format(product.title))
            pt.write("cat_slug : {}\n".format(cat_slug))
            pt.write("group_slug: {}\n".format(group.slug))
            pt.write("slug: {}\n".format(product.slug))
            if pprices:
                pt.write("pricing:\n")
                for v, c in pprices:
                    pt.write("  - variant : {}\n".format(v.strip()))
                    pt.write("    price   : {}\n".format(c.strip()))
            pt.write("---\n")
            pt.write("\n{}\n\n".format(ptext.strip()))
            pt.close()

            if not os.path.exists(ifile):
                ifile = "default.jpg"

            iffile = cpath + product.slug + ".jpg"
            itfile = cpath + product.slug + "_tn.jpg"

            im_orig = Image.open(ifile)

            im = im_orig.resize((640,400), Image.ANTIALIAS)
            im.save(iffile)

            im = im_orig.resize((261,163), Image.ANTIALIAS)
            im.save(itfile)

            used_products.append(product.slug)

        f.write("---\n")

        #
        # Group description
        #

        pfile = os.path.join("products", group.slug+".txt")
        if os.path.exists(pfile):
            pf = open(pfile).read()
            f.write(pf)
            f.close()

        result += "\n"

for p in os.listdir("products"):
    p = os.path.splitext(p)[0]
    if not p in used_products:
        print ("Unused", p)

f = open("../_data/products.yml","w")
f.write(result)
f.close()

<commit_msg>empty groups fixes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

import os
from slugify import slugify
from PIL import Image

CATS = [
    "Koníci",
    "Pejsci a kočičky",
    "Jiná zvířátka",
    "Lidičky a jezdci",
    "Bytový textil",
    "Potahy na autosedačky"
]

class BaseObject:
    def __init__(self, title):
        self.title = title

    @property
    def slug(self):
        return slugify(self.title)

class Product(BaseObject):
   pass

class Group(BaseObject):
    def __init__(self, title):
        BaseObject.__init__(self, title)
        self.products = []

    def append(self, ptitle):
        self.products.append(Product(ptitle))

used_products = []

result = ""

for id_cat, cat_title in enumerate(CATS):
    cat_file = "source_data{}".format(id_cat)
    cat_slug = slugify(cat_title)
    result += "- id    : {}\n".format(id_cat)
    result += "  title : {}\n".format(cat_title)
    result += "  slug  : {}\n".format(cat_slug)

    if not os.path.exists(cat_file):
        continue

    result += "  groups :\n"

    groups = []
    current_group = None

    data_feed = open(cat_file)
    for line in data_feed.readlines():
        line = line.strip()
        if not line:
            continue

        if line.startswith("::"):
            group_title = line.lstrip("::")
            groups.append(Group(group_title))
            continue

        if not groups:
            continue

        groups[-1].append(line)

    for id_group, group in enumerate(groups):
        if not group.products:
            continue
        id_group += 1
        result += "    - id    : {}\n".format(id_group)
        result += "      title : {}\n".format(group.title)
        result += "      slug  : {}\n".format(group.slug)

        if group.products:
            result += "      products :\n"

        cpath = "../{}/{}/".format(cat_slug, group.slug)
        if not os.path.exists(cpath):
            os.makedirs(cpath)
        f = open(cpath + "index.html", "w")
        f.write("---\n")
        f.write("layout: product_group\n")
        f.write("title: {}\n".format(group.title))
        f.write("cat_slug : {}\n".format(cat_slug))
        f.write("group_slug: {}\n".format(group.slug))

        #
        # Group thumbnail (barevne sady)
        #

        ifile = os.path.join("products", group.slug+".jpg")
        if os.path.exists(ifile):
            itgt = cpath + group.slug + ".jpg"
            itgt_tn = cpath + group.slug + "_tn.jpg"
            f.write("group_image: {}\n".format(group.slug+".jpg"))
            im_orig = Image.open(ifile)

            im = im_orig.resize((640,400), Image.ANTIALIAS)
            im.save(itgt)

            im = im_orig.resize((261,163), Image.ANTIALIAS)
            im.save(itgt_tn)

        #
        # Products
        #

        if group.products:
            f.write("products:\n")

        for id_product, product in enumerate(group.products):
            id_product += 1000*id_group
            result += "        - id   : {}\n".format(id_product)
            result += "          title: {}\n".format(product.title)
            result += "          slug : {}\n".format(product.slug)

            f.write("  - id   : {}\n".format(id_product))
            f.write("    title: {}\n".format(product.title))
            f.write("    slug : {}\n".format(product.slug))

            pfile = os.path.join("products", product.slug+".txt")
            ifile = os.path.join("products", product.slug+".jpg")
            ptext = ""
            pprices = []
            if os.path.exists(pfile):
                for pline in open(pfile).readlines():
                    r = pline.split(":")
                    if len(r) == 2 and r[1].strip().isdigit():
                        pprices.append(r)
                        continue
                    ptext += pline
            else:
                print (pfile, "does not exist")
                pff = open(pfile,"w")
                pff.close()

            ptfile = cpath + product.slug + ".md"
            pt = open(ptfile, "w")
            pt.write("---\n")
            pt.write("layout: product\n")
            pt.write("title: {}\n".format(product.title))
            pt.write("cat_slug : {}\n".format(cat_slug))
            pt.write("group_slug: {}\n".format(group.slug))
            pt.write("slug: {}\n".format(product.slug))
            if pprices:
                pt.write("pricing:\n")
                for v, c in pprices:
                    pt.write("  - variant : {}\n".format(v.strip()))
                    pt.write("    price   : {}\n".format(c.strip()))
            pt.write("---\n")
            pt.write("\n{}\n\n".format(ptext.strip()))
            pt.close()

            if not os.path.exists(ifile):
                ifile = "default.jpg"

            iffile = cpath + product.slug + ".jpg"
            itfile = cpath + product.slug + "_tn.jpg"

            im_orig = Image.open(ifile)

            im = im_orig.resize((640,400), Image.ANTIALIAS)
            im.save(iffile)

            im = im_orig.resize((261,163), Image.ANTIALIAS)
            im.save(itfile)

            used_products.append(product.slug)

        f.write("---\n")

        #
        # Group description
        #

        pfile = os.path.join("products", group.slug+".txt")
        if os.path.exists(pfile):
            pf = open(pfile).read()
            f.write(pf)
            f.close()

        result += "\n"

for p in os.listdir("products"):
    p = os.path.splitext(p)[0]
    if not p in used_products:
        print ("Unused", p)

f = open("../_data/products.yml","w")
f.write(result)
f.close()

<|endoftext|>


<commit_before>""" Parent contains the base class for diverter devices."""
# diverter.py
# Mission Pinball Framework
# Written by Brian Madden & Gabe Knuth
# Released under the MIT License. (See license info at the end of this file.)

# Documentation and more info at http://missionpinball.com/mpf

from collections import deque

from mpf.system.device import Device
from mpf.system.tasks import DelayManager
from mpf.system.timing import Timing
from mpf.system.config import Config

class Diverter(Device):
    """Represents a diverter in a pinball machine.

    Args: Same as the Device parent class.
    """

    config_section = 'diverters'
    collection = 'diverters'
    class_label = 'diverter'

    def __init__(self, machine, name, config, collection=None, validate=True):
        super(Diverter, self).__init__(machine, name, config, collection,
                                       validate=validate)

        self.delay = DelayManager()

        # Attributes
        self.active = False
        self.enabled = False
        self.platform = None

        self.divering_ejects_count = 0
        self.eject_state = False
        self.eject_attempt_queue = deque()

        # Create a list of ball device objects when active and inactive. We need
        # this because ball eject attempts pass the target device as an object
        # rather than by name.

        # register for feeder device eject events
        for feeder_device in self.config['feeder_devices']:
            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_attempt',
                                            self._feeder_eject_attempt)

            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_failed',
                                            self._feeder_eject_count_decrease)

            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_success',
                                            self._feeder_eject_count_decrease)

        self.machine.events.add_handler('init_phase_3', self._register_switches)

        self.platform = self.config['activation_coil'].platform

    def _register_switches(self):
                # register for deactivation switches
        for switch in self.config['deactivation_switches']:
            self.machine.switch_controller.add_switch_handler(
                switch.name, self.deactivate)

        # register for disable switches:
        for switch in self.config['disable_switches']:
            self.machine.switch_controller.add_switch_handler(
                switch.name, self.disable)

    def enable(self, auto=False, activations=-1, **kwargs):
        """Enables this diverter.

        Args:
            auto: Boolean value which is used to indicate whether this
                diverter enabled itself automatically. This is passed to the
                event which is posted.
            activations: Integer of how many times you'd like this diverter to
                activate before it will automatically disable itself. Default is
                -1 which is unlimited.

        If an 'activation_switches' is configured, then this method writes a
        hardware autofire rule to the pinball controller which fires the
        diverter coil when the switch is activated.

        If no `activation_switches` is specified, then the diverter is activated
        immediately.
        """

        self.enabled = True

        self.machine.events.post('diverter_' + self.name + '_enabling',
                                 auto=auto)

        if self.config['activation_switches']:
            self.enable_hw_switches()
        else:
            self.activate()

    def disable(self, auto=False, **kwargs):
        """Disables this diverter.

        This method will remove the hardware rule if this diverter is activated
        via a hardware switch.

        Args:
            auto: Boolean value which is used to indicate whether this
                diverter disabled itself automatically. This is passed to the
                event which is posted.
            **kwargs: This is here because this disable method is called by
                whatever event the game programmer specifies in their machine
                configuration file, so we don't know what event that might be
                or whether it has random kwargs attached to it.
        """

        self.enabled = False

        self.machine.events.post('diverter_' + self.name + '_disabling',
                                 auto=auto)

        self.log.debug("Disabling Diverter")
        if self.config['activation_switches']:
            self.disable_hw_switch()
        else:
            self.deactivate()

    def activate(self):
        """Physically activates this diverter's coil."""
        self.log.debug("Activating Diverter")

        self.active = True

        #if self.remaining_activations > 0:
        #    self.remaining_activations -= 1

        self.machine.events.post('diverter_' + self.name + '_activating')
        if self.config['type'] == 'pulse':
            self.config['activation_coil'].pulse()
        elif self.config['type'] == 'hold':
            self.config['activation_coil'].enable()
            self.schedule_deactivation()

    def deactivate(self):
        """Deactivates this diverter.

        This method will disable the activation_coil, and (optionally) if it's
        configured with a deactivation coil, it will pulse it.
        """
        self.log.debug("Deactivating Diverter")

        self.active = False

        self.machine.events.post('diverter_' + self.name + '_deactivating')
        self.config['activation_coil'].disable()

        if self.config['deactivation_coil']:
            self.config['deactivation_coil'].pulse()

        #if self.remaining_activations != 0:
        #    self.enable()
            # todo this will be weird if the diverter is enabled without a hw
            # switch.. wonder if we should check for that here?

    def schedule_deactivation(self, time=None):
        """Schedules a delay to deactivate this diverter.

        Args:
            time: The MPF string time of how long you'd like the delay before
                deactivating the diverter. Default is None which means it uses
                the 'activation_time' setting configured for this diverter. If
                there is no 'activation_time' setting and no delay is passed,
                it will disable the diverter immediately.
        """

        if time is not None:
            delay = Timing.string_to_ms(time)
        elif self.config['activation_time']:
            delay = self.config['activation_time']
        else:
            delay = False

        if delay:
            self.delay.add('disable_held_coil', delay, self.disable_held_coil)
        else:
            self.disable_held_coil()

    def enable_hw_switches(self):
        """Enables the hardware switch rule which causes this diverter to
        activate when the switch is hit.

        This is typically used for diverters on loops and ramps where you don't
        want the diverter to phsyically activate until the ramp entry switch is
        activated.

        If this diverter is configured with a activation_time, this method will
        also set switch handlers which will set a delay to deactivate the
        diverter once the activation activation_time expires.

        If this diverter is configured with a deactivation switch, this method
        will set up the switch handlers to deactivate the diverter when the
        deactivation switch is activated.
        """
        self.log.debug("Enabling Diverter for hw switch: %s",
                       self.config['activation_switches'])

        if self.config['type'] == 'hold':

            for switch in self.config['activation_switches']:

                self.platform.set_hw_rule(
                    sw_name=switch.name,
                    sw_activity='active',
                    coil_name=self.config['activation_coil'].name,
                    coil_action_ms=-1,
                    pulse_ms=self.config['activation_coil'].config['pulse_ms'],
                    pwm_on=self.config['activation_coil'].config['pwm_on'],
                    pwm_off=self.config['activation_coil'].config['pwm_off'],
                    debounced=False)

                # If there's a activation_time then we need to watch for the hw
                # switch to be activated so we can disable the diverter

                if self.config['activation_time']:
                    self.machine.switch_controller.add_switch_handler(
                        switch.name,
                        self.schedule_deactivation)

        elif self.config['type'] == 'pulse':

            for switch in self.config['activation_switches']:

                self.platform.set_hw_rule(
                    sw_name=switch.name,
                    sw_activity='active',
                    coil_name=self.config['activation_coil'].name,
                    coil_action_ms=self.config['activation_coil'].config['pulse_ms'],
                    pulse_ms=self.config['activation_coil'].config['pulse_ms'],
                    debounced=False)

    def disable_hw_switch(self):
        """Removes the hardware rule to disable the hardware activation switch
        for this diverter.
        """

        for switch in self.config['activation_switches']:
            self.platform.clear_hw_rule(switch.name)

        # todo this should not clear all the rules for this switch

    def disable_held_coil(self):
        """Physically disables the coil holding this diverter open."""
        self.config['activation_coil'].disable()

    def _feeder_eject_count_decrease(self, target, **kwargs):
        self.divering_ejects_count -= 1
        if self.divering_ejects_count <= 0:
            self.divering_ejects_count = 0

            # If there are ejects waiting for the other target switch diverter
            if len(self.eject_attempt_queue) > 0:
                if self.eject_state == False:
                    self.eject_state = True
                    self.log.debug("Enabling diverter since eject target is on the "
                                   "active target list")
                    self.enable()
                elif self.eject_state == True:
                    self.eject_state = False
                    self.log.debug("Enabling diverter since eject target is on the "
                                   "inactive target list")
                    self.disable()
            # And perform those ejects
            while len(self.eject_attempt_queue) > 0:
                self.divering_ejects_count += 1
                queue = self.eject_attempt_queue.pop()
                queue.clear()

    def _feeder_eject_attempt(self, queue, target, **kwargs):
        # Event handler which is called when one of this diverter's feeder
        # devices attempts to eject a ball. This is what allows this diverter
        # to get itself in the right position to send the ball to where it needs
        # to go.

        # Since the 'target' kwarg is going to be an object, not a name, we need
        # to figure out if this object is one of the targets of this diverter.

        self.log.debug("Feeder device eject attempt for target: %s", target)

        desired_state = None
        if target in self.config['targets_when_active']:
            desired_state = True

        elif target in self.config['targets_when_inactive']:
            desired_state = False

        if desired_state == None:
            self.log.debug("Feeder device ejects to an unknown target: %s. "
                           "Ignoring!", target.name)
            return

        if self.divering_ejects_count > 0 and self.eject_state != desired_state:
            self.log.debug("Feeder devices tries to eject to a target which "
                           "would require a state change. Postponing that "
                           "because we have an eject to the other side")
            queue.wait()
            self.eject_attempt_queue.append(queue)
            return

        self.divering_ejects_count += 1

        if desired_state == True:
            self.log.debug("Enabling diverter since eject target is on the "
                           "active target list")
            self.eject_state = desired_state
            self.enable()
        elif desired_state == False:
            self.log.debug("Enabling diverter since eject target is on the "
                           "inactive target list")
            self.eject_state = desired_state
            self.disable()

# The MIT License (MIT)

# Copyright (c) 2013-2015 Brian Madden and Gabe Knuth

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
<commit_msg>fix error in diverter<commit_after>""" Parent contains the base class for diverter devices."""
# diverter.py
# Mission Pinball Framework
# Written by Brian Madden & Gabe Knuth
# Released under the MIT License. (See license info at the end of this file.)

# Documentation and more info at http://missionpinball.com/mpf

from collections import deque

from mpf.system.device import Device
from mpf.system.tasks import DelayManager
from mpf.system.timing import Timing
from mpf.system.config import Config

class Diverter(Device):
    """Represents a diverter in a pinball machine.

    Args: Same as the Device parent class.
    """

    config_section = 'diverters'
    collection = 'diverters'
    class_label = 'diverter'

    def __init__(self, machine, name, config, collection=None, validate=True):
        super(Diverter, self).__init__(machine, name, config, collection,
                                       validate=validate)

        self.delay = DelayManager()

        # Attributes
        self.active = False
        self.enabled = False
        self.platform = None

        self.divering_ejects_count = 0
        self.eject_state = False
        self.eject_attempt_queue = deque()

        # Create a list of ball device objects when active and inactive. We need
        # this because ball eject attempts pass the target device as an object
        # rather than by name.

        # register for feeder device eject events
        for feeder_device in self.config['feeder_devices']:
            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_attempt',
                                            self._feeder_eject_attempt)

            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_failed',
                                            self._feeder_eject_count_decrease)

            self.machine.events.add_handler('balldevice_' + feeder_device.name +
                                            '_ball_eject_success',
                                            self._feeder_eject_count_decrease)

        self.machine.events.add_handler('init_phase_3', self._register_switches)

        self.platform = self.config['activation_coil'].platform

    def _register_switches(self):
                # register for deactivation switches
        for switch in self.config['deactivation_switches']:
            self.machine.switch_controller.add_switch_handler(
                switch.name, self.deactivate)

        # register for disable switches:
        for switch in self.config['disable_switches']:
            self.machine.switch_controller.add_switch_handler(
                switch.name, self.disable)

    def enable(self, auto=False, activations=-1, **kwargs):
        """Enables this diverter.

        Args:
            auto: Boolean value which is used to indicate whether this
                diverter enabled itself automatically. This is passed to the
                event which is posted.
            activations: Integer of how many times you'd like this diverter to
                activate before it will automatically disable itself. Default is
                -1 which is unlimited.

        If an 'activation_switches' is configured, then this method writes a
        hardware autofire rule to the pinball controller which fires the
        diverter coil when the switch is activated.

        If no `activation_switches` is specified, then the diverter is activated
        immediately.
        """

        self.enabled = True

        self.machine.events.post('diverter_' + self.name + '_enabling',
                                 auto=auto)

        if self.config['activation_switches']:
            self.enable_hw_switches()
        else:
            self.activate()

    def disable(self, auto=False, **kwargs):
        """Disables this diverter.

        This method will remove the hardware rule if this diverter is activated
        via a hardware switch.

        Args:
            auto: Boolean value which is used to indicate whether this
                diverter disabled itself automatically. This is passed to the
                event which is posted.
            **kwargs: This is here because this disable method is called by
                whatever event the game programmer specifies in their machine
                configuration file, so we don't know what event that might be
                or whether it has random kwargs attached to it.
        """

        self.enabled = False

        self.machine.events.post('diverter_' + self.name + '_disabling',
                                 auto=auto)

        self.log.debug("Disabling Diverter")
        if self.config['activation_switches']:
            self.disable_hw_switch()
        else:
            self.deactivate()

    def activate(self):
        """Physically activates this diverter's coil."""
        self.log.debug("Activating Diverter")

        self.active = True

        #if self.remaining_activations > 0:
        #    self.remaining_activations -= 1

        self.machine.events.post('diverter_' + self.name + '_activating')
        if self.config['type'] == 'pulse':
            self.config['activation_coil'].pulse()
        elif self.config['type'] == 'hold':
            self.config['activation_coil'].enable()
            self.schedule_deactivation()

    def deactivate(self):
        """Deactivates this diverter.

        This method will disable the activation_coil, and (optionally) if it's
        configured with a deactivation coil, it will pulse it.
        """
        self.log.debug("Deactivating Diverter")

        self.active = False

        self.machine.events.post('diverter_' + self.name + '_deactivating')
        self.config['activation_coil'].disable()

        if self.config['deactivation_coil']:
            self.config['deactivation_coil'].pulse()

        #if self.remaining_activations != 0:
        #    self.enable()
            # todo this will be weird if the diverter is enabled without a hw
            # switch.. wonder if we should check for that here?

    def schedule_deactivation(self, time=None):
        """Schedules a delay to deactivate this diverter.

        Args:
            time: The MPF string time of how long you'd like the delay before
                deactivating the diverter. Default is None which means it uses
                the 'activation_time' setting configured for this diverter. If
                there is no 'activation_time' setting and no delay is passed,
                it will disable the diverter immediately.
        """

        if time is not None:
            delay = Timing.string_to_ms(time)
        elif self.config['activation_time']:
            delay = self.config['activation_time']
        else:
            delay = False

        if delay:
            self.delay.add('disable_held_coil', delay, self.disable_held_coil)
        else:
            self.disable_held_coil()

    def enable_hw_switches(self):
        """Enables the hardware switch rule which causes this diverter to
        activate when the switch is hit.

        This is typically used for diverters on loops and ramps where you don't
        want the diverter to phsyically activate until the ramp entry switch is
        activated.

        If this diverter is configured with a activation_time, this method will
        also set switch handlers which will set a delay to deactivate the
        diverter once the activation activation_time expires.

        If this diverter is configured with a deactivation switch, this method
        will set up the switch handlers to deactivate the diverter when the
        deactivation switch is activated.
        """
        self.log.debug("Enabling Diverter for hw switch: %s",
                       self.config['activation_switches'])

        if self.config['type'] == 'hold':

            for switch in self.config['activation_switches']:

                self.platform.set_hw_rule(
                    sw_name=switch.name,
                    sw_activity=1,
                    coil_name=self.config['activation_coil'].name,
                    coil_action_ms=-1,
                    pulse_ms=self.config['activation_coil'].config['pulse_ms'],
                    pwm_on=self.config['activation_coil'].config['pwm_on'],
                    pwm_off=self.config['activation_coil'].config['pwm_off'],
                    debounced=False)

                # If there's a activation_time then we need to watch for the hw
                # switch to be activated so we can disable the diverter

                if self.config['activation_time']:
                    self.machine.switch_controller.add_switch_handler(
                        switch.name,
                        self.schedule_deactivation)

        elif self.config['type'] == 'pulse':

            for switch in self.config['activation_switches']:

                self.platform.set_hw_rule(
                    sw_name=switch.name,
                    sw_activity=1,
                    coil_name=self.config['activation_coil'].name,
                    coil_action_ms=self.config['activation_coil'].config['pulse_ms'],
                    pulse_ms=self.config['activation_coil'].config['pulse_ms'],
                    debounced=False)

    def disable_hw_switch(self):
        """Removes the hardware rule to disable the hardware activation switch
        for this diverter.
        """

        for switch in self.config['activation_switches']:
            self.platform.clear_hw_rule(switch.name)

        # todo this should not clear all the rules for this switch

    def disable_held_coil(self):
        """Physically disables the coil holding this diverter open."""
        self.config['activation_coil'].disable()

    def _feeder_eject_count_decrease(self, target, **kwargs):
        self.divering_ejects_count -= 1
        if self.divering_ejects_count <= 0:
            self.divering_ejects_count = 0

            # If there are ejects waiting for the other target switch diverter
            if len(self.eject_attempt_queue) > 0:
                if self.eject_state == False:
                    self.eject_state = True
                    self.log.debug("Enabling diverter since eject target is on the "
                                   "active target list")
                    self.enable()
                elif self.eject_state == True:
                    self.eject_state = False
                    self.log.debug("Enabling diverter since eject target is on the "
                                   "inactive target list")
                    self.disable()
            # And perform those ejects
            while len(self.eject_attempt_queue) > 0:
                self.divering_ejects_count += 1
                queue = self.eject_attempt_queue.pop()
                queue.clear()

    def _feeder_eject_attempt(self, queue, target, **kwargs):
        # Event handler which is called when one of this diverter's feeder
        # devices attempts to eject a ball. This is what allows this diverter
        # to get itself in the right position to send the ball to where it needs
        # to go.

        # Since the 'target' kwarg is going to be an object, not a name, we need
        # to figure out if this object is one of the targets of this diverter.

        self.log.debug("Feeder device eject attempt for target: %s", target)

        desired_state = None
        if target in self.config['targets_when_active']:
            desired_state = True

        elif target in self.config['targets_when_inactive']:
            desired_state = False

        if desired_state == None:
            self.log.debug("Feeder device ejects to an unknown target: %s. "
                           "Ignoring!", target.name)
            return

        if self.divering_ejects_count > 0 and self.eject_state != desired_state:
            self.log.debug("Feeder devices tries to eject to a target which "
                           "would require a state change. Postponing that "
                           "because we have an eject to the other side")
            queue.wait()
            self.eject_attempt_queue.append(queue)
            return

        self.divering_ejects_count += 1

        if desired_state == True:
            self.log.debug("Enabling diverter since eject target is on the "
                           "active target list")
            self.eject_state = desired_state
            self.enable()
        elif desired_state == False:
            self.log.debug("Enabling diverter since eject target is on the "
                           "inactive target list")
            self.eject_state = desired_state
            self.disable()

# The MIT License (MIT)

# Copyright (c) 2013-2015 Brian Madden and Gabe Knuth

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2011-2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Tails a realtime search using the export endpoint and prints results to
   stdout."""

from pprint import pprint
import sys

from splunklib.client import connect
import splunklib.results as results

import utils

def main():
    usage = "usage: %prog <search>"
    opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage=usage)

    if len(opts.args) != 1:
        utils.error("Search expression required", 2)
    search = opts.args[0]

    service = connect(**opts.kwargs)

    try:
        result = service.get(
            "search/jobs/export",
            search=search,
            earliest_time="rt",
            latest_time="rt",
            search_mode="realtime")

        reader = results.ResultsReader(result.body)
        while True:
            kind = reader.read()
            if kind == None: break
            if kind == results.RESULT:
                event = reader.value
                pprint(event)

    except KeyboardInterrupt:
        print "\nInterrupted."

if __name__ == "__main__":
    main()

<commit_msg>Fixed stail.py in examples to use new ResultsReader interface.<commit_after>#!/usr/bin/env python
#
# Copyright 2011-2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Tails a realtime search using the export endpoint and prints results to
   stdout."""

import sys
sys.path.insert(0, '../')

from pprint import pprint

from splunklib.client import connect
from splunklib.results import ResultsReader

import utils

def main():
    usage = "usage: %prog <search>"
    opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage=usage)

    if len(opts.args) != 1:
        utils.error("Search expression required", 2)
    search = opts.args[0]

    service = connect(**opts.kwargs)

    try:
        result = service.get(
            "search/jobs/export",
            search=search,
            earliest_time="rt",
            latest_time="rt",
            search_mode="realtime")

        for result in ResultsReader(result.body):
            if result is not None:
                print pprint(result)

    except KeyboardInterrupt:
        print "\nInterrupted."

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import print_function

import pytest
import responses
import simplesqlite
from click.testing import CliRunner
from sqlitebiter._enum import ExitCode
from sqlitebiter.sqlitebiter import cmd

from .common import print_traceback
from .dataset import complex_json

class Test_url_subcommand(object):

    db_path = "test.sqlite"

    @responses.activate
    def test_normal_json(self):
        url = "https://example.com/complex_json.json"
        responses.add(
            responses.GET,
            url,
            body=complex_json,
            content_type='text/plain; charset=utf-8',
            status=200)
        runner = CliRunner()

        with runner.isolated_filesystem():
            result = runner.invoke(cmd, ["url", url, "-o", self.db_path])
            print_traceback(result)

            assert result.exit_code == ExitCode.SUCCESS

            con = simplesqlite.SimpleSQLite(self.db_path, "r")
            expected = set([
                'ratings', 'screenshots_4', 'screenshots_3', 'screenshots_5', 'screenshots_1',
                'screenshots_2', 'tags', 'versions', 'root'])

            assert set(con.get_table_name_list()) == expected
<commit_msg>Add test cases<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <tsuyoshi.hombashi@gmail.com>
"""

from __future__ import print_function

import pytest
import responses
import simplesqlite
from click.testing import CliRunner
from sqlitebiter._enum import ExitCode
from sqlitebiter.sqlitebiter import cmd

from .common import print_traceback
from .dataset import complex_json

class Test_url_subcommand(object):

    db_path = "test.sqlite"

    @responses.activate
    def test_normal_json(self):
        url = "https://example.com/complex_json.json"
        responses.add(
            responses.GET,
            url,
            body=complex_json,
            content_type='text/plain; charset=utf-8',
            status=200)
        runner = CliRunner()

        with runner.isolated_filesystem():
            result = runner.invoke(cmd, ["url", url, "-o", self.db_path])
            print_traceback(result)

            assert result.exit_code == ExitCode.SUCCESS

            con = simplesqlite.SimpleSQLite(self.db_path, "r")
            expected = set([
                'ratings', 'screenshots_4', 'screenshots_3', 'screenshots_5', 'screenshots_1',
                'screenshots_2', 'tags', 'versions', 'root'])

            assert set(con.get_table_name_list()) == expected

    @pytest.mark.parametrize(["url", "expected"], [
        [
            "https://en.wikipedia.org/wiki/Comparison_of_firewalls",
            ExitCode.SUCCESS,
        ], [
            "https://en.wikipedia.org/wiki/Comparison_of_firewalls#Firewall_software",
            ExitCode.SUCCESS,
        ],
    ])
    def test_normal_html(self, url, expected):
        runner = CliRunner()

        with runner.isolated_filesystem():
            result = runner.invoke(cmd, ["url", url, "-o", self.db_path])
            print_traceback(result)

            assert result.exit_code == expected
<|endoftext|>


<commit_before>import math
import random
import re
import sys
from hashlib import sha1

from flask import abort
from flask import render_template
from flask import request
from peewee import DoesNotExist
from peewee import ForeignKeyField
from peewee import Model
from peewee import SelectQuery

from flask_peewee._compat import text_type

def get_object_or_404(query_or_model, *query):
    if not isinstance(query_or_model, SelectQuery):
        query_or_model = query_or_model.select()
    try:
        return query_or_model.where(*query).get()
    except DoesNotExist:
        abort(404)

def object_list(template_name, qr, var_name='object_list', **kwargs):
    pq = PaginatedQuery(qr, kwargs.pop('paginate_by', 20))
    kwargs[var_name] = pq.get_list()
    return render_template(template_name, pagination=pq, page=pq.get_page(), **kwargs)

class PaginatedQuery(object):
    page_var = 'page'

    def __init__(self, query_or_model, paginate_by):
        self.paginate_by = paginate_by

        if isinstance(query_or_model, SelectQuery):
            self.query = query_or_model
            self.model = self.query.model_class
        else:
            self.model = query_or_model
            self.query = self.model.select()

    def get_page(self):
        curr_page = request.args.get(self.page_var)
        if curr_page and curr_page.isdigit():
            return int(curr_page)
        return 1

    def get_pages(self):
        return int(math.ceil(float(self.query.count()) / self.paginate_by))

    def get_list(self):
        return self.query.paginate(self.get_page(), self.paginate_by)

def get_next():
    if not request.query_string:
        return request.path
    return '%s?%s' % (request.path, request.query_string)

def slugify(s):
    return re.sub('[^a-z0-9_\-]+', '-', s.lower())

def load_class(s):
    path, klass = s.rsplit('.', 1)
    __import__(path)
    mod = sys.modules[path]
    return getattr(mod, klass)

def get_dictionary_from_model(model, fields=None, exclude=None):
    model_class = type(model)
    data = {}

    fields = fields or {}
    exclude = exclude or {}
    curr_exclude = exclude.get(model_class, [])
    curr_fields = fields.get(model_class, model._meta.sorted_field_names)

    for field_name in curr_fields:
        if field_name in curr_exclude:
            continue
        field_obj = model_class._meta.fields[field_name]
        field_data = model._data.get(field_name)
        if isinstance(field_obj, ForeignKeyField) and field_data and field_obj.rel_model in fields:
            rel_obj = getattr(model, field_name)
            data[field_name] = get_dictionary_from_model(rel_obj, fields, exclude)
        else:
            data[field_name] = field_data
    return data

def get_model_from_dictionary(model, field_dict):
    if isinstance(model, Model):
        model_instance = model
        check_fks = True
    else:
        model_instance = model()
        check_fks = False
    models = [model_instance]
    for field_name, value in field_dict.items():
        field_obj = model._meta.fields[field_name]
        if isinstance(value, dict):
            rel_obj = field_obj.rel_model
            if check_fks:
                try:
                    rel_obj = getattr(model, field_name)
                except field_obj.rel_model.DoesNotExist:
                    pass
                if rel_obj is None:
                    rel_obj = field_obj.rel_model
            rel_inst, rel_models = get_model_from_dictionary(rel_obj, value)
            models.extend(rel_models)
            setattr(model_instance, field_name, rel_inst)
        else:
            setattr(model_instance, field_name, field_obj.python_value(value))
    return model_instance, models

def path_to_models(model, path):
    accum = []
    if '__' in path:
        next, path = path.split('__')
    else:
        next, path = path, ''
    if next in model._meta.rel:
        field = model._meta.rel[next]
        accum.append(field.rel_model)
    else:
        raise AttributeError('%s has no related field named "%s"' % (model, next))
    if path:
        accum.extend(path_to_models(model, path))
    return accum

# borrowing these methods, slightly modified, from django.contrib.auth
def get_hexdigest(salt, raw_password):
    data = salt + raw_password
    return sha1(data.encode('utf8')).hexdigest()

def make_password(raw_password):
    salt = get_hexdigest(text_type(random.random()), text_type(random.random()))[:5]
    hsh = get_hexdigest(salt, raw_password)
    return '%s$%s' % (salt, hsh)

def check_password(raw_password, enc_password):
    salt, hsh = enc_password.split('$', 1)
    return hsh == get_hexdigest(salt, raw_password)
<commit_msg>Cache count()<commit_after>import math
import random
import re
import sys
from hashlib import sha1

from flask import abort
from flask import render_template
from flask import request
from peewee import DoesNotExist
from peewee import ForeignKeyField
from peewee import Model
from peewee import SelectQuery

from flask_peewee._compat import text_type

def get_object_or_404(query_or_model, *query):
    if not isinstance(query_or_model, SelectQuery):
        query_or_model = query_or_model.select()
    try:
        return query_or_model.where(*query).get()
    except DoesNotExist:
        abort(404)

def object_list(template_name, qr, var_name='object_list', **kwargs):
    pq = PaginatedQuery(qr, kwargs.pop('paginate_by', 20))
    kwargs[var_name] = pq.get_list()
    return render_template(template_name, pagination=pq, page=pq.get_page(), **kwargs)

class PaginatedQuery(object):
    page_var = 'page'

    def __init__(self, query_or_model, paginate_by):
        self.paginate_by = paginate_by

        if isinstance(query_or_model, SelectQuery):
            self.query = query_or_model
            self.model = self.query.model_class
        else:
            self.model = query_or_model
            self.query = self.model.select()

    def get_page(self):
        curr_page = request.args.get(self.page_var)
        if curr_page and curr_page.isdigit():
            return int(curr_page)
        return 1

    def get_pages(self):
        if not hasattr(self, '_get_pages'):
            self._get_pages = int(math.ceil(
                float(self.query.count()) / self.paginate_by))
        return self._get_pages

    def get_list(self):
        return self.query.paginate(self.get_page(), self.paginate_by)

def get_next():
    if not request.query_string:
        return request.path
    return '%s?%s' % (request.path, request.query_string)

def slugify(s):
    return re.sub('[^a-z0-9_\-]+', '-', s.lower())

def load_class(s):
    path, klass = s.rsplit('.', 1)
    __import__(path)
    mod = sys.modules[path]
    return getattr(mod, klass)

def get_dictionary_from_model(model, fields=None, exclude=None):
    model_class = type(model)
    data = {}

    fields = fields or {}
    exclude = exclude or {}
    curr_exclude = exclude.get(model_class, [])
    curr_fields = fields.get(model_class, model._meta.sorted_field_names)

    for field_name in curr_fields:
        if field_name in curr_exclude:
            continue
        field_obj = model_class._meta.fields[field_name]
        field_data = model._data.get(field_name)
        if isinstance(field_obj, ForeignKeyField) and field_data and field_obj.rel_model in fields:
            rel_obj = getattr(model, field_name)
            data[field_name] = get_dictionary_from_model(rel_obj, fields, exclude)
        else:
            data[field_name] = field_data
    return data

def get_model_from_dictionary(model, field_dict):
    if isinstance(model, Model):
        model_instance = model
        check_fks = True
    else:
        model_instance = model()
        check_fks = False
    models = [model_instance]
    for field_name, value in field_dict.items():
        field_obj = model._meta.fields[field_name]
        if isinstance(value, dict):
            rel_obj = field_obj.rel_model
            if check_fks:
                try:
                    rel_obj = getattr(model, field_name)
                except field_obj.rel_model.DoesNotExist:
                    pass
                if rel_obj is None:
                    rel_obj = field_obj.rel_model
            rel_inst, rel_models = get_model_from_dictionary(rel_obj, value)
            models.extend(rel_models)
            setattr(model_instance, field_name, rel_inst)
        else:
            setattr(model_instance, field_name, field_obj.python_value(value))
    return model_instance, models

def path_to_models(model, path):
    accum = []
    if '__' in path:
        next, path = path.split('__')
    else:
        next, path = path, ''
    if next in model._meta.rel:
        field = model._meta.rel[next]
        accum.append(field.rel_model)
    else:
        raise AttributeError('%s has no related field named "%s"' % (model, next))
    if path:
        accum.extend(path_to_models(model, path))
    return accum

# borrowing these methods, slightly modified, from django.contrib.auth
def get_hexdigest(salt, raw_password):
    data = salt + raw_password
    return sha1(data.encode('utf8')).hexdigest()

def make_password(raw_password):
    salt = get_hexdigest(text_type(random.random()), text_type(random.random()))[:5]
    hsh = get_hexdigest(salt, raw_password)
    return '%s$%s' % (salt, hsh)

def check_password(raw_password, enc_password):
    salt, hsh = enc_password.split('$', 1)
    return hsh == get_hexdigest(salt, raw_password)
<|endoftext|>