<commit_before>import six
import logging

from uuid import uuid4
from collections import namedtuple, defaultdict
from altgraph.Graph import Graph

from rabix.common.errors import ValidationError, RabixError
from rabix.common.util import wrap_in_list
from rabix.common.models import App, IO, Job
from rabix.schema import JsonSchema

log = logging.getLogger(__name__)

AppNode = namedtuple('AppNode', ['app', 'inputs'])

Relation = namedtuple('Relation', ['src_port', 'dst_port'])
InputRelation = namedtuple('InputRelation', ['dst_port'])
OutputRelation = namedtuple('OutputRelation', ['src_port'])

class Step(object):

    def __init__(self, step_id, app, inputs, outputs=None):
        self.id = step_id
        self.app = app
        self.inputs = inputs
        self.outputs = outputs

    def to_dict(self):
        return {
            "id": self.id,
            "@type": "Step",
            "app": self.app.to_dict(),
            "inputs": self.inputs,
            "outputs": self.outputs
        }

    @classmethod
    def from_dict(cls, context, d):
        return cls(
            d["id"],
            context.from_dict(d['app']),
            context.from_dict(d.get('inputs')),
            context.from_dict(d.get('outputs')))

class WorkflowApp(App):

    def __init__(self, app_id, steps, executor,
                 inputs=None, outputs=None,
                 app_description=None,
                 annotations=None,
                 platform_features=None):
        self.graph = Graph()
        self.inputs = inputs or []
        self.outputs = outputs or []
        self.executor = executor
        self.steps = steps

        for step in steps:
            self.add_node(step.id,  AppNode(step.app, {}))

        for step in steps:
            # inputs
            for input_port, input_val in six.iteritems(step.inputs):
                inp = wrap_in_list(input_val)
                for item in inp:
                    self.add_edge_or_input(step, input_port, item)

            # outputs
            if step.outputs:
                for output_port, output_val in six.iteritems(step.outputs):
                    output_node = step.app.get_output(output_port)
                    output_id = output_val['$to']
                    self.add_node(output_id, output_node)
                    self.graph.add_edge(
                        step.id, output_id, OutputRelation(output_port)
                    )
                    self.outputs.append(output_node)

        if not self.graph.connected():
            pass
            # raise ValidationError('Graph is not connected')

        schema = {
            "@type": "JsonSchema",
            "type": "object",
            "properties": {},
            "required": []
        }

        for inp in self.inputs:
            schema['properties'][inp.id] = inp.validator.schema
            if inp.required:
                schema['required'].append(inp.id)

        super(WorkflowApp, self).__init__(
            app_id, JsonSchema(None, schema), self.outputs,
            app_description=app_description,
            annotations=annotations,
            platform_features=platform_features
        )

    def add_edge_or_input(self, step, input_name, input_val):
        node_id = step.id
        if isinstance(input_val, dict) and '$from' in input_val:
            frm = wrap_in_list(input_val['$from'])
            for inp in frm:
                if '.' in inp:
                    node, outp = inp.split('.')
                    self.graph.add_edge(node, node_id, Relation(outp, input_name))
                else:
                    # TODO: merge input schemas if one input goes to different apps

                    input = step.app.get_input(input_name)
                    self.add_node(inp, input)
                    self.graph.add_edge(
                        inp, node_id, InputRelation(input_name)
                    )
                    self.inputs.append(input)

        else:
            self.graph.node_data(node_id).inputs[input_name] = input_val

    # Graph.add_node silently fails if node already exists
    def add_node(self, node_id, node):
        if node_id in self.graph.nodes:
            raise ValidationError('Duplicate node ID: %s' % node_id)
        self.graph.add_node(node_id, node)

    def hide_nodes(self, type):
        for node_id in self.graph.node_list():
            node = self.graph.node_data(node_id)
            if isinstance(node, type):
                self.graph.hide_node(node_id)

    def run(self, job):
        eg = ExecutionGraph(self, job)
        while eg.has_next():
            next = eg.next_job()
            self.executor.execute(next, eg.job_done)
        return eg.outputs

    def to_dict(self):
        d = super(WorkflowApp, self).to_dict()
        d.update({
            "@type": "Workflow",
            'steps': [step.to_dict() for step in self.steps]
        })
        return d

    @classmethod
    def from_dict(cls, context, d):
        steps = [Step(
            step['id'], context.from_dict(step['app']),
            step['inputs'], step.get('outputs')
        )
            for step in d['steps']]

        return cls(
            d.get('@id', str(uuid4())),
            steps,
            context.executor
        )

def init(context):
    context.add_type('Workflow', WorkflowApp.from_dict)

class PartialJob(object):

    def __init__(self, node_id, app, inputs, input_counts, outputs):
        self.result = None
        self.status = 'WAITING'
        self.node_id = node_id
        self.app = app
        self.inputs = inputs
        self.input_counts = input_counts
        self.outputs = outputs

        self.running = []
        self.resources = None

    @property
    def resolved(self):
        for name, cnt in six.iteritems(self.input_counts):
            if cnt > 0:
                return False
        return True

    def resolve_input(self, input_port, results):
        log.debug("Resolving input '%s' with value %s" % (input_port, results))
        input_count = self.input_counts[input_port]
        if input_count <= 0:
            raise RabixError("Input already satisfied")
        self.input_counts[input_port] = input_count - 1
        # recursive_merge(self.job['inputs'].get(input_port), results)
        prev_result = self.inputs.get(input_port)
        if prev_result is None:
            self.inputs[input_port] = results
        elif isinstance(prev_result, list):
            prev_result.append(results)
        else:
            self.inputs[input_port] = [prev_result, results]
        return self.resolved

    def propagate_result(self, result):
        log.debug("Propagating result: %s" % result)
        self.result = result
        for k, v in six.iteritems(result):
            log.debug("Propagating result: %s, %s" % (k, v))
            self.outputs[k].resolve_input(v)

    def job(self):
        return Job(self.node_id, self.app, self.inputs, {})

class ExecRelation(object):

    def __init__(self, node, input_port):
        self.node = node
        self.input_port = input_port

    def resolve_input(self, result):
        self.node.resolve_input(self.input_port, result)

class OutRelation(object):

    def __init__(self, graph, name):
        self.name = name
        self.graph = graph

    def resolve_input(self, result):
        self.graph.outputs[self.name] = result

class ExecutionGraph(object):

    def __init__(self, workflow, job):
        self.workflow = workflow
        self.executables = {}
        self.ready = {}
        self.job = job
        self.outputs = {}

        graph = workflow.graph

        for node_id in graph.back_topo_sort()[1]:
            executable = self.make_executable(node_id)
            if executable:
                self.executables[node_id] = executable

        workflow.hide_nodes(IO)

        self.order = graph.back_topo_sort()[1]

    def make_executable(self, node_id):
        node = self.graph.node_data(node_id)
        if isinstance(node, IO):
            return None

        out_edges = self.graph.out_edges(node_id)
        in_edges = self.graph.inc_edges(node_id)

        outputs = {}
        input_counts = ExecutionGraph.count_inputs(self.graph, in_edges)
        for out_edge in out_edges:
            rel = self.graph.edge_data(out_edge)
            if isinstance(rel, Relation):
                tail = self.executables[self.graph.tail(out_edge)]
                outputs[rel.src_port] = ExecRelation(tail, rel.dst_port)
            elif isinstance(rel, OutputRelation):
                tail = self.graph.tail(out_edge)
                outputs[rel.src_port] = OutRelation(self, tail)

        executable = PartialJob(
            node_id, node.app, node.inputs, input_counts, outputs
        )

        for in_edge in in_edges:
            rel = self.graph.edge_data(in_edge)
            head = self.graph.head(in_edge)
            if (isinstance(rel, InputRelation) and
                    head in self.job.inputs):

                executable.resolve_input(
                    rel.dst_port, self.job.inputs[head]
                )

        return executable

    @staticmethod
    def count_inputs(graph, in_edges):
        input_counts = defaultdict(lambda: 0)
        for edge in in_edges:
            relation = graph.edge_data(edge)
            input_count = input_counts[relation.dst_port]
            input_counts[relation.dst_port] = input_count + 1
        return input_counts

    def job_done(self, node_id, results):
        ex = self.executables[node_id]
        ex.propagate_result(results)

    def next_job(self):
        if not self.order:
            return None
        return self.executables[self.order.pop()].job()

    def has_next(self):
        return len(self.order) > 0

    @property
    def graph(self):
        return self.workflow.graph

# Smoke test
if __name__ == '__main__':
    from os.path import abspath, join
    from rabix.common.ref_resolver import from_url

    def root_relative(path):
        return abspath(join(__file__, '../../../', path))

    doc = from_url(root_relative('examples/workflow.yml'))

    wf = WorkflowApp(doc['workflows']['add_one_mul_two']['steps'])
    print(wf.graph.forw_topo_sort())

    for edge in wf.graph.edges:
        print(wf.graph.describe_edge(edge))

    print(wf.inputs)
    print(wf.outputs)
<commit_msg>workflow input.id fix<commit_after>import six
import copy
import logging

from uuid import uuid4
from collections import namedtuple, defaultdict
from altgraph.Graph import Graph

from rabix.common.errors import ValidationError, RabixError
from rabix.common.util import wrap_in_list
from rabix.common.models import App, IO, Job
from rabix.schema import JsonSchema

log = logging.getLogger(__name__)

AppNode = namedtuple('AppNode', ['app', 'inputs'])

Relation = namedtuple('Relation', ['src_port', 'dst_port'])
InputRelation = namedtuple('InputRelation', ['dst_port'])
OutputRelation = namedtuple('OutputRelation', ['src_port'])

class Step(object):

    def __init__(self, step_id, app, inputs, outputs=None):
        self.id = step_id
        self.app = app
        self.inputs = inputs
        self.outputs = outputs

    def to_dict(self):
        return {
            "id": self.id,
            "@type": "Step",
            "app": self.app.to_dict(),
            "inputs": self.inputs,
            "outputs": self.outputs
        }

    @classmethod
    def from_dict(cls, context, d):
        return cls(
            d["id"],
            context.from_dict(d['app']),
            context.from_dict(d.get('inputs')),
            context.from_dict(d.get('outputs')))

class WorkflowApp(App):

    def __init__(self, app_id, steps, executor,
                 inputs=None, outputs=None,
                 app_description=None,
                 annotations=None,
                 platform_features=None):
        self.graph = Graph()
        self.inputs = inputs or []
        self.outputs = outputs or []
        self.executor = executor
        self.steps = steps

        for step in steps:
            self.add_node(step.id,  AppNode(step.app, {}))

        for step in steps:
            # inputs
            for input_port, input_val in six.iteritems(step.inputs):
                inp = wrap_in_list(input_val)
                for item in inp:
                    self.add_edge_or_input(step, input_port, item)

            # outputs
            if step.outputs:
                for output_port, output_val in six.iteritems(step.outputs):
                    output_node = step.app.get_output(output_port)
                    output_id = output_val['$to']
                    self.add_node(output_id, output_node)
                    self.graph.add_edge(
                        step.id, output_id, OutputRelation(output_port)
                    )
                    self.outputs.append(output_node)

        if not self.graph.connected():
            pass
            # raise ValidationError('Graph is not connected')

        schema = {
            "@type": "JsonSchema",
            "type": "object",
            "properties": {},
            "required": []
        }

        for inp in self.inputs:
            schema['properties'][inp.id] = inp.validator.schema
            if inp.required:
                schema['required'].append(inp.id)

        super(WorkflowApp, self).__init__(
            app_id, JsonSchema(None, schema), self.outputs,
            app_description=app_description,
            annotations=annotations,
            platform_features=platform_features
        )

    def add_edge_or_input(self, step, input_name, input_val):
        node_id = step.id
        if isinstance(input_val, dict) and '$from' in input_val:
            frm = wrap_in_list(input_val['$from'])
            for inp in frm:
                if '.' in inp:
                    node, outp = inp.split('.')
                    self.graph.add_edge(node, node_id, Relation(outp, input_name))
                else:
                    # TODO: merge input schemas if one input goes to different apps

                    input = step.app.get_input(input_name)
                    self.add_node(inp, input)
                    self.graph.add_edge(
                        inp, node_id, InputRelation(input_name)
                    )
                    wf_input = copy.deepcopy(input)
                    wf_input.id = inp
                    self.inputs.append(wf_input)

        else:
            self.graph.node_data(node_id).inputs[input_name] = input_val

    # Graph.add_node silently fails if node already exists
    def add_node(self, node_id, node):
        if node_id in self.graph.nodes:
            raise ValidationError('Duplicate node ID: %s' % node_id)
        self.graph.add_node(node_id, node)

    def hide_nodes(self, type):
        for node_id in self.graph.node_list():
            node = self.graph.node_data(node_id)
            if isinstance(node, type):
                self.graph.hide_node(node_id)

    def run(self, job):
        eg = ExecutionGraph(self, job)
        while eg.has_next():
            next = eg.next_job()

            self.executor.execute(next, eg.job_done)
        return eg.outputs

    def to_dict(self):
        d = super(WorkflowApp, self).to_dict()
        d.update({
            "@type": "Workflow",
            'steps': [step.to_dict() for step in self.steps]
        })
        return d

    @classmethod
    def from_dict(cls, context, d):
        steps = [Step(
            step['id'], context.from_dict(step['app']),
            step['inputs'], step.get('outputs')
        )
            for step in d['steps']]

        return cls(
            d.get('@id', str(uuid4())),
            steps,
            context.executor
        )

def init(context):
    context.add_type('Workflow', WorkflowApp.from_dict)

class PartialJob(object):

    def __init__(self, node_id, app, inputs, input_counts, outputs):
        self.result = None
        self.status = 'WAITING'
        self.node_id = node_id
        self.app = app
        self.inputs = inputs
        self.input_counts = input_counts
        self.outputs = outputs

        self.running = []
        self.resources = None

    @property
    def resolved(self):
        for name, cnt in six.iteritems(self.input_counts):
            if cnt > 0:
                return False
        return True

    def resolve_input(self, input_port, results):
        log.debug("Resolving input '%s' with value %s" % (input_port, results))
        input_count = self.input_counts[input_port]
        if input_count <= 0:
            raise RabixError("Input already satisfied")
        self.input_counts[input_port] = input_count - 1
        # recursive_merge(self.job['inputs'].get(input_port), results)
        prev_result = self.inputs.get(input_port)
        if prev_result is None:
            self.inputs[input_port] = results
        elif isinstance(prev_result, list):
            prev_result.append(results)
        else:
            self.inputs[input_port] = [prev_result, results]
        return self.resolved

    def propagate_result(self, result):
        log.debug("Propagating result: %s" % result)
        self.result = result
        for k, v in six.iteritems(result):
            log.debug("Propagating result: %s, %s" % (k, v))
            self.outputs[k].resolve_input(v)

    def job(self):
        return Job(self.node_id, self.app, self.inputs, {})

class ExecRelation(object):

    def __init__(self, node, input_port):
        self.node = node
        self.input_port = input_port

    def resolve_input(self, result):
        self.node.resolve_input(self.input_port, result)

class OutRelation(object):

    def __init__(self, graph, name):
        self.name = name
        self.graph = graph

    def resolve_input(self, result):
        self.graph.outputs[self.name] = result

class ExecutionGraph(object):

    def __init__(self, workflow, job):
        self.workflow = workflow
        self.executables = {}
        self.ready = {}
        self.job = job
        self.outputs = {}

        graph = workflow.graph

        for node_id in graph.back_topo_sort()[1]:
            executable = self.make_executable(node_id)
            if executable:
                self.executables[node_id] = executable

        workflow.hide_nodes(IO)

        self.order = graph.back_topo_sort()[1]

    def make_executable(self, node_id):
        node = self.graph.node_data(node_id)
        if isinstance(node, IO):
            return None

        out_edges = self.graph.out_edges(node_id)
        in_edges = self.graph.inc_edges(node_id)

        outputs = {}
        input_counts = ExecutionGraph.count_inputs(self.graph, in_edges)
        for out_edge in out_edges:
            rel = self.graph.edge_data(out_edge)
            if isinstance(rel, Relation):
                tail = self.executables[self.graph.tail(out_edge)]
                outputs[rel.src_port] = ExecRelation(tail, rel.dst_port)
            elif isinstance(rel, OutputRelation):
                tail = self.graph.tail(out_edge)
                outputs[rel.src_port] = OutRelation(self, tail)

        executable = PartialJob(
            node_id, node.app, node.inputs, input_counts, outputs
        )

        for in_edge in in_edges:
            rel = self.graph.edge_data(in_edge)
            head = self.graph.head(in_edge)
            if (isinstance(rel, InputRelation) and
                    head in self.job.inputs):
                executable.resolve_input(
                    rel.dst_port, self.job.inputs[head]
                )

        return executable

    @staticmethod
    def count_inputs(graph, in_edges):
        input_counts = defaultdict(lambda: 0)
        for edge in in_edges:
            relation = graph.edge_data(edge)
            input_count = input_counts[relation.dst_port]
            input_counts[relation.dst_port] = input_count + 1
        return input_counts

    def job_done(self, node_id, results):
        ex = self.executables[node_id]
        ex.propagate_result(results)

    def next_job(self):
        if not self.order:
            return None
        return self.executables[self.order.pop()].job()

    def has_next(self):
        return len(self.order) > 0

    @property
    def graph(self):
        return self.workflow.graph

# Smoke test
if __name__ == '__main__':
    from os.path import abspath, join
    from rabix.common.ref_resolver import from_url

    def root_relative(path):
        p = abspath(join(__file__, '../', path))
        return abspath(join(__file__, '../../../', path))

    doc = from_url(root_relative('tests/workflow.yml'))

    wf = WorkflowApp(doc['workflows']['add_one_mul_two']['steps'])
    print(wf.graph.forw_topo_sort())

    for edge in wf.graph.edges:
        print(wf.graph.describe_edge(edge))

    print(wf.inputs)
    print(wf.outputs)
<|endoftext|>


<commit_before>#!/usr/bin/python2.7
# -*- coding: utf8 -*-

"""
Copyright [2014,2015] [beholder developers]

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import redis
import argparse
import bz2
import Queue
import sys
import os
import time
import ConfigParser
from threading import Thread

from termcolor import colored
from cache import rainbowCache
from compare import rainbowCompare

REDIS_TIMEOUT = 120

class Rainbow:

    def __init__(self, cmdln_args, redis_servers, redis_timeout):
        self.args = cmdln_args
        self.cached = False
        self.redis_timeout = redis_timeout
        self.redis_servers = redis_servers
        # Shared queue to store query results from Redis
        self.output_queue = Queue.Queue()

    def run(self):

        if self.args.available:
            self.redis_get_all_keys()
        else:
            self.redis_get_specific_key(self.args.file)

    def formated_output(self, now, hostname, redis_ts, mtime_ts, line):
        """Colored output of a Redis entry plus metadata"""

        redis_ts_str = time.strftime('%H:%M',
                                     time.localtime(int(redis_ts)))
        mtime_ts_str = time.strftime('%Y-%m-%dT%H:%M',
                                     time.localtime(int(mtime_ts)))
        if self.args.color:
            hostname = colored(hostname, 'cyan')
            diff = now - float(redis_ts)
            if diff < 600:
                redis_ts_str = colored(redis_ts_str, 'green')
            elif diff > 3600:
                redis_ts_str = colored(redis_ts_str, 'red')
            else:
                redis_ts_str = colored(redis_ts_str, 'yellow')

            mtime_ts_str = colored(mtime_ts_str, 'blue')
            line = colored(line, 'white', 'on_grey')

        print "%s (%s/%s) %s" % (hostname, redis_ts_str, mtime_ts_str, line)

    def redis_query(self, redis_server, method, key):
        """ Query a Redis server (within a thread)
            and write results to the shared queue"""

        if self.args.verbose:
            print "Querying_redis_server %s" % redis_server

        r = redis.Redis(host=redis_server, port=6379)

        if method == "GET":
            # Fetch all keys matching the requested pattern
            items = r.keys('*#%s' % key)
            for item in items:
                result = r.get(item)
                if result:
                    # Split the key name, which format is <hostname>#<file>
                    try:
                        hostname, key = item.split('#')
                    except ValueError:
                        print "Warning - Skipping key with invalid \
                                format: %s" % item
                        continue
                    self.output_queue.put((hostname, result))

        elif method == "KEYS":
            # Fetch all available keys
            items = r.keys('*')
            for item in items:
                # Split the key name, which format is <hostname>#<file>
                try:
                    hostname, key = item.split('#')
                except ValueError:
                    print "Warning - Skipping key with invalid \
                            format: %s" % item
                    continue
                self.output_queue.put(key)
        else:
            print "Not supported. You're doing it wrong."

    def redis_get_all_keys(self):
        """ Fetch available keys from all Redis servers"""

        threads = []
        for redis_server in self.redis_servers:
            worker = Thread(target=self.redis_query,
                            args=(redis_server, 'KEYS', '',))
            worker.start()
            threads.append(worker)

        for t in threads:
            t.join(REDIS_TIMEOUT)

        self.display_redis_all_keys()

    def display_redis_all_keys(self):
        """ Print a nicely formatted summary of all keys found on
            all the Redis servers
        """

        uniq_files = {}

        while not self.output_queue.empty():
            i = self.output_queue.get()
            self.output_queue.task_done()
            if i in uniq_files:
                uniq_files[i] += 1
            else:
                uniq_files[i] = 1

        if self.args.color:
            for k in sorted(uniq_files):
                # 13 here because of color escape characters \033[xx(x)
                print "%13s %s" % (colored(uniq_files[k], 'blue'),
                                   colored(k, 'cyan'))
        else:
            for k in sorted(uniq_files):
                print "%5s %s" % (uniq_files[k], k)

    def display_redis_specific_key(self, key):
        """ Display contents of a specific key through all the Redis servers"""

        # Actual timestamp of Redis query
        now = time.time()

        if self.args.logstash:
            self.cached = True
            cache = rainbowCache(key)
            redis_resources = {}

        while not self.output_queue.empty():
            (hostname, result) = self.output_queue.get()
            self.output_queue.task_done()

            # Here we split our custom Redis value format:
            # <base64_encoded_data> <epoch_redis> <epoch_mtime> <file_checksum>

            fields = result.split(' ')
            base64_data = fields[0]
            try:
                epoch_redis = fields[1]
                epoch_mtime = fields[2]
                file_md5 = fields[3]
            except IndexError:
                epoch_redis = 0
                epoch_mtime = 0
                file_md5 = "no_signature"

            if self.cached:
                redis_resources[hostname] = "%s %s %s %s" % (base64_data, epoch_redis, epoch_mtime, file_md5)
                continue

            try:
                c = bz2.decompress(base64_data.decode("base64"))
            except:
                c = "<error while decompressing bz2 data"

            if self.args.signature:
                self.formated_output(now, hostname, epoch_redis,
                                     epoch_mtime, file_md5)
            else:
                for line in c.split('\n'):
                    self.formated_output(now, hostname, epoch_redis,
                                         epoch_mtime, line)
        if self.args.logstash:
            c = rainbowCompare(key)
            c.diff(cache.getResources(), redis_resources)
            cache.update(redis_resources, key)
            cache.dump()

    def redis_get_specific_key(self, key):
        """ Fetch contents of a specific key over all the Redis servers"""

        threads = []
        for redis_server in self.redis_servers:
            worker = Thread(target=self.redis_query,
                            args=(redis_server, 'GET', key,))
            worker.start()
            threads.append(worker)

        for t in threads:
            t.join(REDIS_TIMEOUT)

        keys_found = self.output_queue.qsize()
        self.display_redis_specific_key(key)

        if self.args.verbose:
            print "### Keys found: %s ###" % keys_found
        if keys_found == 0:
            print 'try the -ac switch for a list of all available keys'

def autogen_conf(f):
    config.add_section('query_servers')
    config.set('query_servers', 'hostnames', 'redis1.example.org,'
                                             'redis2.example.org')
    config.write(open(f, 'w'))
    print("Created an empty config file at %s." % f)
    print("Please modify it & re-run this command.")
    sys.exit(1)

def test_default_conf(f):
    if redis_servers[0] == 'redis1.example.org':
        print("Please set your own redis instance in %s file "
              "and re-run this command." % f)
        sys.exit(1)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-f", "--file",
                       help="File to query")
    group.add_argument("-a", "--available",
                       help="Show available files", action='store_true')
    parser.add_argument("-c", "--color",
                        help="Colorize output", action='store_true')
    parser.add_argument("-s", "--signature",
                        help="MD5 of file", action='store_true')
    parser.add_argument("-l", "--logstash",
                        help="compute logstash output", action='store_true')
    parser.add_argument("-v", "--verbose",
                        help="verbose stdout informations", action='store_true')
    parser.add_argument("-C", "--conffile",
                        help="Configuration file to Use")

    args = parser.parse_args()

    config = ConfigParser.RawConfigParser()
    if args.conffile:
        conffile = args.conffile
    else:
        conffile = os.path.abspath(os.path.expanduser('~/.rainbow.cf'))
    if not os.path.exists(conffile):
        autogen_conf(conffile)

    config.read(conffile)
    redis_servers = config.get('query_servers', 'hostnames').split(',')
    test_default_conf(conffile)
    rb = Rainbow(args, redis_servers, REDIS_TIMEOUT)
    rb.run()
<commit_msg>dont need verbosWarning visible on verbose mode<commit_after>#!/usr/bin/python2.7
# -*- coding: utf8 -*-

"""
Copyright [2014,2015] [beholder developers]

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import redis
import argparse
import bz2
import Queue
import sys
import os
import time
import ConfigParser
from threading import Thread

from termcolor import colored
from cache import rainbowCache
from compare import rainbowCompare

REDIS_TIMEOUT = 120

class Rainbow:

    def __init__(self, cmdln_args, redis_servers, redis_timeout):
        self.args = cmdln_args
        self.cached = False
        self.redis_timeout = redis_timeout
        self.redis_servers = redis_servers
        # Shared queue to store query results from Redis
        self.output_queue = Queue.Queue()

    def run(self):

        if self.args.available:
            self.redis_get_all_keys()
        else:
            self.redis_get_specific_key(self.args.file)

    def formated_output(self, now, hostname, redis_ts, mtime_ts, line):
        """Colored output of a Redis entry plus metadata"""

        redis_ts_str = time.strftime('%H:%M',
                                     time.localtime(int(redis_ts)))
        mtime_ts_str = time.strftime('%Y-%m-%dT%H:%M',
                                     time.localtime(int(mtime_ts)))
        if self.args.color:
            hostname = colored(hostname, 'cyan')
            diff = now - float(redis_ts)
            if diff < 600:
                redis_ts_str = colored(redis_ts_str, 'green')
            elif diff > 3600:
                redis_ts_str = colored(redis_ts_str, 'red')
            else:
                redis_ts_str = colored(redis_ts_str, 'yellow')

            mtime_ts_str = colored(mtime_ts_str, 'blue')
            line = colored(line, 'white', 'on_grey')

        print "%s (%s/%s) %s" % (hostname, redis_ts_str, mtime_ts_str, line)

    def redis_query(self, redis_server, method, key):
        """ Query a Redis server (within a thread)
            and write results to the shared queue"""

        if self.args.verbose:
            print "Querying_redis_server %s" % redis_server

        r = redis.Redis(host=redis_server, port=6379)

        if method == "GET":
            # Fetch all keys matching the requested pattern
            items = r.keys('*#%s' % key)
            for item in items:
                result = r.get(item)
                if result:
                    # Split the key name, which format is <hostname>#<file>
                    try:
                        hostname, key = item.split('#')
                    except ValueError:
                        if self.args.verbose:
                            print( "Warning - Skipping key with invalid "
                                   "format: %s" % item )
                        continue
                    self.output_queue.put((hostname, result))

        elif method == "KEYS":
            # Fetch all available keys
            items = r.keys('*')
            for item in items:
                # Split the key name, which format is <hostname>#<file>
                try:
                    hostname, key = item.split('#')
                except ValueError:
                    if self.args.verbose:
                        print( "Warning - Skipping key with invalid "
                               "format: %s" % item )
                    continue
                self.output_queue.put(key)
        else:
            print "Not supported. You're doing it wrong."

    def redis_get_all_keys(self):
        """ Fetch available keys from all Redis servers"""

        threads = []
        for redis_server in self.redis_servers:
            worker = Thread(target=self.redis_query,
                            args=(redis_server, 'KEYS', '',))
            worker.start()
            threads.append(worker)

        for t in threads:
            t.join(REDIS_TIMEOUT)

        self.display_redis_all_keys()

    def display_redis_all_keys(self):
        """ Print a nicely formatted summary of all keys found on
            all the Redis servers
        """

        uniq_files = {}

        while not self.output_queue.empty():
            i = self.output_queue.get()
            self.output_queue.task_done()
            if i in uniq_files:
                uniq_files[i] += 1
            else:
                uniq_files[i] = 1

        if self.args.color:
            for k in sorted(uniq_files):
                # 13 here because of color escape characters \033[xx(x)
                print "%13s %s" % (colored(uniq_files[k], 'blue'),
                                   colored(k, 'cyan'))
        else:
            for k in sorted(uniq_files):
                print "%5s %s" % (uniq_files[k], k)

    def display_redis_specific_key(self, key):
        """ Display contents of a specific key through all the Redis servers"""

        # Actual timestamp of Redis query
        now = time.time()

        if self.args.logstash:
            self.cached = True
            cache = rainbowCache(key)
            redis_resources = {}

        while not self.output_queue.empty():
            (hostname, result) = self.output_queue.get()
            self.output_queue.task_done()

            # Here we split our custom Redis value format:
            # <base64_encoded_data> <epoch_redis> <epoch_mtime> <file_checksum>

            fields = result.split(' ')
            base64_data = fields[0]
            try:
                epoch_redis = fields[1]
                epoch_mtime = fields[2]
                file_md5 = fields[3]
            except IndexError:
                epoch_redis = 0
                epoch_mtime = 0
                file_md5 = "no_signature"

            if self.cached:
                redis_resources[hostname] = "%s %s %s %s" % (base64_data, epoch_redis, epoch_mtime, file_md5)
                continue

            try:
                c = bz2.decompress(base64_data.decode("base64"))
            except:
                c = "<error while decompressing bz2 data"

            if self.args.signature:
                self.formated_output(now, hostname, epoch_redis,
                                     epoch_mtime, file_md5)
            else:
                for line in c.split('\n'):
                    self.formated_output(now, hostname, epoch_redis,
                                         epoch_mtime, line)
        if self.args.logstash:
            c = rainbowCompare(key)
            c.diff(cache.getResources(), redis_resources)
            cache.update(redis_resources, key)
            cache.dump()

    def redis_get_specific_key(self, key):
        """ Fetch contents of a specific key over all the Redis servers"""

        threads = []
        for redis_server in self.redis_servers:
            worker = Thread(target=self.redis_query,
                            args=(redis_server, 'GET', key,))
            worker.start()
            threads.append(worker)

        for t in threads:
            t.join(REDIS_TIMEOUT)

        keys_found = self.output_queue.qsize()
        self.display_redis_specific_key(key)

        if self.args.verbose:
            print "### Keys found: %s ###" % keys_found
        if keys_found == 0:
            print 'try the -ac switch for a list of all available keys'

def autogen_conf(f):
    config.add_section('query_servers')
    config.set('query_servers', 'hostnames', 'redis1.example.org,'
                                             'redis2.example.org')
    config.write(open(f, 'w'))
    print("Created an empty config file at %s." % f)
    print("Please modify it & re-run this command.")
    sys.exit(1)

def test_default_conf(f):
    if redis_servers[0] == 'redis1.example.org':
        print("Please set your own redis instance in %s file "
              "and re-run this command." % f)
        sys.exit(1)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    group = parser.add_mutually_exclusive_group()
    group.add_argument("-f", "--file",
                       help="File to query")
    group.add_argument("-a", "--available",
                       help="Show available files", action='store_true')
    parser.add_argument("-c", "--color",
                        help="Colorize output", action='store_true')
    parser.add_argument("-s", "--signature",
                        help="MD5 of file", action='store_true')
    parser.add_argument("-l", "--logstash",
                        help="compute logstash output", action='store_true')
    parser.add_argument("-v", "--verbose",
                        help="verbose stdout informations", action='store_true')
    parser.add_argument("-C", "--conffile",
                        help="Configuration file to Use")

    args = parser.parse_args()

    config = ConfigParser.RawConfigParser()
    if args.conffile:
        conffile = args.conffile
    else:
        conffile = os.path.abspath(os.path.expanduser('~/.rainbow.cf'))
    if not os.path.exists(conffile):
        autogen_conf(conffile)

    config.read(conffile)
    redis_servers = config.get('query_servers', 'hostnames').split(',')
    test_default_conf(conffile)
    rb = Rainbow(args, redis_servers, REDIS_TIMEOUT)
    rb.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

# stdlib
import string

# installed:
from sh import svn

class MultipleMatchException(Exception):
    pass

class Subversion(object):
    '''
    This class encapsulates interaction with Subversion
    '''

    def __init__(self, config):
        self.config = config
        self.strippers = ''.join(['/', string.whitespace])

    def get_branches(self):
        '''
        Returns a list of branches in the SVN repo based on the 'branch_url'
        given in the configuration file.
        '''

        cmd = svn.ls(self.config['branch_url'])
        return [x.rstrip(self.strippers) for x in cmd]

    def get_unique_branch(self, search_string):
        '''
        Given a search string attempts to find one branch that contains the
        string in it's name. If more than one branch matches then an exception
        is raised.
        '''

        branches = self.get_branches()
        branch = [x.rstrip(self.strippers)
                for x in branches if search_string in x]
        if len(branch) > 1:
            raise MultipleMatchException('more than one branch matches "{}"')

        return branch[0]

    def make_branch(self, name, commit_msg):

        src = self.config['trunk_url']
        dest = '{}/{}'.format(self.config['branch_url'], name)

        process = svn.copy(src, dest, m=commit_msg)
        return process
<commit_msg>wrote routine to generate diff -- and clean it up for Crucible<commit_after>#!/usr/bin/env python

from __future__ import print_function

# stdlib
import string

# installed:
from sh import svn

class MultipleMatchException(Exception):
    pass

class Subversion(object):
    '''
    This class encapsulates interaction with Subversion
    '''

    def __init__(self, config):
        self.config = config
        self.strippers = ''.join(['/', string.whitespace])

    def get_branches(self):
        '''
        Returns a list of branches in the SVN repo based on the 'branch_url'
        given in the configuration file.
        '''

        cmd = svn.ls(self.config['branch_url'])
        return [x.rstrip(self.strippers) for x in cmd]

    def get_unique_branch(self, search_string):
        '''
        Given a search string attempts to find one branch that contains the
        string in it's name. If more than one branch matches then an exception
        is raised.
        '''

        branches = self.get_branches()
        branch = [x.rstrip(self.strippers)
                for x in branches if search_string in x]
        if len(branch) > 1:
            raise MultipleMatchException('more than one branch matches "{}"')

        return branch[0]

    def diff(self, branch):

        trunk = self.config['trunk_url']
        branch = '/'.join([self.config['branch_url'], branch])
        # if filterdiff exists, pipe diff through it to clean up parts that
        # Crucible doesn't like
        try:
            from sh import filterdiff
            proc = filterdiff(svn.diff(trunk, branch, diff_cmd='diff', x='-U 300 -a'), clean=True)

        # if there's no filterdiff then hope for the best:
        except CommandNotFound:
            proc = svn.diff(trunk, branch, diff_cmd='diff', x='-U 300 -a')

        return proc.stdout

    def make_branch(self, name, commit_msg):

        src = self.config['trunk_url']
        dest = '{}/{}'.format(self.config['branch_url'], name)

        process = svn.copy(src, dest, m=commit_msg)
        return process
<|endoftext|>


<commit_before>"""Implements the serial interface protocol of RaspiBot's Attiny firmware."""

from collections import namedtuple

EncoderValues = namedtuple('EncoderValues', 'left right')

# Named byte values for the protocol commands

# Requests
ALIVE = b'\x01'
ENCODERS_RIGHT = b'\x02'
ENCODERS_LEFT = b'\x03'
ENCODERS_BOTH = b'\x04'

# Responses
ACK = b'\x10'
NAK = b'\x14'

class InvalidResponseException(Exception):
    """
    Raised when the protocol implementation reads an unexpected byte.

    This is typically something that is neither an ACK nor a NAK.
    """

    pass

class AttinyProtocol(object):
    """Provides methods for protocol transactions."""

    def __init__(self, serial):
        """Create a protocol instance on a serial interface."""
        super(AttinyProtocol, self).__init__()
        self._serial = serial

    def get_encoders(self):
        """Request both left and right encoder values."""
        self._serial.write(ENCODERS_BOTH)
        response = self._serial.read(4)
        left = int.from_bytes(response[:2], 'little')
        right = int.from_bytes(response[2:], 'little')
        return EncoderValues(left, right)

    def get_left_encoder(self):
        """Request the left encoder value."""
        self._serial.write(ENCODERS_LEFT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'little')
        return value

    def get_right_encoder(self):
        """Request the right encoder value."""
        self._serial.write(ENCODERS_RIGHT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'little')
        return value

    def alive(self):
        """Request an 'alive' signal from the microcontroller."""
        self._serial.write(ALIVE)
        response = self._serial.read(1)
        return response == ACK
<commit_msg>implement serial command "alive": NAK, timeout and undefined cases<commit_after>"""Implements the serial interface protocol of RaspiBot's Attiny firmware."""

from collections import namedtuple

EncoderValues = namedtuple('EncoderValues', 'left right')

# Named byte values for the protocol commands

# Requests
ALIVE = b'\x01'
ENCODERS_RIGHT = b'\x02'
ENCODERS_LEFT = b'\x03'
ENCODERS_BOTH = b'\x04'

# Responses
ACK = b'\x10'
NAK = b'\x14'

class InvalidResponseException(Exception):
    """
    Raised when the protocol implementation reads an unexpected byte.

    This is typically something that is neither an ACK nor a NAK.
    """

    pass

class AttinyProtocol(object):
    """Provides methods for protocol transactions."""

    def __init__(self, serial):
        """Create a protocol instance on a serial interface."""
        super(AttinyProtocol, self).__init__()
        self._serial = serial

    def get_encoders(self):
        """Request both left and right encoder values."""
        self._serial.write(ENCODERS_BOTH)
        response = self._serial.read(4)
        left = int.from_bytes(response[:2], 'little')
        right = int.from_bytes(response[2:], 'little')
        return EncoderValues(left, right)

    def get_left_encoder(self):
        """Request the left encoder value."""
        self._serial.write(ENCODERS_LEFT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'little')
        return value

    def get_right_encoder(self):
        """Request the right encoder value."""
        self._serial.write(ENCODERS_RIGHT)
        response = self._serial.read(2)
        value = int.from_bytes(response, 'little')
        return value

    def alive(self):
        """Request an 'alive' signal from the microcontroller."""
        self._serial.write(ALIVE)
        response = self._serial.read(1)
        if response == ACK:
            return True
        # empty response can happen on a timeout, which we interpret as
        # "not alive"
        elif response == NAK or response == b'':
            return False
        else:
            raise InvalidResponseException()
<|endoftext|>


<commit_before># This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# For more information, please refer to <http://unlicense.org/>

import re
from datetime import datetime

try:
    import rb_exceptions
    import soup as soup_helper
except ImportError:  # No implicit package imports in py3.
    from ratebeer import rb_exceptions
    from ratebeer import soup as soup_helper

class Beer(object):
    """The Beer object. Contains information about an individual beer.

    Args:
        url (string): the URL of the beer you're looking for.

    Returns:
        abv (float): percentage alcohol
        brewery (Brewery object): the beer's brewery
        brewed_at (Brewery object): actual brewery if contract brewed
        calories (float): estimated calories for the beer
        description (string): the beer's description
        img_url (string): a url to an image of the beer
        mean_rating (float): the mean rating for the beer (out of 5)
        name (string): the full name of the beer (may include the brewery name)
        num_ratings (int): the number of reviews*
        overall_rating (int): the overall rating (out of 100)
        seasonal (string): Summer, Winter, Autumn, Spring, Series, Special, None
        style (string): beer style
        style_url (string): beer style URL
        style_rating (int): rating of the beer within its style (out of 100)
        tags (list of strings): tags given to the beer
        url (string): the beer's url
        weighted_avg (float): the beer rating average, weighted using some unknown
            algorithm (out of 5)

        Any attributes not available will be returned as None

    """
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Beer('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        soup = soup_helper._get_soup(self.url)
        # check for 404s
        try:
            soup_rows = soup.find('div', id='container').find('table').find_all('tr')
        except AttributeError:
            raise rb_exceptions.PageNotFound(self.url)
        # ratebeer pages don't actually 404, they just send you to this weird
        # "beer reference" page but the url doesn't actually change, it just
        # seems like it's all getting done server side -- so we have to look
        # for the contents h1 to see if we're looking at the beer reference or
        # not
        if "beer reference" in soup_rows[0].find_all('td')[1].h1.contents:
            raise rb_exceptions.PageNotFound(self.url)

        if "Also known as " in soup_rows[1].find_all('td')[1].div.div.contents:
            raise rb_exceptions.AliasedBeer(self.url, soup_rows[1].find_all('td')[1].div.div.a['href'])

        # General information from the top of the page
        self.name = soup.find(itemprop='name').text.strip()
        breweries = soup.find_all('a', href=re.compile('brewers'))
        self.brewery = Brewery(breweries[1].get('href'))
        self.brewery.name = breweries[1].text
        if len(breweries) == 3:
            self.brewed_at = Brewery(breweries[2].get('href'))
            self.brewed_at.name = breweries[2].text
        else:
            self.brewed_at = None
        try:
            self.overall_rating = int(soup.find('span', text='overall').next_sibling.next_sibling.text)
        except ValueError: # 'n/a'
            self.overall_rating = None
        except AttributeError:
            self.overall_rating = None
        try:
            self.style_rating = int(soup.find('span', text='style').previous_sibling.previous_sibling)
        except ValueError: # 'n/a'
            self.style_rating = None
        except AttributeError:
            self.style_rating = None
        self.style = soup.find(text='Style: ').next_sibling.text
        self.style_url = soup.find(text='Style: ').next_sibling.get('href')
        self.img_url = soup.find(id="beerImg").get('src')
        # Data from the info bar
        self.num_ratings = int(soup.find('span', itemprop="ratingCount").text)
        try:
            self.mean_rating = float(soup.find(text='MEAN: ').next_sibling.text.split('/')[0])
        except ValueError: # Empty mean rating: '/5.0'
            self.mean_rating = None
        except AttributeError: # No mean rating
            self.mean_rating = None
        try:
            self.weighted_avg = float(soup.find(attrs={"name": "real average"}).find('span', itemprop="ratingValue").text)
        except ValueError: # Empty weighted average rating: '/5'
            self.weighted_avg = None
        except AttributeError: # No weighted average rating
            self.weighted_avg = None
        try:
            self.seasonal = soup.find(text=u'\xa0\xa0 SEASONAL: ').next_sibling.text
        except AttributeError:
            self.seasonal = None
        try:
            self.ibu = int(soup.find(title="International Bittering Units - Normally from hops").next_sibling.next_sibling.text)
        except AttributeError:
            self.ibu = None
        try:
            self.calories = int(soup.find(title="Estimated calories for a 12 fluid ounce serving").next_sibling.next_sibling.text)
        except AttributeError:
            self.calories = None
        try:
            self.abv = float(soup.find(title="Alcohol By Volume").next_sibling.next_sibling.text[:-1])
        except ValueError: # Empty ABV: '-'
            self.abv = None
        # Description
        description = soup.find('div',
            style=(
                'border: 1px solid #e0e0e0; background: #fff; '
                'padding: 14px; color: #777;'
            )
        )
        if 'no commercial description' not in description.text.lower():
            # strip ads
            [s.extract() for s in description('small')]
            self.description = ' '.join([s for s in description.strings]).strip()
        self.tags = [t.text[1:] for t in soup.find_all('span', class_="tagLink")]

        self._has_fetched = True

        return self

    def get_reviews(self, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A generator of dictionaries, containing the information about the review.
        """

        if not self._has_fetched:
            self._populate()

        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if not url_flag:
            raise ValueError("Invalid ``review_order``.")

        page_number = 1
        while True:
            complete_url = u'{0}{1}/{2}/'.format(self.url, url_flag, page_number)
            soup = soup_helper._get_soup(complete_url)
            content = soup.find('table', style='padding: 10px;').tr.td
            reviews = content.find_all('div', style='padding: 0px 0px 0px 0px;')
            if len(reviews) < 1:
                raise StopIteration

            for review_soup in reviews:
                yield Review(review_soup)

            page_number += 1

class Review(object):
    """
    Args:
        review_soup (soup): the soup of the review

    Returns:
        appearance (int): rating for appearance (out of 5)
        aroma (int): aroma rating (out of 10)
        date (datetime): review date
        overall (int): overall rating (out of 20, for some reason)
        palate (int): palate rating (out of 5)
        rating (float): another overall rating provided in the review. Not sure how this different from overall.
        taste (int): taste rating (out of 10)
        text (string): actual text of the review.
        user_location (string): writer's location
        user_name (string): writer's username
    """

    def __init__(self, review_soup):
        # get ratings
        # gets every second entry in a list
        raw_ratings = zip(*[iter(review_soup.find('strong').find_all(["big", "small"]))] * 2)
        # strip html and everything else
        for (label, rating) in raw_ratings:
            rating_int = int(rating.text[:rating.text.find("/")])
            setattr(
                self,
                label.text.lower().strip(),
                rating_int
            )
        self.rating = float(review_soup.find_all('div')[1].text)

        # get user information
        userinfo = review_soup.next_sibling
        self.text = userinfo.next_sibling.next_sibling.text.strip()
        self.user_name = re.findall(r'(.*?)\xa0\(\d*?\)', userinfo.a.text)[0]
        self.user_location = re.findall(r'-\s(.*?)\s-', userinfo.a.next_sibling)[0]

        # get date it was posted
        date = re.findall(r'-(?:\s.*?\s-)+\s(.*)', userinfo.a.next_sibling)[0]
        self.date = datetime.strptime(date.strip(), '%b %d, %Y').date()

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.text

class Brewery(object):
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Brewery('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""

        soup = soup_helper._get_soup(self.url)
        s_contents = soup.find_all('div', {'itemtype':'http://schema.org/LocalBusiness'})
        if not s_contents:
            raise rb_exceptions.PageNotFound(self.url)

        self.name = soup.h1.text
        self.type = s_contents[0].find_all('div')[1].text.strip()
        website = s_contents[0].find_all('div',{'class':'media-links'})[0].find_all('a')[0]
        if website:
            self.web = website['href']
        self.telephone = Brewery._find_span(s_contents[0], 'telephone')
        self.street = Brewery._find_span(s_contents[0], 'streetAddress')
        self.city = Brewery._find_span(s_contents[0], 'addressLocality')
        self.state = Brewery._find_span(s_contents[0], 'addressRegion')
        self.country = Brewery._find_span(s_contents[0], 'addressCountry')
        self.postal_code = Brewery._find_span(s_contents[0], 'postalCode')
        self._has_fetched = True

        return self

    @staticmethod
    def _find_span(search_soup, item_prop):
        output = search_soup.find('span', attrs={'itemprop': item_prop})
        output = output.text.strip() if output else None
        return output

    def get_beers(self):
        """Generator that provides Beer objects for the brewery's beers"""
        if not self._has_fetched:
            self._populate()

        page_number = 1
        while True:
            complete_url = u'{0}0/{1}/'.format(self.url, page_number)
            soup = soup_helper._get_soup(complete_url)
            soup_beer_rows = soup.find('table', 'maintable nohover').findAll('tr')

            if len(soup_beer_rows) < 2:
                raise StopIteration

            for row in soup_beer_rows[1:]:
                url = row.a.get('href')
                # Only return rows that are ratable
                if not row.find(class_='rate'):
                    continue
                # Remove any whitespace characters. Rare, but possible.
                url = re.sub(r"\s+", "", url, flags=re.UNICODE)
                beer = Beer(url)
                beer.name = row.a.text.strip()
                # Add attributes from row
                abv = row.findAll('td')[2].text
                weighted_avg = row.findAll('td')[3].text.strip()
                overall_rating = row.findAll('td')[4].text.strip()
                style_rating = row.findAll('td')[5].text.strip()
                num_ratings = row.findAll('td')[6].text.strip()
                if abv:
                    beer.abv = float(abv)
                if weighted_avg:
                    beer.weighted_avg = float(weighted_avg)
                if overall_rating:
                    beer.overall_rating = int(overall_rating)
                if style_rating:
                    beer.style_rating = int(style_rating)
                if num_ratings:
                    beer.num_ratings = int(num_ratings)
                yield beer

            page_number += 1
<commit_msg>Fix get_beers (off of Brewery) for new table layout<commit_after># This is free and unencumbered software released into the public domain.
#
# Anyone is free to copy, modify, publish, use, compile, sell, or
# distribute this software, either in source code form or as a compiled
# binary, for any purpose, commercial or non-commercial, and by any
# means.
#
# In jurisdictions that recognize copyright laws, the author or authors
# of this software dedicate any and all copyright interest in the
# software to the public domain. We make this dedication for the benefit
# of the public at large and to the detriment of our heirs and
# successors. We intend this dedication to be an overt act of
# relinquishment in perpetuity of all present and future rights to this
# software under copyright law.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# For more information, please refer to <http://unlicense.org/>

import re
from datetime import datetime

try:
    import rb_exceptions
    import soup as soup_helper
except ImportError:  # No implicit package imports in py3.
    from ratebeer import rb_exceptions
    from ratebeer import soup as soup_helper

class Beer(object):
    """The Beer object. Contains information about an individual beer.

    Args:
        url (string): the URL of the beer you're looking for.

    Returns:
        abv (float): percentage alcohol
        brewery (Brewery object): the beer's brewery
        brewed_at (Brewery object): actual brewery if contract brewed
        calories (float): estimated calories for the beer
        description (string): the beer's description
        img_url (string): a url to an image of the beer
        mean_rating (float): the mean rating for the beer (out of 5)
        name (string): the full name of the beer (may include the brewery name)
        num_ratings (int): the number of reviews*
        overall_rating (int): the overall rating (out of 100)
        seasonal (string): Summer, Winter, Autumn, Spring, Series, Special, None
        style (string): beer style
        style_url (string): beer style URL
        style_rating (int): rating of the beer within its style (out of 100)
        tags (list of strings): tags given to the beer
        url (string): the beer's url
        weighted_avg (float): the beer rating average, weighted using some unknown
            algorithm (out of 5)

        Any attributes not available will be returned as None

    """
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Beer('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        soup = soup_helper._get_soup(self.url)
        # check for 404s
        try:
            soup_rows = soup.find('div', id='container').find('table').find_all('tr')
        except AttributeError:
            raise rb_exceptions.PageNotFound(self.url)
        # ratebeer pages don't actually 404, they just send you to this weird
        # "beer reference" page but the url doesn't actually change, it just
        # seems like it's all getting done server side -- so we have to look
        # for the contents h1 to see if we're looking at the beer reference or
        # not
        if "beer reference" in soup_rows[0].find_all('td')[1].h1.contents:
            raise rb_exceptions.PageNotFound(self.url)

        if "Also known as " in soup_rows[1].find_all('td')[1].div.div.contents:
            raise rb_exceptions.AliasedBeer(self.url, soup_rows[1].find_all('td')[1].div.div.a['href'])

        # General information from the top of the page
        self.name = soup.find(itemprop='name').text.strip()
        breweries = soup.find_all('a', href=re.compile('brewers'))
        self.brewery = Brewery(breweries[1].get('href'))
        self.brewery.name = breweries[1].text
        if len(breweries) == 3:
            self.brewed_at = Brewery(breweries[2].get('href'))
            self.brewed_at.name = breweries[2].text
        else:
            self.brewed_at = None
        try:
            self.overall_rating = int(soup.find('span', text='overall').next_sibling.next_sibling.text)
        except ValueError: # 'n/a'
            self.overall_rating = None
        except AttributeError:
            self.overall_rating = None
        try:
            self.style_rating = int(soup.find('span', text='style').previous_sibling.previous_sibling)
        except ValueError: # 'n/a'
            self.style_rating = None
        except AttributeError:
            self.style_rating = None
        self.style = soup.find(text='Style: ').next_sibling.text
        self.style_url = soup.find(text='Style: ').next_sibling.get('href')
        self.img_url = soup.find(id="beerImg").get('src')
        # Data from the info bar
        self.num_ratings = int(soup.find('span', itemprop="ratingCount").text)
        try:
            self.mean_rating = float(soup.find(text='MEAN: ').next_sibling.text.split('/')[0])
        except ValueError: # Empty mean rating: '/5.0'
            self.mean_rating = None
        except AttributeError: # No mean rating
            self.mean_rating = None
        try:
            self.weighted_avg = float(soup.find(attrs={"name": "real average"}).find('span', itemprop="ratingValue").text)
        except ValueError: # Empty weighted average rating: '/5'
            self.weighted_avg = None
        except AttributeError: # No weighted average rating
            self.weighted_avg = None
        try:
            self.seasonal = soup.find(text=u'\xa0\xa0 SEASONAL: ').next_sibling.text
        except AttributeError:
            self.seasonal = None
        try:
            self.ibu = int(soup.find(title="International Bittering Units - Normally from hops").next_sibling.next_sibling.text)
        except AttributeError:
            self.ibu = None
        try:
            self.calories = int(soup.find(title="Estimated calories for a 12 fluid ounce serving").next_sibling.next_sibling.text)
        except AttributeError:
            self.calories = None
        try:
            self.abv = float(soup.find(title="Alcohol By Volume").next_sibling.next_sibling.text[:-1])
        except ValueError: # Empty ABV: '-'
            self.abv = None
        # Description
        description = soup.find('div',
            style=(
                'border: 1px solid #e0e0e0; background: #fff; '
                'padding: 14px; color: #777;'
            )
        )
        if 'no commercial description' not in description.text.lower():
            # strip ads
            [s.extract() for s in description('small')]
            self.description = ' '.join([s for s in description.strings]).strip()
        self.tags = [t.text[1:] for t in soup.find_all('span', class_="tagLink")]

        self._has_fetched = True

        return self

    def get_reviews(self, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A generator of dictionaries, containing the information about the review.
        """

        if not self._has_fetched:
            self._populate()

        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if not url_flag:
            raise ValueError("Invalid ``review_order``.")

        page_number = 1
        while True:
            complete_url = u'{0}{1}/{2}/'.format(self.url, url_flag, page_number)
            soup = soup_helper._get_soup(complete_url)
            content = soup.find('table', style='padding: 10px;').tr.td
            reviews = content.find_all('div', style='padding: 0px 0px 0px 0px;')
            if len(reviews) < 1:
                raise StopIteration

            for review_soup in reviews:
                yield Review(review_soup)

            page_number += 1

class Review(object):
    """
    Args:
        review_soup (soup): the soup of the review

    Returns:
        appearance (int): rating for appearance (out of 5)
        aroma (int): aroma rating (out of 10)
        date (datetime): review date
        overall (int): overall rating (out of 20, for some reason)
        palate (int): palate rating (out of 5)
        rating (float): another overall rating provided in the review. Not sure how this different from overall.
        taste (int): taste rating (out of 10)
        text (string): actual text of the review.
        user_location (string): writer's location
        user_name (string): writer's username
    """

    def __init__(self, review_soup):
        # get ratings
        # gets every second entry in a list
        raw_ratings = zip(*[iter(review_soup.find('strong').find_all(["big", "small"]))] * 2)
        # strip html and everything else
        for (label, rating) in raw_ratings:
            rating_int = int(rating.text[:rating.text.find("/")])
            setattr(
                self,
                label.text.lower().strip(),
                rating_int
            )
        self.rating = float(review_soup.find_all('div')[1].text)

        # get user information
        userinfo = review_soup.next_sibling
        self.text = userinfo.next_sibling.next_sibling.text.strip()
        self.user_name = re.findall(r'(.*?)\xa0\(\d*?\)', userinfo.a.text)[0]
        self.user_location = re.findall(r'-\s(.*?)\s-', userinfo.a.next_sibling)[0]

        # get date it was posted
        date = re.findall(r'-(?:\s.*?\s-)+\s(.*)', userinfo.a.next_sibling)[0]
        self.date = datetime.strptime(date.strip(), '%b %d, %Y').date()

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.text

class Brewery(object):
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Brewery('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""

        soup = soup_helper._get_soup(self.url)
        s_contents = soup.find_all('div', {'itemtype':'http://schema.org/LocalBusiness'})
        if not s_contents:
            raise rb_exceptions.PageNotFound(self.url)

        self.name = soup.h1.text
        self.type = s_contents[0].find_all('div')[1].text.strip()
        website = s_contents[0].find_all('div',{'class':'media-links'})[0].find_all('a')[0]
        if website:
            self.web = website['href']
        self.telephone = Brewery._find_span(s_contents[0], 'telephone')
        self.street = Brewery._find_span(s_contents[0], 'streetAddress')
        self.city = Brewery._find_span(s_contents[0], 'addressLocality')
        self.state = Brewery._find_span(s_contents[0], 'addressRegion')
        self.country = Brewery._find_span(s_contents[0], 'addressCountry')
        self.postal_code = Brewery._find_span(s_contents[0], 'postalCode')
        self._has_fetched = True

        return self

    @staticmethod
    def _find_span(search_soup, item_prop):
        output = search_soup.find('span', attrs={'itemprop': item_prop})
        output = output.text.strip() if output else None
        return output

    def get_beers(self):
        """Generator that provides Beer objects for the brewery's beers"""
        if not self._has_fetched:
            self._populate()

        _id = self.url.split('/')[-2]
        complete_url = u'/Ratings/Beer/ShowBrewerBeers.asp?BrewerID={0}'.format(_id)
        soup = soup_helper._get_soup(complete_url)
        soup_beer_rows = soup.find('table', id='brewer-beer-table').findAll('tr')

        for row in soup_beer_rows[1:]:
            url = row.a.get('href')
            # Only return rows that are ratable
            if not row.find('a',title="Rate this beer"):
                continue
            # Remove any whitespace characters. Rare, but possible.
            url = re.sub(r"\s+", "", url, flags=re.UNICODE)
            beer = Beer(url)
            beer.name = row.a.text.strip()
            # Add attributes from row
            abv = row.findAll('td')[1].text
            weighted_avg = row.findAll('td')[4].text.strip()
            overall_rating = row.findAll('td')[5].text.strip()
            style_rating = row.findAll('td')[6].text.strip()
            num_ratings = row.findAll('td')[7].text.strip()
            if abv:
                beer.abv = float(abv)
            if weighted_avg:
                beer.weighted_avg = float(weighted_avg)
            if overall_rating:
                beer.overall_rating = int(overall_rating)
            if style_rating:
                beer.style_rating = int(style_rating)
            if num_ratings:
                beer.num_ratings = int(num_ratings)
            yield beer
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class Part(scrapy.Item):
    name = scrapy.Field()
    site = scrapy.Field()
    manufacturer = scrapy.Field()
    sku = scrapy.Field()
    weight = scrapy.Field()
    # url, price, quantity, stock, description, location, timestamp
    variants = scrapy.Field()
<commit_msg>Update comment to be more specific about stock fields<commit_after># -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class Part(scrapy.Item):
    name = scrapy.Field()
    site = scrapy.Field()
    manufacturer = scrapy.Field()
    sku = scrapy.Field()
    weight = scrapy.Field()
    # url, price, quantity, stock_state, stock_text, description, location, timestamp
    variants = scrapy.Field()
<|endoftext|>


<commit_before>from functools import wraps
import threading

from flask import Flask, render_template, request, redirect, url_for, Response
from flask_bower import Bower

from rcblog import git
from rcblog import utils
from rcblog.db import DataBase

database = DataBase()

app = Flask(__name__)
Bower(app)

def check_auth(username, password):
    """This function is called to check if a username /
    password combination is valid.
    """
    return username == 'admin' and password == 'secret'

def authenticate():
    """Sends a 401 response that enables basic auth"""
    return Response(
        'Could not verify your access level for that URL.\n'
        'You have to login with proper credentials', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)

    return decorated

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/posts/<int:post_id>')
def show_post(post_id):
    selected_language = request.args.get('lang', 'eng')
    post = get_post_by_id_test(post_id)
    language_codes = []
    for language, translation in post['translations'].items():
        language_codes.append(language)
        translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())
    languages = database.get_languages_by_codes(*language_codes)
    return render_template('post.html',
                           post=post,
                           selected_language=selected_language,
                           languages=languages,
                           current_address='/posts/{}'.format(post_id))

@app.route('/drafts/<int:draft_id>')
def show_draft(draft_id):
    selected_language = request.args.get('lang', 'eng')
    post = get_post_by_id_test(draft_id)
    language_codes = []
    for language, translation in post['translations'].items():
        language_codes.append(language)
        translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())
        translation['post'] = open(translation['markdown_file']).read()
    languages = database.get_languages_by_codes(*language_codes)
    all_languages = database.get_all_languages()
    remaining_languages = utils.complement_of_lists_of_dictionaries(all_languages, languages)
    for language in remaining_languages:
        post['translations'][language] = {'title': '', 'post': ''}
    import pprint; pprint.pprint(post)
    return render_template('draft.html',
                           post=post,
                           selected_language=selected_language,
                           languages=all_languages,
                           values=post['translations'])

@app.route('/posts/add')
@requires_auth
def add_post():
    return render_template('add_post.html', languages=database.get_all_languages())

@app.route('/posts')
def posts_list():
    posts = get_all_posts_test()
    for post in posts:
        for language, translation in post['translations'].items():
            translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())

    return render_template('posts.html', posts=posts)

@app.route('/drafts')
def drafts_list():
    drafts = get_all_drafts_test()
    for draft in drafts:
        for language, translation in draft['translations'].items():
            translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())

    return render_template('drafts.html', posts=drafts)

@app.route('/posts', methods=['POST'])
def commit_post():
    post = {'translations': {}}
    language_codes = set()
    for key in request.form:
        if '$' in key:
            language_codes.add(key.split('$')[0])

    files_to_commit = []

    for language_code in language_codes:
        title = request.form.get(language_code + '$title', '')
        md = request.form.get(language_code + '$post', '')
        if title and md:
            file_name = '{}_{}.md'.format(title, language_code)
            file_path = utils.get_repository_path() / file_name
            files_to_commit.append(file_path)
            with open(str(file_path), 'w') as f:
                f.write(md)
            post['translations'][language_code] = {
                'title': title,
                'markdown_file': file_name
            }
    draft = bool(request.form.get('draft', False))
    database.add_post(post['translations'], [], draft)
    threading.Thread(target=git.commit, args=(utils.get_repository_path(),
                                              files_to_commit,
                                              'Add {}'.format('new post')))
    return redirect(url_for('index'))

def get_all_posts_test():
    return [
        {
            'id': 1,
            'draft': False,
            'translations': {
                'eng': {
                    'title': 'How to do something',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        },
        {
            'id': 2,
            'draft': False,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        },
        {
            'id': 3,
            'draft': True,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        }
    ]

def get_all_drafts_test():
    return [
        {
            'id': 3,
            'draft': True,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        }
    ]

def get_post_by_id_test(*args):
    return {
        'id': 1,
        'draft': False,
        'translations': {
            'eng': {
                'title': 'How to do something',
                'markdown_file': 'rcblog/templates/test.md'
            },
            'rus': {
                'title': 'Как сделать что-то',
                'markdown_file': 'rcblog/templates/test.md'
            }
        },
    }
<commit_msg>Fix bug when posts weren't commited<commit_after>from functools import wraps
import threading

from flask import Flask, render_template, request, redirect, url_for, Response
from flask_bower import Bower

from rcblog import git
from rcblog import utils
from rcblog.db import DataBase

database = DataBase()

app = Flask(__name__)
Bower(app)

def check_auth(username, password):
    """This function is called to check if a username /
    password combination is valid.
    """
    return username == 'admin' and password == 'secret'

def authenticate():
    """Sends a 401 response that enables basic auth"""
    return Response(
        'Could not verify your access level for that URL.\n'
        'You have to login with proper credentials', 401,
        {'WWW-Authenticate': 'Basic realm="Login Required"'})

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)

    return decorated

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/posts/<int:post_id>')
def show_post(post_id):
    selected_language = request.args.get('lang', 'eng')
    post = get_post_by_id_test(post_id)
    language_codes = []
    for language, translation in post['translations'].items():
        language_codes.append(language)
        translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())
    languages = database.get_languages_by_codes(*language_codes)
    return render_template('post.html',
                           post=post,
                           selected_language=selected_language,
                           languages=languages,
                           current_address='/posts/{}'.format(post_id))

@app.route('/drafts/<int:draft_id>')
def show_draft(draft_id):
    selected_language = request.args.get('lang', 'eng')
    post = get_post_by_id_test(draft_id)
    language_codes = []
    for language, translation in post['translations'].items():
        language_codes.append(language)
        translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())
        translation['post'] = open(translation['markdown_file']).read()
    languages = database.get_languages_by_codes(*language_codes)
    all_languages = database.get_all_languages()
    remaining_languages = utils.complement_of_lists_of_dictionaries(all_languages, languages)
    for language in remaining_languages:
        post['translations'][language] = {'title': '', 'post': ''}
    import pprint; pprint.pprint(post)
    return render_template('draft.html',
                           post=post,
                           selected_language=selected_language,
                           languages=all_languages,
                           values=post['translations'])

@app.route('/posts/add')
@requires_auth
def add_post():
    return render_template('add_post.html', languages=database.get_all_languages())

@app.route('/posts')
def posts_list():
    posts = get_all_posts_test()
    for post in posts:
        for language, translation in post['translations'].items():
            translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())

    return render_template('posts.html', posts=posts)

@app.route('/drafts')
def drafts_list():
    drafts = get_all_drafts_test()
    for draft in drafts:
        for language, translation in draft['translations'].items():
            translation['html'] = utils.md_to_html(open(translation['markdown_file']).read())

    return render_template('drafts.html', posts=drafts)

@app.route('/posts', methods=['POST'])
def commit_post():
    post = {'translations': {}}
    language_codes = set()
    for key in request.form:
        if '$' in key:
            language_codes.add(key.split('$')[0])

    files_to_commit = []

    for language_code in language_codes:
        title = request.form.get(language_code + '$title', '')
        md = request.form.get(language_code + '$post', '')
        if title and md:
            file_name = '{}_{}.md'.format(title, language_code)
            file_path = utils.get_repository_path() / file_name
            files_to_commit.append(file_path)
            with open(str(file_path), 'w') as f:
                f.write(md)
            post['translations'][language_code] = {
                'title': title,
                'markdown_file': file_name
            }
    draft = bool(request.form.get('draft', False))
    database.add_post(post['translations'], [], draft)
    threading.Thread(target=git.commit, args=(utils.get_repository_path(),
                                              files_to_commit,
                                              'Add {}'.format('new post'))).run()
    return redirect(url_for('index'))

def get_all_posts_test():
    return [
        {
            'id': 1,
            'draft': False,
            'translations': {
                'eng': {
                    'title': 'How to do something',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        },
        {
            'id': 2,
            'draft': False,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        },
        {
            'id': 3,
            'draft': True,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        }
    ]

def get_all_drafts_test():
    return [
        {
            'id': 3,
            'draft': True,
            'translations': {
                'eng': {
                    'title': 'How to do something 2',
                    'markdown_file': 'rcblog/templates/test.md'
                },
                'rus': {
                    'title': 'Как сделать что-то 2',
                    'markdown_file': 'rcblog/templates/test.md'
                }
            },
        }
    ]

def get_post_by_id_test(*args):
    return {
        'id': 1,
        'draft': False,
        'translations': {
            'eng': {
                'title': 'How to do something',
                'markdown_file': 'rcblog/templates/test.md'
            },
            'rus': {
                'title': 'Как сделать что-то',
                'markdown_file': 'rcblog/templates/test.md'
            }
        },
    }
<|endoftext|>