<commit_before>"""Beam pipeline for generating random graph samples.
"""
import argparse
import json
import logging
import os
import setuptools

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions

class SampleSbmDoFn(beam.DoFn):

    def __init__(self, nvertex_min, nvertex_max, nedges_min, nedges_max,
                 feature_center_distance_max):
        self._nvertex_min = nvertex_min
        self._nvertex_max = nvertex_max
        self._nedges_min = nedges_min
        self._nedges_max = nedges_max
        self._feature_center_distance_max = feature_center_distance_max

    def process(self, sample_id):
        """Sample and save SMB outputs given a configuration filepath.
        """
        # Avoid save_main_session in Pipeline args so the controller doesn't
        # have to import the same libraries as the workers which may be using
        # a custom container. The import will execute once then the sys.modeules
        # will be referenced to further calls.
        import numpy as np
        from sbm.sbm_simulator import GenerateStochasticBlockModelWithFeatures

        # Parameterize me...
        edge_center_distance_min = 1.0
        edge_center_distance_max = 10.0
        feature_dim = 16
        edge_center_distance = 2.0
        edge_feature_dim = 4

        num_vertices = np.random.randint(self._nvertex_min, self._nvertex_max)
        num_edges = np.random.randint(self._nedges_min, self._nedges_max)
        feature_center_distance = np.random.uniform(
                0.0, self._feature_center_distance_max)

        generator_config = {
            'generator_name': 'StochasticBlockModel',
            'num_verticies': num_vertices,
            'num_edges': num_edges,
            'feature_dim': feature_dim,
            'feature_center_distance': 2.0,
            'edge_center_distance': edge_center_distance,
            'edge_feature_dim': 4
        }

        data = GenerateStochasticBlockModelWithFeatures(
            num_vertices = num_vertices,
            num_edges = num_edges,
            pi = np.array([0.25, 0.25, 0.25, 0.25]),
            prop_mat = np.ones((4, 4)) + 9.0 * np.diag([1,1,1,1]),
            feature_center_distance = generator_config['feature_center_distance'],
            feature_dim=generator_config['feature_dim'],
            edge_center_distance = generator_config['edge_center_distance'],
            edge_feature_dim = generator_config['edge_feature_dim']
        )

        yield {'sample_id': sample_id,
               'generator_config': generator_config,
               'data': data}

class WriteSbmDoFn(beam.DoFn):

    def __init__(self, output_path):
        self._output_path = output_path

    def process(self, element):
        import apache_beam
        import numpy as np

        sample_id = element['sample_id']
        config = element['generator_config']
        data = element['data']

        text_mime = 'text/plain'
        prefix = '{0:05}'.format(sample_id)
        config_object_name = os.path.join(self._output_path, prefix + '_config.txt')
        with apache_beam.io.filesystems.FileSystems.create(config_object_name, text_mime) as f:
            buf = bytes(json.dumps(config), 'utf-8')
            f.write(buf)
            f.close()

        graph_object_name = os.path.join(self._output_path, prefix + '_graph.gt')
        with apache_beam.io.filesystems.FileSystems.create(graph_object_name) as f:
            data.graph.save(f)
            f.close()

        graph_memberships_object_name = os.path.join(
            self._output_path, prefix + '_graph_memberships.txt')
        with apache_beam.io.filesystems.FileSystems.create(graph_memberships_object_name, text_mime) as f:
            np.savetxt(f, data.graph_memberships)
            f.close()

        node_features_object_name = os.path.join(
            self._output_path, prefix + '_node_features.txt')
        with apache_beam.io.filesystems.FileSystems.create(node_features_object_name, text_mime) as f:
            np.savetxt(f, data.node_features)
            f.close()

        feature_memberships_object_name = os.path.join(
            self._output_path, prefix + '_feature_membership.txt')
        with apache_beam.io.filesystems.FileSystems.create(feature_memberships_object_name, text_mime) as f:
            np.savetxt(f, data.feature_memberships)
            f.close()

        edge_features_object_name = os.path.join(
            self._output_path, prefix + '_edge_features.txt')
        with apache_beam.io.filesystems.FileSystems.create(edge_features_object_name, text_mime) as f:
            for edge_tuple, features in data.edge_features.items():
                buf = bytes('{0},{1},{2}'.format(edge_tuple[0], edge_tuple[1], features), 'utf-8')
                f.write(buf)
            f.close()

class ConvertToTorchGeoDataParDo(beam.DoFn):
    def __init__(self, output_path):
        self._output_path = output_path

    def process(self, element):
        import logging
        import apache_beam
        import numpy as np
        from sbm.utils import sbm_data_to_torchgeo_data, get_kclass_masks
        sample_id = element['sample_id']
        sbm_data = element['data']

        out = {
            'sample_id': sample_id,
            'torch_data': None,
            'masks': None,
            'skipped': False
        }

        try:
            torch_data = sbm_data_to_torchgeo_data(sbm_data)
            out['torch_data'] = sbm_data_to_torchgeo_data(sbm_data)

            torchgeo_stats = {
                'nodes': torch_data.num_nodes,
                'edges': torch_data.num_edges,
                'average_node_degree': torch_data.num_edges / torch_data.num_nodes,
                # 'contains_isolated_nodes': torchgeo_data.contains_isolated_nodes(),
                # 'contains_self_loops': torchgeo_data.contains_self_loops(),
                # 'undirected': bool(torchgeo_data.is_undirected())
            }
            stats_object_name = os.path.join(self._output_path, '{0:05}_torch_stats.txt'.format(sample_id))
            with apache_beam.io.filesystems.FileSystems.create(stats_object_name, 'text/plain') as f:
                buf = bytes(json.dumps(torchgeo_stats), 'utf-8')
                f.write(buf)
                f.close()
        except:
            out['skipped'] = True
            print(f'faied convert {sample_id}')
            logging.info(f'Failed to convert sbm_data to torchgeo for sample id {sample_id}')
            yield out

        try:
            out['masks'] = get_kclass_masks(sbm_data)

            masks_object_name = os.path.join(self._output_path, '{0:05}_masks.txt'.format(sample_id))
            with apache_beam.io.filesystems.FileSystems.create(masks_object_name, 'text/plain') as f:
                for mask in out['masks']:
                    np.savetxt(f, np.atleast_2d(mask.numpy()), fmt='%i', delimiter=' ')
                f.close()
        except:
            out['skipped'] = True
            print(f'failed masks {sample_id}')
            logging.info(f'Failed to sample masks for sample id {sample_id}')
            yield out

        yield out

class BenchmarkSimpleGCNParDo(beam.DoFn):
    def __init__(self, output_path, num_features, num_classes, hidden_channels, epochs):
        self._output_path = output_path
        self._num_features = num_features
        self._num_classes = num_classes
        self._hidden_channels = hidden_channels
        self._epochs = epochs

    def process(self, element):
        import logging
        import apache_beam
        from sbm.models import LinearGCN
        sample_id = element['sample_id']
        torch_data = element['torch_data']
        masks = element['masks']
        skipped = element['skipped']

        out = {
            'skipped': skipped,
            'results': None
        }

        if skipped:
            logging.info(f'Skipping benchmark for sample id {sample_id}')
            return

        train_mask, val_mask, test_mask = masks
        linear_model = LinearGCN(
            self._num_features,
            self._num_classes,
            self._hidden_channels,
            train_mask,
            val_mask,
            test_mask)

        losses = linear_model.train(self._epochs, torch_data)
        test_accuracy = None
        try:
            # Divide by zero somesimtes happens with the ksample masks.
            test_accuracy = linear_model.test(torch_data)
        except:
            logging.info(f'Failed to compute test accuracy for sample id {sample_id}')

        results = {
            'sample_id': sample_id,
            'losses': losses,
            'test_accuracy': test_accuracy
        }

        results_object_name = os.path.join(self._output_path, '{0:05}_results.txt'.format(sample_id))
        with apache_beam.io.filesystems.FileSystems.create(results_object_name, 'text/plain') as f:
            buf = bytes(json.dumps(results), 'utf-8')
            f.write(buf)
            f.close()

        yield results

def main(argv=None):
    parser = argparse.ArgumentParser()

    parser.add_argument('--output',
                        dest='output',
                        default='/tmp/graph_configs.json',
                        help='Location to write output files.')

    parser.add_argument('--nsamples',
                        dest='nsamples',
                        type=int,
                        default=100,
                        help='The number of graph samples.')

    parser.add_argument('--nvertex_min',
                        dest='nvertex_min',
                        type=int,
                        default=5,
                        help='Minimum number of nodes in graph samples.')

    parser.add_argument('--nvertex_max',
                        dest='nvertex_max',
                        type=int,
                        default=50,
                        help='Maximum number of nodes in the graph samples.')

    parser.add_argument('--nedges_min',
                        dest='nedges_min',
                        type=int,
                        default=10,
                        help='Minimum number of edges in the graph samples.')

    parser.add_argument('--nedges_max',
                        dest='nedges_max',
                        type=int,
                        default=100,
                        help='Maximum number of edges in the graph samples.')

    parser.add_argument('--feature_center_distance_max',
                        dest='feature_center_distance_max',
                        type=float,
                        default=10.0,
                        help=('Maximum cluster mean feature distance in the'
                              'graph samples.'))

    args, pipeline_args = parser.parse_known_args(argv)

    logging.info(f'Pipeline Args: {pipeline_args}')

    pipeline_options = PipelineOptions(pipeline_args)
    pipeline_options.view_as(SetupOptions).save_main_session = False

    # Parameterize
    num_features = 16
    num_classes = 4
    hidden_channels = 8
    epochs = 256

    with beam.Pipeline(options=pipeline_options) as p:

        graph_samples = (
            p
            | 'Create Sample Ids' >> beam.Create(range(args.nsamples))
            | 'Sample Graphs' >> beam.ParDo(
                SampleSbmDoFn(args.nvertex_min, args.nvertex_max,
                              args.nedges_min, args.nedges_max,
                              args.feature_center_distance_max))
        )

        graph_samples | 'Write Sampled Graph' >> beam.ParDo(WriteSbmDoFn(args.output))

        torch_data = (
            graph_samples | 'Convert to torchgeo data.' >> beam.ParDo(ConvertToTorchGeoDataParDo(args.output)))

        (torch_data | 'Filter skipped conversions' >> beam.Filter(lambda el: el['skipped'])
                    | 'Extract skipped sample ids' >> beam.Map(lambda el: el['sample_id'])
                    | 'Write skipped text file' >> beam.io.WriteToText(os.path.join(args.output, 'skipped.txt')))

        torch_data | 'Benchmark Simple GCN.' >> beam.ParDo(BenchmarkSimpleGCNParDo(
                args.output, num_features, num_classes, hidden_channels, epochs))

if __name__ == '__main__':
    main(None)
<commit_msg>First stab at beam.dataframe csv saving.<commit_after>"""Beam pipeline for generating random graph samples.
"""
import argparse
import json
import logging
import os
import setuptools

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.dataframe.convert import to_dataframe

class SampleSbmDoFn(beam.DoFn):

    def __init__(self, nvertex_min, nvertex_max, nedges_min, nedges_max,
                 feature_center_distance_max):
        self._nvertex_min = nvertex_min
        self._nvertex_max = nvertex_max
        self._nedges_min = nedges_min
        self._nedges_max = nedges_max
        self._feature_center_distance_max = feature_center_distance_max

    def process(self, sample_id):
        """Sample and save SMB outputs given a configuration filepath.
        """
        # Avoid save_main_session in Pipeline args so the controller doesn't
        # have to import the same libraries as the workers which may be using
        # a custom container. The import will execute once then the sys.modeules
        # will be referenced to further calls.
        import numpy as np
        from sbm.sbm_simulator import GenerateStochasticBlockModelWithFeatures

        # Parameterize me...
        edge_center_distance_min = 1.0
        edge_center_distance_max = 10.0
        feature_dim = 16
        edge_center_distance = 2.0
        edge_feature_dim = 4

        num_vertices = np.random.randint(self._nvertex_min, self._nvertex_max)
        num_edges = np.random.randint(self._nedges_min, self._nedges_max)
        feature_center_distance = np.random.uniform(
                0.0, self._feature_center_distance_max)

        generator_config = {
            'generator_name': 'StochasticBlockModel',
            'num_verticies': num_vertices,
            'num_edges': num_edges,
            'feature_dim': feature_dim,
            'feature_center_distance': 2.0,
            'edge_center_distance': edge_center_distance,
            'edge_feature_dim': 4
        }

        data = GenerateStochasticBlockModelWithFeatures(
            num_vertices = num_vertices,
            num_edges = num_edges,
            pi = np.array([0.25, 0.25, 0.25, 0.25]),
            prop_mat = np.ones((4, 4)) + 9.0 * np.diag([1,1,1,1]),
            feature_center_distance = generator_config['feature_center_distance'],
            feature_dim=generator_config['feature_dim'],
            edge_center_distance = generator_config['edge_center_distance'],
            edge_feature_dim = generator_config['edge_feature_dim']
        )

        yield {'sample_id': sample_id,
               'generator_config': generator_config,
               'data': data}

class WriteSbmDoFn(beam.DoFn):

    def __init__(self, output_path):
        self._output_path = output_path

    def process(self, element):
        import apache_beam
        import numpy as np

        sample_id = element['sample_id']
        config = element['generator_config']
        data = element['data']

        text_mime = 'text/plain'
        prefix = '{0:05}'.format(sample_id)
        config_object_name = os.path.join(self._output_path, prefix + '_config.txt')
        with apache_beam.io.filesystems.FileSystems.create(config_object_name, text_mime) as f:
            buf = bytes(json.dumps(config), 'utf-8')
            f.write(buf)
            f.close()

        graph_object_name = os.path.join(self._output_path, prefix + '_graph.gt')
        with apache_beam.io.filesystems.FileSystems.create(graph_object_name) as f:
            data.graph.save(f)
            f.close()

        graph_memberships_object_name = os.path.join(
            self._output_path, prefix + '_graph_memberships.txt')
        with apache_beam.io.filesystems.FileSystems.create(graph_memberships_object_name, text_mime) as f:
            np.savetxt(f, data.graph_memberships)
            f.close()

        node_features_object_name = os.path.join(
            self._output_path, prefix + '_node_features.txt')
        with apache_beam.io.filesystems.FileSystems.create(node_features_object_name, text_mime) as f:
            np.savetxt(f, data.node_features)
            f.close()

        feature_memberships_object_name = os.path.join(
            self._output_path, prefix + '_feature_membership.txt')
        with apache_beam.io.filesystems.FileSystems.create(feature_memberships_object_name, text_mime) as f:
            np.savetxt(f, data.feature_memberships)
            f.close()

        edge_features_object_name = os.path.join(
            self._output_path, prefix + '_edge_features.txt')
        with apache_beam.io.filesystems.FileSystems.create(edge_features_object_name, text_mime) as f:
            for edge_tuple, features in data.edge_features.items():
                buf = bytes('{0},{1},{2}'.format(edge_tuple[0], edge_tuple[1], features), 'utf-8')
                f.write(buf)
            f.close()

class ConvertToTorchGeoDataParDo(beam.DoFn):
    def __init__(self, output_path):
        self._output_path = output_path

    def process(self, element):
        import logging
        import apache_beam
        import numpy as np
        from sbm.utils import sbm_data_to_torchgeo_data, get_kclass_masks
        sample_id = element['sample_id']
        sbm_data = element['data']

        out = {
            'sample_id': sample_id,
            'torch_data': None,
            'masks': None,
            'skipped': False
        }

        try:
            torch_data = sbm_data_to_torchgeo_data(sbm_data)
            out['torch_data'] = sbm_data_to_torchgeo_data(sbm_data)
            out['generator_config'] = element['generator_config']

            torchgeo_stats = {
                'nodes': torch_data.num_nodes,
                'edges': torch_data.num_edges,
                'average_node_degree': torch_data.num_edges / torch_data.num_nodes,
                # 'contains_isolated_nodes': torchgeo_data.contains_isolated_nodes(),
                # 'contains_self_loops': torchgeo_data.contains_self_loops(),
                # 'undirected': bool(torchgeo_data.is_undirected())
            }
            stats_object_name = os.path.join(self._output_path, '{0:05}_torch_stats.txt'.format(sample_id))
            with apache_beam.io.filesystems.FileSystems.create(stats_object_name, 'text/plain') as f:
                buf = bytes(json.dumps(torchgeo_stats), 'utf-8')
                f.write(buf)
                f.close()
        except:
            out['skipped'] = True
            print(f'faied convert {sample_id}')
            logging.info(f'Failed to convert sbm_data to torchgeo for sample id {sample_id}')
            yield out

        try:
            out['masks'] = get_kclass_masks(sbm_data)

            masks_object_name = os.path.join(self._output_path, '{0:05}_masks.txt'.format(sample_id))
            with apache_beam.io.filesystems.FileSystems.create(masks_object_name, 'text/plain') as f:
                for mask in out['masks']:
                    np.savetxt(f, np.atleast_2d(mask.numpy()), fmt='%i', delimiter=' ')
                f.close()
        except:
            out['skipped'] = True
            print(f'failed masks {sample_id}')
            logging.info(f'Failed to sample masks for sample id {sample_id}')
            yield out

        yield out

class BenchmarkSimpleGCNParDo(beam.DoFn):
    def __init__(self, output_path, num_features, num_classes, hidden_channels, epochs):
        self._output_path = output_path
        self._num_features = num_features
        self._num_classes = num_classes
        self._hidden_channels = hidden_channels
        self._epochs = epochs

    def process(self, element):
        import logging
        import apache_beam
        from sbm.models import LinearGCN
        sample_id = element['sample_id']
        torch_data = element['torch_data']
        masks = element['masks']
        skipped = element['skipped']

        out = {
            'skipped': skipped,
            'results': None
        }

        if skipped:
            logging.info(f'Skipping benchmark for sample id {sample_id}')
            return

        train_mask, val_mask, test_mask = masks
        linear_model = LinearGCN(
            self._num_features,
            self._num_classes,
            self._hidden_channels,
            train_mask,
            val_mask,
            test_mask)

        losses = linear_model.train(self._epochs, torch_data)
        test_accuracy = None
        try:
            # Divide by zero somesimtes happens with the ksample masks.
            test_accuracy = linear_model.test(torch_data)
        except:
            logging.info(f'Failed to compute test accuracy for sample id {sample_id}')

        results = {
            'sample_id': sample_id,
            'losses': losses,
            'test_accuracy': test_accuracy,
            'generator_config': element['generator_config']
        }

        results_object_name = os.path.join(self._output_path, '{0:05}_results.txt'.format(sample_id))
        with apache_beam.io.filesystems.FileSystems.create(results_object_name, 'text/plain') as f:
            buf = bytes(json.dumps(results), 'utf-8')
            f.write(buf)
            f.close()

        yield results

def main(argv=None):
    parser = argparse.ArgumentParser()

    parser.add_argument('--output',
                        dest='output',
                        default='/tmp/graph_configs.json',
                        help='Location to write output files.')

    parser.add_argument('--nsamples',
                        dest='nsamples',
                        type=int,
                        default=100,
                        help='The number of graph samples.')

    parser.add_argument('--nvertex_min',
                        dest='nvertex_min',
                        type=int,
                        default=5,
                        help='Minimum number of nodes in graph samples.')

    parser.add_argument('--nvertex_max',
                        dest='nvertex_max',
                        type=int,
                        default=50,
                        help='Maximum number of nodes in the graph samples.')

    parser.add_argument('--nedges_min',
                        dest='nedges_min',
                        type=int,
                        default=10,
                        help='Minimum number of edges in the graph samples.')

    parser.add_argument('--nedges_max',
                        dest='nedges_max',
                        type=int,
                        default=100,
                        help='Maximum number of edges in the graph samples.')

    parser.add_argument('--feature_center_distance_max',
                        dest='feature_center_distance_max',
                        type=float,
                        default=10.0,
                        help=('Maximum cluster mean feature distance in the'
                              'graph samples.'))

    args, pipeline_args = parser.parse_known_args(argv)

    logging.info(f'Pipeline Args: {pipeline_args}')

    pipeline_options = PipelineOptions(pipeline_args)
    pipeline_options.view_as(SetupOptions).save_main_session = False

    # Parameterize
    num_features = 16
    num_classes = 4
    hidden_channels = 8
    epochs = 256

    def ConvertToRowDict(benchmark_result):
        row_dict = {'test_accuracy': benchmark_result['test_accuracy']}
        row_dict.update(benchmark_result['generator_config'])
        return row_dict

    with beam.Pipeline(options=pipeline_options) as p:

        graph_samples = (
            p
            | 'Create Sample Ids' >> beam.Create(range(args.nsamples))
            | 'Sample Graphs' >> beam.ParDo(
                SampleSbmDoFn(args.nvertex_min, args.nvertex_max,
                              args.nedges_min, args.nedges_max,
                              args.feature_center_distance_max))
        )

        graph_samples | 'Write Sampled Graph' >> beam.ParDo(WriteSbmDoFn(args.output))

        torch_data = (
            graph_samples | 'Convert to torchgeo data.' >> beam.ParDo(ConvertToTorchGeoDataParDo(args.output)))

        (torch_data | 'Filter skipped conversions' >> beam.Filter(lambda el: el['skipped'])
                    | 'Extract skipped sample ids' >> beam.Map(lambda el: el['sample_id'])
                    | 'Write skipped text file' >> beam.io.WriteToText(os.path.join(args.output, 'skipped.txt')))

        dataframe_rows = (
            torch_data | 'Benchmark Simple GCN.' >> beam.ParDo(BenchmarkSimpleGCNParDo(
                args.output, num_features, num_classes, hidden_channels, epochs))
                       | 'Convert to dataframe rows.' >> beam.Map(
                            lambda result: beam.Row(test_accuracy=result["test_accuracy"])))

        to_dataframe(dataframe_rows).to_csv(os.path.join(args.output, 'results_df.csv'))

if __name__ == '__main__':
    main(None)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Rickshaw
-------

Python Pandas + Rickshaw.js

'''

from __future__ import print_function
from __future__ import division
import time
import json
import os
from collections import defaultdict
from pkg_resources import resource_string
import pandas as pd
import numpy as np
from jinja2 import Environment, PackageLoader

class Chart(object):
    '''Visualize Pandas Timeseries with Rickshaw.js'''

    def __init__(self, data=None, width=750, height=400, plt_type='line',
                 colors=None, x_time=True, y_zero=False, **kwargs):
        '''Generate a Rickshaw time series visualization with Pandas
        Series and DataFrames.

         The bearcart Chart generates the Rickshaw visualization of a Pandas
         timeseries Series or DataFrame. The only required parameters are
         data, width, height, and type. Colors is an optional parameter;
         bearcart will default to the Rickshaw spectrum14 color palette if
         none are passed. Keyword arguments can be passed to disable the
         following components:
            - x_axis
            - y_axis
            - hover
            - legend

        Parameters
        ----------
        data: Pandas Series or DataFrame, default None
             The Series or Dataframe must have a Datetime index.
        width: int, default 960
            Width of the chart in pixels
        height: int, default 500
            Height of the chart in pixels
        plt_type: string, default 'line'
            Must be one of 'line', 'area', 'scatterplot' or 'bar'
        colors: dict, default None
            Dict with keys matching DataFrame or Series column names, and hex
            strings for colors
        x_time: boolean, default True
            If passed as False, the x-axis will have non-time values
        y_zero: boolean, default False
            The y-axis defaults to auto scaling. Pass True to set the min
            y-axis value to 0.
        kwargs:
            Keyword arguments that, if passed as False, will disable the
            following components: x_axis, y_axis, hover, legend

        Returns
        -------
        Bearcart object

        Examples
        --------
        >>>vis = bearcart.Chart(data=df, width=800, height=300, type='area')
        >>>vis = bearcart.Chart(data=series,type='scatterplot',
                                colors={'Data 1': '#25aeb0',
                                        'Data 2': '#114e4f'})
        #Disable x_axis and legend
        >>>vis = bearcart.Chart(data=df, x_axis=False, legend=False)

        '''

        self.defaults = {'x_axis': True, 'y_axis': True, 'hover': True,
                         'legend': True}

        self.env = Environment(loader=PackageLoader('bearcart', 'templates'))

        #Colors need to be js strings
        if colors:
            self.colors = {key: "'{0}'".format(value)
                           for key, value in colors.iteritems()}
        else:
            self.colors = None

        self.x_axis_time = x_time
        self.renderer = plt_type
        self.width = width
        self.height = height
        self.y_zero = y_zero
        self.template_vars = {}

        #Update defaults for passed kwargs
        for key, value in kwargs.iteritems():
            self.defaults[key] = value

        #Get templates for graph elements
        for att, val in self.defaults.iteritems():
            render_vars = {}
            if val:
                if not self.x_axis_time:
                    if att == 'x_axis':
                        att = 'x_axis_num'
                    elif att == 'hover':
                        render_vars = {'x_hover': 'xFormatter: function(x)'
                                       '{return Math.floor(x / 10) * 10}'}
                temp = self.env.get_template(att + '.js')
                self.template_vars.update({att: temp.render(render_vars)})

        #Transform data into Rickshaw-happy JSON format
        if data is not None:
            self.transform_data(data)

    def transform_data(self, data):
        '''Transform Pandas Timeseries into JSON format

        Parameters
        ----------
        data: DataFrame or Series
            Pandas DataFrame or Series must have datetime index

        Returns
        -------
        JSON to object.json_data

        Example
        -------
        >>>vis.transform_data(df)
        >>>vis.json_data

        '''

        def type_check(value):
            '''Type check values for JSON serialization. Native Python JSON
            serialization will not recognize some Numpy data types properly,
            so they must be explictly converted.'''
            if pd.isnull(value):
                return None
            elif (isinstance(value, pd.tslib.Timestamp) or
                  isinstance(value, pd.Period)):
                return time.mktime(value.timetuple())
            elif isinstance(value, (int, np.integer)):
                return int(value)
            elif isinstance(value, (float, np.float_)):
                return float(value)
            elif isinstance(value, str):
                return str(value)
            else:
                return value

        objectify = lambda dat: [{"x": type_check(x), "y": type_check(y)}
                                 for x, y in dat.iteritems()]

        self.raw_data = data
        if isinstance(data, pd.Series):
            data.name = data.name or 'data'
            self.json_data = [{'name': data.name, 'data': objectify(data)}]
        elif isinstance(data, pd.DataFrame):
            self.json_data = [{'name': x[0], 'data': objectify(x[1])}
                              for x in data.iteritems()]

    def _build_graph(self):
        '''Build Rickshaw graph syntax with all data'''

        #Set palette colors if necessary
        if not self.colors:
            self.palette = self.env.get_template('palette.js')
            self.template_vars.update({'palette': self.palette.render()})
            self.colors = {x['name']: 'palette.color()' for x in self.json_data}

        template_vars = []
        for index, dataset in enumerate(self.json_data):
            group = 'datagroup' + str(index)
            template_vars.append({'name': str(dataset['name']),
                                  'color': self.colors[dataset['name']],
                                  'data': 'json[{0}].data'.format(index)})

        variables = {'dataset': template_vars, 'width': self.width,
                     'height': self.height, 'render': self.renderer}
        if not self.y_zero:
            variables.update({'min': "min: 'auto',"})

        graph = self.env.get_template('graph.js')
        self.template_vars.update({'graph': graph.render(variables)})

    def create_chart(self, html_path='index.html', data_path='data.json',
                     js_path=None, css_path=None):
        '''Save bearcart output to HTML and JSON.

        Parameters
        ----------
        html_path: string, default 'index.html'
            Path for html output
        data_path: string, default 'data.json'
            Path for data JSON output
        js_path: string, default None
            If passed, the Rickshaw javascript library will be saved to the
            path. The file must be named "rickshaw.min.js"
        css_path: string, default None
            If passed, the Rickshaw css library will be saved to the
            path. The file must be named "rickshaw.min.css"

        Returns
        -------
        HTML, JSON, JS, and CSS

        Example
        --------
        >>>vis.create_chart(html_path='myvis.html', data_path='visdata.json'),
                            js_path='rickshaw.min.js',
                            cs_path='rickshaw.min.css')
        '''

        self.template_vars.update({'data_path': str(data_path)})

        self._build_graph()
        html = self.env.get_template('bcart_template.html')
        self.HTML = html.render(self.template_vars)

        with open(html_path, 'w') as f:
            f.write(self.HTML)

        with open(data_path, 'w') as f:
            json.dump(self.json_data, f, sort_keys=True, indent=4,
                      separators=(',', ': '))

        if js_path:
            js = resource_string('bearcart', 'rickshaw.min.js')
            with open(js_path, 'w') as f:
                f.write(js)
        if css_path:
            css = resource_string('bearcart', 'rickshaw.min.css')
            with open(css_path, 'w') as f:
                    f.write(css)
<commit_msg>create_chart js_path and css_path now default to rickshaw js and css, rather than None<commit_after># -*- coding: utf-8 -*-
'''
Rickshaw
-------

Python Pandas + Rickshaw.js

'''

from __future__ import print_function
from __future__ import division
import time
import json
import os
from collections import defaultdict
from pkg_resources import resource_string
import pandas as pd
import numpy as np
from jinja2 import Environment, PackageLoader

class Chart(object):
    '''Visualize Pandas Timeseries with Rickshaw.js'''

    def __init__(self, data=None, width=750, height=400, plt_type='line',
                 colors=None, x_time=True, y_zero=False, **kwargs):
        '''Generate a Rickshaw time series visualization with Pandas
        Series and DataFrames.

         The bearcart Chart generates the Rickshaw visualization of a Pandas
         timeseries Series or DataFrame. The only required parameters are
         data, width, height, and type. Colors is an optional parameter;
         bearcart will default to the Rickshaw spectrum14 color palette if
         none are passed. Keyword arguments can be passed to disable the
         following components:
            - x_axis
            - y_axis
            - hover
            - legend

        Parameters
        ----------
        data: Pandas Series or DataFrame, default None
             The Series or Dataframe must have a Datetime index.
        width: int, default 960
            Width of the chart in pixels
        height: int, default 500
            Height of the chart in pixels
        plt_type: string, default 'line'
            Must be one of 'line', 'area', 'scatterplot' or 'bar'
        colors: dict, default None
            Dict with keys matching DataFrame or Series column names, and hex
            strings for colors
        x_time: boolean, default True
            If passed as False, the x-axis will have non-time values
        y_zero: boolean, default False
            The y-axis defaults to auto scaling. Pass True to set the min
            y-axis value to 0.
        kwargs:
            Keyword arguments that, if passed as False, will disable the
            following components: x_axis, y_axis, hover, legend

        Returns
        -------
        Bearcart object

        Examples
        --------
        >>>vis = bearcart.Chart(data=df, width=800, height=300, type='area')
        >>>vis = bearcart.Chart(data=series,type='scatterplot',
                                colors={'Data 1': '#25aeb0',
                                        'Data 2': '#114e4f'})
        #Disable x_axis and legend
        >>>vis = bearcart.Chart(data=df, x_axis=False, legend=False)

        '''

        self.defaults = {'x_axis': True, 'y_axis': True, 'hover': True,
                         'legend': True}

        self.env = Environment(loader=PackageLoader('bearcart', 'templates'))

        #Colors need to be js strings
        if colors:
            self.colors = {key: "'{0}'".format(value)
                           for key, value in colors.iteritems()}
        else:
            self.colors = None

        self.x_axis_time = x_time
        self.renderer = plt_type
        self.width = width
        self.height = height
        self.y_zero = y_zero
        self.template_vars = {}

        #Update defaults for passed kwargs
        for key, value in kwargs.iteritems():
            self.defaults[key] = value

        #Get templates for graph elements
        for att, val in self.defaults.iteritems():
            render_vars = {}
            if val:
                if not self.x_axis_time:
                    if att == 'x_axis':
                        att = 'x_axis_num'
                    elif att == 'hover':
                        render_vars = {'x_hover': 'xFormatter: function(x)'
                                       '{return Math.floor(x / 10) * 10}'}
                temp = self.env.get_template(att + '.js')
                self.template_vars.update({att: temp.render(render_vars)})

        #Transform data into Rickshaw-happy JSON format
        if data is not None:
            self.transform_data(data)

    def transform_data(self, data):
        '''Transform Pandas Timeseries into JSON format

        Parameters
        ----------
        data: DataFrame or Series
            Pandas DataFrame or Series must have datetime index

        Returns
        -------
        JSON to object.json_data

        Example
        -------
        >>>vis.transform_data(df)
        >>>vis.json_data

        '''

        def type_check(value):
            '''Type check values for JSON serialization. Native Python JSON
            serialization will not recognize some Numpy data types properly,
            so they must be explictly converted.'''
            if pd.isnull(value):
                return None
            elif (isinstance(value, pd.tslib.Timestamp) or
                  isinstance(value, pd.Period)):
                return time.mktime(value.timetuple())
            elif isinstance(value, (int, np.integer)):
                return int(value)
            elif isinstance(value, (float, np.float_)):
                return float(value)
            elif isinstance(value, str):
                return str(value)
            else:
                return value

        objectify = lambda dat: [{"x": type_check(x), "y": type_check(y)}
                                 for x, y in dat.iteritems()]

        self.raw_data = data
        if isinstance(data, pd.Series):
            data.name = data.name or 'data'
            self.json_data = [{'name': data.name, 'data': objectify(data)}]
        elif isinstance(data, pd.DataFrame):
            self.json_data = [{'name': x[0], 'data': objectify(x[1])}
                              for x in data.iteritems()]

    def _build_graph(self):
        '''Build Rickshaw graph syntax with all data'''

        #Set palette colors if necessary
        if not self.colors:
            self.palette = self.env.get_template('palette.js')
            self.template_vars.update({'palette': self.palette.render()})
            self.colors = {x['name']: 'palette.color()' for x in self.json_data}

        template_vars = []
        for index, dataset in enumerate(self.json_data):
            group = 'datagroup' + str(index)
            template_vars.append({'name': str(dataset['name']),
                                  'color': self.colors[dataset['name']],
                                  'data': 'json[{0}].data'.format(index)})

        variables = {'dataset': template_vars, 'width': self.width,
                     'height': self.height, 'render': self.renderer}
        if not self.y_zero:
            variables.update({'min': "min: 'auto',"})

        graph = self.env.get_template('graph.js')
        self.template_vars.update({'graph': graph.render(variables)})

    def create_chart(self, html_path='index.html', data_path='data.json',
                     js_path='rickshaw.min.js', css_path='rickshaw.min.css'):
        '''Save bearcart output to HTML and JSON.

        Parameters
        ----------
        html_path: string, default 'index.html'
            Path for html output
        data_path: string, default 'data.json'
            Path for data JSON output
        js_path: string, default None
            If passed, the Rickshaw javascript library will be saved to the
            path. The file must be named "rickshaw.min.js"
        css_path: string, default None
            If passed, the Rickshaw css library will be saved to the
            path. The file must be named "rickshaw.min.css"

        Returns
        -------
        HTML, JSON, JS, and CSS

        Example
        --------
        >>>vis.create_chart(html_path='myvis.html', data_path='visdata.json'),
                            js_path='rickshaw.min.js',
                            cs_path='rickshaw.min.css')
        '''

        self.template_vars.update({'data_path': str(data_path)})

        self._build_graph()
        html = self.env.get_template('bcart_template.html')
        self.HTML = html.render(self.template_vars)

        with open(html_path, 'w') as f:
            f.write(self.HTML)

        with open(data_path, 'w') as f:
            json.dump(self.json_data, f, sort_keys=True, indent=4,
                      separators=(',', ': '))

        if js_path:
            js = resource_string('bearcart', 'rickshaw.min.js')
            with open(js_path, 'w') as f:
                f.write(js)
        if css_path:
            css = resource_string('bearcart', 'rickshaw.min.css')
            with open(css_path, 'w') as f:
                    f.write(css)
<|endoftext|>


<commit_before>'''
Created on Mar 7, 2011

@author: al
'''

class BatchIndexer(object):
    def __init__(self, solr_conn, db_conn, index_query, batch_size=1000):
        self.batch_size = batch_size
        self.solr_conn = solr_conn
        self.db_conn = db_conn.cursor()
        self.index_query = index_query

    def exec_query(self):
        self.db_conn.execute(self.index_query)

    def next_batch(self):
        self.batch = self.db_conn.fetchmany(self.batch_size)
        return self.batch

    def index_batch(self, batch):
        self.solr_conn.add(batch)

    def index_all(self):
        count = 0
        while self.next_batch():
            count += 1
            print "Indexing batch", count
            self.index_batch(self.batch)

if __name__ == '__main__':
    import MySQLdb
    from MySQLdb import cursors
    import pysolr

    solr_conn = pysolr.Solr('http://localhost:8983/solr')
    solr_conn.delete(q="*:*")

    db_conn = MySQLdb.connect(host='localhost', user='root', db='test',
                              cursorclass=cursors.SSDictCursor
                              )

    indexer = BatchIndexer(solr_conn, db_conn, "select * from solr_test", batch_size=1000)
    print "Executing query..."
    indexer.exec_query()
    print "Running indexer..."
    indexer.index_all()<commit_msg>[data import] Revamped the API for batch data import, provided an extensible BatchIndexer class and more ways to customize its internals. Moving the DB-API cursor semantics into a subclass called DBAPIBatchIndexer, which is the default<commit_after>'''
Created on Mar 7, 2011

@author: al
'''

from util import NotGiven

import logging

log = logging.getLogger('bebop')

class BatchIndexer(object):
    COMMIT_PER_BATCH = 'batch'
    COMMIT_PER_INDEX = 'index'
    COMMIT_MANUALLY = 'manual'

    _query = None
    setup_func = None
    setup_func_args = None
    setup_func_kwargs = None
    _batch_size = 1000
    _solr_conn = None
    _commit_interval = COMMIT_PER_BATCH

    def __init__(self, solr_conn, solr_index):
        self.solr_conn = solr_conn
        self.solr_index = solr_index

    def setup(self, func, *args, **kw):
        self.setup_func = func
        self.setup_func_args = args
        self.setup_func_kwargs = kw
        return self

    def query(self, q):
        self._query = q
        return self

    def batch_size(self, size):
        self._batch_size = size
        return self

    # Template for _batch_getter function, can be set in batch_getter method
    def get_batch(self, limit=NotGiven, offset=NotGiven):
        raise NotImplementedError

    def batch_getter(self, f):
        """
        @param f: function object which takes two parameters, limit and offset
        """
        self._batch_getter = f
        return self

    def handle_row(self, row):
        return self.solr_index(row)

    def row_handler(self, f):
        """
        @param f: function object which takes one parameters "row", the row returned
        """
        self.handle_row = f
        return self

    def commit_interval(self, commit_interval):
        self._commit_interval = commit_interval
        return self

    def execute(self):
        raise NotImplementedError

    def index_batch(self, batch):
        self.solr_conn.add_all(batch)

    def index_all(self):
        docs = []
        self.offset = 0
        while True:
            batch = [self.handle_row(row) for row in self.get_batch(self._batch_size, self.offset)]
            if not batch:
                break
            self.offset += self._batch_size
            log.info("Indexing documents %d - %d" % (self.offset, self.offset+self._batch_size - 1))
            self.index_batch(batch)
            if self._commit_interval == BatchIndexer.COMMIT_PER_BATCH:
                self.solr_conn.commit()

        if self._commit_interval == BatchIndexer.COMMIT_PER_INDEX:
            self.solr_conn.commit()
        return self

class DBAPIBatchIndexer(BatchIndexer):
    def cursor(self, cur):
        self.db_cursor = cur
        return self

    def get_batch(self, limit, offset):
        return self.db_cursor.fetchmany(limit)

    def execute(self, query):
        self.db_cursor.execute(query)
        return self

if __name__ == '__main__':
    import MySQLdb
    from MySQLdb import cursors
    import pysolr
    from bebop.test.test_model import Foo

    solr_conn = pysolr.Solr('http://localhost:8983/solr')
    solr_conn.delete(q="*:*")

    db_conn = MySQLdb.connect(host='localhost', user='root', db='test',
                              cursorclass=cursors.SSDictCursor
                              )

    indexer = DBAPIBatchIndexer(solr_conn, Foo).cursor(db_conn.cursor()).batch_size(1000)
    print "Executing query..."
    indexer.execute("select * from solr_test")
    print "Running indexer..."
    indexer.index_all()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import types

import inflect

import requests

from .clients import HTTPHypermediaClient
from .constants import DEFAULT_VALID_STATUS_CODES
from .exceptions import BadURLException

if sys.version_info[0] == 3:
    # Py3
    from urllib.parse import urlparse
else:
    # Py2
    from urlparse import urlparse

class BaseResource(object):
    """
    A simple representation of a resource.

    Usage:

        from beckett.resources import BaseResource

        class Product(BaseResource):

            class Meta:
                name = 'Product'
                identifier = 'slug'
                attributes = (
                    'slug',
                    'name',
                    'price',
                    'discount'
                )

        # Data will usually come straight from the client method
        >>> data = {'name': 'Tasty product', 'slug': 'sluggy'}
        >>> product = Product(**data)
        <Product | sluggy>
        >>> product.name
        'Tasty product'

    """

    class Meta:
        # The name of this resource, used in __str__ methods.
        name = 'Resource'
        # The name of the resource used in the URL, i.e. 'resources'
        resource_name = None
        # The key with which you uniquely identify this resource.
        identifier = 'id'
        # Acceptable attributes that you want to display in this resource.
        attributes = (identifier,)
        # subresources are complex attributes within a resource
        subresources = {}
        # HTTP status codes that are considered "acceptable"
        # when calling this resource
        valid_status_codes = DEFAULT_VALID_STATUS_CODES
        # HTTP Methods that work on this resource
        methods = (
            'get',
        )
        # When receiving paginated results, use this key to render instances.
        pagination_key = 'results'

    def __init__(self, **kwargs):
        self._subresource_map = getattr(self.Meta, 'subresources', {})
        self.set_attributes(**kwargs)

    def __str__(self):
        """
        Returns a string representation based on the `self.Meta.name`
        and `self.Meta.identifier` attribute value.
        """
        return '<{} | {}>'.format(
            self.Meta.name, getattr(self, self.Meta.identifier))

    def set_subresources(self, **kwargs):
        """
        For each subresource assigned to this resource, generate the
        subresource instance and set it as an attribute on this instance.
        """
        for attribute_name, resource in self._subresource_map.items():
            sub_attr = kwargs.get(attribute_name)
            if isinstance(sub_attr, list):
                # A list of subresources is supported
                value = [resource(**x) for x in sub_attr]
            else:
                # So is a single resource
                value = resource(**sub_attr)
            setattr(self, attribute_name, value)

    def set_attributes(self, **kwargs):
        """
        Set the resource attributes from the kwargs.
        Only sets items in the `self.Meta.attributes` white list.

        Subclass this method to customise attributes.

        Args:
            kwargs: Keyword arguements passed into the init of this class
        """
        if self._subresource_map:
            self.set_subresources(**kwargs)
            for key in self._subresource_map.keys():
                # Don't let these attributes be overridden later
                kwargs.pop(key, None)
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)

    @classmethod
    def get_resource_url(cls, resource, base_url):
        """
        Construct the URL for talking to this resource.

        i.e.:

        http://myapi.com/api/resource

        Note that this is NOT the method for calling individual instances i.e.

        http://myapi.com/api/resource/1

        Args:
            resource: The resource class instance
            base_url: The Base URL of this API service.
        returns:
            resource_url: The URL for this resource
        """
        if resource.Meta.resource_name:
            url = '{}/{}'.format(base_url, resource.Meta.resource_name)
        else:
            p = inflect.engine()
            plural_name = p.plural(resource.Meta.name.lower())
            url = '{}/{}'.format(base_url, plural_name)
        return cls._parse_url_and_validate(url)

    @classmethod
    def get_url(cls, url, uid, **kwargs):
        """
        Construct the URL for talking to an individual resource.

        http://myapi.com/api/resource/1

        Args:
            url: The url for this resource
            uid: The unique identifier for an individual resource
            kwargs: Additional keyword argueents
        returns:
            final_url: The URL for this individual resource
        """
        if uid:
            url = '{}/{}'.format(url, uid)
        else:
            url = url
        return cls._parse_url_and_validate(url)

    @staticmethod
    def get_method_name(resource, method_type):
        """
        Generate a method name for this resource based on the method type.
        """
        return '{}_{}'.format(method_type.lower(), resource.Meta.name.lower())

    @classmethod
    def _parse_url_and_validate(cls, url):
        """
        Recieves a URL string and validates it using urlparse.

        Args:
            url: A URL string
        Returns:
            parsed_url: A validated URL
        Raises:
            BadURLException
        """
        parsed_url = urlparse(url)
        if parsed_url.scheme and parsed_url.netloc:
            final_url = parsed_url.geturl()
        else:
            raise BadURLException
        return final_url

class HypermediaResource(BaseResource, HTTPHypermediaClient):
    """
    A HypermediaResource is similar to a BaseResource except
    it understands relationships between attributes that
    are URLs to related registered resources.
    """
    class Meta(BaseResource.Meta):
        # HypermediaResource requires a base_url attribute
        base_url = NotImplemented
        related_resources = ()

    def __init__(self, *args, **kwargs):
        super(HypermediaResource, self).__init__(*args, **kwargs)
        self.session = requests.Session()

    def set_related_method(self, resource, full_resource_url):
        """
        Using reflection, generate the related method and return it.
        """
        method_name = self.get_method_name(resource, 'get')

        def get(self, **kwargs):
            return self._call_api_single_related_resource(
                resource, full_resource_url, method_name, **kwargs
            )

        def get_list(self, **kwargs):
            return self._call_api_many_related_resources(
                resource, full_resource_url, method_name, **kwargs
            )

        if isinstance(full_resource_url, list):
            setattr(
                self, method_name,
                types.MethodType(get_list, self)
            )
        else:
            setattr(
                self, method_name,
                types.MethodType(get, self)
            )

    def match_urls_to_resources(self, url_values):
        """
        For the list of valid URLs, try and match them up
        to resources in the related_resources attribute.

        Args:
            url_values: A dictionary of keys and URL strings that
                        could be related resources.
        Returns:
            valid_values: The values that are valid
        """
        valid_values = {}
        for resource in self.Meta.related_resources:
            for k, v in url_values.items():
                resource_url = resource.get_resource_url(
                    resource, resource.Meta.base_url)
                if isinstance(v, list):
                    if all([resource_url in i for i in v]):
                        self.set_related_method(resource, v)
                        valid_values[k] = v
                elif resource_url in v:
                    self.set_related_method(resource, v)
                    valid_values[k] = v
        return valid_values

    def set_attributes(self, **kwargs):
        """
        Similar to BaseResource.set_attributes except
        it will attempt to match URL strings with registered
        related resources, and build their get_* method and
        attach it to this resource.
        """
        if not self.Meta.related_resources:
            # Just do what the normal BaseResource does
            super(HypermediaResource, self).set_attributes(**kwargs)
            return

        # Extract all the values that are URLs
        url_values = {}
        for k, v in kwargs.items():
            try:
                if isinstance(v, list):
                    [self._parse_url_and_validate(i) for i in v]
                else:
                    self._parse_url_and_validate(v)
                url_values[k] = v
            except BadURLException:
                # This is a badly formed URL or not a URL at all, so skip
                pass
        # Assign the valid method values and then remove them from the kwargs
        assigned_values = self.match_urls_to_resources(url_values)
        for k in assigned_values.keys():
            kwargs.pop(k, None)
        # Assign the rest as attributes.
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)

class SubResource(object):
    """
    A "mini resource" within a larger resource. Similarly to BaseResource but
    minus some features.

    Consider the following JSON for a "Book" resource:

    ```json
    {
        "author": {
            "name": "Earnest"
        },
        "title": "A Farewell to Arms"
    }
    ```
    I can use this class to transform the "author" attribute into
    a typed resource.
    """

    class Meta:
        # The name of this Subresource, used in __str__ methods.
        name = 'SubResource'
        # The key with which you uniquely identify this resource.
        identifier = 'id'
        # Acceptable attributes that you want to display in this resource.
        attributes = (identifier,)

    def __init__(self, **kwargs):
        self.set_attributes(**kwargs)

    def __str__(self):
        """
        Returns a string representation based on the `self.Meta.name`
        and `self.Meta.identifier` attribute value.
        """
        return '<{} | {}>'.format(
            self.Meta.name, getattr(self, self.Meta.identifier, ''))

    def set_attributes(self, **kwargs):
        """
        Set the resource attributes from the kwargs.
        Only sets items in the `self.Meta.attributes` white list.

        Args:
            kwargs: Keyword arguements passed into the init of this class
        """
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)
<commit_msg>Fixes "type object argument after ** must be a mapping, not NoneType" (#92)<commit_after># -*- coding: utf-8 -*-

import sys
import types

import inflect

import requests

from .clients import HTTPHypermediaClient
from .constants import DEFAULT_VALID_STATUS_CODES
from .exceptions import BadURLException

if sys.version_info[0] == 3:
    # Py3
    from urllib.parse import urlparse
else:
    # Py2
    from urlparse import urlparse

class BaseResource(object):
    """
    A simple representation of a resource.

    Usage:

        from beckett.resources import BaseResource

        class Product(BaseResource):

            class Meta:
                name = 'Product'
                identifier = 'slug'
                attributes = (
                    'slug',
                    'name',
                    'price',
                    'discount'
                )

        # Data will usually come straight from the client method
        >>> data = {'name': 'Tasty product', 'slug': 'sluggy'}
        >>> product = Product(**data)
        <Product | sluggy>
        >>> product.name
        'Tasty product'

    """

    class Meta:
        # The name of this resource, used in __str__ methods.
        name = 'Resource'
        # The name of the resource used in the URL, i.e. 'resources'
        resource_name = None
        # The key with which you uniquely identify this resource.
        identifier = 'id'
        # Acceptable attributes that you want to display in this resource.
        attributes = (identifier,)
        # subresources are complex attributes within a resource
        subresources = {}
        # HTTP status codes that are considered "acceptable"
        # when calling this resource
        valid_status_codes = DEFAULT_VALID_STATUS_CODES
        # HTTP Methods that work on this resource
        methods = (
            'get',
        )
        # When receiving paginated results, use this key to render instances.
        pagination_key = 'results'

    def __init__(self, **kwargs):
        self._subresource_map = getattr(self.Meta, 'subresources', {})
        self.set_attributes(**kwargs)

    def __str__(self):
        """
        Returns a string representation based on the `self.Meta.name`
        and `self.Meta.identifier` attribute value.
        """
        return '<{} | {}>'.format(
            self.Meta.name, getattr(self, self.Meta.identifier))

    def set_subresources(self, **kwargs):
        """
        For each subresource assigned to this resource, generate the
        subresource instance and set it as an attribute on this instance.
        """
        for attribute_name, resource in self._subresource_map.items():
            sub_attr = kwargs.get(attribute_name)
            if sub_attr is None:
                # Attribute was not found or is null
                value = None
            elif isinstance(sub_attr, list):
                # A list of subresources is supported
                value = [resource(**x) for x in sub_attr]
            else:
                # So is a single resource
                value = resource(**sub_attr)
            setattr(self, attribute_name, value)

    def set_attributes(self, **kwargs):
        """
        Set the resource attributes from the kwargs.
        Only sets items in the `self.Meta.attributes` white list.

        Subclass this method to customise attributes.

        Args:
            kwargs: Keyword arguements passed into the init of this class
        """
        if self._subresource_map:
            self.set_subresources(**kwargs)
            for key in self._subresource_map.keys():
                # Don't let these attributes be overridden later
                kwargs.pop(key, None)
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)

    @classmethod
    def get_resource_url(cls, resource, base_url):
        """
        Construct the URL for talking to this resource.

        i.e.:

        http://myapi.com/api/resource

        Note that this is NOT the method for calling individual instances i.e.

        http://myapi.com/api/resource/1

        Args:
            resource: The resource class instance
            base_url: The Base URL of this API service.
        returns:
            resource_url: The URL for this resource
        """
        if resource.Meta.resource_name:
            url = '{}/{}'.format(base_url, resource.Meta.resource_name)
        else:
            p = inflect.engine()
            plural_name = p.plural(resource.Meta.name.lower())
            url = '{}/{}'.format(base_url, plural_name)
        return cls._parse_url_and_validate(url)

    @classmethod
    def get_url(cls, url, uid, **kwargs):
        """
        Construct the URL for talking to an individual resource.

        http://myapi.com/api/resource/1

        Args:
            url: The url for this resource
            uid: The unique identifier for an individual resource
            kwargs: Additional keyword argueents
        returns:
            final_url: The URL for this individual resource
        """
        if uid:
            url = '{}/{}'.format(url, uid)
        else:
            url = url
        return cls._parse_url_and_validate(url)

    @staticmethod
    def get_method_name(resource, method_type):
        """
        Generate a method name for this resource based on the method type.
        """
        return '{}_{}'.format(method_type.lower(), resource.Meta.name.lower())

    @classmethod
    def _parse_url_and_validate(cls, url):
        """
        Recieves a URL string and validates it using urlparse.

        Args:
            url: A URL string
        Returns:
            parsed_url: A validated URL
        Raises:
            BadURLException
        """
        parsed_url = urlparse(url)
        if parsed_url.scheme and parsed_url.netloc:
            final_url = parsed_url.geturl()
        else:
            raise BadURLException
        return final_url

class HypermediaResource(BaseResource, HTTPHypermediaClient):
    """
    A HypermediaResource is similar to a BaseResource except
    it understands relationships between attributes that
    are URLs to related registered resources.
    """
    class Meta(BaseResource.Meta):
        # HypermediaResource requires a base_url attribute
        base_url = NotImplemented
        related_resources = ()

    def __init__(self, *args, **kwargs):
        super(HypermediaResource, self).__init__(*args, **kwargs)
        self.session = requests.Session()

    def set_related_method(self, resource, full_resource_url):
        """
        Using reflection, generate the related method and return it.
        """
        method_name = self.get_method_name(resource, 'get')

        def get(self, **kwargs):
            return self._call_api_single_related_resource(
                resource, full_resource_url, method_name, **kwargs
            )

        def get_list(self, **kwargs):
            return self._call_api_many_related_resources(
                resource, full_resource_url, method_name, **kwargs
            )

        if isinstance(full_resource_url, list):
            setattr(
                self, method_name,
                types.MethodType(get_list, self)
            )
        else:
            setattr(
                self, method_name,
                types.MethodType(get, self)
            )

    def match_urls_to_resources(self, url_values):
        """
        For the list of valid URLs, try and match them up
        to resources in the related_resources attribute.

        Args:
            url_values: A dictionary of keys and URL strings that
                        could be related resources.
        Returns:
            valid_values: The values that are valid
        """
        valid_values = {}
        for resource in self.Meta.related_resources:
            for k, v in url_values.items():
                resource_url = resource.get_resource_url(
                    resource, resource.Meta.base_url)
                if isinstance(v, list):
                    if all([resource_url in i for i in v]):
                        self.set_related_method(resource, v)
                        valid_values[k] = v
                elif resource_url in v:
                    self.set_related_method(resource, v)
                    valid_values[k] = v
        return valid_values

    def set_attributes(self, **kwargs):
        """
        Similar to BaseResource.set_attributes except
        it will attempt to match URL strings with registered
        related resources, and build their get_* method and
        attach it to this resource.
        """
        if not self.Meta.related_resources:
            # Just do what the normal BaseResource does
            super(HypermediaResource, self).set_attributes(**kwargs)
            return

        # Extract all the values that are URLs
        url_values = {}
        for k, v in kwargs.items():
            try:
                if isinstance(v, list):
                    [self._parse_url_and_validate(i) for i in v]
                else:
                    self._parse_url_and_validate(v)
                url_values[k] = v
            except BadURLException:
                # This is a badly formed URL or not a URL at all, so skip
                pass
        # Assign the valid method values and then remove them from the kwargs
        assigned_values = self.match_urls_to_resources(url_values)
        for k in assigned_values.keys():
            kwargs.pop(k, None)
        # Assign the rest as attributes.
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)

class SubResource(object):
    """
    A "mini resource" within a larger resource. Similarly to BaseResource but
    minus some features.

    Consider the following JSON for a "Book" resource:

    ```json
    {
        "author": {
            "name": "Earnest"
        },
        "title": "A Farewell to Arms"
    }
    ```
    I can use this class to transform the "author" attribute into
    a typed resource.
    """

    class Meta:
        # The name of this Subresource, used in __str__ methods.
        name = 'SubResource'
        # The key with which you uniquely identify this resource.
        identifier = 'id'
        # Acceptable attributes that you want to display in this resource.
        attributes = (identifier,)

    def __init__(self, **kwargs):
        self.set_attributes(**kwargs)

    def __str__(self):
        """
        Returns a string representation based on the `self.Meta.name`
        and `self.Meta.identifier` attribute value.
        """
        return '<{} | {}>'.format(
            self.Meta.name, getattr(self, self.Meta.identifier, ''))

    def set_attributes(self, **kwargs):
        """
        Set the resource attributes from the kwargs.
        Only sets items in the `self.Meta.attributes` white list.

        Args:
            kwargs: Keyword arguements passed into the init of this class
        """
        for field, value in kwargs.items():
            if field in self.Meta.attributes:
                setattr(self, field, value)
<|endoftext|>


<commit_before># vim: set ts=4 sw=4 et: coding=UTF-8

# Copyright (c) 2015, SUSE LINUX Products GmbH, Nuernberg, Germany
# All rights reserved.
# See COPYING for details.

import os
import sys
import argparse

from .rpmexception import RpmWrongArgs, RpmException
from .rpmcleaner import RpmSpecCleaner

__version__ = '0.8.6'

def process_args(argv):
    """
    Process the parsed arguments and return the result
    :param argv: passed arguments
    """

    parser = argparse.ArgumentParser(prog='spec-cleaner',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                                     description='Cleans the given spec file according to style guide and returns the result.')

    # Make the -d, -i, and -o exclusive as we can do only one of those
    output_group = parser.add_mutually_exclusive_group()

    parser.add_argument('spec', metavar='SPEC', type=str,
                        help='spec file to beautify')
    output_group.add_argument('-d', '--diff', action='store_true', default=False,
                              help='run the diff program to show differences between new and orginal specfile.')
    parser.add_argument('--diff-prog', default='vimdiff',
                        help='specify the diff binary to call with diff option.')
    parser.add_argument('-f', '--force', action='store_true', default=False,
                        help='overwrite the output file if already exist.')
    output_group.add_argument('-i', '--inline', action='store_true', default=False,
                              help='inline the changes directly to the parsed file.')
    parser.add_argument('-m', '--minimal', action='store_true', default=False,
                        help='run in minimal mode that does not do anything intrusive (ie. just sets the Copyright)')
    parser.add_argument('--no-copyright', action='store_true', default=False,
                        help='do not include official SUSE copyright hear and just keep what is present')
    output_group.add_argument('-o', '--output', default='',
                              help='specify the output file for the cleaned spec content.')
    parser.add_argument('-p', '--pkgconfig', action='store_true', default=False,
                        help='convert dependencies to their pkgconfig counterparts, requires bit more of cleanup in spec afterwards.')
    parser.add_argument('-v', '--version', action='version', version=__version__,
                        help='show package version and exit')

    # print help if there is no argument
    if len(argv) < 1:
        parser.print_help()
        sys.exit(0)

    options = parser.parse_args(args=argv)

    # the spec must exist for us to do anything
    if not os.path.exists(options.spec):
        raise RpmWrongArgs('{0} does not exist.'.format(options.spec))

    # the path for output must exist and the file must not be there unless
    # force is specified
    if options.output:
        options.output = os.path.expanduser(options.output)
        if not options.force and os.path.exists(options.output):
            raise RpmWrongArgs('{0} already exists.'.format(options.output))

    # convert options to dict
    options_dict = {
        'specfile': options.spec,
        'output': options.output,
        'pkgconfig': options.pkgconfig,
        'inline': options.inline,
        'diff': options.diff,
        'diff_prog': options.diff_prog,
        'minimal': options.minimal,
        'no_copyright': options.no_copyright,
    }

    return options_dict

def main():
    """
    Main function that calls argument parsing ensures their sanity
    and then creates RpmSpecCleaner object that works with passed spec file.
    """

    try:
        options = process_args(sys.argv[1:])
    except RpmWrongArgs as exception:
        sys.stderr.write('ERROR: {0}\n'.format(exception))
        return 1

    try:
        cleaner = RpmSpecCleaner(options)
        cleaner.run()
    except RpmException as exception:
        sys.stderr.write('ERROR: {0}\n'.format(exception))
        return 1

    return 0
<commit_msg>Post release version bump<commit_after># vim: set ts=4 sw=4 et: coding=UTF-8

# Copyright (c) 2015, SUSE LINUX Products GmbH, Nuernberg, Germany
# All rights reserved.
# See COPYING for details.

import os
import sys
import argparse

from .rpmexception import RpmWrongArgs, RpmException
from .rpmcleaner import RpmSpecCleaner

__version__ = '0.8.7'

def process_args(argv):
    """
    Process the parsed arguments and return the result
    :param argv: passed arguments
    """

    parser = argparse.ArgumentParser(prog='spec-cleaner',
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
                                     description='Cleans the given spec file according to style guide and returns the result.')

    # Make the -d, -i, and -o exclusive as we can do only one of those
    output_group = parser.add_mutually_exclusive_group()

    parser.add_argument('spec', metavar='SPEC', type=str,
                        help='spec file to beautify')
    output_group.add_argument('-d', '--diff', action='store_true', default=False,
                              help='run the diff program to show differences between new and orginal specfile.')
    parser.add_argument('--diff-prog', default='vimdiff',
                        help='specify the diff binary to call with diff option.')
    parser.add_argument('-f', '--force', action='store_true', default=False,
                        help='overwrite the output file if already exist.')
    output_group.add_argument('-i', '--inline', action='store_true', default=False,
                              help='inline the changes directly to the parsed file.')
    parser.add_argument('-m', '--minimal', action='store_true', default=False,
                        help='run in minimal mode that does not do anything intrusive (ie. just sets the Copyright)')
    parser.add_argument('--no-copyright', action='store_true', default=False,
                        help='do not include official SUSE copyright hear and just keep what is present')
    output_group.add_argument('-o', '--output', default='',
                              help='specify the output file for the cleaned spec content.')
    parser.add_argument('-p', '--pkgconfig', action='store_true', default=False,
                        help='convert dependencies to their pkgconfig counterparts, requires bit more of cleanup in spec afterwards.')
    parser.add_argument('-v', '--version', action='version', version=__version__,
                        help='show package version and exit')

    # print help if there is no argument
    if len(argv) < 1:
        parser.print_help()
        sys.exit(0)

    options = parser.parse_args(args=argv)

    # the spec must exist for us to do anything
    if not os.path.exists(options.spec):
        raise RpmWrongArgs('{0} does not exist.'.format(options.spec))

    # the path for output must exist and the file must not be there unless
    # force is specified
    if options.output:
        options.output = os.path.expanduser(options.output)
        if not options.force and os.path.exists(options.output):
            raise RpmWrongArgs('{0} already exists.'.format(options.output))

    # convert options to dict
    options_dict = {
        'specfile': options.spec,
        'output': options.output,
        'pkgconfig': options.pkgconfig,
        'inline': options.inline,
        'diff': options.diff,
        'diff_prog': options.diff_prog,
        'minimal': options.minimal,
        'no_copyright': options.no_copyright,
    }

    return options_dict

def main():
    """
    Main function that calls argument parsing ensures their sanity
    and then creates RpmSpecCleaner object that works with passed spec file.
    """

    try:
        options = process_args(sys.argv[1:])
    except RpmWrongArgs as exception:
        sys.stderr.write('ERROR: {0}\n'.format(exception))
        return 1

    try:
        cleaner = RpmSpecCleaner(options)
        cleaner.run()
    except RpmException as exception:
        sys.stderr.write('ERROR: {0}\n'.format(exception))
        return 1

    return 0
<|endoftext|>


<commit_before>__author__ = 'jmartinez'
__version__ = '1.0.0'

import os
import click
import requests
import requirements
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile

from datetime import date
from glob import glob as ls
from jinja2 import Environment, FileSystemLoader
from uuid import uuid4

from pkg_resources import RequirementParseError

def find_dependencies(directory):
    cwd = os.getcwd()
    os.chdir(directory)
    dependencies = set()
    single_version_externally_managed = True
    _requirements = ['{}/requirements.txt'.format(directory)]
    try:
        subprocess.check_output([sys.executable, 'setup.py', 'egg_info'])
        _requirements.extend(ls('{}/*.egg-info/requires.txt'.format(directory)))
    except subprocess.CalledProcessError:
        click.echo('egg_info not supported by {}'.format(directory))
        single_version_externally_managed = False

    try:
        for _requirement in _requirements:
            if not os.path.isfile(_requirement):
                continue

            click.echo('Reading requirements file {}'.format(_requirement))
            with open(_requirement, 'r') as f:
                try:
                    for req in requirements.parse(f):
                        if req.specs:
                            dependencies.add('{} {}'.format(req.name,
                                ''.join([ '{} {}'.format(o, v)
                                          for o,v in req.specs])))
                        else:
                            dependencies.add(req.name)
                except RequirementParseError:
                    pass

    finally:
        os.chdir(cwd)
        shutil.rmtree(directory)
        return dependencies, single_version_externally_managed

def extract_files(_file):
    def extract_tar(_file, tmpdir):
        tar = tarfile.open(_file)
        os.makedirs(tmpdir)
        os.chdir(tmpdir)
        tar.extractall()

    def extract_zip(_file, tmpdir):
        with zipfile.ZipFile(_file) as _zip:
            os.makedirs(tmpdir)
            os.chdir(tmpdir)
            _zip.extractall()

    name, extension = os.path.splitext(_file)
    if extension == '.gz':
        _, _extension = os.path.splitext(name)
        extension = '{}{}'.format(_extension, extension)
        name = name.replace(_extension, '')

    tmpdir = "{}/{}".format(tempfile.gettempdir(), uuid4())
    cwd = os.getcwd()

    if extension in ['.tar.gz', '.tgz']:
        extract_tar(_file, tmpdir)
    elif extension == '.zip':
        extract_zip(_file, tmpdir)

    os.chdir(cwd)
    return os.path.join(tmpdir, name)

def create(meta):
    os.mkdir(meta['package_name'])
    click.echo('Downloading {}'.format(meta['source']))
    cwd = os.getcwd()
    os.chdir(meta['package_name'])
    _file = download_file(meta['source'])
    click.echo('Extracting {}'.format(_file))
    extracted = extract_files(_file)
    click.echo('Searching for extra dependencies on {}'.format(extracted))
    extra_deps, single_version_externally_managed = find_dependencies(extracted)
    if not single_version_externally_managed:
        meta.update({'single_version_externally_managed': False})

    meta['requires'].extend(extra_deps)
    os.chdir(cwd)

    env = Environment(loader=FileSystemLoader('{}/templates'.format(
            os.path.dirname(os.path.abspath(__file__)))))
    spec = env.get_template('python-spec.tmpl')
    rendered = spec.render(meta)
    with open('{name}/{name}.spec'.format(
            name=meta['package_name']), 'w') as spec:
        spec.write(rendered)

def download_file(url):
    local_filename = url.split('/')[-1]
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                f.flush()
    return local_filename

def build_metadata(pypi):
    click.echo('Building metadata for the package genaration...')
    meta = {'name': pypi['info']['name']}
    meta.update({'source':
        next((url['url'] for url in pypi['urls']
            if 'source' in url['python_version']
            and ('tar.gz' in url['url'] or
                 'zip' in url['url'] or
                 'tar.bz2' in url['url'])), '')
    })
    if meta['source'] == '':
        click.echo("Cannot determine download URL... "
              "Check Pypi: https://pypi.python.org/pypi/{}/"
              .format(meta['name']))
        sys.exit(3)

    meta.update(
        {'version': (meta['source'].split('-')[-1].
                     replace('.tar.gz', '').
                     replace('.zip', '').
                     replace('.tar.bz2', ''))}
    )
    meta.update({'url': pypi['info']['home_page']})
    meta.update({'summary': pypi['info']['summary']})
    meta.update({'license': pypi['info']['license']})
    meta.update({'requires': pypi['info'].get('requires', [])})
    meta.update({'author': pypi['info']['author']})
    meta.update({'description': pypi['info'].get(
        'description', meta['summary'])}
    )
    meta.update({'me': os.getenv('PACKAGER_NAME')})
    meta.update({'email': os.getenv('PACKAGER_EMAIL')})
    meta.update({'date': date.today().strftime("%a %b %d %Y")})
    meta.update({'single_version_externally_managed': True})
    return meta

def read_pypi(name, version=None):
    click.echo('Trying to fetch pypi information about {}...'.format(name))
    if not version:
        url = "https://pypi.python.org/pypi/{}/json".format(name)
    else:
        url = "https://pypi.python.org/pypi/{}/{}/json".format(name, version)
    result = requests.get(url)
    if result.status_code == 200:
        pypi = result.json()
    else:
        click.echo("Package or release not found on {}".format(url))
        sys.exit(2)

    return build_metadata(pypi)

def run(name, python_prefix, recursive):
    meta = read_pypi(name)
    package_name = '{}-{}'.format(python_prefix, meta['name'])
    if os.path.isdir(package_name):
        click.echo('{} already exists, skipping...'.format(package_name))
        return
    meta.update({'python_prefix': python_prefix})
    meta.update({'package_name': package_name})
    create(meta)
    if recursive:
        for dep in meta['requires']:
            ## Todo:
            # Refactor here to actually enfore the version
            dep = dep.replace('>', ' ').replace('<', ' ').\
                    replace('=', '/').split()[0]
            run(dep, python_prefix, recursive)

@click.command()
@click.argument('name')
@click.option('--python_prefix', '-p', default='python35')
@click.option('--recursive', '-r', is_flag=True)
def cli(name, python_prefix, recursive):
    if os.path.isdir(name):
        click.echo("Package {} alreayd exists".format(name))
        sys.exit(1)

    run(name, python_prefix, recursive)
    sys.exit(0)

if __name__ == '__main__':
    cli()
<commit_msg>adds support to bz2<commit_after>__author__ = 'jmartinez'
__version__ = '1.0.0'

import os
import click
import requests
import requirements
import shutil
import subprocess
import sys
import tarfile
import tempfile
import zipfile

from datetime import date
from glob import glob as ls
from jinja2 import Environment, FileSystemLoader
from uuid import uuid4

from pkg_resources import RequirementParseError

def find_dependencies(directory):
    cwd = os.getcwd()
    os.chdir(directory)
    dependencies = set()
    single_version_externally_managed = True
    _requirements = ['{}/requirements.txt'.format(directory)]
    try:
        subprocess.check_output([sys.executable, 'setup.py', 'egg_info'])
        _requirements.extend(ls('{}/*.egg-info/requires.txt'.format(directory)))
    except subprocess.CalledProcessError:
        click.echo('egg_info not supported by {}'.format(directory))
        single_version_externally_managed = False

    try:
        for _requirement in _requirements:
            if not os.path.isfile(_requirement):
                continue

            click.echo('Reading requirements file {}'.format(_requirement))
            with open(_requirement, 'r') as f:
                try:
                    for req in requirements.parse(f):
                        if req.specs:
                            dependencies.add('{} {}'.format(req.name,
                                ''.join([ '{} {}'.format(o, v)
                                          for o,v in req.specs])))
                        else:
                            dependencies.add(req.name)
                except RequirementParseError:
                    pass

    finally:
        os.chdir(cwd)
        shutil.rmtree(directory)
        return dependencies, single_version_externally_managed

def extract_files(_file):
    def extract_tar(_file, tmpdir):
        tar = tarfile.open(_file)
        os.makedirs(tmpdir)
        os.chdir(tmpdir)
        tar.extractall()

    def extract_zip(_file, tmpdir):
        with zipfile.ZipFile(_file) as _zip:
            os.makedirs(tmpdir)
            os.chdir(tmpdir)
            _zip.extractall()

    name, extension = os.path.splitext(_file)
    if extension == '.gz':
        _, _extension = os.path.splitext(name)
        extension = '{}{}'.format(_extension, extension)
        name = name.replace(_extension, '')

    tmpdir = "{}/{}".format(tempfile.gettempdir(), uuid4())
    cwd = os.getcwd()

    if extension in ['.tar.gz', '.tgz', '.tar.bz2']:
        extract_tar(_file, tmpdir)
    elif extension == '.zip':
        extract_zip(_file, tmpdir)

    os.chdir(cwd)
    return os.path.join(tmpdir, name)

def create(meta):
    os.mkdir(meta['package_name'])
    click.echo('Downloading {}'.format(meta['source']))
    cwd = os.getcwd()
    os.chdir(meta['package_name'])
    _file = download_file(meta['source'])
    click.echo('Extracting {}'.format(_file))
    extracted = extract_files(_file)
    click.echo('Searching for extra dependencies on {}'.format(extracted))
    extra_deps, single_version_externally_managed = find_dependencies(extracted)
    if not single_version_externally_managed:
        meta.update({'single_version_externally_managed': False})

    meta['requires'].extend(extra_deps)
    os.chdir(cwd)

    env = Environment(loader=FileSystemLoader('{}/templates'.format(
            os.path.dirname(os.path.abspath(__file__)))))
    spec = env.get_template('python-spec.tmpl')
    rendered = spec.render(meta)
    with open('{name}/{name}.spec'.format(
            name=meta['package_name']), 'w') as spec:
        spec.write(rendered)

def download_file(url):
    local_filename = url.split('/')[-1]
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                f.flush()
    return local_filename

def build_metadata(pypi):
    click.echo('Building metadata for the package genaration...')
    meta = {'name': pypi['info']['name']}
    meta.update({'source':
        next((url['url'] for url in pypi['urls']
            if 'source' in url['python_version']
            and ('tar.gz' in url['url'] or
                 'zip' in url['url'] or
                 'tar.bz2' in url['url'])), '')
    })
    if meta['source'] == '':
        click.echo("Cannot determine download URL... "
              "Check Pypi: https://pypi.python.org/pypi/{}/"
              .format(meta['name']))
        sys.exit(3)

    meta.update(
        {'version': (meta['source'].split('-')[-1].
                     replace('.tar.gz', '').
                     replace('.zip', '').
                     replace('.tar.bz2', ''))}
    )
    meta.update({'url': pypi['info']['home_page']})
    meta.update({'summary': pypi['info']['summary']})
    meta.update({'license': pypi['info']['license']})
    meta.update({'requires': pypi['info'].get('requires', [])})
    meta.update({'author': pypi['info']['author']})
    meta.update({'description': pypi['info'].get(
        'description', meta['summary'])}
    )
    meta.update({'me': os.getenv('PACKAGER_NAME')})
    meta.update({'email': os.getenv('PACKAGER_EMAIL')})
    meta.update({'date': date.today().strftime("%a %b %d %Y")})
    meta.update({'single_version_externally_managed': True})
    return meta

def read_pypi(name, version=None):
    click.echo('Trying to fetch pypi information about {}...'.format(name))
    if not version:
        url = "https://pypi.python.org/pypi/{}/json".format(name)
    else:
        url = "https://pypi.python.org/pypi/{}/{}/json".format(name, version)
    result = requests.get(url)
    if result.status_code == 200:
        pypi = result.json()
    else:
        click.echo("Package or release not found on {}".format(url))
        sys.exit(2)

    return build_metadata(pypi)

def run(name, python_prefix, recursive):
    meta = read_pypi(name)
    package_name = '{}-{}'.format(python_prefix, meta['name'])
    if os.path.isdir(package_name):
        click.echo('{} already exists, skipping...'.format(package_name))
        return
    meta.update({'python_prefix': python_prefix})
    meta.update({'package_name': package_name})
    create(meta)
    if recursive:
        for dep in meta['requires']:
            ## Todo:
            # Refactor here to actually enfore the version
            dep = dep.replace('>', ' ').replace('<', ' ').\
                    replace('=', '/').split()[0]
            run(dep, python_prefix, recursive)

@click.command()
@click.argument('name')
@click.option('--python_prefix', '-p', default='python35')
@click.option('--recursive', '-r', is_flag=True)
def cli(name, python_prefix, recursive):
    if os.path.isdir(name):
        click.echo("Package {} alreayd exists".format(name))
        sys.exit(1)

    run(name, python_prefix, recursive)
    sys.exit(0)

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>
class BaseWrapper(object):

    def set_up():
        pass

    def get_key(key, default=None):
        raise NotImplementedError()

    def get_key_list(keys, default=None):
        return [ get_key(key, default) for key in keys ]

    def set_key_list(key_value_tuples):
        return [ set_key(key, value) for key, value in key_value_tuples ]

    def set_key(key, value):
        raise NotImplementedError()
<commit_msg>Add selfs<commit_after>
class BaseWrapper(object):

    def set_up(self):
        pass

    def get_key(self, key, default=None):
        raise NotImplementedError()

    def get_key_list(self, keys, default=None):
        return [ self.get_key(key, default) for key in keys ]

    def set_key_list(self, key_value_tuples):
        return [ self.set_key(key, value) for key, value in key_value_tuples ]

    def set_key(self, key, value):
        raise NotImplementedError()
<|endoftext|>