<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Classes to read, validate and work with the input data files.
"""
import logging
from collections import OrderedDict
from itertools import chain
from pathlib import Path
import urllib.parse
from astropy.table import Table
from .info import gammacat_info
from .utils import load_yaml, NA, validate_schema
from .sed import SEDList
from .lightcurve import LightcurveList

__all__ = [
    'BasicSourceInfo',
    'BasicSourceList',
    'DatasetSourceInfo',
    'InputData',
    'InputDataset',
    'InputDatasetCollection',
]

log = logging.getLogger(__name__)

class BasicSourceInfo:
    """All basic info for a source.
    """
    schema = load_yaml(gammacat_info.base_dir / 'input/schemas/basic_source_info.schema.yaml')

    def __init__(self, data, path):
        self.data = data
        self.path = path

    @classmethod
    def read(cls, path):
        path = Path(path)
        data = load_yaml(path)
        return cls(data=data, path=path)

    def __repr__(self):
        return 'BasicSourceInfo(source_id={})'.format(repr(self.data['source_id']))

    def to_dict(self, filled=False):
        """Data as a flat OrderedDict that can be stored in a table row."""
        data = OrderedDict()

        if filled:
            for name, spec in self.schema['properties'].items():
                try:
                    datatype = spec['type']
                except KeyError:
                    datatype = 'string'

                # TODO: write code to handle position
                if name == 'pos':
                    continue

                try:
                    data[name] = NA.fill_value[datatype]
                except TypeError:
                    data[name] = NA.fill_value[datatype[0]]

        data.update(self.data)

        if data['reference_ids'] is None or data['reference_ids'][0] is None:
            data['reference_ids'] = ''
        else:
            data['reference_ids'] = ','.join(data['reference_ids'])

        # TODO: write code to handle position
        data.pop('pos', None)

        return data

    def validate(self):
        validate_schema(path=self.path, data=self.data, schema=self.schema)

class DatasetSourceInfo:
    """All info from one dataset for one source.
    """
    schema = load_yaml(gammacat_info.base_dir / 'input/schemas/dataset_source_info.schema.yaml')

    def __init__(self, data, path):
        self.data = data
        self.path = path

    @classmethod
    def read(cls, path):
        path = Path(path)
        data = load_yaml(path)
        return cls(data=data, path=path)

    def __repr__(self):
        return 'DatasetSourceInfo(source_id={}, reference_id={})'.format(
            repr(self.data['source_id']),
            repr(self.data['reference_id']),
        )

    def validate(self):
        validate_schema(path=self.path, data=self.data, schema=self.schema)

class InputDataset:
    """All info for one dataset.
    """

    def __init__(self, reference_id, path, sources):
        self.reference_id = reference_id
        self.path = path
        self.sources = sources
        # TODO: remove this cache
        _source_ids = [source.data['source_id'] for source in sources]
        self._sources_by_id = dict(zip(_source_ids, sources))

    @classmethod
    def read(cls, path):
        path = Path(path)
        reference_id = urllib.parse.unquote(path.parts[-1])

        # TODO: maybe just use an OrderedDict
        sources = []
        for source_path in sorted(path.glob('*.yaml')):
            source_info = DatasetSourceInfo.read(source_path)
            sources.append(source_info)

        path = '/'.join(path.parts[-2:])
        return cls(reference_id=reference_id, path=path, sources=sources)

    def to_json(self):
        sources = []
        for source in self.sources:
            data = OrderedDict()
            data['source_id'] = source.data['source_id']
            data['reference_id'] = source.data['reference_id']
            sources.append(data)

        # TODO: This would give the full information from the input files.
        # sources = [dict(_.data for _ in self.sources]

        # This is what it takes to build the URL on Github
        url = self.path.replace('%26', '%2526')

        data = OrderedDict()
        data['reference_id'] = self.reference_id
        data['path'] = self.path
        data['url'] = url
        data['sources'] = sources

        return data

    def __repr__(self):
        return 'InputDataset(reference_id={})'.format(repr(self.reference_id))

    def validate(self):
        [_.validate() for _ in self.sources]

    def get_source_by_id(self, source_id):
        # returning empty DatasetSourceInfo makes sense, because it leads to a key
        # error later and will be treated as missing info
        missing = DatasetSourceInfo(data={}, path='')
        return self._sources_by_id.get(source_id, missing)

class BasicSourceList:
    """
    List of `BasicSourceInfo` objects.
    """
    column_spec = load_yaml(gammacat_info.base_dir / 'input/schemas/basic_source_list.schema.yaml')

    def __init__(self, data):
        self.data = data

    @property
    def source_ids(self):
        return [source.data['source_id'] for source in self.data]

    def get_source_by_id(self, source_id):
        idx = self.source_ids.index(source_id)
        return self.data[idx]

    @classmethod
    def read(cls):
        path = gammacat_info.base_dir / 'input/sources'
        paths = path.glob('*.yaml')

        data = []
        for path in paths:
            if path.name in {'example.yaml'}:
                continue
            info = BasicSourceInfo.read(path)
            data.append(info)

        return cls(data=data)

    def to_json(self):
        """Return data in format that can be written to JSON.

        A dict with `data` key.
        """
        return OrderedDict(data=self.data_per_row(filled=True))

    def data_per_row(self, filled=False):
        """Data as list of dicts (per-row)"""
        return [
            source.to_dict(filled=filled)
            for source in self.data
            ]

    def validate(self):
        # TODO: validate `column_spec` schema?

        log.info('Validating YAML files in `input/sources`')
        [_.validate() for _ in self.data]

class InputDatasetCollection:
    """
    Represents all data in ``input/data``.
    """

    def __init__(self, data):
        self.data = data

    @classmethod
    def read(cls, internal=False):
        path = gammacat_info.base_dir / 'input/data'
        paths = list(path.glob('*/*'))

        if internal:
            path_internal = gammacat_info.internal_dir
            paths = chain(paths, [path_internal])

        data = []
        for path in paths:
            info = InputDataset.read(path)
            data.append(info)
        return cls(data=data)

    @property
    def reference_ids(self):
        return [dataset.reference_id for dataset in self.data]

    def to_table(self):
        """Convert info of `sources` list into a Table.
        """
        meta = OrderedDict()
        meta['name'] = 'todo'
        meta['version'] = 'todo'
        meta['url'] = 'todo'

        # rows = self.data_per_row(filled=True)
        rows = [OrderedDict(spam=99)]
        return Table(rows=rows, meta=meta, masked=True)

    def to_json(self):
        """Return data in format that can be written to JSON.

        A dict with `data` key.
        """
        data = []
        for dataset in self.data:
            data.append(dataset.to_json())
        return OrderedDict(data=data)

    def validate(self):
        log.info('Validating YAML files in `input/data`')
        for dataset in self.data:
            dataset.validate()

    def get_dataset_by_reference_id(self, reference_id):
        """Get dataset for a given reference_id
        """
        # TODO: this is not a good way to handle things.
        # Remove once gamma-cat script is set up in a better way.
        if reference_id is None:
            return InputDataset(reference_id=None, path=None, sources=[])

        idx = self.reference_ids.index(reference_id)
        return self.data[idx]

class Schemas:
    def __init__(self, data):
        self.data = data

    @classmethod
    def read(cls):
        path = gammacat_info.base_dir / 'input/schemas'
        paths = path.glob('*.yaml')

        data = []
        for path in paths:
            info = load_yaml(path)
            data.append(info)

        return cls(data=data)

    def validate(self):
        log.info('Validating YAML files in `input/schemas')
        # For now (and maybe always) we don't have any checks here.
        # This just shows that the schema files can be parsed OK.
        # for schema in self.data:
        #     log.debug(schema)

class InputData:
    """
    Read all data from the `input` folder.

    Expose it as Python objects that can be validated and used.
    """

    def __init__(self, schemas=None, sources=None, datasets=None,
                 seds=None, lightcurves=None, gammacat_dataset_config=None):
        self.path = gammacat_info.base_dir / 'input'
        self.schemas = schemas
        self.sources = sources
        self.datasets = datasets
        self.seds = seds
        self.lightcurves = lightcurves
        self.gammacat_dataset_config = gammacat_dataset_config

    @classmethod
    def read(cls, internal=False):
        """Read all data from disk.
        """
        # Delayed import to avoid circular dependency
        from .cat import GammaCatDataSetConfig
        schemas = Schemas.read()
        sources = BasicSourceList.read()
        datasets = InputDatasetCollection.read(internal=internal)
        seds = SEDList.read(internal=internal)
        lightcurves = LightcurveList.read()
        gammacat_dataset_config = GammaCatDataSetConfig.read()
        return cls(
            schemas=schemas,
            sources=sources,
            datasets=datasets,
            seds=seds,
            lightcurves=lightcurves,
            gammacat_dataset_config=gammacat_dataset_config,
        )

    def __str__(self):
        ss = 'Input data summary:\n'
        ss += 'Path: {}\n'.format(self.path)
        ss += 'Number of schemas: {}\n'.format(len(self.schemas.data))
        ss += '\n'
        ss += 'Number of YAML files in `input/sources`: {}\n'.format(len(self.sources.data))
        ss += 'Number of entries in `input/gammacat/gamma_cat_dataset.yaml`: {}\n'.format(
            len(self.gammacat_dataset_config.data))
        ss += '\n'
        ss += 'Number of folders in `input/data`: {}\n'.format(len(self.datasets.data))
        ss += 'Number of total datasets in `input/gammacat/gamma_cat_dataset.yaml`: {}\n'.format(
            len(self.gammacat_dataset_config.reference_ids))
        ss += '\n'
        ss += 'Number of SEDs: {}\n'.format(len(self.seds.data))
        ss += 'Number of lightcurves: {}\n'.format(len(self.lightcurves.data))
        return ss

    def validate(self):
        log.info('Validating input data ...')
        self.schemas.validate()
        self.sources.validate()
        self.datasets.validate()
        self.seds.validate()
        # self.lightcurves.validate()
        self.gammacat_dataset_config.validate(self)
<commit_msg>Fix internal glob call in input.py<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Classes to read, validate and work with the input data files.
"""
import logging
from collections import OrderedDict
from itertools import chain
from pathlib import Path
import urllib.parse
from astropy.table import Table
from .info import gammacat_info
from .utils import load_yaml, NA, validate_schema
from .sed import SEDList
from .lightcurve import LightcurveList

__all__ = [
    'BasicSourceInfo',
    'BasicSourceList',
    'DatasetSourceInfo',
    'InputData',
    'InputDataset',
    'InputDatasetCollection',
]

log = logging.getLogger(__name__)

class BasicSourceInfo:
    """All basic info for a source.
    """
    schema = load_yaml(gammacat_info.base_dir / 'input/schemas/basic_source_info.schema.yaml')

    def __init__(self, data, path):
        self.data = data
        self.path = path

    @classmethod
    def read(cls, path):
        path = Path(path)
        data = load_yaml(path)
        return cls(data=data, path=path)

    def __repr__(self):
        return 'BasicSourceInfo(source_id={})'.format(repr(self.data['source_id']))

    def to_dict(self, filled=False):
        """Data as a flat OrderedDict that can be stored in a table row."""
        data = OrderedDict()

        if filled:
            for name, spec in self.schema['properties'].items():
                try:
                    datatype = spec['type']
                except KeyError:
                    datatype = 'string'

                # TODO: write code to handle position
                if name == 'pos':
                    continue

                try:
                    data[name] = NA.fill_value[datatype]
                except TypeError:
                    data[name] = NA.fill_value[datatype[0]]

        data.update(self.data)

        if data['reference_ids'] is None or data['reference_ids'][0] is None:
            data['reference_ids'] = ''
        else:
            data['reference_ids'] = ','.join(data['reference_ids'])

        # TODO: write code to handle position
        data.pop('pos', None)

        return data

    def validate(self):
        validate_schema(path=self.path, data=self.data, schema=self.schema)

class DatasetSourceInfo:
    """All info from one dataset for one source.
    """
    schema = load_yaml(gammacat_info.base_dir / 'input/schemas/dataset_source_info.schema.yaml')

    def __init__(self, data, path):
        self.data = data
        self.path = path

    @classmethod
    def read(cls, path):
        path = Path(path)
        data = load_yaml(path)
        return cls(data=data, path=path)

    def __repr__(self):
        return 'DatasetSourceInfo(source_id={}, reference_id={})'.format(
            repr(self.data['source_id']),
            repr(self.data['reference_id']),
        )

    def validate(self):
        validate_schema(path=self.path, data=self.data, schema=self.schema)

class InputDataset:
    """All info for one dataset.
    """

    def __init__(self, reference_id, path, sources):
        self.reference_id = reference_id
        self.path = path
        self.sources = sources
        # TODO: remove this cache
        _source_ids = [source.data['source_id'] for source in sources]
        self._sources_by_id = dict(zip(_source_ids, sources))

    @classmethod
    def read(cls, path):
        path = Path(path)
        reference_id = urllib.parse.unquote(path.parts[-1])

        # TODO: maybe just use an OrderedDict
        sources = []
        for source_path in sorted(path.glob('tev-*.yaml')):
            source_info = DatasetSourceInfo.read(source_path)
            sources.append(source_info)

        path = '/'.join(path.parts[-2:])
        return cls(reference_id=reference_id, path=path, sources=sources)

    def to_json(self):
        sources = []
        for source in self.sources:
            data = OrderedDict()
            data['source_id'] = source.data['source_id']
            data['reference_id'] = source.data['reference_id']
            sources.append(data)

        # TODO: This would give the full information from the input files.
        # sources = [dict(_.data for _ in self.sources]

        # This is what it takes to build the URL on Github
        url = self.path.replace('%26', '%2526')

        data = OrderedDict()
        data['reference_id'] = self.reference_id
        data['path'] = self.path
        data['url'] = url
        data['sources'] = sources

        return data

    def __repr__(self):
        return 'InputDataset(reference_id={})'.format(repr(self.reference_id))

    def validate(self):
        [_.validate() for _ in self.sources]

    def get_source_by_id(self, source_id):
        # returning empty DatasetSourceInfo makes sense, because it leads to a key
        # error later and will be treated as missing info
        missing = DatasetSourceInfo(data={}, path='')
        return self._sources_by_id.get(source_id, missing)

class BasicSourceList:
    """
    List of `BasicSourceInfo` objects.
    """
    column_spec = load_yaml(gammacat_info.base_dir / 'input/schemas/basic_source_list.schema.yaml')

    def __init__(self, data):
        self.data = data

    @property
    def source_ids(self):
        return [source.data['source_id'] for source in self.data]

    def get_source_by_id(self, source_id):
        idx = self.source_ids.index(source_id)
        return self.data[idx]

    @classmethod
    def read(cls):
        path = gammacat_info.base_dir / 'input/sources'
        paths = path.glob('*.yaml')

        data = []
        for path in paths:
            if path.name in {'example.yaml'}:
                continue
            info = BasicSourceInfo.read(path)
            data.append(info)

        return cls(data=data)

    def to_json(self):
        """Return data in format that can be written to JSON.

        A dict with `data` key.
        """
        return OrderedDict(data=self.data_per_row(filled=True))

    def data_per_row(self, filled=False):
        """Data as list of dicts (per-row)"""
        return [
            source.to_dict(filled=filled)
            for source in self.data
            ]

    def validate(self):
        # TODO: validate `column_spec` schema?

        log.info('Validating YAML files in `input/sources`')
        [_.validate() for _ in self.data]

class InputDatasetCollection:
    """
    Represents all data in ``input/data``.
    """

    def __init__(self, data):
        self.data = data

    @classmethod
    def read(cls, internal=False):
        path = gammacat_info.base_dir / 'input/data'
        paths = list(path.glob('*/*'))

        if internal:
            path_internal = gammacat_info.internal_dir
            paths = chain(paths, [path_internal])

        data = []
        for path in paths:
            info = InputDataset.read(path)
            data.append(info)
        return cls(data=data)

    @property
    def reference_ids(self):
        return [dataset.reference_id for dataset in self.data]

    def to_table(self):
        """Convert info of `sources` list into a Table.
        """
        meta = OrderedDict()
        meta['name'] = 'todo'
        meta['version'] = 'todo'
        meta['url'] = 'todo'

        # rows = self.data_per_row(filled=True)
        rows = [OrderedDict(spam=99)]
        return Table(rows=rows, meta=meta, masked=True)

    def to_json(self):
        """Return data in format that can be written to JSON.

        A dict with `data` key.
        """
        data = []
        for dataset in self.data:
            data.append(dataset.to_json())
        return OrderedDict(data=data)

    def validate(self):
        log.info('Validating YAML files in `input/data`')
        for dataset in self.data:
            dataset.validate()

    def get_dataset_by_reference_id(self, reference_id):
        """Get dataset for a given reference_id
        """
        # TODO: this is not a good way to handle things.
        # Remove once gamma-cat script is set up in a better way.
        if reference_id is None:
            return InputDataset(reference_id=None, path=None, sources=[])

        idx = self.reference_ids.index(reference_id)
        return self.data[idx]

class Schemas:
    def __init__(self, data):
        self.data = data

    @classmethod
    def read(cls):
        path = gammacat_info.base_dir / 'input/schemas'
        paths = path.glob('*.yaml')

        data = []
        for path in paths:
            info = load_yaml(path)
            data.append(info)

        return cls(data=data)

    def validate(self):
        log.info('Validating YAML files in `input/schemas')
        # For now (and maybe always) we don't have any checks here.
        # This just shows that the schema files can be parsed OK.
        # for schema in self.data:
        #     log.debug(schema)

class InputData:
    """
    Read all data from the `input` folder.

    Expose it as Python objects that can be validated and used.
    """

    def __init__(self, schemas=None, sources=None, datasets=None,
                 seds=None, lightcurves=None, gammacat_dataset_config=None):
        self.path = gammacat_info.base_dir / 'input'
        self.schemas = schemas
        self.sources = sources
        self.datasets = datasets
        self.seds = seds
        self.lightcurves = lightcurves
        self.gammacat_dataset_config = gammacat_dataset_config

    @classmethod
    def read(cls, internal=False):
        """Read all data from disk.
        """
        # Delayed import to avoid circular dependency
        from .cat import GammaCatDataSetConfig
        schemas = Schemas.read()
        sources = BasicSourceList.read()
        datasets = InputDatasetCollection.read(internal=internal)
        seds = SEDList.read(internal=internal)
        lightcurves = LightcurveList.read()
        gammacat_dataset_config = GammaCatDataSetConfig.read()
        return cls(
            schemas=schemas,
            sources=sources,
            datasets=datasets,
            seds=seds,
            lightcurves=lightcurves,
            gammacat_dataset_config=gammacat_dataset_config,
        )

    def __str__(self):
        ss = 'Input data summary:\n'
        ss += 'Path: {}\n'.format(self.path)
        ss += 'Number of schemas: {}\n'.format(len(self.schemas.data))
        ss += '\n'
        ss += 'Number of YAML files in `input/sources`: {}\n'.format(len(self.sources.data))
        ss += 'Number of entries in `input/gammacat/gamma_cat_dataset.yaml`: {}\n'.format(
            len(self.gammacat_dataset_config.data))
        ss += '\n'
        ss += 'Number of folders in `input/data`: {}\n'.format(len(self.datasets.data))
        ss += 'Number of total datasets in `input/gammacat/gamma_cat_dataset.yaml`: {}\n'.format(
            len(self.gammacat_dataset_config.reference_ids))
        ss += '\n'
        ss += 'Number of SEDs: {}\n'.format(len(self.seds.data))
        ss += 'Number of lightcurves: {}\n'.format(len(self.lightcurves.data))
        return ss

    def validate(self):
        log.info('Validating input data ...')
        self.schemas.validate()
        self.sources.validate()
        self.datasets.validate()
        self.seds.validate()
        # self.lightcurves.validate()
        self.gammacat_dataset_config.validate(self)
<|endoftext|>


<commit_before>import random

import discord
from discord.ext import commands

class Fun:
  def __init__(self, bot):
    self.bot = bot
    self.ua = {
            'User-Agent': 'lolbot (discord.py/aiohttp)'
        }

  @commands.command()
  async def cat(self, ctx):
    """Random cat images. Awww, so cute! Powered by random.cat"""
    async with self.bot.session.get('https://random.cat/meow', headers=self.ua) as r:
      if r.status == 200:
        js = await r.json()
        em = discord.Embed(name='random.cat', colour=0x690E8)
        em.set_image(url=js['file'])
        await ctx.send(embed=em)

  @commands.command()
  async def httpcat(self, ctx, *, http_id: str):
    """http.cat images - ^httpcat <http code>"""
    httpcat_em = discord.Embed(name='http.cat', colour=0x690E8)
    httpcat_em.set_image(url='https://http.cat/' + http_id + '.jpg')
    httpcat_em.set_footer(text='If there is no image, it doesn\'t exist.')
    await ctx.send(embed=httpcat_em)

  @commands.command()
  async def dog(self, ctx):
    """Random dogs, by random.dog"""
    async with self.bot.session.get('https://random.dog/woof', headers=self.ua) as shibeGet:
        if shibeGet.status == 200:
            shibeImg = await shibeGet.text()
            shibeURL = 'https://random.dog/' + shibeImg
            if '.mp4' in str(shibeURL):
                await ctx.send('mp4 file: ' + shibeURL)
            else:
                shibeEmbed = discord.Embed(colour=0x690E8)
                shibeEmbed.set_image(url=shibeURL)
                await ctx.send(embed=shibeEmbed)
        else:
            await ctx.send(f'Something happened while fetching the picture (HTTP code {shibeGet.status})')

    @commands.command()
    async def lizard(self, ctx):
        """Gets a random lizard"""
        async with self.bot.session.get('https://nekos.life/api/lizard', headers=self.ua) as lizard:
            if lizard.status == 200:
                lizJS = lizard.json()
                lizEm = discord.Embed(colour=0x690E8)
                lizEm.set_image(lizJS['lizard'])
                await ctx.send(embed=lizEm)
            else:
                await ctx.send(f'Something happened while fetching the lizard (HTTP code {lizard.status})')

    @commands.command()
    async def why(self, ctx):
        """Why _____?"""
        async with self.bot.session.get('https://nekos.life/api/why', headers=self.ua) as why:
            if why.status == 200:
                whyJS = why.json()
                whyEm = discord.Embed(title=f'{ctx.author.name} wonders...',
                        description=whyJS['why'], colour=0x690E8)
                await ctx.send(embed=whyEm)
            else:
                await ctx.send(f'I wasn\'t able to fetch the sentence. (HTTP code {why.status})')

    @commands.command()
    async def k(self, ctx):
        """k"""
        await ctx.send('k')

    @commands.command()
    async def fuck(self, ctx):
        """fuck"""
        await ctx.send('fuck')

    @commands.command(name='8ball')
    async def an8ball(self, ctx, *, question: str):
        pool = ['It is certain', 'Outlook good', 'You may rely on it', 'Ask again '
        'later', 'Concentrate and ask again', 'Reply hazy, try again', 'My reply is '
        'no', 'My sources say no']
        ans = random.choice(pool)
        emb = discord.Embed(title='The Magic 8-ball', description='**Question: ' +
        str(question) + '**\nAnswer: ' + str(ans), colour=0x690E8)
        await ctx.send(embed=emb)

def setup(bot):
    bot.add_cog(Fun(bot))
<commit_msg>cogs[fun]: Ultimate facepalm<commit_after>import random

import discord
from discord.ext import commands

class Fun:
  def __init__(self, bot):
    self.bot = bot
    self.ua = {
            'User-Agent': 'lolbot (discord.py/aiohttp)'
        }

  @commands.command()
  async def cat(self, ctx):
    """Random cat images. Awww, so cute! Powered by random.cat"""
    async with self.bot.session.get('https://random.cat/meow', headers=self.ua) as r:
      if r.status == 200:
        js = await r.json()
        em = discord.Embed(name='random.cat', colour=0x690E8)
        em.set_image(url=js['file'])
        await ctx.send(embed=em)

  @commands.command()
  async def httpcat(self, ctx, *, http_id: str):
    """http.cat images - ^httpcat <http code>"""
    httpcat_em = discord.Embed(name='http.cat', colour=0x690E8)
    httpcat_em.set_image(url='https://http.cat/' + http_id + '.jpg')
    httpcat_em.set_footer(text='If there is no image, it doesn\'t exist.')
    await ctx.send(embed=httpcat_em)

  @commands.command()
  async def dog(self, ctx):
    """Random dogs, by random.dog"""
    async with self.bot.session.get('https://random.dog/woof', headers=self.ua) as shibeGet:
        if shibeGet.status == 200:
            shibeImg = await shibeGet.text()
            shibeURL = 'https://random.dog/' + shibeImg
            if '.mp4' in str(shibeURL):
                await ctx.send('mp4 file: ' + shibeURL)
            else:
                shibeEmbed = discord.Embed(colour=0x690E8)
                shibeEmbed.set_image(url=shibeURL)
                await ctx.send(embed=shibeEmbed)
        else:
            await ctx.send(f'Something happened while fetching the picture (HTTP code {shibeGet.status})')

    @commands.command()
    async def lizard(self, ctx):
        """Gets a random lizard"""
        async with self.bot.session.get('https://nekos.life/api/lizard', headers=self.ua) as lizard:
            if lizard.status == 200:
                lizJS = lizard.json()
                lizEm = discord.Embed(colour=0x690E8)
                lizEm.set_image(url=lizJS['lizard'])
                await ctx.send(embed=lizEm)
            else:
                await ctx.send(f'Something happened while fetching the lizard (HTTP code {lizard.status})')

    @commands.command()
    async def why(self, ctx):
        """Why _____?"""
        async with self.bot.session.get('https://nekos.life/api/why', headers=self.ua) as why:
            if why.status == 200:
                whyJS = why.json()
                whyEm = discord.Embed(title=f'{ctx.author.name} wonders...',
                        description=whyJS['why'], colour=0x690E8)
                await ctx.send(embed=whyEm)
            else:
                await ctx.send(f'I wasn\'t able to fetch the sentence. (HTTP code {why.status})')

    @commands.command()
    async def k(self, ctx):
        """k"""
        await ctx.send('k')

    @commands.command()
    async def fuck(self, ctx):
        """fuck"""
        await ctx.send('fuck')

    @commands.command(name='8ball')
    async def an8ball(self, ctx, *, question: str):
        pool = ['It is certain', 'Outlook good', 'You may rely on it', 'Ask again '
        'later', 'Concentrate and ask again', 'Reply hazy, try again', 'My reply is '
        'no', 'My sources say no']
        ans = random.choice(pool)
        emb = discord.Embed(title='The Magic 8-ball', description='**Question: ' +
        str(question) + '**\nAnswer: ' + str(ans), colour=0x690E8)
        await ctx.send(embed=emb)

def setup(bot):
    bot.add_cog(Fun(bot))
<|endoftext|>


<commit_before># Python 2/3 compatibility

try:
    from urllib.error import URLError, HTTPError
    from urllib.request import urlopen, urlretrieve
    from io import BytesIO

    def bytes_decode(bytes_, encoding='utf-8'):
        return bytes_.decode(encoding)
except ImportError:
    # Python 2.X
    from urllib2 import URLError, HTTPError, urlopen
    from urllib import urlretrieve
    from StringIO import StringIO as BytesIO

    def bytes_decode(bytes_, encoding='utf-8'):
        return bytes_

try:
    from functools import lru_cache
except ImportError:
    # Python 2.X: function not available
    def lru_cache(maxsize=128, typed=False):
        return lambda y: y
<commit_msg>MAINT: remove unused Py2/3 compat<commit_after># Python 2/3 compatibility

try:
    from urllib.error import URLError, HTTPError
    from urllib.request import urlopen, urlretrieve
    from io import BytesIO

    def bytes_decode(bytes_, encoding='utf-8'):
        return bytes_.decode(encoding)
except ImportError:
    # Python 2.X
    from urllib2 import URLError, HTTPError, urlopen
    from urllib import urlretrieve
    from StringIO import StringIO as BytesIO

    def bytes_decode(bytes_, encoding='utf-8'):
        return bytes_
<|endoftext|>


<commit_before>import os
import mimetypes
import paste.fileapp
import pylons.config as config

import ckan.plugins.toolkit as toolkit
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.lib.uploader as uploader
from ckan.common import _, request, c, response

from ckanext.s3filestore.uploader import S3Uploader

import logging
log = logging.getLogger(__name__)

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
get_action = logic.get_action
abort = base.abort
redirect = toolkit.redirect_to

class S3Controller(base.BaseController):

    def resource_download(self, id, resource_id, filename=None):
        '''
        Provide a download by either redirecting the user to the url stored or
        downloading the uploaded file from S3.
        '''
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author, 'auth_user_obj': c.userobj}

        try:
            rsc = get_action('resource_show')(context, {'id': resource_id})
            get_action('package_show')(context, {'id': id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % id)

        if rsc.get('url_type') == 'upload':
            upload = uploader.get_resource_uploader(rsc)
            bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
            region = config.get('ckanext.s3filestore.region_name')
            bucket = upload.get_s3_bucket(bucket_name)

            if filename is None:
                filename = os.path.basename(rsc['url'])
            key_path = upload.get_path(rsc['id'], filename)
            key = filename

            if key is None:
                log.warn('Key \'{0}\' not found in bucket \'{1}\''
                         .format(key_path, bucket_name))

                # attempt fallback
                if config.get('ckanext.s3filestore.filesystem_download_fallback',
                              False):
                    log.info('Attempting filesystem fallback for resource {0}'
                             .format(resource_id))
                    url = toolkit.url_for(controller='ckanext.s3filestore.controller:S3Controller',
                                          action='filesystem_resource_download',
                                          id=id,
                                          resource_id=resource_id,
                                          filename=filename)
                    redirect(url)

                abort(404, _('Resource data not found'))

            obj = bucket.Object(key_path)
            contents = str(obj.get()['Body'].read())

            dataapp = paste.fileapp.DataApp(contents)

            try:
                status, headers, app_iter = request.call_application(dataapp)
            except OSError:
                abort(404, _('Resource data not found'))

            response.headers.update(dict(headers))
            response.status = status
            content_type, x = mimetypes.guess_type(rsc.get('url', ''))
            if content_type:
                response.headers['Content-Type'] = content_type
            return app_iter

        elif 'url' not in rsc:
            abort(404, _('No download is available'))
        redirect(rsc['url'])

    def filesystem_resource_download(self, id, resource_id, filename=None):
        """
        A fallback controller action to download resources from the
        filesystem. A copy of the action from
        `ckan.controllers.package:PackageController.resource_download`.

        Provide a direct download by either redirecting the user to the url
        stored or downloading an uploaded file directly.
        """
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author, 'auth_user_obj': c.userobj}

        try:
            rsc = get_action('resource_show')(context, {'id': resource_id})
            get_action('package_show')(context, {'id': id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % id)

        if rsc.get('url_type') == 'upload':
            upload = uploader.ResourceUpload(rsc)
            filepath = upload.get_path(rsc['id'])
            fileapp = paste.fileapp.FileApp(filepath)
            try:
                status, headers, app_iter = request.call_application(fileapp)
            except OSError:
                abort(404, _('Resource data not found'))
            response.headers.update(dict(headers))
            content_type, content_enc = mimetypes.guess_type(rsc.get('url',
                                                                     ''))
            if content_type:
                response.headers['Content-Type'] = content_type
            response.status = status
            return app_iter
        elif 'url' not in rsc:
            abort(404, _('No download is available'))
        redirect(rsc['url'])

    def uploaded_file_redirect(self, upload_to, filename):
        '''Redirect static file requests to their location on S3.'''
        host_name = config.get('ckanext.s3filestore.host_name')
        storage_path = S3Uploader.get_storage_path(upload_to)
        filepath = os.path.join(storage_path, filename)
        #host = config.get('ckanext.s3.filestore.hostname')
        # redirect_url = 'https://{bucket_name}.minio.omc.ckan.io/{filepath}' \
        #     .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'),
        #             filepath=filepath)
        redirect_url = '{host_name}/{bucket_name}/{filepath}'\
                          .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'),
                          filepath=filepath,
                          host_name=host_name)
        redirect(redirect_url)
<commit_msg>Add check for trailing slash in s3filestore.host_name<commit_after>import os
import mimetypes
import paste.fileapp
import pylons.config as config

import ckan.plugins.toolkit as toolkit
import ckan.logic as logic
import ckan.lib.base as base
import ckan.model as model
import ckan.lib.uploader as uploader
from ckan.common import _, request, c, response

from ckanext.s3filestore.uploader import S3Uploader

import logging
log = logging.getLogger(__name__)

NotFound = logic.NotFound
NotAuthorized = logic.NotAuthorized
get_action = logic.get_action
abort = base.abort
redirect = toolkit.redirect_to

class S3Controller(base.BaseController):

    def resource_download(self, id, resource_id, filename=None):
        '''
        Provide a download by either redirecting the user to the url stored or
        downloading the uploaded file from S3.
        '''
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author, 'auth_user_obj': c.userobj}

        try:
            rsc = get_action('resource_show')(context, {'id': resource_id})
            get_action('package_show')(context, {'id': id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % id)

        if rsc.get('url_type') == 'upload':
            upload = uploader.get_resource_uploader(rsc)
            bucket_name = config.get('ckanext.s3filestore.aws_bucket_name')
            region = config.get('ckanext.s3filestore.region_name')
            bucket = upload.get_s3_bucket(bucket_name)

            if filename is None:
                filename = os.path.basename(rsc['url'])
            key_path = upload.get_path(rsc['id'], filename)
            key = filename

            if key is None:
                log.warn('Key \'{0}\' not found in bucket \'{1}\''
                         .format(key_path, bucket_name))

                # attempt fallback
                if config.get('ckanext.s3filestore.filesystem_download_fallback',
                              False):
                    log.info('Attempting filesystem fallback for resource {0}'
                             .format(resource_id))
                    url = toolkit.url_for(controller='ckanext.s3filestore.controller:S3Controller',
                                          action='filesystem_resource_download',
                                          id=id,
                                          resource_id=resource_id,
                                          filename=filename)
                    redirect(url)

                abort(404, _('Resource data not found'))

            obj = bucket.Object(key_path)
            contents = str(obj.get()['Body'].read())

            dataapp = paste.fileapp.DataApp(contents)

            try:
                status, headers, app_iter = request.call_application(dataapp)
            except OSError:
                abort(404, _('Resource data not found'))

            response.headers.update(dict(headers))
            response.status = status
            content_type, x = mimetypes.guess_type(rsc.get('url', ''))
            if content_type:
                response.headers['Content-Type'] = content_type
            return app_iter

        elif 'url' not in rsc:
            abort(404, _('No download is available'))
        redirect(rsc['url'])

    def filesystem_resource_download(self, id, resource_id, filename=None):
        """
        A fallback controller action to download resources from the
        filesystem. A copy of the action from
        `ckan.controllers.package:PackageController.resource_download`.

        Provide a direct download by either redirecting the user to the url
        stored or downloading an uploaded file directly.
        """
        context = {'model': model, 'session': model.Session,
                   'user': c.user or c.author, 'auth_user_obj': c.userobj}

        try:
            rsc = get_action('resource_show')(context, {'id': resource_id})
            get_action('package_show')(context, {'id': id})
        except NotFound:
            abort(404, _('Resource not found'))
        except NotAuthorized:
            abort(401, _('Unauthorized to read resource %s') % id)

        if rsc.get('url_type') == 'upload':
            upload = uploader.ResourceUpload(rsc)
            filepath = upload.get_path(rsc['id'])
            fileapp = paste.fileapp.FileApp(filepath)
            try:
                status, headers, app_iter = request.call_application(fileapp)
            except OSError:
                abort(404, _('Resource data not found'))
            response.headers.update(dict(headers))
            content_type, content_enc = mimetypes.guess_type(rsc.get('url',
                                                                     ''))
            if content_type:
                response.headers['Content-Type'] = content_type
            response.status = status
            return app_iter
        elif 'url' not in rsc:
            abort(404, _('No download is available'))
        redirect(rsc['url'])

    def uploaded_file_redirect(self, upload_to, filename):
        '''Redirect static file requests to their location on S3.'''
        host_name = config.get('ckanext.s3filestore.host_name')
        # Remove last characted if it's a slash
        if host_name[-1] == '/':
            host_name = host_name[:-1]
        storage_path = S3Uploader.get_storage_path(upload_to)
        filepath = os.path.join(storage_path, filename)
        #host = config.get('ckanext.s3.filestore.hostname')
        # redirect_url = 'https://{bucket_name}.minio.omc.ckan.io/{filepath}' \
        #     .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'),
        #             filepath=filepath)
        redirect_url = '{host_name}/{bucket_name}/{filepath}'\
                          .format(bucket_name=config.get('ckanext.s3filestore.aws_bucket_name'),
                          filepath=filepath,
                          host_name=host_name)
        redirect(redirect_url)
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals
import os
import sys
from ConfigParser import ConfigParser, NoSectionError

import subprocess

LOOKUP_TABLE = [
    "/etc/gammu-dispatch.conf"
    "~/gammu-dispatch.conf",
    "./gammu-dispatch.conf",
]

def open_config():
    cfg = ConfigParser()
    cfg.read(LOOKUP_TABLE)
    return cfg

def parse_sms():
    if 'SMS_MESSAGES' not in os.environ:
        return {}

    numparts = int(os.environ['SMS_MESSAGES'])

    # Are there any decoded parts?
    if numparts == 0:
        return {}

    # Get all text parts
    text = ''
    for i in range(1, numparts + 1):
        varname = 'SMS_%d_TEXT' % i
        if varname in os.environ:
            text = text + os.environ[varname]

    return {
        'phone': os.environ['SMS_1_NUMBER'],
        'message': text.strip()
    }

def exec_cmd(syscmd, parsed):
    args = syscmd.split(' ')
    args.append(parsed['phone'])
    args.append(parsed['message'])

    if subprocess.call(args) != 0:
        sys.exit(1)

def main(argv):
    cfg = open_config()

    try:
        cmds = cfg.items('Commands')

    except NoSectionError:
        sys.exit(1)

    parsed = parse_sms()

    if len(parsed) == 0:
        sys.exit(1)

    for (order, syscmd) in cmds:
        if parsed["message"].lower() == order:
            exec_cmd(syscmd, parsed)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>Different exit code depending on the situation<commit_after>from __future__ import print_function, unicode_literals
import os
import sys
from ConfigParser import ConfigParser, NoSectionError

import subprocess

LOOKUP_TABLE = [
    "/etc/gammu-dispatch.conf"
    "~/gammu-dispatch.conf",
    "./gammu-dispatch.conf",
]

def open_config():
    cfg = ConfigParser()
    cfg.read(LOOKUP_TABLE)
    return cfg

def parse_sms():
    if 'SMS_MESSAGES' not in os.environ:
        return {}

    numparts = int(os.environ['SMS_MESSAGES'])

    # Are there any decoded parts?
    if numparts == 0:
        return {}

    # Get all text parts
    text = ''
    for i in range(1, numparts + 1):
        varname = 'SMS_%d_TEXT' % i
        if varname in os.environ:
            text = text + os.environ[varname]

    return {
        'phone': os.environ['SMS_1_NUMBER'],
        'message': text.strip()
    }

def exec_cmd(syscmd, parsed):
    args = syscmd.split(' ')
    args.append(parsed['phone'])
    args.append(parsed['message'])

    if subprocess.call(args) != 0:
        sys.exit(3)

def main(argv):
    cfg = open_config()

    try:
        cmds = cfg.items('Commands')

    except NoSectionError:
        sys.exit(1)

    parsed = parse_sms()

    if len(parsed) == 0:
        sys.exit(2)

    for (order, syscmd) in cmds:
        if parsed["message"].lower() == order:
            exec_cmd(syscmd, parsed)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>from vega_datasets._dataset import Dataset

class DataLoader(object):
    """Load a dataset from a local file or remote URL.

    There are two ways to call this; for example to load the iris dataset, you
    can call this object and pass the dataset name by string:

        >>> from vega_datasets import data
        >>> df = data('iris')

    or you can call the associated named method:

        >>> df = data.iris()

    Optionally, additional parameters can be passed to either of these

    Optional parameters
    -------------------
    return_raw : boolean
        If True, then return the raw string or bytes.
        If False (default), then return a pandas dataframe.
    use_local : boolean
        If True (default), then attempt to load the dataset locally. If
        False or if the dataset is not available locally, then load the
        data from an external URL.
    **kwargs :
        additional keyword arguments are passed to the pandas parsing function,
        either ``read_csv()`` or ``read_json()`` depending on the data format.
    """
    _datasets = {name.replace('-', '_'): name for name in Dataset.list_datasets()}

    def _method_factory(self, name):
        def get_dataset(return_raw=False, use_local=True, **kwargs):
            return self._load_dataset(self._datasets[name],
                                      return_raw=return_raw,
                                      use_local=use_local,
                                      **kwargs)
        return get_dataset

    def _load_dataset(self, name, **kwargs):
        return Dataset(name).load(**kwargs)

    def __call__(self, name, return_raw=False, use_local=True, **kwargs):
        loader = getattr(self, name.replace('-', '_'))
        return loader(return_raw=False, use_local=True, **kwargs)

    def __getattr__(self, attr):
        if attr in self._datasets:
            return self._method_factory(attr)
        else:
            raise AttributeError("No dataset named '{0}'".format(attr))

    def __dir__(self):
        return list(self._datasets.keys())

    #--------------------------------------------------------------------------
    # specialized methods for individual datasets that need specific processing
    def stocks(self, pivoted=False, return_raw=False, use_local=True,
               parse_dates=('date',), **kwargs):
        """A time-series of stock prices from several tech companies

        Parameters
        ----------
        pivoted : boolean
            If True, then pivot the data so each company is in its own column
        """
        if type(parse_dates) is tuple:
            parse_dates=list(parse_dates)
        data = self._load_dataset('stocks',
                                  return_raw=return_raw,
                                  use_local=use_local,
                                  parse_dates=parse_dates, **kwargs)
        if pivoted:
            data = data.pivot(index='date', columns='symbol', values='price')
        return data
<commit_msg>BUG: don't pivot raw data<commit_after>from vega_datasets._dataset import Dataset

class DataLoader(object):
    """Load a dataset from a local file or remote URL.

    There are two ways to call this; for example to load the iris dataset, you
    can call this object and pass the dataset name by string:

        >>> from vega_datasets import data
        >>> df = data('iris')

    or you can call the associated named method:

        >>> df = data.iris()

    Optionally, additional parameters can be passed to either of these

    Optional parameters
    -------------------
    return_raw : boolean
        If True, then return the raw string or bytes.
        If False (default), then return a pandas dataframe.
    use_local : boolean
        If True (default), then attempt to load the dataset locally. If
        False or if the dataset is not available locally, then load the
        data from an external URL.
    **kwargs :
        additional keyword arguments are passed to the pandas parsing function,
        either ``read_csv()`` or ``read_json()`` depending on the data format.
    """
    _datasets = {name.replace('-', '_'): name for name in Dataset.list_datasets()}

    def _method_factory(self, name):
        def get_dataset(return_raw=False, use_local=True, **kwargs):
            return self._load_dataset(self._datasets[name],
                                      return_raw=return_raw,
                                      use_local=use_local,
                                      **kwargs)
        return get_dataset

    def _load_dataset(self, name, **kwargs):
        return Dataset(name).load(**kwargs)

    def __call__(self, name, return_raw=False, use_local=True, **kwargs):
        loader = getattr(self, name.replace('-', '_'))
        return loader(return_raw=False, use_local=True, **kwargs)

    def __getattr__(self, attr):
        if attr in self._datasets:
            return self._method_factory(attr)
        else:
            raise AttributeError("No dataset named '{0}'".format(attr))

    def __dir__(self):
        return list(self._datasets.keys())

    #--------------------------------------------------------------------------
    # specialized methods for individual datasets that need specific processing
    def stocks(self, pivoted=False, return_raw=False, use_local=True,
               parse_dates=('date',), **kwargs):
        """A time-series of stock prices from several tech companies

        Parameters
        ----------
        pivoted : boolean
            If True, then pivot the data so each company is in its own column.
            Not referenced if return_raw is True.
        """
        if type(parse_dates) is tuple:
            parse_dates=list(parse_dates)
        data = self._load_dataset('stocks',
                                  return_raw=return_raw,
                                  use_local=use_local,
                                  parse_dates=parse_dates, **kwargs)
        if pivoted and not return_raw:
            data = data.pivot(index='date', columns='symbol', values='price')
        return data
<|endoftext|>


<commit_before>from django.conf import settings
from django.urls import reverse
from rest_framework import serializers

from falmer.content.models import StaffMemberSnippet
from falmer.matte.models import MatteImage

def generate_image_url(image, filter_spec):
    from wagtail.wagtailimages.views.serve import generate_signature
    signature = generate_signature(image.id, filter_spec)
    url = reverse('wagtailimages_serve', args=(signature, image.id, filter_spec))

    # Append image's original filename to the URL (optional)
    # url += image.file.name[len('original_images/'):]

    return settings.PUBLIC_HOST + url

class WagtailImageSerializer(serializers.ModelSerializer):
    wagtail_image = serializers.SerializerMethodField()
    resource = serializers.SerializerMethodField()

    class Meta:
        model = MatteImage
        fields = ('id', 'wagtail_image', 'resource')

    def get_wagtail_image(self, image):
        return generate_image_url(image, 'fill-400x400')

    def get_resource(self, image):
        return image.file.name

class SnippetSerializer(serializers.ModelSerializer):
    photo = WagtailImageSerializer()

    class Meta:
        model = StaffMemberSnippet
        fields = ('name', 'job_title', 'email', 'office_phone_number', 'mobile_phone_number', 'job_description', 'office_location', 'photo')
<commit_msg>Remove wagtail_image from image resources<commit_after>from django.conf import settings
from django.urls import reverse
from rest_framework import serializers

from falmer.content.models import StaffMemberSnippet
from falmer.matte.models import MatteImage

def generate_image_url(image, filter_spec):
    from wagtail.wagtailimages.views.serve import generate_signature
    signature = generate_signature(image.id, filter_spec)
    url = reverse('wagtailimages_serve', args=(signature, image.id, filter_spec))

    # Append image's original filename to the URL (optional)
    # url += image.file.name[len('original_images/'):]

    return settings.PUBLIC_HOST + url

class WagtailImageSerializer(serializers.ModelSerializer):
    resource = serializers.SerializerMethodField()

    class Meta:
        model = MatteImage
        fields = ('id', 'resource')

    def get_resource(self, image):
        return image.file.name

class SnippetSerializer(serializers.ModelSerializer):
    photo = WagtailImageSerializer()

    class Meta:
        model = StaffMemberSnippet
        fields = ('name', 'job_title', 'email', 'office_phone_number', 'mobile_phone_number', 'job_description', 'office_location', 'photo')
<|endoftext|>


<commit_before>import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.uploader as uploader
import logging
import lxml.etree as etree
import urllib2
import os
import os.path
import mimetypes
import six

log = logging.getLogger(__name__)

def open_resource(resource):
    if resource.get('url_type') == 'upload':
        upload = uploader.ResourceUpload(resource)
        filepath = upload.get_path(resource['id'])
        return open(filepath)
    else:
        return urllib2.urlopen(resource['url'])

def render_wsdl_resource(wsdl_to_html):
    ERROR_HTML = '
%s
'

    def render(resource):
        try:
            resource_file = open_resource(resource)
            wsdl_content = etree.parse(resource_file)

            xml_namespaces = {
                    'soap-env': 'http://schemas.xmlsoap.org/soap/envelope/',
                    'xsd': 'http://www.w3.org/2001/XMLSchema'}

            # FIXME: Should be handled at harvester, temporary cosmetic fix
            soap_faults = wsdl_content.xpath('//soap-env:Fault', namespaces=xml_namespaces)
            if len(soap_faults) > 0:
                try:
                    soap_fault = soap_faults[0]
                    faultcode = soap_fault.find('faultcode', namespaces=xml_namespaces).text
                    faultstring = soap_fault.find('faultstring', namespaces=xml_namespaces).text
                    error_form = ERROR_HTML % 'SOAP error harvesting WSDL: (faultcode: "%s", faultstring: "%s")'
                    error_message = error_form % (faultcode, faultstring)
                except Exception:
                    error_message = ERROR_HTML % 'SOAP error harvesting WSDL'
                return error_message

            # FIXME: Remove XSD imports and includes in WSDL documents until dependency resolution can be made
            for element in wsdl_content.xpath('//xsd:import|//xsd:include', namespaces=xml_namespaces):
                element.getparent().remove(element)
            html_content = wsdl_to_html(wsdl_content)
            return etree.tostring(html_content, pretty_print=True, method='html', encoding=six.text_type)
        except urllib2.URLError:
            return ERROR_HTML % 'Invalid URL'
        except urllib2.HTTPError as e:
            return ERROR_HTML % 'HTTP error: %s' % e
        except etree.XMLSyntaxError as e:
            return ERROR_HTML % 'XML syntax error: %s' % e
        except etree.XSLTApplyError as e:
            return ERROR_HTML % 'Preview error: %s' % e
        except OSError:
            return ERROR_HTML % 'Server error: uploaded file not found'
        except Exception:
            return ERROR_HTML % 'Invalid WSDL'

    return render

class WSDL_ViewPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IResourceView, inherit=True)
    plugins.implements(plugins.ITemplateHelpers)

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'wsdl_view')
        relpath = "./xslt/wsdl_to_html.xslt"
        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), relpath)
        self.wsdl_to_html = etree.XSLT(etree.parse(path))
        mimetypes.add_type('wsdl', '.wsdl')

    # IResourceView

    def info(self):
        return {'name': 'wsdl_view',
                'title': toolkit._('WSDL'),
                'icon': 'file-text-alt',
                'default_title': toolkit._('WSDL'),
                }

    def can_view(self, data_dict):
        resource = data_dict['resource']
        format_lower = resource.get('format', '').lower()
        return format_lower in ['wsdl', 'wsdl+xml']

    def view_template(self, context, data_dict):
        return 'wsdl_view.html'

    # ITemplateHelpers

    def get_helpers(self):
        return {'render_wsdl_resource': render_wsdl_resource(self.wsdl_to_html)}

    def setup_template_variables(self, context, data_dict):
        return {'resource': data_dict['resource']}
<commit_msg>LIKA-333: py3 compatability<commit_after>import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import ckan.lib.uploader as uploader
import logging
import lxml.etree as etree
import os
import os.path
import mimetypes
import six
import requests

log = logging.getLogger(__name__)

def open_resource(resource):
    if resource.get('url_type') == 'upload':
        upload = uploader.ResourceUpload(resource)
        filepath = upload.get_path(resource['id'])
        return open(filepath)
    else:
        return requests.get(resource['url'])

def render_wsdl_resource(wsdl_to_html):
    ERROR_HTML = '
%s
'

    def render(resource):
        try:
            resource_file = open_resource(resource)
            wsdl_content = etree.parse(resource_file)

            xml_namespaces = {
                    'soap-env': 'http://schemas.xmlsoap.org/soap/envelope/',
                    'xsd': 'http://www.w3.org/2001/XMLSchema'}

            # FIXME: Should be handled at harvester, temporary cosmetic fix
            soap_faults = wsdl_content.xpath('//soap-env:Fault', namespaces=xml_namespaces)
            if len(soap_faults) > 0:
                try:
                    soap_fault = soap_faults[0]
                    faultcode = soap_fault.find('faultcode', namespaces=xml_namespaces).text
                    faultstring = soap_fault.find('faultstring', namespaces=xml_namespaces).text
                    error_form = ERROR_HTML % 'SOAP error harvesting WSDL: (faultcode: "%s", faultstring: "%s")'
                    error_message = error_form % (faultcode, faultstring)
                except Exception:
                    error_message = ERROR_HTML % 'SOAP error harvesting WSDL'
                return error_message

            # FIXME: Remove XSD imports and includes in WSDL documents until dependency resolution can be made
            for element in wsdl_content.xpath('//xsd:import|//xsd:include', namespaces=xml_namespaces):
                element.getparent().remove(element)
            html_content = wsdl_to_html(wsdl_content)
            return etree.tostring(html_content, pretty_print=True, method='html', encoding=six.text_type)
        except requests.ConnectionError:
            return ERROR_HTML % 'Invalid URL'
        except requests.HTTPError as e:
            return ERROR_HTML % 'HTTP error: %s' % e
        except etree.XMLSyntaxError as e:
            return ERROR_HTML % 'XML syntax error: %s' % e
        except etree.XSLTApplyError as e:
            return ERROR_HTML % 'Preview error: %s' % e
        except OSError:
            return ERROR_HTML % 'Server error: uploaded file not found'
        except Exception:
            return ERROR_HTML % 'Invalid WSDL'

    return render

class WSDL_ViewPlugin(plugins.SingletonPlugin):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IResourceView, inherit=True)
    plugins.implements(plugins.ITemplateHelpers)

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'wsdl_view')
        relpath = "./xslt/wsdl_to_html.xslt"
        path = os.path.join(os.path.dirname(os.path.realpath(__file__)), relpath)
        self.wsdl_to_html = etree.XSLT(etree.parse(path))
        mimetypes.add_type('wsdl', '.wsdl')

    # IResourceView

    def info(self):
        return {'name': 'wsdl_view',
                'title': toolkit._('WSDL'),
                'icon': 'file-text-alt',
                'default_title': toolkit._('WSDL'),
                }

    def can_view(self, data_dict):
        resource = data_dict['resource']
        format_lower = resource.get('format', '').lower()
        return format_lower in ['wsdl', 'wsdl+xml']

    def view_template(self, context, data_dict):
        return 'wsdl_view.html'

    # ITemplateHelpers

    def get_helpers(self):
        return {'render_wsdl_resource': render_wsdl_resource(self.wsdl_to_html)}

    def setup_template_variables(self, context, data_dict):
        return {'resource': data_dict['resource']}
<|endoftext|>


<commit_before>import contextlib
from subprocess import run, CalledProcessError, PIPE
import os

from Datasnakes.Tools.sge import basejobids, writecodefile, import_temp, file2str
from Datasnakes.Tools.sge.sgeconfig import __DEFAULT__, __CUSTOM__

class BaseSGEJob(object):
    """Base class for simple jobs."""
    def __init__(self, jobname):
        """Initialize job attributes."""
        self.default_job_attributes = __DEFAULT__
        self.custom_job_attributes = __CUSTOM__
        self.file2str = file2str
        self.job_id, self.base_jobname = self._configure(jobname)

    def _configure(self, jobname):
        """Configure job attributes or set it up."""
        baseid, base = basejobids(jobname)
        return baseid, base

    @classmethod
    def _cleanup(jobname):
        """Clean up job scripts."""
        os.remove(jobname + '.pbs')
        os.remove(jobname + '.py')

    def dryrun(self):
        pass

class SGEJob(BaseSGEJob):
    """Create a qsub/pbs job & script for the job to execute."""
    def __init__(self, jobname):
        super().__init__(jobname)

    def submit(self, code, cleanup=False, default=True):
        """Create and submit a qsub job.

        Submit python code."""
        # TIP If python is in your environment as only 'python' update that.
        # TODO-SDH add a default parameter or custom parameter
        # If default, a python file will be created from code that is used.

        # Allow user input to be a python file
        if os.path.isfile(code) and str(code).endswith('.py'):
            code_str = self.file2str(code)
        elif type(code) == str:
            code_str = code

        if default:
            writecodefile(filename=self.base_jobname, code=code_str,
                          language='python')

            # Create the pbs script from the template or dict
            pbstemp = import_temp('temp.pbs')

            attributes = self.default_job_attributes

            with open(self.base_jobname + '.pbs', 'w') as pbsfile:
                pbsfile.write(pbstemp.substitute(attributes))
                pbsfile.close()
        else:
            raise NotImplementedError('Custom qsub jobs are forbidden.')
            # TODO Improve custom job creation
        #            pbstemp = self.import_temp('temp.pbs')
        #            with open(base + '.pbs', 'w') as pbsfile:
        #                pbsfile.write(pbstemp.substitute(self.pbs_dict))
        #                pbsfile.close()

        with contextlib.suppress(CalledProcessError):
            cmd = ['qsub ' + self.base_jobname + '.pbs']  # this is the command
            # Shell MUST be True
            cmd_status = run(cmd, stdout=PIPE, stderr=PIPE, shell=True)

            if cmd_status.returncode == 0:  # Command was successful.
                print('Job submitted.\n')
                # TODO add a check to for job errors or check for error file.

            else:  # Unsuccessful. Stdout will be '1'
                print("PBS job not submitted.")

        # TODO Introduce a wait function w/qstat to check for job completion
<commit_msg>Implemented logging in SGEJob<commit_after>import contextlib
from subprocess import run, CalledProcessError, PIPE
import os
from pkg_resources import resource_filename
import json

from Datasnakes.Tools.logit import LogIt
from Datasnakes.Tools.sge import basejobids, writecodefile, import_temp, file2str
from Datasnakes.Tools.sge.sgeconfig import __DEFAULT__, __CUSTOM__
from Datasnakes.Manager.config import templates

class BaseSGEJob(object):
    """Base class for simple jobs."""
    def __init__(self, length, base_jobname):
        """Initialize job attributes."""
        self.default_job_attributes = __DEFAULT__
        self.custom_job_attributes = __CUSTOM__
        self.file2str = file2str
        self.job_id, self.jobname = self._configure(length, base_jobname)
        self.sgejob_log = LogIt().default(logname="SGE JOB", logfile=None)

        # Import the temp.pbs file using pkg_resources
        self.temp_pbs = resource_filename(templates.__name__, "temp.pbs")

    @classmethod
    def _configure(cls, length, base_jobname):
        """Configure job attributes or set it up."""
        baseid, base = basejobids(length, base_jobname)
        return baseid, base

    @classmethod
    def updatedict(dictionary, new_attributes):
        dict_input = json.dumps(dictionary)
        dict_input.format(new_attributes)
        dict_output = json.loads(dict_input)

        return dict_output

    @classmethod
    def _cleanup(cls, jobname):
        """Clean up job scripts."""
        os.remove(jobname + '.pbs')
        os.remove(jobname + '.py')

class SGEJob(BaseSGEJob):
    """Create a qsub/pbs job & script for the job to execute."""
    def __init__(self, base_jobname):
        super().__init__(length=5, base_jobname=base_jobname)

    def debug(self, code):
        """Debug the SGEJob."""
        self.submit(code, debug=True)

    def submit(self, code, cleanup=False, default=True, debug=False):
        """Create and submit a qsub job.

        Submit python code."""
        if debug:
            self.sgejob_log.info('You decided to debug your SGEJob')

        # TIP If python is in your environment as only 'python' update that.
        # If default, a python file will be created from code that is used.
        # Allow user input to be a python file
        if os.path.isfile(code) and str(code).endswith('.py'):
            code_str = self.file2str(code)
            self.sgejob_log.info('%s converted to string.' % code)
        elif type(code) == str:
            code_str = code

        if default:
            self.sgejob_log.info('You are running a job with default attributes.')
            writecodefile(filename=self.jobname, code=code_str, language='python')
            pyfilename = self.jobname + '.py'
            self.sgejob_log.info('%s python file has been created.' % pyfilename)

            # Create the pbs script from the template or dict
            pbstemp = import_temp(self.temp_pbs)

            attributes = self.default_job_attributes
            pbsfilename = self.jobname + '.pbs'
            with open(pbsfilename, 'w') as pbsfile:
                pbsfile.write(pbstemp.substitute(attributes))
                pbsfile.close()
            self.sgejob_log.info('%s has been created.' % pbsfilename)
        else:
            msg = 'Custom SGEJob creation is not yet implemented.'
            self.sgejob_log.error(msg)
            raise NotImplementedError(msg)
            # TODO Add custom job creation

        if debug:
            with contextlib.suppress(CalledProcessError):
                cmd = 'qsub ' + self.jobname + '.pbs'  # this is the command
                self.sgejob_log.info('`%s` will be run.' % cmd)
                self.sgejob_log.info('Job submitted.')
                self.cleanup(self.jobname)

        else:
            with contextlib.suppress(CalledProcessError):
                cmd = ['qsub ' + self.jobname + '.pbs']  # this is the command
                # Shell MUST be True
                cmd_status = run(cmd, stdout=PIPE, stderr=PIPE, shell=True)

                if cmd_status.returncode == 0:  # Command was successful.
                    self.sgejob_log.info('Job submitted.')
                    # TODO add a check to for job errors or check for error file.

                else:  # Unsuccessful. Stdout will be '1'
                    self.sgejob_log.error('PBS job not submitted.')
        # TODO Introduce a wait function w/qstat to check for job completion
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#

import logging
import urllib2
import httplib

from feedservice.parse.models import Feed
from feedservice import urlstore
from feedservice.parse import feed, youtube, soundcloud, fm4, vimeo

logger = logging.getLogger(__name__)

class FetchFeedException(Exception):
    """ raised when there's an error while fetching the podcast feed """

PARSER_CLASSES = (
        youtube.YoutubeParser,
        vimeo.VimeoParser,
        soundcloud.SoundcloudParser,
        soundcloud.SoundcloudFavParser,
        fm4.FM4OnDemandPlaylistParser,
        feed.Feedparser, # fallback, has to be the last entry
    )

def parse_feeds(feed_urls, mod_since_utc=None,
        text_processor=None, cache=None, **kwargs):
    """ Parses the specified feeds and returns their JSON representations

    RSS-Redirects are followed automatically by including both feeds in the
    result. """

    visited_urls = set()
    result = []

    for url in feed_urls:

        try:
            feed = parse_feed(url, text_processor, mod_since_utc)

        except FetchFeedException as ffe:
            feed = Feed()
            feed.urls = [url]
            feed.new_location = None
            feed.add_error('fetch-feed', str(ffe))

        if not feed:
            continue

        visited  = feed.urls
        new_loc  = feed.new_location

        # we follow RSS-redirects automatically
        if new_loc and new_loc not in (list(visited_urls) + feed_urls):
            feed_urls.append(new_loc)

        visited_urls.add(url)

        result.append(feed)

    return result

def get_parser_cls(url):
    feed_cls = None

    for cls in PARSER_CLASSES:
        if cls.handles_url(url):
            return cls

    raise ValueError('no feed can handle %s' % url)

def parse_feed(feed_url, text_processor, mod_since_utc=None, base_url=None,
        cache=None, **kwargs):
    """ Parses a feed and returns its JSON object

    mod_since_utc: feeds that have not changed since this timestamp are ignored
    base_url: base url of the service -- used for pubsub subscriptions
    text_processor: class to pre-process text contents
    cache: cache or None to disable caching
    """

    parser_cls = get_parser_cls(feed_url)

    try:
        resp = urlstore.fetch_url(feed_url, mod_since_utc)
        assert resp

    except urlstore.NotModified:
        return None

    except (httplib.InvalidURL, urllib2.URLError, urllib2.HTTPError,
            httplib.BadStatusLine, ValueError) as ex:
        raise FetchFeedException(ex)

    parser = parser_cls(feed_url, resp, text_processor=text_processor)

    return parser.get_feed()
<commit_msg>catch and handle socket errors<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#

import logging
import urllib2
import httplib
import socket

from feedservice.parse.models import Feed
from feedservice import urlstore
from feedservice.parse import feed, youtube, soundcloud, fm4, vimeo

logger = logging.getLogger(__name__)

class FetchFeedException(Exception):
    """ raised when there's an error while fetching the podcast feed """

PARSER_CLASSES = (
        youtube.YoutubeParser,
        vimeo.VimeoParser,
        soundcloud.SoundcloudParser,
        soundcloud.SoundcloudFavParser,
        fm4.FM4OnDemandPlaylistParser,
        feed.Feedparser, # fallback, has to be the last entry
    )

def parse_feeds(feed_urls, mod_since_utc=None,
        text_processor=None, cache=None, **kwargs):
    """ Parses the specified feeds and returns their JSON representations

    RSS-Redirects are followed automatically by including both feeds in the
    result. """

    visited_urls = set()
    result = []

    for url in feed_urls:

        try:
            feed = parse_feed(url, text_processor, mod_since_utc)

        except FetchFeedException as ffe:
            feed = Feed()
            feed.urls = [url]
            feed.new_location = None
            feed.add_error('fetch-feed', str(ffe))

        if not feed:
            continue

        visited  = feed.urls
        new_loc  = feed.new_location

        # we follow RSS-redirects automatically
        if new_loc and new_loc not in (list(visited_urls) + feed_urls):
            feed_urls.append(new_loc)

        visited_urls.add(url)

        result.append(feed)

    return result

def get_parser_cls(url):
    feed_cls = None

    for cls in PARSER_CLASSES:
        if cls.handles_url(url):
            return cls

    raise ValueError('no feed can handle %s' % url)

def parse_feed(feed_url, text_processor, mod_since_utc=None, base_url=None,
        cache=None, **kwargs):
    """ Parses a feed and returns its JSON object

    mod_since_utc: feeds that have not changed since this timestamp are ignored
    base_url: base url of the service -- used for pubsub subscriptions
    text_processor: class to pre-process text contents
    cache: cache or None to disable caching
    """

    parser_cls = get_parser_cls(feed_url)

    try:
        resp = urlstore.fetch_url(feed_url, mod_since_utc)
        assert resp

    except urlstore.NotModified:
        return None

    except (httplib.InvalidURL, urllib2.URLError, urllib2.HTTPError,
            httplib.BadStatusLine, ValueError, socket.error) as ex:
        raise FetchFeedException(ex)

    parser = parser_cls(feed_url, resp, text_processor=text_processor)

    return parser.get_feed()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py

"""
Parallel clang-tidy runner
==========================

Runs clang-tidy over all files in a compilation database. Requires clang-tidy
and clang-apply-replacements in $PATH.

Example invocations.
- Run clang-tidy on all files in the current working directory with a default
  set of checks and show warnings in the cpp files and all project headers.
    run-clang-tidy.py $PWD

- Fix all header guards.
    run-clang-tidy.py -fix -checks=-*,llvm-header-guard

- Fix all header guards included from clang-tidy and header guards
  for clang-tidy headers.
    run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
                      -header-filter=extra/clang-tidy

Compilation database setup:
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
"""

from __future__ import print_function

import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback

try:
  import yaml
except ImportError:
  yaml = None

is_py2 = sys.version[0] == '2'

if is_py2:
    import Queue as queue
else:
    import queue as queue

def find_compilation_database(path):
  """Adjusts the directory until a compilation database is found."""
  result = './'
  while not os.path.isfile(os.path.join(result, path)):
    if os.path.realpath(result) == '/':
      print('Error: could not find compilation database.')
      sys.exit(1)
    result += '../'
  return os.path.realpath(result)

def make_absolute(f, directory):
  if os.path.isabs(f):
    return f
  return os.path.normpath(os.path.join(directory, f))

def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
                        header_filter, extra_arg, extra_arg_before, quiet,
                        config):
  """Gets a command line for clang-tidy."""
  start = [clang_tidy_binary]
  if header_filter is not None:
    start.append('-header-filter=' + header_filter)
  else:
    # Show warnings in all in-project headers by default.
    start.append('-header-filter=^' + build_path + '/.*')
  if checks:
    start.append('-checks=' + checks)
  if tmpdir is not None:
    start.append('-export-fixes')
    # Get a temporary file. We immediately close the handle so clang-tidy can
    # overwrite it.
    (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
    os.close(handle)
    start.append(name)
  for arg in extra_arg:
      start.append('-extra-arg=%s' % arg)
  for arg in extra_arg_before:
      start.append('-extra-arg-before=%s' % arg)
  start.append('-p=' + build_path)
  if quiet:
      start.append('-quiet')
  if config:
      start.append('-config=' + config)
  start.append(f)
  return start

def merge_replacement_files(tmpdir, mergefile):
  """Merge all replacement files in a directory into a single file"""
  # The fixes suggested by clang-tidy >= 4.0.0 are given under
  # the top level key 'Diagnostics' in the output yaml files
  mergekey="Diagnostics"
  merged=[]
  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
    content = yaml.safe_load(open(replacefile, 'r'))
    if not content:
      continue # Skip empty files.
    merged.extend(content.get(mergekey, []))

  if merged:
    # MainSourceFile: The key is required by the definition inside
    # include/clang/Tooling/ReplacementsYaml.h, but the value
    # is actually never used inside clang-apply-replacements,
    # so we set it to '' here.
    output = { 'MainSourceFile': '', mergekey: merged }
    with open(mergefile, 'w') as out:
      yaml.safe_dump(output, out)
  else:
    # Empty the file:
    open(mergefile, 'w').close()

def check_clang_apply_replacements_binary(args):
  """Checks if invoking supplied clang-apply-replacements binary works."""
  try:
    subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
  except:
    print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
          'binary correctly specified?', file=sys.stderr)
    traceback.print_exc()
    sys.exit(1)

def apply_fixes(args, tmpdir):
  """Calls clang-apply-fixes on a given directory."""
  invocation = [args.clang_apply_replacements_binary]
  if args.format:
    invocation.append('-format')
  if args.style:
    invocation.append('-style=' + args.style)
  invocation.append(tmpdir)
  subprocess.call(invocation)

def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
  """Takes filenames out of queue and runs clang-tidy on them."""
  while True:
    name = queue.get()
    invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
                                     tmpdir, build_path, args.header_filter,
                                     args.extra_arg, args.extra_arg_before,
                                     args.quiet, args.config)

    proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, err = proc.communicate()
    if proc.returncode != 0:
      failed_files.append(name)
    with lock:
      sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8') + '\n')
      if len(err) > 0:
        sys.stdout.flush()
        sys.stderr.write(err.decode('utf-8') + '\n')
    queue.task_done()

def main():
  parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
                                   'in a compilation database. Requires '
                                   'clang-tidy and clang-apply-replacements in '
                                   '$PATH.')
  parser.add_argument('-clang-tidy-binary', metavar='PATH',
                      default='clang-tidy',
                      help='path to clang-tidy binary')
  parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
                      default='clang-apply-replacements',
                      help='path to clang-apply-replacements binary')
  parser.add_argument('-checks', default=None,
                      help='checks filter, when not specified, use clang-tidy '
                      'default')
  parser.add_argument('-config', default=None,
                      help='Specifies a configuration in YAML/JSON format: '
                      '  -config="{Checks: \'*\', '
                      '                       CheckOptions: [{key: x, '
                      '                                       value: y}]}" '
                      'When the value is empty, clang-tidy will '
                      'attempt to find a file named .clang-tidy for '
                      'each source file in its parent directories.')
  parser.add_argument('-header-filter', default=None,
                      help='regular expression matching the names of the '
                      'headers to output diagnostics from. Diagnostics from '
                      'the main file of each translation unit are always '
                      'displayed.')
  if yaml:
    parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
                        help='Create a yaml file to store suggested fixes in, '
                        'which can be applied with clang-apply-replacements.')
  parser.add_argument('-j', type=int, default=0,
                      help='number of tidy instances to be run in parallel.')
  parser.add_argument('files', nargs='*', default=['.*'],
                      help='files to be processed (regex on path)')
  parser.add_argument('-fix', action='store_true', help='apply fix-its')
  parser.add_argument('-format', action='store_true', help='Reformat code '
                      'after applying fixes')
  parser.add_argument('-style', default='file', help='The style of reformat '
                      'code after applying fixes')
  parser.add_argument('-p', dest='build_path',
                      help='Path used to read a compile command database.')
  parser.add_argument('-extra-arg', dest='extra_arg',
                      action='append', default=[],
                      help='Additional argument to append to the compiler '
                      'command line.')
  parser.add_argument('-extra-arg-before', dest='extra_arg_before',
                      action='append', default=[],
                      help='Additional argument to prepend to the compiler '
                      'command line.')
  parser.add_argument('-quiet', action='store_true',
                      help='Run clang-tidy in quiet mode')
  args = parser.parse_args()

  db_path = 'compile_commands.json'

  if args.build_path is not None:
    build_path = args.build_path
  else:
    # Find our database
    build_path = find_compilation_database(db_path)

  try:
    invocation = [args.clang_tidy_binary, '-list-checks']
    invocation.append('-p=' + build_path)
    if args.checks:
      invocation.append('-checks=' + args.checks)
    invocation.append('-')
    if args.quiet:
      # Even with -quiet we still want to check if we can call clang-tidy.
      with open(os.devnull, 'w') as dev_null:
        subprocess.check_call(invocation, stdout=dev_null)
    else:
      subprocess.check_call(invocation)
  except:
    print("Unable to run clang-tidy.", file=sys.stderr)
    sys.exit(1)

  # Load the database and extract all files.
  database = json.load(open(os.path.join(build_path, db_path)))
  files = [make_absolute(entry['file'], entry['directory'])
           for entry in database]

  max_task = args.j
  if max_task == 0:
    max_task = multiprocessing.cpu_count()

  tmpdir = None
  if args.fix or (yaml and args.export_fixes):
    check_clang_apply_replacements_binary(args)
    tmpdir = tempfile.mkdtemp()

  # Build up a big regexy filter from all command line arguments.
  file_name_re = re.compile('|'.join(args.files))

  return_code = 0
  try:
    # Spin up a bunch of tidy-launching threads.
    task_queue = queue.Queue(max_task)
    # List of files with a non-zero return code.
    failed_files = []
    lock = threading.Lock()
    for _ in range(max_task):
      t = threading.Thread(target=run_tidy,
                           args=(args, tmpdir, build_path, task_queue, lock, failed_files))
      t.daemon = True
      t.start()

    # Fill the queue with files.
    for name in files:
      if file_name_re.search(name):
        task_queue.put(name)

    # Wait for all threads to be done.
    task_queue.join()
    if len(failed_files):
      return_code = 1

  except KeyboardInterrupt:
    # This is a sad hack. Unfortunately subprocess goes
    # bonkers with ctrl-c and we start forking merrily.
    print('\nCtrl-C detected, goodbye.')
    if tmpdir:
      shutil.rmtree(tmpdir)
    os.kill(0, 9)

  if yaml and args.export_fixes:
    print('Writing fixes to ' + args.export_fixes + ' ...')
    try:
      merge_replacement_files(tmpdir, args.export_fixes)
    except:
      print('Error exporting fixes.\n', file=sys.stderr)
      traceback.print_exc()
      return_code=1

  if args.fix:
    print('Applying fixes ...')
    try:
      apply_fixes(args, tmpdir)
    except:
      print('Error applying fixes.\n', file=sys.stderr)
      traceback.print_exc()
      return_code=1

  if tmpdir:
    shutil.rmtree(tmpdir)
  sys.exit(return_code)

if __name__ == '__main__':
  main()
<commit_msg>[clang-tidy] Removed superfluous and slightly annoying newlines in run-clang-tidy's output.<commit_after>#!/usr/bin/env python
#
#===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===------------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py

"""
Parallel clang-tidy runner
==========================

Runs clang-tidy over all files in a compilation database. Requires clang-tidy
and clang-apply-replacements in $PATH.

Example invocations.
- Run clang-tidy on all files in the current working directory with a default
  set of checks and show warnings in the cpp files and all project headers.
    run-clang-tidy.py $PWD

- Fix all header guards.
    run-clang-tidy.py -fix -checks=-*,llvm-header-guard

- Fix all header guards included from clang-tidy and header guards
  for clang-tidy headers.
    run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
                      -header-filter=extra/clang-tidy

Compilation database setup:
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
"""

from __future__ import print_function

import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback

try:
  import yaml
except ImportError:
  yaml = None

is_py2 = sys.version[0] == '2'

if is_py2:
    import Queue as queue
else:
    import queue as queue

def find_compilation_database(path):
  """Adjusts the directory until a compilation database is found."""
  result = './'
  while not os.path.isfile(os.path.join(result, path)):
    if os.path.realpath(result) == '/':
      print('Error: could not find compilation database.')
      sys.exit(1)
    result += '../'
  return os.path.realpath(result)

def make_absolute(f, directory):
  if os.path.isabs(f):
    return f
  return os.path.normpath(os.path.join(directory, f))

def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
                        header_filter, extra_arg, extra_arg_before, quiet,
                        config):
  """Gets a command line for clang-tidy."""
  start = [clang_tidy_binary]
  if header_filter is not None:
    start.append('-header-filter=' + header_filter)
  else:
    # Show warnings in all in-project headers by default.
    start.append('-header-filter=^' + build_path + '/.*')
  if checks:
    start.append('-checks=' + checks)
  if tmpdir is not None:
    start.append('-export-fixes')
    # Get a temporary file. We immediately close the handle so clang-tidy can
    # overwrite it.
    (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
    os.close(handle)
    start.append(name)
  for arg in extra_arg:
      start.append('-extra-arg=%s' % arg)
  for arg in extra_arg_before:
      start.append('-extra-arg-before=%s' % arg)
  start.append('-p=' + build_path)
  if quiet:
      start.append('-quiet')
  if config:
      start.append('-config=' + config)
  start.append(f)
  return start

def merge_replacement_files(tmpdir, mergefile):
  """Merge all replacement files in a directory into a single file"""
  # The fixes suggested by clang-tidy >= 4.0.0 are given under
  # the top level key 'Diagnostics' in the output yaml files
  mergekey="Diagnostics"
  merged=[]
  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
    content = yaml.safe_load(open(replacefile, 'r'))
    if not content:
      continue # Skip empty files.
    merged.extend(content.get(mergekey, []))

  if merged:
    # MainSourceFile: The key is required by the definition inside
    # include/clang/Tooling/ReplacementsYaml.h, but the value
    # is actually never used inside clang-apply-replacements,
    # so we set it to '' here.
    output = { 'MainSourceFile': '', mergekey: merged }
    with open(mergefile, 'w') as out:
      yaml.safe_dump(output, out)
  else:
    # Empty the file:
    open(mergefile, 'w').close()

def check_clang_apply_replacements_binary(args):
  """Checks if invoking supplied clang-apply-replacements binary works."""
  try:
    subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
  except:
    print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
          'binary correctly specified?', file=sys.stderr)
    traceback.print_exc()
    sys.exit(1)

def apply_fixes(args, tmpdir):
  """Calls clang-apply-fixes on a given directory."""
  invocation = [args.clang_apply_replacements_binary]
  if args.format:
    invocation.append('-format')
  if args.style:
    invocation.append('-style=' + args.style)
  invocation.append(tmpdir)
  subprocess.call(invocation)

def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
  """Takes filenames out of queue and runs clang-tidy on them."""
  while True:
    name = queue.get()
    invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
                                     tmpdir, build_path, args.header_filter,
                                     args.extra_arg, args.extra_arg_before,
                                     args.quiet, args.config)

    proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output, err = proc.communicate()
    if proc.returncode != 0:
      failed_files.append(name)
    with lock:
      sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
      if len(err) > 0:
        sys.stdout.flush()
        sys.stderr.write(err.decode('utf-8'))
    queue.task_done()

def main():
  parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
                                   'in a compilation database. Requires '
                                   'clang-tidy and clang-apply-replacements in '
                                   '$PATH.')
  parser.add_argument('-clang-tidy-binary', metavar='PATH',
                      default='clang-tidy',
                      help='path to clang-tidy binary')
  parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
                      default='clang-apply-replacements',
                      help='path to clang-apply-replacements binary')
  parser.add_argument('-checks', default=None,
                      help='checks filter, when not specified, use clang-tidy '
                      'default')
  parser.add_argument('-config', default=None,
                      help='Specifies a configuration in YAML/JSON format: '
                      '  -config="{Checks: \'*\', '
                      '                       CheckOptions: [{key: x, '
                      '                                       value: y}]}" '
                      'When the value is empty, clang-tidy will '
                      'attempt to find a file named .clang-tidy for '
                      'each source file in its parent directories.')
  parser.add_argument('-header-filter', default=None,
                      help='regular expression matching the names of the '
                      'headers to output diagnostics from. Diagnostics from '
                      'the main file of each translation unit are always '
                      'displayed.')
  if yaml:
    parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
                        help='Create a yaml file to store suggested fixes in, '
                        'which can be applied with clang-apply-replacements.')
  parser.add_argument('-j', type=int, default=0,
                      help='number of tidy instances to be run in parallel.')
  parser.add_argument('files', nargs='*', default=['.*'],
                      help='files to be processed (regex on path)')
  parser.add_argument('-fix', action='store_true', help='apply fix-its')
  parser.add_argument('-format', action='store_true', help='Reformat code '
                      'after applying fixes')
  parser.add_argument('-style', default='file', help='The style of reformat '
                      'code after applying fixes')
  parser.add_argument('-p', dest='build_path',
                      help='Path used to read a compile command database.')
  parser.add_argument('-extra-arg', dest='extra_arg',
                      action='append', default=[],
                      help='Additional argument to append to the compiler '
                      'command line.')
  parser.add_argument('-extra-arg-before', dest='extra_arg_before',
                      action='append', default=[],
                      help='Additional argument to prepend to the compiler '
                      'command line.')
  parser.add_argument('-quiet', action='store_true',
                      help='Run clang-tidy in quiet mode')
  args = parser.parse_args()

  db_path = 'compile_commands.json'

  if args.build_path is not None:
    build_path = args.build_path
  else:
    # Find our database
    build_path = find_compilation_database(db_path)

  try:
    invocation = [args.clang_tidy_binary, '-list-checks']
    invocation.append('-p=' + build_path)
    if args.checks:
      invocation.append('-checks=' + args.checks)
    invocation.append('-')
    if args.quiet:
      # Even with -quiet we still want to check if we can call clang-tidy.
      with open(os.devnull, 'w') as dev_null:
        subprocess.check_call(invocation, stdout=dev_null)
    else:
      subprocess.check_call(invocation)
  except:
    print("Unable to run clang-tidy.", file=sys.stderr)
    sys.exit(1)

  # Load the database and extract all files.
  database = json.load(open(os.path.join(build_path, db_path)))
  files = [make_absolute(entry['file'], entry['directory'])
           for entry in database]

  max_task = args.j
  if max_task == 0:
    max_task = multiprocessing.cpu_count()

  tmpdir = None
  if args.fix or (yaml and args.export_fixes):
    check_clang_apply_replacements_binary(args)
    tmpdir = tempfile.mkdtemp()

  # Build up a big regexy filter from all command line arguments.
  file_name_re = re.compile('|'.join(args.files))

  return_code = 0
  try:
    # Spin up a bunch of tidy-launching threads.
    task_queue = queue.Queue(max_task)
    # List of files with a non-zero return code.
    failed_files = []
    lock = threading.Lock()
    for _ in range(max_task):
      t = threading.Thread(target=run_tidy,
                           args=(args, tmpdir, build_path, task_queue, lock, failed_files))
      t.daemon = True
      t.start()

    # Fill the queue with files.
    for name in files:
      if file_name_re.search(name):
        task_queue.put(name)

    # Wait for all threads to be done.
    task_queue.join()
    if len(failed_files):
      return_code = 1

  except KeyboardInterrupt:
    # This is a sad hack. Unfortunately subprocess goes
    # bonkers with ctrl-c and we start forking merrily.
    print('\nCtrl-C detected, goodbye.')
    if tmpdir:
      shutil.rmtree(tmpdir)
    os.kill(0, 9)

  if yaml and args.export_fixes:
    print('Writing fixes to ' + args.export_fixes + ' ...')
    try:
      merge_replacement_files(tmpdir, args.export_fixes)
    except:
      print('Error exporting fixes.\n', file=sys.stderr)
      traceback.print_exc()
      return_code=1

  if args.fix:
    print('Applying fixes ...')
    try:
      apply_fixes(args, tmpdir)
    except:
      print('Error applying fixes.\n', file=sys.stderr)
      traceback.print_exc()
      return_code=1

  if tmpdir:
    shutil.rmtree(tmpdir)
  sys.exit(return_code)

if __name__ == '__main__':
  main()
<|endoftext|>