<commit_before>""" Utilities related to FRB galaxies"""

import os
import glob
from IPython import embed
from pkg_resources import resource_filename
import numpy as np

try:
    from specdb.specdb import SpecDB
except ImportError:
    flg_specdb = False
else:
    flg_specdb = True

from astropy.coordinates import SkyCoord
import pandas as pd
import extinction
from linetools.spectra import xspectrum1d

from frb import frb

def deredden_spec(spectrum:xspectrum1d.XSpectrum1D, ebv:float):

    # Correct for Galactic extinction
    AV = ebv * 3.1  # RV
    Al = extinction.ccm89(spectrum.wavelength.value, AV, 3.1)
    # New spec
    new_flux = spectrum.flux * 10**(Al/2.5)
    new_sig = spectrum.sig * 10**(Al/2.5)
    new_spec = xspectrum1d.XSpectrum1D.from_tuple((spectrum.wavelength, new_flux, new_sig))

    # Return
    return new_spec

def load_specdb(specdb_file=None):
    """
    Automatically load the specDB file from $SPECDB/FRB_specDB.hdf5

    Args:
        specdb_file (str, optional):
            Over-ride the default file

    Returns:
        specdb.specdb.SpecDB:

    """
    if not flg_specdb:
        raise IOError("You must install the specdb package first!")
        return
    if specdb_file is None:
        if os.getenv('SPECDB') is None:
            raise IOError("You must set the SPECDB environmental variable")
        specdb_files = glob.glob(os.path.join(os.getenv('SPECDB'), 'FRB_specDB_*.hdf5'))
        if len(specdb_files) > 0:
            specdb_file = specdb_files[0]
            print("Loading spectra from {:s}".format(specdb_file))
        else:
            raise IOError("There are no FRB_specdb.hdf5 files in your SPECDB folder")
    # Load it up
    specDB = SpecDB(db_file=specdb_file)
    # Return
    return specDB

def list_of_hosts():
    """
    Scan through the Repo and generate a list of FRB Host galaxies

    Also returns a list of the FRBs

    Returns:
        list, list:

    """
    # FRB files
    frb_data = resource_filename('frb', 'data')
    frb_files = glob.glob(os.path.join(frb_data, 'FRBs', 'FRB*.json'))
    frb_files.sort()

    hosts = []
    frbs = []
    for ifile in frb_files:
        # Parse
        name = ifile.split('.')[-2]
        ifrb = frb.FRB.by_name(name)
        host = ifrb.grab_host()
        if host is not None:
            hosts.append(host)
            frbs.append(ifrb)
    # Return
    return frbs, hosts

def build_table_of_hosts():
    """
    Generate a Pandas table of FRB Host galaxy data.  These are slurped
    from the 'derived', 'photom', and 'neb_lines' dicts of each host object

    Warning:  As standard, missing values are given NaN in the Pandas table
        Be careful!

    Note:
        RA, DEC are given as RA_host, DEC_host to avoid conflict with the FRB table

    Returns:
        pd.DataFrame, dict:  Table of data on FRB host galaxies,  dict of their units

    """
    frbs, hosts = list_of_hosts()
    nhosts = len(hosts)

    # Table
    host_tbl = pd.DataFrame({'Host': [host.name for host in hosts]})
    frb_names = [host.frb.frb_name for host in hosts]
    host_tbl['FRBname'] = frb_names
    tbl_units = {}

    # Coordinates
    coords = SkyCoord([host.coord for host in hosts])
    # Named to faciliate merging with an FRB table
    host_tbl['RA_host'] = coords.ra.value
    host_tbl['DEC_host'] = coords.dec.value
    tbl_units['RA_host'] = 'deg'
    tbl_units['DEC_host'] = 'deg'

    # FRBs
    host_tbl['FRBobj'] = frbs

    # Loop on all the main dicts
    for attr in ['derived', 'photom', 'neb_lines','offsets','morphology','redshift']:
        # Load up the dicts
        dicts = [getattr(host, attr) for host in hosts]

        # Photometry
        all_keys = []
        for idict in dicts:
            all_keys += list(idict.keys())
            #all_keys += list(host.photom.keys())
        #
        all_keys = np.array(all_keys)
        uni_keys = np.unique(all_keys)

        # Slurp using Nan's for missing values
        tbl_dict = {}
        for key in uni_keys:
            tbl_dict[key] = np.array([np.nan]*nhosts)
        for ss in range(nhosts): #, host in enumerate(hosts):
            for pkey in dicts[ss].keys(): #host.photom.keys():
                tbl_dict[pkey][ss] = dicts[ss][pkey]
        for key in tbl_dict.keys():
            # Error check
            if key in host_tbl.keys():
                raise IOError("Duplicate items!!")
            # Set
            host_tbl[key] = tbl_dict[key]
            tbl_units[key] = 'See galaxies.defs.py'

    # Return
    return host_tbl, tbl_units

<commit_msg>doc str<commit_after>""" Utilities related to FRB galaxies"""

import os
import glob
from IPython import embed
from pkg_resources import resource_filename
import numpy as np

try:
    from specdb.specdb import SpecDB
except ImportError:
    flg_specdb = False
else:
    flg_specdb = True

from astropy.coordinates import SkyCoord
import pandas as pd
import extinction
from linetools.spectra import xspectrum1d

from frb import frb

def deredden_spec(spectrum:xspectrum1d.XSpectrum1D, ebv:float):
    """ Deredden the input spectrum using the input EBV value

    Args:
        spectrum (xspectrum1d.XSpectrum1D): Spectrum
        ebv (float): Galactic reddening

    Returns:
        xspectrum1d.XSpectrum1D: De-reddened spectrum
    """

    # Correct for Galactic extinction
    AV = ebv * 3.1  # RV
    Al = extinction.ccm89(spectrum.wavelength.value, AV, 3.1)
    # New spec
    new_flux = spectrum.flux * 10**(Al/2.5)
    new_sig = spectrum.sig * 10**(Al/2.5)
    new_spec = xspectrum1d.XSpectrum1D.from_tuple((spectrum.wavelength, new_flux, new_sig))

    # Return
    return new_spec

def load_specdb(specdb_file=None):
    """
    Automatically load the specDB file from $SPECDB/FRB_specDB.hdf5

    Args:
        specdb_file (str, optional):
            Over-ride the default file

    Returns:
        specdb.specdb.SpecDB:

    """
    if not flg_specdb:
        raise IOError("You must install the specdb package first!")
        return
    if specdb_file is None:
        if os.getenv('SPECDB') is None:
            raise IOError("You must set the SPECDB environmental variable")
        specdb_files = glob.glob(os.path.join(os.getenv('SPECDB'), 'FRB_specDB_*.hdf5'))
        if len(specdb_files) > 0:
            specdb_file = specdb_files[0]
            print("Loading spectra from {:s}".format(specdb_file))
        else:
            raise IOError("There are no FRB_specdb.hdf5 files in your SPECDB folder")
    # Load it up
    specDB = SpecDB(db_file=specdb_file)
    # Return
    return specDB

def list_of_hosts():
    """
    Scan through the Repo and generate a list of FRB Host galaxies

    Also returns a list of the FRBs

    Returns:
        list, list:

    """
    # FRB files
    frb_data = resource_filename('frb', 'data')
    frb_files = glob.glob(os.path.join(frb_data, 'FRBs', 'FRB*.json'))
    frb_files.sort()

    hosts = []
    frbs = []
    for ifile in frb_files:
        # Parse
        name = ifile.split('.')[-2]
        ifrb = frb.FRB.by_name(name)
        host = ifrb.grab_host()
        if host is not None:
            hosts.append(host)
            frbs.append(ifrb)
    # Return
    return frbs, hosts

def build_table_of_hosts():
    """
    Generate a Pandas table of FRB Host galaxy data.  These are slurped
    from the 'derived', 'photom', and 'neb_lines' dicts of each host object

    Warning:  As standard, missing values are given NaN in the Pandas table
        Be careful!

    Note:
        RA, DEC are given as RA_host, DEC_host to avoid conflict with the FRB table

    Returns:
        pd.DataFrame, dict:  Table of data on FRB host galaxies,  dict of their units

    """
    frbs, hosts = list_of_hosts()
    nhosts = len(hosts)

    # Table
    host_tbl = pd.DataFrame({'Host': [host.name for host in hosts]})
    frb_names = [host.frb.frb_name for host in hosts]
    host_tbl['FRBname'] = frb_names
    tbl_units = {}

    # Coordinates
    coords = SkyCoord([host.coord for host in hosts])
    # Named to faciliate merging with an FRB table
    host_tbl['RA_host'] = coords.ra.value
    host_tbl['DEC_host'] = coords.dec.value
    tbl_units['RA_host'] = 'deg'
    tbl_units['DEC_host'] = 'deg'

    # FRBs
    host_tbl['FRBobj'] = frbs

    # Loop on all the main dicts
    for attr in ['derived', 'photom', 'neb_lines','offsets','morphology','redshift']:
        # Load up the dicts
        dicts = [getattr(host, attr) for host in hosts]

        # Photometry
        all_keys = []
        for idict in dicts:
            all_keys += list(idict.keys())
            #all_keys += list(host.photom.keys())
        #
        all_keys = np.array(all_keys)
        uni_keys = np.unique(all_keys)

        # Slurp using Nan's for missing values
        tbl_dict = {}
        for key in uni_keys:
            tbl_dict[key] = np.array([np.nan]*nhosts)
        for ss in range(nhosts): #, host in enumerate(hosts):
            for pkey in dicts[ss].keys(): #host.photom.keys():
                tbl_dict[pkey][ss] = dicts[ss][pkey]
        for key in tbl_dict.keys():
            # Error check
            if key in host_tbl.keys():
                raise IOError("Duplicate items!!")
            # Set
            host_tbl[key] = tbl_dict[key]
            tbl_units[key] = 'See galaxies.defs.py'

    # Return
    return host_tbl, tbl_units

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import hashlib
import base64

from .privacy import Privacy
from .branch import Branch

class Message:
    def __init__(self, data=None, key=None, source=None, verified=False,
                 signature=None, public_key=None, private_key=None,
                 privacy=Privacy.friends, timestamp=None,
                 in_reply_to=None, **scrap):

        self.data = data  #: The actual data, as a byte string
        self.key = key  #: The hash key for this message
        self.source = source  #: The global public key of the source of the message
        self.verified = verified  #: Whether or not the source is verified
        self.signature = signature  #: Signature of the message
        self.public_key = public_key  #: The public key for this message
        self.private_key = private_key  #: Your private key for this message
        self.privacy = privacy  #: Verification level
        self.timestamp = timestamp  #: non-trusted timestamp (unix epoch)
        self.in_reply_to = in_reply_to  #: key of a message in the same thread

        # Links, run-time only
        self.branch = None  #: the Branch this message is part of

    def __hash__(self):
        return hash(self.key)

    def __repr__(self):
        return '<Message %s>' % (self.key or ('?'))

    @classmethod
    def from_http(self, body, headers):
        d = {key.lower().replace('-', '_'): value for key, value in headers.items()}
        if 'public_key' in d:
            d['public_key'] = base64.b64decode(d['public_key'])
        if 'signature' in d:
            d['signature'] = base64.b64decode(d['signature'])
        if 'source' in d:
            d['source'] = base64.b64decode(d['source'])
        if d.get('in_reply_to', None) == 'null':
            d['in_reply_to'] = None
        return Message(data=body, **d)

    def to_http(self, for_sharing=False, friends=None):

        friends = friends or {}
        source_alias = None
        if self.source and self.source in friends:
            source_alias = '@' + friends.get(self.source)

        headers = {
            'Key': self.key if self.key else None,
            'Source': base64.b64encode(self.source).decode() if self.source else None,
            'Source-Alias': source_alias,
            'Verified': 'yes' if self.verified else 'no',
            'Signature': base64.b64encode(self.signature).decode() if self.signature else None,
            'Public-Key': base64.b64encode(self.public_key).decode() if self.public_key else None,
            'Timestamp': str(self.timestamp) if self.timestamp else None,
            'In-Reply-To': self.in_reply_to if self.in_reply_to else None,
        }

        if not for_sharing:
            headers.update({
                'Privacy': self.privacy.name,
            })

        headers = {key: value for key, value in headers.items() if value is not None}

        return self.data, headers

    def calculate_key(self):
        if self.data is None:
            raise ValueError("Message data can't be None")

        # key should be a product of data, source, public_key, in_reply_to
        string = self.data\
            + (self.public_key.encode('utf8') if self.public_key else bytes())\
            + (self.source.encode('utf8') if self.source else bytes())\
            + (self.in_reply_to.encode('utf8') if self.in_reply_to else bytes())
        self.key = hashlib.md5(string).hexdigest()

        # create a new branch if none exists
        if self.branch is None:
            self.branch = Branch()
            self.branch.insert(self)

    def to_dict(self, for_sharing=False):
        d = {
            'data': self.data,
            'key': self.key,
            'source': self.source,
            'verified': self.verified,
            'signature': self.signature,
            'public_key': self.public_key,
            'timestamp': self.timestamp,
            'in_reply_to': self.in_reply_to,
        }

        if not for_sharing:
            d.update({
                'private_key': self.private_key,
                'privacy': self.privacy,
            })

        return d

    def is_written_by_me(self):
        return self.private_key is not None

    def copy(self):
        """
        Perform a shallow copy of the message.
        """
        message = Message(**(self.to_dict()))
        message.branch = self.branch
        return message
<commit_msg>Messages have Content-Type<commit_after># -*- coding: utf-8 -*-

import hashlib
import base64

from .privacy import Privacy
from .branch import Branch

class Message:
    def __init__(self, data=None, key=None, source=None, verified=False,
                 signature=None, public_key=None, private_key=None,
                 privacy=Privacy.friends, timestamp=None,
                 in_reply_to=None, content_type='text/plain', **scrap):

        self.data = data  #: The actual data, as a byte string
        self.key = key  #: The hash key for this message
        self.source = source  #: The global public key of the source of the message
        self.verified = verified  #: Whether or not the source is verified
        self.signature = signature  #: Signature of the message
        self.public_key = public_key  #: The public key for this message
        self.private_key = private_key  #: Your private key for this message
        self.privacy = privacy  #: Verification level
        self.timestamp = timestamp  #: non-trusted timestamp (unix epoch)
        self.in_reply_to = in_reply_to  #: key of a message in the same thread
        self.content_type = content_type  #: MIME type of the message data

        # Links, run-time only
        self.branch = None  #: the Branch this message is part of

    def __hash__(self):
        return hash(self.key)

    def __repr__(self):
        return '<Message %s>' % (self.key or ('?'))

    @classmethod
    def from_http(self, body, headers):
        d = {key.lower().replace('-', '_'): value for key, value in headers.items()}
        if 'public_key' in d:
            d['public_key'] = base64.b64decode(d['public_key'])
        if 'signature' in d:
            d['signature'] = base64.b64decode(d['signature'])
        if 'source' in d:
            d['source'] = base64.b64decode(d['source'])
        if d.get('in_reply_to', None) == 'null':
            d['in_reply_to'] = None
        return Message(data=body, **d)

    def to_http(self, for_sharing=False, friends=None):

        friends = friends or {}
        source_alias = None
        if self.source and self.source in friends:
            source_alias = '@' + friends.get(self.source)

        headers = {
            'Key': self.key if self.key else None,
            'Source': base64.b64encode(self.source).decode() if self.source else None,
            'Source-Alias': source_alias,
            'Verified': 'yes' if self.verified else 'no',
            'Signature': base64.b64encode(self.signature).decode() if self.signature else None,
            'Public-Key': base64.b64encode(self.public_key).decode() if self.public_key else None,
            'Timestamp': str(self.timestamp) if self.timestamp else None,
            'In-Reply-To': self.in_reply_to if self.in_reply_to else None,
            'Content-Type': self.conrent_type or 'text/plain',
        }

        if not for_sharing:
            headers.update({
                'Privacy': self.privacy.name,
            })

        headers = {key: value for key, value in headers.items() if value is not None}

        return self.data, headers

    def calculate_key(self):
        if self.data is None:
            raise ValueError("Message data can't be None")

        # key should be a product of data, source, public_key, in_reply_to
        string = self.data\
            + (self.public_key.encode('utf8') if self.public_key else bytes())\
            + (self.source.encode('utf8') if self.source else bytes())\
            + (self.in_reply_to.encode('utf8') if self.in_reply_to else bytes())
        self.key = hashlib.md5(string).hexdigest()

        # create a new branch if none exists
        if self.branch is None:
            self.branch = Branch()
            self.branch.insert(self)

    def to_dict(self, for_sharing=False):
        d = {
            'data': self.data,
            'key': self.key,
            'source': self.source,
            'verified': self.verified,
            'signature': self.signature,
            'public_key': self.public_key,
            'timestamp': self.timestamp,
            'in_reply_to': self.in_reply_to,
            'content_type': self.content_type,
        }

        if not for_sharing:
            d.update({
                'private_key': self.private_key,
                'privacy': self.privacy,
            })

        return d

    def is_written_by_me(self):
        return self.private_key is not None

    def copy(self):
        """
        Perform a shallow copy of the message.
        """
        message = Message(**(self.to_dict()))
        message.branch = self.branch
        return message
<|endoftext|>


<commit_before># coding=utf-8
from django.test import TestCase
from frigg.builds.models import Build

class SimpleTest(TestCase):
    def test_parse_github_repo(self):
        project = Build.objects.create(git_repository="git@github.com:tind/balder.git",
                                       pull_request_id=1, branch="master")

        self.assertEqual(project.get_git_repo_owner_and_name()[0], "tind")
        self.assertEqual(project.get_git_repo_owner_and_name()[1], "balder")
<commit_msg>Remove test of code that is removed<commit_after><|endoftext|>


<commit_before>import logging
from datetime import datetime
from manager import ModuleWarning

class Entry(dict):

    """
        Represents one item in feed. Must have url and title fields.
        See. http://flexget.com/wiki/DevelopersEntry

        Internally stored original_url is neccessary because
        modules (ie. resolvers) may change this into something else
        and otherwise that information would be lost.
    """

    def __init__(self, *args):
        if len(args) == 2:
            self['title'] = args[0]
            self['url'] = args[1]

    def __setitem__(self, key, value):
        if key == 'url':
            if not self.has_key('original_url'):
                self['original_url'] = value
        dict.__setitem__(self, key, value)

    def safe_str(self):
        return "%s | %s" % (self['title'], self['url'])

    def isvalid(self):
        """Return True if entry is valid. Return False if this cannot be used."""
        if not self.has_key('title'):
            return False
        if not self.has_key('url'):
            return True
        return True

class ModuleCache:

    """
        Provides dictionary-like persistent storage for modules, allows saving key value pair
        for n number of days. Purges old entries to keep storage size in reasonable sizes.
    """

    log = logging.getLogger('modulecache')

    def __init__(self, name, storage):
        self.__storage = storage.setdefault(name, {})
        self._cache = None
        self.__namespace = None

    def set_namespace(self, name):
        self._cache = self.__storage.setdefault(name, {})
        self.__namespace = name
        self.__purge()

    def get_namespace(self):
        return self.__namespace

    def get_namespaces(self):
        """Return array of known namespaces in this cache"""
        return self.__storage.keys()

    def store(self, key, value, days=45):
        """Stores key value pair for number of days. Non yaml compatible values are not saved."""
        item = {}
        item['stored'] = datetime.today().strftime('%Y-%m-%d')
        item['days'] = days
        item['value'] = value
        self._cache[key] = item

    def storedefault(self, key, value, days=45):
        """Identical to dictionary setdefault"""
        undefined = object()
        item = self.get(key, undefined)
        if item is undefined:
            self.log.debug('storing default for %s, value %s' % (key, value))
            self.store(key, value, days)
            return self.get(key)
        else:
            return item

    def get(self, key, default=None):
        """Return value by key from cache. Return None or default if not found"""
        item = self._cache.get(key)
        if item is None:
            return default
        else:
            # reading value should update "stored" date .. TODO: refactor stored -> access & days -> keep?
            item['stored'] = datetime.today().strftime('%Y-%m-%d')
            return item['value']

    def __purge(self):
        """Remove all values from cache that have passed their expiry date"""
        now = datetime.today()
        for key in self._cache.keys():
            item = self._cache[key]
            y,m,d = item['stored'].split('-')
            stored = datetime(int(y), int(m), int(d))
            delta = now - stored
            if delta.days > item['days']:
                self.log.debug('Purging from cache %s' % (str(item)))
                self._cache.pop(key)

class Feed:

    def __init__(self, manager, name, config):
        """
            Represents one feed in configuration.

            name - name of the feed
            config - yaml configuration (dict)
        """
        self.name = name
        self.config = config
        self.manager = manager

        self.cache = ModuleCache(name, manager.session.setdefault('cache', {}))
        self.shared_cache = ModuleCache('_shared_', manager.session.setdefault('cache', {}))

        self.entries = []

        # You should NOT change these arrays at any point, and in most cases not even read them!
        # Mainly public since unittest needs these

        self.accepted = [] # accepted entries are always accepted, filtering does not affect them
        self.filtered = [] # filtered entries
        self.rejected = [] # rejected entries are removed unconditionally, even if accepted
        self.failed = []

        # flags and counters
        self.unittest = False
        self.__abort = False
        self.__purged = 0

        # state
        self.__current_event = None
        self.__current_module = None

    def _purge(self):
        """Purge filtered entries from feed. Call this from module only if you know what you're doing."""
        self.__purge(self.filtered, self.accepted)

    def __purge_failed(self):
        """Purge failed entries from feed."""
        self.__purge(self.failed, [], False)

    def __purge_rejected(self):
        """Purge rejected entries from feed."""
        self.__purge(self.rejected)

    def __purge(self, list, not_in_list=[], count=True):
        """Purge entries in list from feed.entries"""
        for entry in self.entries[:]:
            if entry in list and entry not in not_in_list:
                logging.debug('Purging entry %s' % entry.safe_str())
                self.entries.remove(entry)
                if count:
                    self.__purged += 1

    def accept(self, entry):
        """Accepts this entry."""
        if not entry in self.accepted:
            self.accepted.append(entry)
            if entry in self.filtered:
                self.filtered.remove(entry)
                self.verbose_details('Accepted previously filtered %s' % entry['title'])
            else:
                self.verbose_details('Accepted %s' % entry['title'])

    def filter(self, entry):
        """Mark entry to be filtered unless told otherwise. Entry may still be accepted."""
        # accepted checked only because it makes more sense when verbosing details
        if not entry in self.filtered and not entry in self.accepted:
            self.filtered.append(entry)
            self.verbose_details('Filtered %s' % entry['title'])

    def reject(self, entry):
        """Reject this entry immediattely and permanently."""
        # schedule immediately filtering after this module has done execution
        if not entry in self.rejected:
            self.rejected.append(entry)
            self.verbose_details('Rejected %s' % entry['title'])

    def fail(self, entry):
        """Mark entry as failed."""
        logging.debug("Marking entry '%s' as failed" % entry['title'])
        if not entry in self.failed:
            self.failed.append(entry)
            self.manager.add_failed(entry)
            self.verbose_details('Failed %s' % entry['title'])

    def abort(self, **kwargs):
        """Abort this feed execution, no more modules will be executed."""
        if not self.__abort and not kwargs.get('silent', False):
            logging.info('Aborting feed %s' % self.name)
        self.__abort = True
        self.__run_event('abort')

    def get_input_url(self, keyword):
        # TODO: move to better place?
        """
            Helper method for modules. Return url for a specified keyword.
            Supports configuration in following forms:
                <keyword>: <address>
            and
                <keyword>:
                    url: <address>
        """
        if isinstance(self.config[keyword], dict):
            if not self.config[keyword].has_key('url'):
                raise Exception('Input %s has invalid configuration, url is missing.' % keyword)
            return self.config[keyword]['url']
        else:
            return self.config[keyword]

    def log_once(self, s, log=logging):
        """Log string s once"""
        import md5
        m = md5.new()
        m.update(s)
        sum = m.hexdigest()
        seen = self.shared_cache.get('log-%s' % sum, False)
        if (seen): return
        self.shared_cache.store('log-%s' % sum, True, 30)
        log.info(s)

    # TODO: all these verbose methods are confusing
    def verbose_progress(self, s):
        """Verboses progress, outputs only in non quiet mode."""
        # TODO: implement trough own logger?
        if not self.manager.options.quiet and not self.unittest:
          logging.info(s)

    def verbose_details(self, s):
        """Verbose if details option is enabled"""
        # TODO: implement trough own logger?
        if self.manager.options.details:
            try:
                print "+ %-8s %-12s %s" % (self.__current_event, self.__current_module, s)
            except:
                print "+ %-8s %-12s ERROR: Unable to print %s" % (self.__current_event, self.__current_module, repr(s))

    def verbose_details_entries(self):
        """If details option is enabled, print all produced entries"""
        if self.manager.options.details:
            for entry in self.entries:
                self.verbose_details('%s' % entry['title'])

    def __get_priority(self, module, event):
        """Return order for module in this feed. Uses default value if no value is configured."""
        priority = module.get('priorities', {}).get('event', 0)
        keyword = module['name']
        if self.config.has_key(keyword):
            if isinstance(self.config[keyword], dict):
                priority = self.config[keyword].get('priority', priority)
        return priority

    def __sort_modules(self, a, b, event):
        a = self.__get_priority(a, event)
        b = self.__get_priority(b, event)
        return cmp(a, b)

    def __set_namespace(self, name):
        """Switch namespace in session"""
        self.cache.set_namespace(name)
        self.shared_cache.set_namespace(name)

    def __run_event(self, event):
        """Execute module events if module is configured for this feed."""
        modules = self.manager.get_modules_by_event(event)
        # Sort modules based on module event priority
        # Priority can be also configured in which case given value overwrites module default.
        modules.sort(lambda x,y: self.__sort_modules(x,y, event), reverse=True)

        for module in modules:
            keyword = module['name']
            if self.config.has_key(keyword) or (module['builtin'] and not self.config.get('disable_builtins', False)):
                # set cache namespaces to this module realm
                self.__set_namespace(keyword)
                # store execute info
                self.__current_event = event
                self.__current_module = keyword
                # call the module
                try:
                    method = self.manager.event_methods[event]
                    getattr(module['instance'], method)(self)
                except ModuleWarning, m:
                    # this warning should be logged only once (may keep repeating)
                    if m.kwargs.get('log_once', False):
                        self.log_once(m, m.log)
                    else:
                        m.log.warning(m)
                except Exception, e:
                    logging.exception('Unhandled error in module %s: %s' % (keyword, e))
                    self.abort()
                # remove entries
                self.__purge_rejected()
                self.__purge_failed()
                # check for priority operations
                if self.__abort: return

    def execute(self):
        """Execute this feed, runs events in order of events array."""
        # validate configuration
        errors = self.validate()
        if errors:
            logging.critical('Feed \'%s\' has configuration errors:' % self.name)
            for error in errors:
                logging.error(error)
            if not self.manager.options.validate:
                self.abort()
            return
        # do not execute in validate mode
        if self.manager.options.validate:
            if not errors:
                logging.info('Feed \'%s\' passed' % self.name)
            return
        # run events
        for event in self.manager.events:
            # when learning, skip few events
            if self.manager.options.learn:
                if event in ['download', 'output']:
                    # log keywords not executed
                    modules = self.manager.get_modules_by_event(event)
                    for module in modules:
                        if self.config.has_key(module['keyword']):
                            logging.info('Feed %s keyword %s is not executed because of learn/reset.' % (self.name, module['keyword']))
                    continue
            # run all modules with this event
            self.__run_event(event)
            # purge filtered entries between events
            # rejected and failed entries are purged between modules
            self._purge()
            # verbose some progress
            if event == 'input':
                self.verbose_details_entries()
                self.verbose_progress('Feed %s produced %s entries.' % (self.name, len(self.entries)))
            if event == 'filter':
                self.verbose_progress('Feed %s filtered %s entries (%s remains).' % (self.name, self.__purged, len(self.entries)))
            # if abort flag has been set feed should be aborted now
            if self.__abort:
                return

    def terminate(self):
        """Execute terminate event for this feed"""
        if self.__abort: return
        self.__run_event('terminate')

    def validate(self):
        """Module configuration validation. Return array of error messages that were detected."""
        validate_errors = []
        # validate all modules
        for kw, value in self.config.iteritems():
            module = self.manager.modules.get(kw)
            if not module:
                validate_errors.append('unknown keyword \'%s\'' % kw)
                continue
            if hasattr(module['instance'], 'validate'):
                errors = module['instance'].validate(self.config[kw])
                if errors:
                    for error in errors:
                        validate_errors.append('%s %s' % (kw, error))
            else:
                logging.warning('Used module %s does not support validating. Please notify author!' % kw)
        return validate_errors<commit_msg>Fixes --learn<commit_after>import logging
from datetime import datetime
from manager import ModuleWarning

class Entry(dict):

    """
        Represents one item in feed. Must have url and title fields.
        See. http://flexget.com/wiki/DevelopersEntry

        Internally stored original_url is neccessary because
        modules (ie. resolvers) may change this into something else
        and otherwise that information would be lost.
    """

    def __init__(self, *args):
        if len(args) == 2:
            self['title'] = args[0]
            self['url'] = args[1]

    def __setitem__(self, key, value):
        if key == 'url':
            if not self.has_key('original_url'):
                self['original_url'] = value
        dict.__setitem__(self, key, value)

    def safe_str(self):
        return "%s | %s" % (self['title'], self['url'])

    def isvalid(self):
        """Return True if entry is valid. Return False if this cannot be used."""
        if not self.has_key('title'):
            return False
        if not self.has_key('url'):
            return True
        return True

class ModuleCache:

    """
        Provides dictionary-like persistent storage for modules, allows saving key value pair
        for n number of days. Purges old entries to keep storage size in reasonable sizes.
    """

    log = logging.getLogger('modulecache')

    def __init__(self, name, storage):
        self.__storage = storage.setdefault(name, {})
        self._cache = None
        self.__namespace = None

    def set_namespace(self, name):
        self._cache = self.__storage.setdefault(name, {})
        self.__namespace = name
        self.__purge()

    def get_namespace(self):
        return self.__namespace

    def get_namespaces(self):
        """Return array of known namespaces in this cache"""
        return self.__storage.keys()

    def store(self, key, value, days=45):
        """Stores key value pair for number of days. Non yaml compatible values are not saved."""
        item = {}
        item['stored'] = datetime.today().strftime('%Y-%m-%d')
        item['days'] = days
        item['value'] = value
        self._cache[key] = item

    def storedefault(self, key, value, days=45):
        """Identical to dictionary setdefault"""
        undefined = object()
        item = self.get(key, undefined)
        if item is undefined:
            self.log.debug('storing default for %s, value %s' % (key, value))
            self.store(key, value, days)
            return self.get(key)
        else:
            return item

    def get(self, key, default=None):
        """Return value by key from cache. Return None or default if not found"""
        item = self._cache.get(key)
        if item is None:
            return default
        else:
            # reading value should update "stored" date .. TODO: refactor stored -> access & days -> keep?
            item['stored'] = datetime.today().strftime('%Y-%m-%d')
            return item['value']

    def __purge(self):
        """Remove all values from cache that have passed their expiry date"""
        now = datetime.today()
        for key in self._cache.keys():
            item = self._cache[key]
            y,m,d = item['stored'].split('-')
            stored = datetime(int(y), int(m), int(d))
            delta = now - stored
            if delta.days > item['days']:
                self.log.debug('Purging from cache %s' % (str(item)))
                self._cache.pop(key)

class Feed:

    def __init__(self, manager, name, config):
        """
            Represents one feed in configuration.

            name - name of the feed
            config - yaml configuration (dict)
        """
        self.name = name
        self.config = config
        self.manager = manager

        self.cache = ModuleCache(name, manager.session.setdefault('cache', {}))
        self.shared_cache = ModuleCache('_shared_', manager.session.setdefault('cache', {}))

        self.entries = []

        # You should NOT change these arrays at any point, and in most cases not even read them!
        # Mainly public since unittest needs these

        self.accepted = [] # accepted entries are always accepted, filtering does not affect them
        self.filtered = [] # filtered entries
        self.rejected = [] # rejected entries are removed unconditionally, even if accepted
        self.failed = []

        # flags and counters
        self.unittest = False
        self.__abort = False
        self.__purged = 0

        # state
        self.__current_event = None
        self.__current_module = None

    def _purge(self):
        """Purge filtered entries from feed. Call this from module only if you know what you're doing."""
        self.__purge(self.filtered, self.accepted)

    def __purge_failed(self):
        """Purge failed entries from feed."""
        self.__purge(self.failed, [], False)

    def __purge_rejected(self):
        """Purge rejected entries from feed."""
        self.__purge(self.rejected)

    def __purge(self, list, not_in_list=[], count=True):
        """Purge entries in list from feed.entries"""
        for entry in self.entries[:]:
            if entry in list and entry not in not_in_list:
                logging.debug('Purging entry %s' % entry.safe_str())
                self.entries.remove(entry)
                if count:
                    self.__purged += 1

    def accept(self, entry):
        """Accepts this entry."""
        if not entry in self.accepted:
            self.accepted.append(entry)
            if entry in self.filtered:
                self.filtered.remove(entry)
                self.verbose_details('Accepted previously filtered %s' % entry['title'])
            else:
                self.verbose_details('Accepted %s' % entry['title'])

    def filter(self, entry):
        """Mark entry to be filtered unless told otherwise. Entry may still be accepted."""
        # accepted checked only because it makes more sense when verbosing details
        if not entry in self.filtered and not entry in self.accepted:
            self.filtered.append(entry)
            self.verbose_details('Filtered %s' % entry['title'])

    def reject(self, entry):
        """Reject this entry immediattely and permanently."""
        # schedule immediately filtering after this module has done execution
        if not entry in self.rejected:
            self.rejected.append(entry)
            self.verbose_details('Rejected %s' % entry['title'])

    def fail(self, entry):
        """Mark entry as failed."""
        logging.debug("Marking entry '%s' as failed" % entry['title'])
        if not entry in self.failed:
            self.failed.append(entry)
            self.manager.add_failed(entry)
            self.verbose_details('Failed %s' % entry['title'])

    def abort(self, **kwargs):
        """Abort this feed execution, no more modules will be executed."""
        if not self.__abort and not kwargs.get('silent', False):
            logging.info('Aborting feed %s' % self.name)
        self.__abort = True
        self.__run_event('abort')

    def get_input_url(self, keyword):
        # TODO: move to better place?
        """
            Helper method for modules. Return url for a specified keyword.
            Supports configuration in following forms:
                <keyword>: <address>
            and
                <keyword>:
                    url: <address>
        """
        if isinstance(self.config[keyword], dict):
            if not self.config[keyword].has_key('url'):
                raise Exception('Input %s has invalid configuration, url is missing.' % keyword)
            return self.config[keyword]['url']
        else:
            return self.config[keyword]

    def log_once(self, s, log=logging):
        """Log string s once"""
        import md5
        m = md5.new()
        m.update(s)
        sum = m.hexdigest()
        seen = self.shared_cache.get('log-%s' % sum, False)
        if (seen): return
        self.shared_cache.store('log-%s' % sum, True, 30)
        log.info(s)

    # TODO: all these verbose methods are confusing
    def verbose_progress(self, s):
        """Verboses progress, outputs only in non quiet mode."""
        # TODO: implement trough own logger?
        if not self.manager.options.quiet and not self.unittest:
          logging.info(s)

    def verbose_details(self, s):
        """Verbose if details option is enabled"""
        # TODO: implement trough own logger?
        if self.manager.options.details:
            try:
                print "+ %-8s %-12s %s" % (self.__current_event, self.__current_module, s)
            except:
                print "+ %-8s %-12s ERROR: Unable to print %s" % (self.__current_event, self.__current_module, repr(s))

    def verbose_details_entries(self):
        """If details option is enabled, print all produced entries"""
        if self.manager.options.details:
            for entry in self.entries:
                self.verbose_details('%s' % entry['title'])

    def __get_priority(self, module, event):
        """Return order for module in this feed. Uses default value if no value is configured."""
        priority = module.get('priorities', {}).get('event', 0)
        keyword = module['name']
        if self.config.has_key(keyword):
            if isinstance(self.config[keyword], dict):
                priority = self.config[keyword].get('priority', priority)
        return priority

    def __sort_modules(self, a, b, event):
        a = self.__get_priority(a, event)
        b = self.__get_priority(b, event)
        return cmp(a, b)

    def __set_namespace(self, name):
        """Switch namespace in session"""
        self.cache.set_namespace(name)
        self.shared_cache.set_namespace(name)

    def __run_event(self, event):
        """Execute module events if module is configured for this feed."""
        modules = self.manager.get_modules_by_event(event)
        # Sort modules based on module event priority
        # Priority can be also configured in which case given value overwrites module default.
        modules.sort(lambda x,y: self.__sort_modules(x,y, event), reverse=True)

        for module in modules:
            keyword = module['name']
            if self.config.has_key(keyword) or (module['builtin'] and not self.config.get('disable_builtins', False)):
                # set cache namespaces to this module realm
                self.__set_namespace(keyword)
                # store execute info
                self.__current_event = event
                self.__current_module = keyword
                # call the module
                try:
                    method = self.manager.event_methods[event]
                    getattr(module['instance'], method)(self)
                except ModuleWarning, m:
                    # this warning should be logged only once (may keep repeating)
                    if m.kwargs.get('log_once', False):
                        self.log_once(m, m.log)
                    else:
                        m.log.warning(m)
                except Exception, e:
                    logging.exception('Unhandled error in module %s: %s' % (keyword, e))
                    self.abort()
                # remove entries
                self.__purge_rejected()
                self.__purge_failed()
                # check for priority operations
                if self.__abort: return

    def execute(self):
        """Execute this feed, runs events in order of events array."""
        # validate configuration
        errors = self.validate()
        if errors:
            logging.critical('Feed \'%s\' has configuration errors:' % self.name)
            for error in errors:
                logging.error(error)
            if not self.manager.options.validate:
                self.abort()
            return
        # do not execute in validate mode
        if self.manager.options.validate:
            if not errors:
                logging.info('Feed \'%s\' passed' % self.name)
            return
        # run events
        for event in self.manager.events:
            # when learning, skip few events
            if self.manager.options.learn:
                if event in ['download', 'output']:
                    # log keywords not executed
                    modules = self.manager.get_modules_by_event(event)
                    for module in modules:
                        if self.config.has_key(module['name']):
                            logging.info('Feed %s keyword %s is not executed because of learn/reset.' % (self.name, module['name']))
                    continue
            # run all modules with this event
            self.__run_event(event)
            # purge filtered entries between events
            # rejected and failed entries are purged between modules
            self._purge()
            # verbose some progress
            if event == 'input':
                self.verbose_details_entries()
                self.verbose_progress('Feed %s produced %s entries.' % (self.name, len(self.entries)))
            if event == 'filter':
                self.verbose_progress('Feed %s filtered %s entries (%s remains).' % (self.name, self.__purged, len(self.entries)))
            # if abort flag has been set feed should be aborted now
            if self.__abort:
                return

    def terminate(self):
        """Execute terminate event for this feed"""
        if self.__abort: return
        self.__run_event('terminate')

    def validate(self):
        """Module configuration validation. Return array of error messages that were detected."""
        validate_errors = []
        # validate all modules
        for kw, value in self.config.iteritems():
            module = self.manager.modules.get(kw)
            if not module:
                validate_errors.append('unknown keyword \'%s\'' % kw)
                continue
            if hasattr(module['instance'], 'validate'):
                errors = module['instance'].validate(self.config[kw])
                if errors:
                    for error in errors:
                        validate_errors.append('%s %s' % (kw, error))
            else:
                logging.warning('Used module %s does not support validating. Please notify author!' % kw)
        return validate_errors<|endoftext|>


<commit_before>import itertools
import subprocess
from collections import OrderedDict
from datetime import timedelta

import django_filters
from django.http.response import StreamingHttpResponse, JsonResponse
from django.utils.html import escape
from django.views import View
from django_filters.rest_framework import DjangoFilterBackend, filters
from rest_framework import serializers
from rest_framework import viewsets
from rest_framework.decorators import api_view
from rest_framework.request import Request
from rest_framework.serializers import Serializer

from config.config import PHP_MAIN_SCRIPT
from frontend.api.models import TweetCountCache, Cluster
from frontend.api.serializer import TweetCountCacheSerializer, ClusterSerializer
from frontend.base.pagination import LimitedPagenumbrPagination

def stream_command_output_git():
    return itertools.chain(stream_command_output('git pull'), '---', stream_command_output('git status'))

def stream_command_output(command):
    yield "
"
    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    while True:
        out = process.stdout.read(1024).decode("utf-8")
        if out == '' and process.poll() is not None:
            break
        if out != '':
            yield escape(out)
    yield "
"

class TestPhpOutputView(View):
    def get(self, request, *args, **kwargs):
        php_args = 'test_command'
        return StreamingHttpResponse(stream_command_output(PHP_MAIN_SCRIPT + ' ' + php_args))

# Tweet Counts ################################################################
class TweetCountCacheViewSet(viewsets.ReadOnlyModelViewSet):
    """
    ### Get the tweet count per date.
    If jQuery is used the array should be passed after `$.ajaxSettings.traditional = true;`

    The endpoint of tweet counts per week is in GET [tweet_count_week](/tweet_count_week)
    You do have to supply start and end parameters formatted as "YEAR weeknumber", such as: [tweet_count_week/?start=2016 1&end=2016 3](/tweet_count_week/?start=2016%201&end=2016%203)
    """
    serializer_class = TweetCountCacheSerializer
    queryset = TweetCountCache.objects.all()

class TweetCountWeeksValidator(Serializer):
    start = serializers.DateField(input_formats=['%Y %W %w'])  # DateField
    end = serializers.DateField(input_formats=['%Y %W %w'])  # DateField

@api_view(['GET'])
def get_tweet_counts_week(request: Request):
    data = request.query_params.copy()
    if not data['start'].endswith(' 0'):
        data['start'] += ' 0'
    if not data['end'].endswith(' 0'):
        data['end'] += ' 0'
    v = TweetCountWeeksValidator(data=data)
    v.is_valid(raise_exception=True)
    start = v.validated_data['start']
    end = v.validated_data['end']

    result = OrderedDict()

    week_start = start
    while week_start <= end:
        result_key = week_start.strftime('%Y %W')

        n = 0
        d = week_start
        next_week_start = week_start + timedelta(weeks=1)
        while d < next_week_start:
            tc = TweetCountCache.get_or_create(day=d)
            n += tc.count

            d += timedelta(days=1)

        result[result_key] = n

        week_start = next_week_start

    return JsonResponse(result)

class TweetCountDaysValidator(Serializer):
    start = serializers.DateField(input_formats=['%d-%m-%Y'])  # DateField
    end = serializers.DateField(input_formats=['%d-%m-%Y'])  # DateField

@api_view(['GET'])
def get_tweet_counts_day(request: Request):
    data = request.query_params.copy()
    v = TweetCountDaysValidator(data=data)
    v.is_valid(raise_exception=True)
    start = v.validated_data['start']
    end = v.validated_data['end']

    result = OrderedDict()
    #result = {}

    n = 0
    d = start
    while d <= end:
        result_key = d.strftime('%d-%m-%Y')
        tc = TweetCountCache.get_or_create(day=d)
        n += tc.count
        d += timedelta(days=1)

        result[result_key] = n

    return JsonResponse(result)

# Tweet Clusters ################################################################
class ArticleFilter(django_filters.rest_framework.FilterSet):
    article = django_filters.CharFilter(name="article__article_id")

    class Meta:
        model = Cluster
        fields = ['article']

class ClusterViewSet(viewsets.ReadOnlyModelViewSet):
    """
    These are all the clusters that we have
    """
    serializer_class = ClusterSerializer
    # queryset = Cluster.objects.filter(article__article_id='r405155958')
    queryset = Cluster.objects.all()
    pagination_class = LimitedPagenumbrPagination
    filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
    filter_class = ArticleFilter

class ClusterManualViewSet(viewsets.ReadOnlyModelViewSet):
    """
    Get all tweet clusters that have not yet been manually rated
    """
    serializer_class = ClusterSerializer
    queryset = Cluster.objects.filter()
    pagination_class = LimitedPagenumbrPagination
<commit_msg>fixed bug in GET /tweet_count_day<commit_after>import itertools
import subprocess
from collections import OrderedDict
from datetime import timedelta

import django_filters
from django.http.response import StreamingHttpResponse, JsonResponse
from django.utils.html import escape
from django.views import View
from django_filters.rest_framework import DjangoFilterBackend, filters
from rest_framework import serializers
from rest_framework import viewsets
from rest_framework.decorators import api_view
from rest_framework.request import Request
from rest_framework.serializers import Serializer

from config.config import PHP_MAIN_SCRIPT
from frontend.api.models import TweetCountCache, Cluster
from frontend.api.serializer import TweetCountCacheSerializer, ClusterSerializer
from frontend.base.pagination import LimitedPagenumbrPagination

def stream_command_output_git():
    return itertools.chain(stream_command_output('git pull'), '---', stream_command_output('git status'))

def stream_command_output(command):
    yield "
"
    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    while True:
        out = process.stdout.read(1024).decode("utf-8")
        if out == '' and process.poll() is not None:
            break
        if out != '':
            yield escape(out)
    yield "
"

class TestPhpOutputView(View):
    def get(self, request, *args, **kwargs):
        php_args = 'test_command'
        return StreamingHttpResponse(stream_command_output(PHP_MAIN_SCRIPT + ' ' + php_args))

# Tweet Counts ################################################################
class TweetCountCacheViewSet(viewsets.ReadOnlyModelViewSet):
    """
    ### Get the tweet count per date.
    If jQuery is used the array should be passed after `$.ajaxSettings.traditional = true;`

    The endpoint of tweet counts per week is in GET [tweet_count_week](/tweet_count_week)
    You do have to supply start and end parameters formatted as "YEAR weeknumber", such as: [tweet_count_week/?start=2016 1&end=2016 3](/tweet_count_week/?start=2016%201&end=2016%203)
    """
    serializer_class = TweetCountCacheSerializer
    queryset = TweetCountCache.objects.all()

class TweetCountWeeksValidator(Serializer):
    start = serializers.DateField(input_formats=['%Y %W %w'])  # DateField
    end = serializers.DateField(input_formats=['%Y %W %w'])  # DateField

@api_view(['GET'])
def get_tweet_counts_week(request: Request):
    data = request.query_params.copy()
    if not data['start'].endswith(' 0'):
        data['start'] += ' 0'
    if not data['end'].endswith(' 0'):
        data['end'] += ' 0'
    v = TweetCountWeeksValidator(data=data)
    v.is_valid(raise_exception=True)
    start = v.validated_data['start']
    end = v.validated_data['end']

    result = OrderedDict()

    week_start = start
    while week_start <= end:
        result_key = week_start.strftime('%Y %W')

        n = 0
        d = week_start
        next_week_start = week_start + timedelta(weeks=1)
        while d < next_week_start:
            tc = TweetCountCache.get_or_create(day=d)
            n += tc.count

            d += timedelta(days=1)

        result[result_key] = n

        week_start = next_week_start

    return JsonResponse(result)

class TweetCountDaysValidator(Serializer):
    start = serializers.DateField(input_formats=['%d-%m-%Y'])  # DateField
    end = serializers.DateField(input_formats=['%d-%m-%Y'])  # DateField

@api_view(['GET'])
def get_tweet_counts_day(request: Request):
    data = request.query_params.copy()
    v = TweetCountDaysValidator(data=data)
    v.is_valid(raise_exception=True)
    start = v.validated_data['start']
    end = v.validated_data['end']

    result = OrderedDict()
    #result = {}

    d = start
    while d <= end:
        result_key = d.strftime('%d-%m-%Y')
        tc = TweetCountCache.get_or_create(day=d)
        d += timedelta(days=1)

        result[result_key] = tc.count

    return JsonResponse(result)

# Tweet Clusters ################################################################
class ArticleFilter(django_filters.rest_framework.FilterSet):
    article = django_filters.CharFilter(name="article__article_id")

    class Meta:
        model = Cluster
        fields = ['article']

class ClusterViewSet(viewsets.ReadOnlyModelViewSet):
    """
    These are all the clusters that we have
    """
    serializer_class = ClusterSerializer
    # queryset = Cluster.objects.filter(article__article_id='r405155958')
    queryset = Cluster.objects.all()
    pagination_class = LimitedPagenumbrPagination
    filter_backends = (django_filters.rest_framework.DjangoFilterBackend,)
    filter_class = ArticleFilter

class ClusterManualViewSet(viewsets.ReadOnlyModelViewSet):
    """
    Get all tweet clusters that have not yet been manually rated
    """
    serializer_class = ClusterSerializer
    queryset = Cluster.objects.filter()
    pagination_class = LimitedPagenumbrPagination
<|endoftext|>


<commit_before>import os
import re
from typing import Dict, Optional

from commitizen import factory, git, out
from commitizen.config import BaseConfig
from commitizen.exceptions import (
    InvalidCommandArgumentError,
    InvalidCommitMessageError,
    NoCommitsFoundError,
)

class Check:
    """Check if the current commit msg matches the commitizen format."""

    def __init__(self, config: BaseConfig, arguments: Dict[str, str], cwd=os.getcwd()):
        """Initial check command.

        Args:
            config: The config object required for the command to perform its action
            arguments: All the flags provided by the user
            cwd: Current work directory
        """
        self.commit_msg_file: Optional[str] = arguments.get("commit_msg_file")
        self.rev_range: Optional[str] = arguments.get("rev_range")

        self._valid_command_argument()

        self.config: BaseConfig = config
        self.cz = factory.commiter_factory(self.config)

    def _valid_command_argument(self):
        if bool(self.commit_msg_file) is bool(self.rev_range):
            raise InvalidCommandArgumentError(
                (
                    "One and only one argument is required for check command! "
                    "See 'cz check -h' for more information"
                )
            )

    def __call__(self):
        """Validate if commit messages follows the conventional pattern.

        Raises:
            InvalidCommitMessageError: if the commit provided not follows the conventional pattern
        """
        commits = self._get_commits()
        if not commits:
            raise NoCommitsFoundError(f"No commit found with range: '{self.rev_range}'")

        pattern = self.cz.schema_pattern()
        ill_formated_commits = [
            commit
            for commit in commits
            if not Check.validate_commit_message(commit.message, pattern)
        ]
        displayed_msgs_content = "\n".join(
            [
                f"commit {commit.rev}: {commit.message}"
                for commit in ill_formated_commits
            ]
        )
        if displayed_msgs_content:
            raise InvalidCommitMessageError(
                "commit validation: failed!\n"
                "please enter a commit message in the commitizen format.\n"
                f"{displayed_msgs_content}\n"
                f"pattern: {pattern}"
            )
        out.success("Commit validation: successful!")

    def _get_commits(self):
        # Get commit message from file (--commit-msg-file)
        if self.commit_msg_file:
            with open(self.commit_msg_file, "r") as commit_file:
                commit_title = commit_file.readline()
                commit_body = commit_file.read()
            return [git.GitCommit(rev="", title=commit_title, body=commit_body)]

        # Get commit messages from git log (--rev-range)
        return git.get_commits(end=self.rev_range)

    @staticmethod
    def validate_commit_message(commit_msg: str, pattern: str) -> bool:
        if commit_msg.startswith("Merge") or commit_msg.startswith("Revert"):
            return True
        return bool(re.match(pattern, commit_msg))
<commit_msg>style(cz_check): Add quote to displaying rev and msg<commit_after>import os
import re
from typing import Dict, Optional

from commitizen import factory, git, out
from commitizen.config import BaseConfig
from commitizen.exceptions import (
    InvalidCommandArgumentError,
    InvalidCommitMessageError,
    NoCommitsFoundError,
)

class Check:
    """Check if the current commit msg matches the commitizen format."""

    def __init__(self, config: BaseConfig, arguments: Dict[str, str], cwd=os.getcwd()):
        """Initial check command.

        Args:
            config: The config object required for the command to perform its action
            arguments: All the flags provided by the user
            cwd: Current work directory
        """
        self.commit_msg_file: Optional[str] = arguments.get("commit_msg_file")
        self.rev_range: Optional[str] = arguments.get("rev_range")

        self._valid_command_argument()

        self.config: BaseConfig = config
        self.cz = factory.commiter_factory(self.config)

    def _valid_command_argument(self):
        if bool(self.commit_msg_file) is bool(self.rev_range):
            raise InvalidCommandArgumentError(
                (
                    "One and only one argument is required for check command! "
                    "See 'cz check -h' for more information"
                )
            )

    def __call__(self):
        """Validate if commit messages follows the conventional pattern.

        Raises:
            InvalidCommitMessageError: if the commit provided not follows the conventional pattern
        """
        commits = self._get_commits()
        if not commits:
            raise NoCommitsFoundError(f"No commit found with range: '{self.rev_range}'")

        pattern = self.cz.schema_pattern()
        ill_formated_commits = [
            commit
            for commit in commits
            if not Check.validate_commit_message(commit.message, pattern)
        ]
        displayed_msgs_content = "\n".join(
            [
                f'commit "{commit.rev}": "{commit.message}"'
                for commit in ill_formated_commits
            ]
        )
        if displayed_msgs_content:
            raise InvalidCommitMessageError(
                "commit validation: failed!\n"
                "please enter a commit message in the commitizen format.\n"
                f"{displayed_msgs_content}\n"
                f"pattern: {pattern}"
            )
        out.success("Commit validation: successful!")

    def _get_commits(self):
        # Get commit message from file (--commit-msg-file)
        if self.commit_msg_file:
            with open(self.commit_msg_file, "r") as commit_file:
                commit_title = commit_file.readline()
                commit_body = commit_file.read()
            return [git.GitCommit(rev="", title=commit_title, body=commit_body)]

        # Get commit messages from git log (--rev-range)
        return git.get_commits(end=self.rev_range)

    @staticmethod
    def validate_commit_message(commit_msg: str, pattern: str) -> bool:
        if commit_msg.startswith("Merge") or commit_msg.startswith("Revert"):
            return True
        return bool(re.match(pattern, commit_msg))
<|endoftext|>


<commit_before>import os
import io
from typing import Dict, Optional, Tuple, Union

import yamldown

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

class GoRef:
    def __init__(self, goref_path: str) -> None:
        self.yd_path = goref_path

    @property
    def goref_path(self):
        return os.path.join(self.yd_path)

    @goref_path.setter
    def goref_path(self, goref_path: str):
        self.yd_path = goref_path

    def parse(
        self, portion: Optional[str] = None
    ) -> Union[Tuple[Dict, str], Dict, str]:
        with open(self.goref_path, "r") as file:
            self.yd_content = file.read()

        f = io.StringIO(self.yd_content)
        yaml, md = yamldown.load(f)

        if portion == "yaml":
            return yaml
        elif portion == "md":
            return md

        return (yaml, md)
<commit_msg>change instance variable in goref_path property setter<commit_after>import os
import io
from typing import Dict, Optional, Tuple, Union

import yamldown

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

class GoRef:
    def __init__(self, goref_path: str) -> None:
        self.yd_path = goref_path

    @property
    def goref_path(self):
        return os.path.join(self.yd_path)

    @goref_path.setter
    def goref_path(self, goref_path: str):
        self.yd_path = goref_path

    def parse(
        self, portion: Optional[str] = None
    ) -> Union[Tuple[Dict, str], Dict, str]:
        with open(self.yd_path, "r") as file:
            yd_content = file.read()

        f = io.StringIO(yd_content)
        yaml, md = yamldown.load(f)

        if portion == "yaml":
            return yaml
        elif portion == "md":
            return md

        return (yaml, md)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## $Id: taskDaemon.py,v 1.16 2008/08/13 13:31:38 jose Exp $
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import sys, getopt, os, time
from datetime import timedelta, datetime
from pytz import timezone
try:
    from MaKaC.conference import CategoryManager
    from MaKaC.common import DBMgr
    from MaKaC.common.timerExec import timer, HelperTaskList, StatisticsUpdater, task
    from MaKaC.common.timezoneUtils import nowutc
except ImportError, e:
    print "ImportError:%s"%e
    sys.exit(0)

duration = 15*60 # seconds
linuxPIDFile="/var/run/IndicoTaskDaemon.pid"
logFile="/opt/indico/log/taskDaemon.log"

def _isPosix():
    return os.name == "posix"

def _isWindows():
    return os.name == "nt"

def _savePID(pid):
    try:
        f=open(linuxPIDFile, "w")
        f.write(str(pid))
        f.close()
    except IOError, e:
        print "Exception while trying to store the daemon PID in the file '%s':\n%s"%(linuxPIDFile, e)

def _getPID(printExceptions=True):
    try:
        f=open(linuxPIDFile, "r")
        pid=f.read()
        pid=pid.strip()
        pid=int(pid)
        f.close()
        return pid
    except IOError, e:
        if printExceptions:
            print "Exception while opening the file '%s' with the daemon PID:\n%s"%(linuxPIDFile, e)
    except ValueError, e:
        if printExceptions:
            print "Exception while reading the file '%s' with the daemon PID:\nThere is not a PID in the file"%(linuxPIDFile)
    return -1

def _killProcess():
    if _isPosix():
        pid=_getPID()
        if pid != -1:
            try:
                os.kill(pid, 9)
            except OSError, e:
                print e
                return False
            os.remove(linuxPIDFile)
            return True
    return False

def initDefaultTasks():
    print "\nTrying to connect with the database...\nIf you don't see any following message, please make sure that you have already started the database because you might have a connection problem with it.\n"
    DBMgr.getInstance().startRequest()
    print "Database connection established."
    # ------------------------ STATISTICS ------------------------
    tl = HelperTaskList.getTaskListInstance()
    catRoot = CategoryManager().getRoot()
    try:
        if catRoot.statsUpdater != None:
            tl.removeTask(catRoot.statsUpdater)
            catRoot.statsUpdater = None
    except:
        catRoot.statsUpdater = None
    su = StatisticsUpdater(catRoot)
    su.setId("StatisticsUpdater")
    ta = task()
    d = nowutc() + timedelta(days=1)
    ta.setStartDate(datetime(d.year, d.month, d.day, 0, 0, tzinfo=timezone('UTC')))
    ta.setInterval(timedelta(days=1))
    ta.addObj(su)
    tl.addTask(ta)
    catRoot.statsUpdater = ta
    print "Added statistics updater task to the tasks list."
    # --------------------- END STATISTICS ---------------------
    DBMgr.getInstance().endRequest()
    print ""

def fileExists(f):
     try:
        fi = open(f,'a')
        fi.close()
     except IOError:
         exists = False
     else:
         exists = True
     return exists

def run(log=None):
    if _isPosix() and (not log or not fileExists(log)):
        print "\nPlease enter an existing log file, and start again\nDaemon stopped!\n"
        os.remove(linuxPIDFile)
        sys.exit(2)
    t = timer(duration, log)
    initDefaultTasks()
    t.start()

def startTaskDaemon():
    """
    Run the function listed in the toExec instance each 'duration' seconds
    """
    oldPID=_getPID(False)
    if oldPID!=-1:
        print "Task daemon is already running"
        sys.exit(0)
    if _isPosix():
        pid=os.fork()
        if pid==0:
            #Child
            run(log=logFile)
        else:
            #Parent
            _savePID(pid)
            print "Daemon started! (pid=%s)\n"%pid
            time.sleep(3)
    elif _isWindows():
        print "Close this window to stop the server"
        run()
    else:
        print "Impossible to start the daemon"

def stopTaskDaemon():
    if _killProcess():
        print "Daemon stopped!"
    else:
        print "Impossible to stop the process 'taskDaemon', please do it manually"

def restartTaskDaemon():
    _killProcess()
    print "Daemon stopped!"
    startTaskDaemon()

def main():
    "Main"
    try:
        opts, args = getopt.getopt(sys.argv[1:],"hstr", \
                 [ \
                   "help", \
                   "start", \
		   "stop", \
		   "restart" \
                 ]
        )
    except getopt.GetoptError, e:
        help()
        sys.exit(2)

    arg=None
    if len(args) == 1:
        arg=args[0]
    elif len(opts)==1:
        arg=opts[0][0]
    if arg is not None:
        if   arg in ("start", "s", "-s"):
            startTaskDaemon()
        elif arg in ("stop", "t", "-t"):
            stopTaskDaemon()
        elif arg in ("restart", "r", "-r"):
            restartTaskDaemon()
        elif arg in ("help", "h", "-h", "--help"):
            help()
            sys.exit()
        else:
            help()
            sys.exit(2)
        return
    else:
        help()
        sys.exit(2)

def help():
        "Print out info"

        print "\nUsage: python taskDaemon.py  start|stop|restart [-h]\n"
        print "  -h                    Help"
        print "  -s                    Start the task daemon"
        print "  -t                    Stop the task daemon"
        print "  -r                    Restart the task daemon"

if __name__ == "__main__":
    main()
<commit_msg>[FIXED] Problem with taskdaemon<commit_after># -*- coding: utf-8 -*-
##
## $Id: taskDaemon.py,v 1.16 2008/08/13 13:31:38 jose Exp $
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

import sys, getopt, os, time

duration = 15*60 # seconds
linuxPIDFile="/var/run/IndicoTaskDaemon.pid"
logFile="/opt/indico/log/taskDaemon.log"

def _isPosix():
    return os.name == "posix"

def _isWindows():
    return os.name == "nt"

def _savePID(pid):
    try:
        f=open(linuxPIDFile, "w")
        f.write(str(pid))
        f.close()
    except IOError, e:
        print "Exception while trying to store the daemon PID in the file '%s':\n%s"%(linuxPIDFile, e)

def _getPID(printExceptions=True):
    try:
        f=open(linuxPIDFile, "r")
        pid=f.read()
        pid=pid.strip()
        pid=int(pid)
        f.close()
        return pid
    except IOError, e:
        if printExceptions:
            print "Exception while opening the file '%s' with the daemon PID:\n%s"%(linuxPIDFile, e)
    except ValueError, e:
        if printExceptions:
            print "Exception while reading the file '%s' with the daemon PID:\nThere is not a PID in the file"%(linuxPIDFile)
    return -1

def _killProcess():
    if _isPosix():
        pid=_getPID()
        if pid != -1:
            try:
                os.kill(pid, 9)
            except OSError, e:
                print e
                return False
            os.remove(linuxPIDFile)
            return True
    return False

def initDefaultTasks():
    from datetime import timedelta, datetime
    from pytz import timezone
    try:
        from MaKaC.conference import CategoryManager
        from MaKaC.common import DBMgr
        from MaKaC.common.timerExec import HelperTaskList, StatisticsUpdater, task
        from MaKaC.common.timezoneUtils import nowutc
    except ImportError, e:
        print "ImportError:%s"%e
        sys.exit(0)

    print "\nTrying to connect with the database...\nIf you don't see any following message, please make sure that you have already started the database because you might have a connection problem with it.\n"
    DBMgr.getInstance().startRequest()
    print "Database connection established."
    # ------------------------ STATISTICS ------------------------
    tl = HelperTaskList.getTaskListInstance()
    catRoot = CategoryManager().getRoot()
    try:
        if catRoot.statsUpdater != None:
            tl.removeTask(catRoot.statsUpdater)
            catRoot.statsUpdater = None
    except:
        catRoot.statsUpdater = None
    su = StatisticsUpdater(catRoot)
    su.setId("StatisticsUpdater")
    ta = task()
    d = nowutc() + timedelta(days=1)
    ta.setStartDate(datetime(d.year, d.month, d.day, 0, 0, tzinfo=timezone('UTC')))
    ta.setInterval(timedelta(days=1))
    ta.addObj(su)
    tl.addTask(ta)
    catRoot.statsUpdater = ta
    print "Added statistics updater task to the tasks list."
    # --------------------- END STATISTICS ---------------------
    DBMgr.getInstance().endRequest()
    print ""

def fileExists(f):
     try:
        fi = open(f,'a')
        fi.close()
     except IOError:
         exists = False
     else:
         exists = True
     return exists

def run(log=None):
    from MaKaC.common.timerExec import timer

    if _isPosix() and (not log or not fileExists(log)):
        print "\nPlease enter an existing log file, and start again\nDaemon stopped!\n"
        os.remove(linuxPIDFile)
        sys.exit(2)
    t = timer(duration, log)
    initDefaultTasks()
    t.start()

def startTaskDaemon():
    """
    Run the function listed in the toExec instance each 'duration' seconds
    """
    oldPID=_getPID(False)
    if oldPID!=-1:
        print "Task daemon is already running"
        sys.exit(0)
    if _isPosix():
        pid=os.fork()
        if pid==0:
            #Child
            run(log=logFile)
        else:
            #Parent
            _savePID(pid)
            print "Daemon started! (pid=%s)\n"%pid
            time.sleep(3)
    elif _isWindows():
        print "Close this window to stop the server"
        run()
    else:
        print "Impossible to start the daemon"

def stopTaskDaemon():
    if _killProcess():
        print "Daemon stopped!"
    else:
        print "Impossible to stop the process 'taskDaemon', please do it manually"

def restartTaskDaemon():
    _killProcess()
    print "Daemon stopped!"
    startTaskDaemon()

def main():
    "Main"
    try:
        opts, args = getopt.getopt(sys.argv[1:],"hstr", \
                 [ \
                   "help", \
                   "start", \
		   "stop", \
		   "restart" \
                 ]
        )
    except getopt.GetoptError, e:
        help()
        sys.exit(2)

    arg=None
    if len(args) == 1:
        arg=args[0]
    elif len(opts)==1:
        arg=opts[0][0]
    if arg is not None:
        if   arg in ("start", "s", "-s"):
            startTaskDaemon()
        elif arg in ("stop", "t", "-t"):
            stopTaskDaemon()
        elif arg in ("restart", "r", "-r"):
            restartTaskDaemon()
        elif arg in ("help", "h", "-h", "--help"):
            help()
            sys.exit()
        else:
            help()
            sys.exit(2)
        return
    else:
        help()
        sys.exit(2)

def help():
        "Print out info"

        print "\nUsage: python taskDaemon.py  start|stop|restart [-h]\n"
        print "  -h                    Help"
        print "  -s                    Start the task daemon"
        print "  -t                    Stop the task daemon"
        print "  -r                    Restart the task daemon"

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for interacting with packages defined by pom files
"""

import logging

from artifactory.pompackage import createPomPackage
from conary.lib import graph
import artifactory

from ..config import MavenCoordinateGlobList
from ..lib import util

log = logging.getLogger('updatebot.pkgsource')

def loaded(func):
    def wrapper(other, *args, **kwargs):
        if other._loaded:
            return
        return func(other, *args, **kwargs)
    return wrapper

class PomSource(object):
    """
    Creates a map of packages from a pom file
    """

    def __init__(self, cfg, ui):
        self._cfg = cfg
        self._ui = ui

        self._exclusions = cfg.excludePoms.copy()
        self._inclusions = MavenCoordinateGlobList()

        # process excludePackages
        if cfg.packageAll:
            # cfg.package is an additional exclude list
            self._exclusions.extend(cfg.package)
        else:
            # cfg.package is an explicity include list
            self._inclusions.extend(cfg.package)

        self._pkgMap = {}  # map group, artifact, version to package
        self.pkgQueue = graph.DirectedGraph()
        self._loaded = False

    def _iterPackages(self, client, repo=None):
        searchKwargs = {}
        if repo:
            searchKwargs = {'repos': repo}
        if self._inclusions:
            searchFunc = client.gavc_search
            searchArgs = [package.split(':') for package in self._cfg.package]
        else:
            searchFunc = client.quick_search
            searchArgs = [ ["*.pom"], ]

        for args in searchArgs:
            for result in searchFunc(*args, **searchKwargs):
                if result["mimeType"] == "application/x-maven-pom+xml":
                    yield result

    def _includeResult(self, group, artifact, version):
        """Return whether to include result

        A result should be included if the artifact name is not in
        cfg.excludePackages. If the artifact is in cfg.package, then it should
        be included if cfg.packageAll is False, otherwise it should not be
        included.

        :param path: path to filter
        :type path: str
        :return: True if result should be included
        :rtype: bool
        """
        coordinate = ':'.join((group, artifact, version))
        if coordinate in self._exclusions:
            return False

        if self._cfg.packageAll:
            # cfg.package already mapped into exclusions
            return True
        else:
            return coordinate in self._inclusions

    @loaded
    def finalize(self):
        for node in self.pkgQueue.iterNodes():
            self.pkgQueue.addEdges((node, dep, 1) for dep in node.dependencies)

    @loaded
    def load(self):
        client = artifactory.Client(self._cfg)
        log.info('loading repository data')
        self.loadFromClient(client)
        self.finalize()
        self._loaded = True

    def loadFromClient(self, client):
        for result in self._iterPackages(client):
            log.debug('loading %s', result['path'])
            # process path into group, artifact, verstion tuple
            path = result['path'][1:]  # strip the leading /
            # split path and strip file
            group, artifact, version = path.rsplit('/', 3)[:-1]
            group = group.replace('/', '.')  # replace / with . in group
            gav = (group, artifact, version)

            if not self._includeResult(*gav):
                log.debug('excluding %s from package source', ':'.join(gav))
                continue

            if gav not in self._pkgMap:
                # this is a new package
                pkg = createPomPackage(*gav, client=client, cache=self._pkgMap)
            else:
                log.debug("already processed %s", ':'.join(gav))
                pkg = self._pkgMap[gav]

            if not pkg:
                continue

            self.pkgQueue.addNode(pkg)
            for dep in util.recurseDeps(pkg):
                self.pkgQueue.addNode(dep)
<commit_msg>Log pom loading and exclusion as INFO<commit_after>#
# Copyright (c) SAS Institute, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Module for interacting with packages defined by pom files
"""

import logging

from artifactory.pompackage import createPomPackage
from conary.lib import graph
import artifactory

from ..config import MavenCoordinateGlobList
from ..lib import util

log = logging.getLogger('updatebot.pkgsource')

def loaded(func):
    def wrapper(other, *args, **kwargs):
        if other._loaded:
            return
        return func(other, *args, **kwargs)
    return wrapper

class PomSource(object):
    """
    Creates a map of packages from a pom file
    """

    def __init__(self, cfg, ui):
        self._cfg = cfg
        self._ui = ui

        self._exclusions = cfg.excludePoms.copy()
        self._inclusions = MavenCoordinateGlobList()

        # process excludePackages
        if cfg.packageAll:
            # cfg.package is an additional exclude list
            self._exclusions.extend(cfg.package)
        else:
            # cfg.package is an explicity include list
            self._inclusions.extend(cfg.package)

        self._pkgMap = {}  # map group, artifact, version to package
        self.pkgQueue = graph.DirectedGraph()
        self._loaded = False

    def _iterPackages(self, client, repo=None):
        searchKwargs = {}
        if repo:
            searchKwargs = {'repos': repo}
        if self._inclusions:
            searchFunc = client.gavc_search
            searchArgs = [package.split(':') for package in self._cfg.package]
        else:
            searchFunc = client.quick_search
            searchArgs = [ ["*.pom"], ]

        for args in searchArgs:
            for result in searchFunc(*args, **searchKwargs):
                if result["mimeType"] == "application/x-maven-pom+xml":
                    yield result

    def _includeResult(self, group, artifact, version):
        """Return whether to include result

        A result should be included if the artifact name is not in
        cfg.excludePackages. If the artifact is in cfg.package, then it should
        be included if cfg.packageAll is False, otherwise it should not be
        included.

        :param path: path to filter
        :type path: str
        :return: True if result should be included
        :rtype: bool
        """
        coordinate = ':'.join((group, artifact, version))
        if coordinate in self._exclusions:
            return False

        if self._cfg.packageAll:
            # cfg.package already mapped into exclusions
            return True
        else:
            return coordinate in self._inclusions

    @loaded
    def finalize(self):
        for node in self.pkgQueue.iterNodes():
            self.pkgQueue.addEdges((node, dep, 1) for dep in node.dependencies)

    @loaded
    def load(self):
        client = artifactory.Client(self._cfg)
        log.info('loading repository data')
        self.loadFromClient(client)
        self.finalize()
        self._loaded = True

    def loadFromClient(self, client):
        for result in self._iterPackages(client):
            log.info('loading %s', result['path'])
            # process path into group, artifact, verstion tuple
            path = result['path'][1:]  # strip the leading /
            # split path and strip file
            group, artifact, version = path.rsplit('/', 3)[:-1]
            group = group.replace('/', '.')  # replace / with . in group
            gav = (group, artifact, version)

            if not self._includeResult(*gav):
                log.info('excluding %s from package source', ':'.join(gav))
                continue

            if gav not in self._pkgMap:
                # this is a new package
                pkg = createPomPackage(*gav, client=client, cache=self._pkgMap)
            else:
                log.debug("already processed %s", ':'.join(gav))
                pkg = self._pkgMap[gav]

            if not pkg:
                continue

            self.pkgQueue.addNode(pkg)
            for dep in util.recurseDeps(pkg):
                self.pkgQueue.addNode(dep)
<|endoftext|>


<commit_before>from dtest import Tester, debug
from tools import *
from assertions import *
from ccmlib.cluster import Cluster
from ccmlib.node import TimeoutError
import random
import os

from tools import ThriftConnection

versions = (
    '1.1.9', '1.2.2'
)

class TestUpgradeThroughVersions(Tester):
    """
    upgrades a 3-node cluster through each of the above versions.
    """

    def __init__(self, *args, **kwargs):
        # Forcing cluster version on purpose
        os.environ['CASSANDRA_VERSION'] = versions[0]
        # Force cluster options that are common among versions:
        kwargs['cluster_options'] = {'partitioner':'org.apache.cassandra.dht.RandomPartitioner'}
        Tester.__init__(self, *args, **kwargs)

    # def upgrade_test(self):
    #     self.upgrade_scenario()

    def upgrade_test_mixed(self):
        """Only upgrade part of the cluster, so we have mixed versions when
        we're done."""
        self.upgrade_scenario(mixed_version=True)

    def upgrade_scenario(self, mixed_version=False):
        self.num_rows = 0
        cluster = self.cluster

        # Create a ring
        cluster.populate(3)
        cluster.start()
        node1, node2, node3 = cluster.nodelist()
        self.node2 = node2

        node1.watch_log_for('Listening for thrift clients...')
        conn = ThriftConnection(node1)
        conn.create_ks()
        conn.create_cf()
        time.sleep(.5)
        self._write_values()

        # upgrade through versions
        for version in versions[1:]:
            if mixed_version:
                self.upgrade_to_version(version, mixed_version=True, nodes=(node1,))
            else:
                self.upgrade_to_version(version)

    def upgrade_to_version(self, version, mixed_version=False, nodes=None):
        """Upgrade Nodes - if *mixed_version* is True, only upgrade those nodes
        that are specified by *nodes*, otherwise ignore *nodes* specified
        and upgrade all nodes.
        """
        debug('Upgrading to ' + version)
        if not mixed_version:
            nodes = self.cluster.nodelist()

        for node in nodes:
            debug('Prepping node for shutdown: ' + node.name)
            node.flush()
            self._check_values()

        for node in nodes:
            debug('Shutting down node: ' + node.name)
            time.sleep(.5)
            node.stop(wait_other_notice=False)

        for node in nodes:
            debug('Upgrading node: ' + node.name)
            node.set_cassandra_dir(cassandra_version=version)
            node.start(wait_other_notice=True)
            time.sleep(.5)
            if not mixed_version:
                node.nodetool('upgradesstables')

        for node in nodes:
            debug('Checking node: ' + node.name)
            if not mixed_version:
                self._write_values()
            self._check_values()

        if not mixed_version:
            # Check we can bootstrap a new node on the upgraded cluster:
            debug("Adding a node to the cluster")
            self.cluster.set_cassandra_dir(cassandra_version=version)
            nnode = new_node(self.cluster, remote_debug_port=str(2000+len(self.cluster.nodes)))
            nnode.start(no_wait=False)
            nnode.watch_log_for("Bootstrap completed!")
            self._check_values()

    def _write_values(self, consistency_level='ALL'):
        self.num_rows += 2
        conn = ThriftConnection(self.node2).use_ks()
        conn.insert_columns(self.num_rows, consistency_level)

    def _check_values(self, consistency_level='ALL'):
        for node in self.cluster.nodelist():
            conn = ThriftConnection(node).use_ks()
            conn.query_columns(self.num_rows, consistency_level)

<commit_msg>Enables whole-cluster upgrade test and tests against casandra-1.2 HEAD.<commit_after>from dtest import Tester, debug
from tools import *
from assertions import *
from ccmlib.cluster import Cluster
from ccmlib.node import TimeoutError
import random
import os

from tools import ThriftConnection

versions = (
    '1.1.9', 'git:cassandra-1.2'
)

class TestUpgradeThroughVersions(Tester):
    """
    upgrades a 3-node cluster through each of the above versions.
    """

    def __init__(self, *args, **kwargs):
        # Forcing cluster version on purpose
        os.environ['CASSANDRA_VERSION'] = versions[0]
        # Force cluster options that are common among versions:
        kwargs['cluster_options'] = {'partitioner':'org.apache.cassandra.dht.RandomPartitioner'}
        Tester.__init__(self, *args, **kwargs)

    def upgrade_test(self):
        self.upgrade_scenario()

    def upgrade_test_mixed(self):
        """Only upgrade part of the cluster, so we have mixed versions when
        we're done."""
        self.upgrade_scenario(mixed_version=True)

    def upgrade_scenario(self, mixed_version=False):
        self.num_rows = 0
        cluster = self.cluster

        # Create a ring
        cluster.populate(3)
        cluster.start()
        node1, node2, node3 = cluster.nodelist()
        self.node2 = node2

        node1.watch_log_for('Listening for thrift clients...')
        conn = ThriftConnection(node1)
        conn.create_ks()
        conn.create_cf()
        time.sleep(.5)
        self._write_values()

        # upgrade through versions
        for version in versions[1:]:
            if mixed_version:
                self.upgrade_to_version(version, mixed_version=True, nodes=(node1,))
            else:
                self.upgrade_to_version(version)

    def upgrade_to_version(self, version, mixed_version=False, nodes=None):
        """Upgrade Nodes - if *mixed_version* is True, only upgrade those nodes
        that are specified by *nodes*, otherwise ignore *nodes* specified
        and upgrade all nodes.
        """
        debug('Upgrading to ' + version)
        if not mixed_version:
            nodes = self.cluster.nodelist()

        for node in nodes:
            debug('Prepping node for shutdown: ' + node.name)
            node.flush()
            self._check_values()

        for node in nodes:
            debug('Shutting down node: ' + node.name)
            time.sleep(.5)
            node.stop(wait_other_notice=False)

        for node in nodes:
            debug('Upgrading node: ' + node.name)
            node.set_cassandra_dir(cassandra_version=version)
            node.start(wait_other_notice=True)
            time.sleep(.5)
            if not mixed_version:
                node.nodetool('upgradesstables')

        for node in nodes:
            debug('Checking node: ' + node.name)
            if not mixed_version:
                self._write_values()
            self._check_values()

        if not mixed_version:
            # Check we can bootstrap a new node on the upgraded cluster:
            debug("Adding a node to the cluster")
            self.cluster.set_cassandra_dir(cassandra_version=version)
            nnode = new_node(self.cluster, remote_debug_port=str(2000+len(self.cluster.nodes)))
            nnode.start(no_wait=False)
            nnode.watch_log_for("Bootstrap completed!")
            self._check_values()

    def _write_values(self, consistency_level='ALL'):
        self.num_rows += 2
        conn = ThriftConnection(self.node2).use_ks()
        conn.insert_columns(self.num_rows, consistency_level)

    def _check_values(self, consistency_level='ALL'):
        for node in self.cluster.nodelist():
            conn = ThriftConnection(node).use_ks()
            conn.query_columns(self.num_rows, consistency_level)

<|endoftext|>