<commit_before>"""
Utility functions.
"""

from contextlib import contextmanager
import warnings

# Python 2/3 independant dict iteration
iteritems = getattr(dict, 'iteritems', dict.items)
itervalues = getattr(dict, 'itervalues', dict.values)

class LRUCache(dict):
    """
    A simple LRU cache.
    """

    def __init__(self, *args, **kwargs):
        """
        :param capacity: How many items to store before cleaning up old items
                         or ``None`` for an unlimited cache size
        """

        self.capacity = kwargs.pop('capacity', None) or float('nan')
        self.lru = []

        super(LRUCache, self).__init__(*args, **kwargs)

    def refresh(self, key):
        """
        Push a key to the tail of the LRU queue
        """
        if key in self.lru:
            self.lru.remove(key)
        self.lru.append(key)

    def get(self, key, default=None):
        item = super(LRUCache, self).get(key, default)
        self.refresh(key)

        return item

    def __getitem__(self, key):
        item = super(LRUCache, self).__getitem__(key)
        self.refresh(key)

        return item

    def __setitem__(self, key, value):
        super(LRUCache, self).__setitem__(key, value)

        self.refresh(key)

        # Check, if the cache is full and we have to remove old items
        # If the queue is of unlimited size, self.capacity is NaN and
        # x > NaN is always False in Python and the cache won't be cleared.
        if len(self) > self.capacity:
            self.pop(self.lru.pop(0))

    def __delitem__(self, key):
        super(LRUCache, self).__delitem__(key)
        self.lru.remove(key)

    def clear(self):
        super(LRUCache, self).clear()
        del self.lru[:]

# Source: https://github.com/PythonCharmers/python-future/blob/466bfb2dfa36d865285dc31fe2b0c0a53ff0f181/future/utils/__init__.py#L102-L134
def with_metaclass(meta, *bases):
    """
    Function from jinja2/_compat.py. License: BSD.

    Use it like this::

        class BaseForm(object):
            pass

        class FormType(type):
            pass

        class Form(with_metaclass(FormType, BaseForm)):
            pass

    This requires a bit of explanation: the basic idea is to make a
    dummy metaclass for one level of class instantiation that replaces
    itself with the actual metaclass.  Because of internal type checks
    we also need to make sure that we downgrade the custom metaclass
    for one level to something closer to type (that's why __call__ and
    __init__ comes back from type etc.).

    This has the advantage over six.with_metaclass of not introducing
    dummy classes into the final MRO.
    """

    class Metaclass(meta):
        __call__ = type.__call__
        __init__ = type.__init__

        def __new__(cls, name, this_bases, d):
            if this_bases is None:
                return type.__new__(cls, name, (), d)
            return meta(name, bases, d)

    return Metaclass('temporary_class', None, {})

@contextmanager
def catch_warning(warning_cls):
    with warnings.catch_warnings():
        warnings.filterwarnings('error', category=warning_cls)

        yield

class FrozenDict(dict):
    def __hash__(self):
        return hash(frozenset(self))

    def __setitem__(self, key, value):
        raise TypeError('\'FrozenDict\' object does not support item assignment')

def freeze(obj):
    if isinstance(obj, dict):
        return FrozenDict((k, freeze(v)) for k, v in obj.items())
    elif isinstance(obj, list):
        return tuple(freeze(el) for el in obj)
<commit_msg>Handle sets when freezing and cover more methods on FrozenDict<commit_after>"""
Utility functions.
"""

from contextlib import contextmanager
import warnings

# Python 2/3 independant dict iteration
iteritems = getattr(dict, 'iteritems', dict.items)
itervalues = getattr(dict, 'itervalues', dict.values)

class LRUCache(dict):
    """
    A simple LRU cache.
    """

    def __init__(self, *args, **kwargs):
        """
        :param capacity: How many items to store before cleaning up old items
                         or ``None`` for an unlimited cache size
        """

        self.capacity = kwargs.pop('capacity', None) or float('nan')
        self.lru = []

        super(LRUCache, self).__init__(*args, **kwargs)

    def refresh(self, key):
        """
        Push a key to the tail of the LRU queue
        """
        if key in self.lru:
            self.lru.remove(key)
        self.lru.append(key)

    def get(self, key, default=None):
        item = super(LRUCache, self).get(key, default)
        self.refresh(key)

        return item

    def __getitem__(self, key):
        item = super(LRUCache, self).__getitem__(key)
        self.refresh(key)

        return item

    def __setitem__(self, key, value):
        super(LRUCache, self).__setitem__(key, value)

        self.refresh(key)

        # Check, if the cache is full and we have to remove old items
        # If the queue is of unlimited size, self.capacity is NaN and
        # x > NaN is always False in Python and the cache won't be cleared.
        if len(self) > self.capacity:
            self.pop(self.lru.pop(0))

    def __delitem__(self, key):
        super(LRUCache, self).__delitem__(key)
        self.lru.remove(key)

    def clear(self):
        super(LRUCache, self).clear()
        del self.lru[:]

# Source: https://github.com/PythonCharmers/python-future/blob/466bfb2dfa36d865285dc31fe2b0c0a53ff0f181/future/utils/__init__.py#L102-L134
def with_metaclass(meta, *bases):
    """
    Function from jinja2/_compat.py. License: BSD.

    Use it like this::

        class BaseForm(object):
            pass

        class FormType(type):
            pass

        class Form(with_metaclass(FormType, BaseForm)):
            pass

    This requires a bit of explanation: the basic idea is to make a
    dummy metaclass for one level of class instantiation that replaces
    itself with the actual metaclass.  Because of internal type checks
    we also need to make sure that we downgrade the custom metaclass
    for one level to something closer to type (that's why __call__ and
    __init__ comes back from type etc.).

    This has the advantage over six.with_metaclass of not introducing
    dummy classes into the final MRO.
    """

    class Metaclass(meta):
        __call__ = type.__call__
        __init__ = type.__init__

        def __new__(cls, name, this_bases, d):
            if this_bases is None:
                return type.__new__(cls, name, (), d)
            return meta(name, bases, d)

    return Metaclass('temporary_class', None, {})

@contextmanager
def catch_warning(warning_cls):
    with warnings.catch_warnings():
        warnings.filterwarnings('error', category=warning_cls)

        yield

class FrozenDict(dict):
    def __hash__(self):
        return hash(frozenset(self))

    def _immutable(self, *args, **kws):
        raise TypeError('object is immutable')

    __setitem__ = _immutable
    __delitem__ = _immutable
    clear = _immutable
    update = _immutable
    setdefault = _immutable
    pop = _immutable
    popitem = _immutable

def freeze(obj):
    if isinstance(obj, dict):
        return FrozenDict((k, freeze(v)) for k, v in obj.items())
    elif isinstance(obj, list):
        return tuple(freeze(el) for el in obj)
    elif isinstance(obj, set):
        return frozenset(obj)
<|endoftext|>


<commit_before>######################################
# File Name : indeed_scrape.py
# Author : Daniel Cuneo
# Creation Date : 11-05-2015
######################################
#TODO: add logger
#TODO: use MRJob mapper for parsing
import ConfigParser
import json
import pandas as pd
import urllib2
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS
from sklearn.metrics.pairwise import cosine_distances
from sklearn.cluster import KMeans
from sklearn.metrics import euclidean_distances
import matplotlib.pyplot as plt
import numpy as np
from nltk import stem
from nltk import tokenize
import re
import os

toker = tokenize.word_tokenize
stemmer = stem.SnowballStemmer('english')

class Indeed(object):
    def __init__(self):
        repo_dir = os.getenv("OPENSHIFT_REPO_DIR")

        self.num_urls = 1000
        self.add_loc = None
        self.stop_words = None
        self.num_samp = 100
        self.zip_code_file = os.path.join(repo_dir, 'us_postal_codes.csv')
        self.df = pd.DataFrame(columns=['url'])
        self.config_path = os.path.join(repo_dir, "tokens.cfg")
        self.query = None
        self.locations = None

    def add_stop_words(self):
        if self.stop_words is not None:
            words = self.stop_words.split(" ")
            self.stop_words = ENGLISH_STOP_WORDS.union(words)

    def build_api_string(self):
        if self.query is None:
            print "query cannot be empty"
            raise ValueError

        # beware of escaped %
        self.format_query(self.query)
        prefix = 'http://api.indeed.com/ads/apisearch?'
        pub = 'publisher=%(pub_id)s'
        chan = '&chnl=%(channel_name)s'
        loc = '&l=%(loc)s'
        query = '&q=title%%3A%%28%(query)s%%29'
        #query = '&q=%s' % query
        start = '&start=0'
        frm = '&fromage=30'
        limit = '&limit=25'
        site = '&st=jobsite'
        format = '&format=json'
        sort = '&sort=0'
        country = '&co=us'
        radius = '&radius=1'
        suffix = '&userip=1.2.3.4&useragent=Mozilla/%%2F4.0%%28Firefox%%29&v=2'

        self.api = prefix + pub + chan + loc + query + start + frm + limit + \
                   site + format + country + sort + radius + suffix

    def format_query(self, query):
         #return "%%20".join(query.split(" "))
         self.query = "+".join(query.split(" "))

    def load_config(self):
        '''loads a config file that contains tokens'''
        config_parser = ConfigParser.RawConfigParser()
        config_parser.read(self.config_path)
        self.pub_id = config_parser.get("id", "pub_id")
        self.channel_name = config_parser.get("channel", "channel_name")

    def load_zipcodes(self):
        '''loads the zip code file and returnes a list of zip codes'''
        self.df_zip = pd.read_csv(self.zip_code_file, dtype=str)
        self.df_zip.dropna(inplace=True, how='all')
        locations = self.df_zip['Postal Code'].dropna(how='any')

        return locations

    def end_url_loop(self):
        df = self.df.dropna(how='any').drop_duplicates(subset=['url', 'company'])
        if df.url.count() >= self.num_urls:
            return True
        else:
            return False

    def get_city_url_content_stem(self):
        ind = 0
        for zipcode in self.locations:
            url_city_title = self.get_url(zipcode)
            if url_city_title is None:
                continue
            for item in url_city_title:
                self.df.loc[ind, 'zipcode'] = str(zipcode)
                self.df.loc[ind, 'url'] = item[0]
                self.df.loc[ind, 'city'] = item[1]
                self.df.loc[ind, 'jobtitle'] = item[2]
                self.df.loc[ind, 'company'] = item[3]
                content = self.parse_content(item[0])
                self.df.loc[ind, 'summary'] = content
                self.df.loc[ind, 'summary_stem'] = self.stemmer_(content)
                ind += 1
                if np.mod(ind, self.num_urls) == 0:# check periodically
                    if self.end_url_loop():
                        return

    def get_urls(self):
        urls = []
        for item in self.locations:
            url = self.get_url(item)
            if url is not None:
                urls.extend(url)

        return urls

    def get_url(self, location):
        '''method good for use with MapReduce'''

        api = self.api %{'pub_id':self.pub_id,
                         'loc':location,
                         'channel_name':self.channel_name,
                         'query':self.query
                        }

        try:
            response = urllib2.urlopen(api)
            data = json.loads(response.read())
            response.close()

            urls = []
            urls.extend([ (item['url'], item['city'], item['jobtitle'], item['company']) for item in data['results']])

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

        return urls

    def get_content(self, url):
        if url is None:
            return None

        try:
            response = urllib2.urlopen(url)
            content = response.read()
            response.close()

            return content

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

    def len_tester(self, word_list):
        new_list = []
        for word in word_list:
            if len(word) < 3:
                continue
            else:
                new_list.append(word)

        return new_list

    def stemmer_(self, string):

        if string is None:
            return None

        words = toker(string)
        words = self.len_tester(words)
        words = map(stemmer.stem, words)

        return " ".join(words)

    def parse_content(self, url):
        content = self.get_content(url)

        if content is None:
            return None

        content = content.decode("ascii", "ignore")
        soup = BeautifulSoup(content, 'html.parser')

        try:
            summary = soup.find('span', {'summary'})

        except AttributeError:
            summary = soup.find_all('span')

        if summary is None:
            return None

        bullets = summary.find_all("li")

        if bullets is not None and len(bullets) > 2:
            skills = bullets
        else:
            skills = summary

        try:
            output = [item.get_text() for item in skills]

        except AttributeError:
            return None

        if len(output) > 0:
            return " ".join(output)
        else:
            return None

    def parse_zipcode_beg(self):
        '''locs are zipcode prefixes, like:902, provided as string'''
        pat = '^%s' %self.add_loc
        obj = re.compile(pat)

        self.df_zip['include'] = self.df_zip['Postal Code'].apply(lambda x: 1 if obj.match(x) else 0)
        zips = self.df_zip[self.df_zip['include']==1]['Postal Code']

        return zips.tolist()

    def handle_locations(self):
        '''main method for setting up the locations for the API call'''
        locations = self.load_zipcodes()
        locations = locations.sample(self.num_samp).tolist()
        if self.add_loc is not None:
            for loc in self.parse_zipcode_beg():
                locations.append(loc)

        locations = np.unique(locations)
        np.random.shuffle(locations)
        # if we stop the program and take the avaiable output we'd like it to
        # be radomly sampled

        return locations

    def save_data(self):

        self.df.drop_duplicates(subset=['url'], inplace=True)
        self.df.dropna(inplace=True, how='any', axis=0)

        self.df.to_csv('/home/daniel/git/Python2.7/DataScience/indeed/data_frame.csv',
                        index=False,
                        encoding='utf-8')

    def main(self):
        '''Run all the steps to collect data and produce a bi-gram
        bar plot of the keyword counts. Terminating the program execution
        with control C, will save whatever data was collected. To make this
        save-on-quit feature more usable, the locations are shuffled prior to
        getting the content.'''

        self.load_config()
        self.build_api_string()
        self.add_stop_words()
        # allow the user to specify locations
        if self.locations is None:
            self.locations = self.handle_locations()

        self.get_city_url_content_stem()

    def vectorizer(self, corpus, max_features=200, max_df=0.8, min_df=0.1, n_min=2):
        vectorizer = CountVectorizer(max_features=max_features,
                                    max_df=max_df,
                                    min_df=min_df,
                                    lowercase=True,
                                    stop_words=self.stop_words,
                                    ngram_range=(n_min, 3),
                                    analyzer='word',
                                    decode_error='ignore',
                                    strip_accents='unicode'
                                    )

        matrix = vectorizer.fit_transform(corpus)
        features = vectorizer.get_feature_names()

        return matrix, features

    def plot_sorted_subsection(self, fea, mat, start, end):
        '''Plot the next n highest counts in sorted order from n to m'''

        m = mat.toarray().sum(axis=0)

        ind_sort = np.argsort(m)
        m = m[ind_sort]

        f = np.array(fea)
        f = f[ind_sort]

        range_ = end - start
        if range_ < 0:
            raise ValueError

        x = np.arange(range_)

        plt.xticks(x, f, rotation=90, fontsize=14)
        plt.bar(x, m[start:end], align='center')

    def plot_features(self, features, matrix):
        x = np.arange(len(features))
        tot = np.array(matrix.sum(axis=0))
        tot = np.squeeze(tot)

        plt.bar(x, tot, align='center', alpha=0.5)
        plt.xticks(x, features, rotation='vertical', fontsize=15)
        plt.grid(True)
        plt.xlim(-1, len(x))

        plt.ylabel("Counts", fontsize=15)
        string = "Key Word Count: %i Features" %len(features)
        plt.title(string,  fontsize=15)
        plt.tight_layout()

        plt.show()

    def cluster(self, matrix):

        #cos_dist = cosine_distances(matrix)
        euc_dist = euclidean_distances(matrix)

        km = KMeans(n_clusters=3,
                    #init='random',
                    init='k-means++',
                    n_init=100,
                    max_iter=1000,
                    tol=1e-7,
                    precompute_distances=False,
                    verbose=0,
                    random_state=1,
                    copy_x=True,
                    n_jobs=1
                    )

        assignments = km.fit_predict(euc_dist)

        return assignments

    def find_words_in_radius(self, series, keyword, radius):

        words_in_radius = []

        for string in series:
            test = string.decode('ascii', 'ignore')
            test = tokenize.word_tokenize(test)
            test = np.array(test)

            kw_ind = np.where(test == keyword)[0]
            if len(kw_ind) == 0: #empty
                continue

            src_range = self._find_words_in_radius(kw_ind, test, radius)

            temp = " ".join(test[src_range])
            words_in_radius.append(temp)

        return words_in_radius

    def _find_words_in_radius(self, kw_ind, test, radius):
            # can be more than one kw in a string
            lower = kw_ind - radius
            upper = kw_ind + radius

            src_range_tot = np.empty(0, dtype=np.int16)

            for i in range(lower.shape[0]):
                src_range = np.arange(lower[i], upper[i])

                # truncate search range to valid values
                # this operation also flattens the array
                src_range = src_range[(src_range >= 0) & (src_range < test.size)]
                src_range_tot = np.hstack((src_range_tot, src_range))

            return np.unique(src_range_tot)

if __name__ == "__main__":
    ind = Indeed()
    ind.main()
<commit_msg>opened search radius<commit_after>######################################
# File Name : indeed_scrape.py
# Author : Daniel Cuneo
# Creation Date : 11-05-2015
######################################
#TODO: add logger
#TODO: use MRJob mapper for parsing
import ConfigParser
import json
import pandas as pd
import urllib2
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS
from sklearn.metrics.pairwise import cosine_distances
from sklearn.cluster import KMeans
from sklearn.metrics import euclidean_distances
import matplotlib.pyplot as plt
import numpy as np
from nltk import stem
from nltk import tokenize
import re
import os

toker = tokenize.word_tokenize
stemmer = stem.SnowballStemmer('english')

class Indeed(object):
    def __init__(self):
        repo_dir = os.getenv("OPENSHIFT_REPO_DIR")

        self.num_urls = 1000
        self.add_loc = None
        self.stop_words = None
        self.num_samp = 100
        self.zip_code_file = os.path.join(repo_dir, 'us_postal_codes.csv')
        self.df = pd.DataFrame(columns=['url'])
        self.config_path = os.path.join(repo_dir, "tokens.cfg")
        self.query = None
        self.locations = None

    def add_stop_words(self):
        if self.stop_words is not None:
            words = self.stop_words.split(" ")
            self.stop_words = ENGLISH_STOP_WORDS.union(words)

    def build_api_string(self):
        if self.query is None:
            print "query cannot be empty"
            raise ValueError

        # beware of escaped %
        self.format_query(self.query)
        prefix = 'http://api.indeed.com/ads/apisearch?'
        pub = 'publisher=%(pub_id)s'
        chan = '&chnl=%(channel_name)s'
        loc = '&l=%(loc)s'
        query = '&q=title%%3A%%28%(query)s%%29'
        #query = '&q=%s' % query
        start = '&start=0'
        frm = '&fromage=30'
        limit = '&limit=25'
        site = '&st=jobsite'
        format = '&format=json'
        sort = '&sort=0'
        country = '&co=us'
        radius = '&radius=10'
        suffix = '&userip=1.2.3.4&useragent=Mozilla/%%2F4.0%%28Firefox%%29&v=2'

        self.api = prefix + pub + chan + loc + query + start + frm + limit + \
                   site + format + country + sort + radius + suffix

    def format_query(self, query):
         #return "%%20".join(query.split(" "))
         self.query = "+".join(query.split(" "))

    def load_config(self):
        '''loads a config file that contains tokens'''
        config_parser = ConfigParser.RawConfigParser()
        config_parser.read(self.config_path)
        self.pub_id = config_parser.get("id", "pub_id")
        self.channel_name = config_parser.get("channel", "channel_name")

    def load_zipcodes(self):
        '''loads the zip code file and returnes a list of zip codes'''
        self.df_zip = pd.read_csv(self.zip_code_file, dtype=str)
        self.df_zip.dropna(inplace=True, how='all')
        locations = self.df_zip['Postal Code'].dropna(how='any')

        return locations

    def end_url_loop(self):
        df = self.df.dropna(how='any').drop_duplicates(subset=['url', 'company'])
        if df.url.count() >= self.num_urls:
            return True
        else:
            return False

    def get_city_url_content_stem(self):
        ind = 0
        for zipcode in self.locations:
            url_city_title = self.get_url(zipcode)
            if url_city_title is None:
                continue
            for item in url_city_title:
                self.df.loc[ind, 'zipcode'] = str(zipcode)
                self.df.loc[ind, 'url'] = item[0]
                self.df.loc[ind, 'city'] = item[1]
                self.df.loc[ind, 'jobtitle'] = item[2]
                self.df.loc[ind, 'company'] = item[3]
                content = self.parse_content(item[0])
                self.df.loc[ind, 'summary'] = content
                self.df.loc[ind, 'summary_stem'] = self.stemmer_(content)
                ind += 1
                if np.mod(ind, self.num_urls) == 0:# check periodically
                    if self.end_url_loop():
                        return

    def get_urls(self):
        urls = []
        for item in self.locations:
            url = self.get_url(item)
            if url is not None:
                urls.extend(url)

        return urls

    def get_url(self, location):
        '''method good for use with MapReduce'''

        api = self.api %{'pub_id':self.pub_id,
                         'loc':location,
                         'channel_name':self.channel_name,
                         'query':self.query
                        }

        try:
            response = urllib2.urlopen(api)
            data = json.loads(response.read())
            response.close()

            urls = []
            urls.extend([ (item['url'], item['city'], item['jobtitle'], item['company']) for item in data['results']])

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

        return urls

    def get_content(self, url):
        if url is None:
            return None

        try:
            response = urllib2.urlopen(url)
            content = response.read()
            response.close()

            return content

        except urllib2.HTTPError, err:
            print err
            return None

        except Exception, err:
            print err
            return None

    def len_tester(self, word_list):
        new_list = []
        for word in word_list:
            if len(word) < 3:
                continue
            else:
                new_list.append(word)

        return new_list

    def stemmer_(self, string):

        if string is None:
            return None

        words = toker(string)
        words = self.len_tester(words)
        words = map(stemmer.stem, words)

        return " ".join(words)

    def parse_content(self, url):
        content = self.get_content(url)

        if content is None:
            return None

        content = content.decode("ascii", "ignore")
        soup = BeautifulSoup(content, 'html.parser')

        try:
            summary = soup.find('span', {'summary'})

        except AttributeError:
            summary = soup.find_all('span')

        if summary is None:
            return None

        bullets = summary.find_all("li")

        if bullets is not None and len(bullets) > 2:
            skills = bullets
        else:
            skills = summary

        try:
            output = [item.get_text() for item in skills]

        except AttributeError:
            return None

        if len(output) > 0:
            return " ".join(output)
        else:
            return None

    def parse_zipcode_beg(self):
        '''locs are zipcode prefixes, like:902, provided as string'''
        pat = '^%s' %self.add_loc
        obj = re.compile(pat)

        self.df_zip['include'] = self.df_zip['Postal Code'].apply(lambda x: 1 if obj.match(x) else 0)
        zips = self.df_zip[self.df_zip['include']==1]['Postal Code']

        return zips.tolist()

    def handle_locations(self):
        '''main method for setting up the locations for the API call'''
        locations = self.load_zipcodes()
        locations = locations.sample(self.num_samp).tolist()
        if self.add_loc is not None:
            for loc in self.parse_zipcode_beg():
                locations.append(loc)

        locations = np.unique(locations)
        np.random.shuffle(locations)
        # if we stop the program and take the avaiable output we'd like it to
        # be radomly sampled

        return locations

    def save_data(self):

        self.df.drop_duplicates(subset=['url'], inplace=True)
        self.df.dropna(inplace=True, how='any', axis=0)

        self.df.to_csv('/home/daniel/git/Python2.7/DataScience/indeed/data_frame.csv',
                        index=False,
                        encoding='utf-8')

    def main(self):
        '''Run all the steps to collect data and produce a bi-gram
        bar plot of the keyword counts. Terminating the program execution
        with control C, will save whatever data was collected. To make this
        save-on-quit feature more usable, the locations are shuffled prior to
        getting the content.'''

        self.load_config()
        self.build_api_string()
        self.add_stop_words()
        # allow the user to specify locations
        if self.locations is None:
            self.locations = self.handle_locations()

        self.get_city_url_content_stem()

    def vectorizer(self, corpus, max_features=200, max_df=0.8, min_df=0.1, n_min=2):
        vectorizer = CountVectorizer(max_features=max_features,
                                    max_df=max_df,
                                    min_df=min_df,
                                    lowercase=True,
                                    stop_words=self.stop_words,
                                    ngram_range=(n_min, 3),
                                    analyzer='word',
                                    decode_error='ignore',
                                    strip_accents='unicode'
                                    )

        matrix = vectorizer.fit_transform(corpus)
        features = vectorizer.get_feature_names()

        return matrix, features

    def plot_sorted_subsection(self, fea, mat, start, end):
        '''Plot the next n highest counts in sorted order from n to m'''

        m = mat.toarray().sum(axis=0)

        ind_sort = np.argsort(m)
        m = m[ind_sort]

        f = np.array(fea)
        f = f[ind_sort]

        range_ = end - start
        if range_ < 0:
            raise ValueError

        x = np.arange(range_)

        plt.xticks(x, f, rotation=90, fontsize=14)
        plt.bar(x, m[start:end], align='center')

    def plot_features(self, features, matrix):
        x = np.arange(len(features))
        tot = np.array(matrix.sum(axis=0))
        tot = np.squeeze(tot)

        plt.bar(x, tot, align='center', alpha=0.5)
        plt.xticks(x, features, rotation='vertical', fontsize=15)
        plt.grid(True)
        plt.xlim(-1, len(x))

        plt.ylabel("Counts", fontsize=15)
        string = "Key Word Count: %i Features" %len(features)
        plt.title(string,  fontsize=15)
        plt.tight_layout()

        plt.show()

    def cluster(self, matrix):

        #cos_dist = cosine_distances(matrix)
        euc_dist = euclidean_distances(matrix)

        km = KMeans(n_clusters=3,
                    #init='random',
                    init='k-means++',
                    n_init=100,
                    max_iter=1000,
                    tol=1e-7,
                    precompute_distances=False,
                    verbose=0,
                    random_state=1,
                    copy_x=True,
                    n_jobs=1
                    )

        assignments = km.fit_predict(euc_dist)

        return assignments

    def find_words_in_radius(self, series, keyword, radius):

        words_in_radius = []

        for string in series:
            test = string.decode('ascii', 'ignore')
            test = tokenize.word_tokenize(test)
            test = np.array(test)

            kw_ind = np.where(test == keyword)[0]
            if len(kw_ind) == 0: #empty
                continue

            src_range = self._find_words_in_radius(kw_ind, test, radius)

            temp = " ".join(test[src_range])
            words_in_radius.append(temp)

        return words_in_radius

    def _find_words_in_radius(self, kw_ind, test, radius):
            # can be more than one kw in a string
            lower = kw_ind - radius
            upper = kw_ind + radius

            src_range_tot = np.empty(0, dtype=np.int16)

            for i in range(lower.shape[0]):
                src_range = np.arange(lower[i], upper[i])

                # truncate search range to valid values
                # this operation also flattens the array
                src_range = src_range[(src_range >= 0) & (src_range < test.size)]
                src_range_tot = np.hstack((src_range_tot, src_range))

            return np.unique(src_range_tot)

if __name__ == "__main__":
    ind = Indeed()
    ind.main()
<|endoftext|>


<commit_before>''' Command Line Arguments '''
import argparse

parser = argparse.ArgumentParser(description='CLI')
parser.add_argument('--input', '-i', type='str', required=True)
args = parser.parse_args()
print("Input:", args.input)

''' Read a gzipped csv file '''
import gzip
import csv

with gzip.open('file.csv.gz', 'rb') as fp:
  reader = csv.reader(fp)
  for row in reader:
    print(row)

''' Vector multiplication (dot product) '''
import numpy as np

w = np.array([2,2,2,2])
x = np.array([1,2,3,4])
y = np.dot(w,x)

''' Generator MxN random numbers '''
import numpy as np

np.random.random((M,N))

''' Time a function '''
import time

tic = time.time()
# A function!
toc = time.time()
print(toc - tic)

''' Remove list from list '''
new_list = [i for i in old_list if i not in list_of_items_to_remove]

''' Read csv header only '''
import csv

with open(fpath, 'r') as fp:
    reader = csv.DictReader(fp)
    fieldnames = reader.fieldnames

''' Unpack a dictionary '''
# 2.x
d = {}
for k,v in d.iteritems():
	print(k,v)
# 3.x
d = {}
for k,v in d.items():
    print(k,v)

''' List files in directory '''
import os
print os.listdir(<directory_path>)

''' Initialize empty dictionary '''
keys = ['k1', 'k2', 'k3']
d = dict.fromkeys(keys)
# Initialize values to empty lists
d = dict.fromkeys(keys, [])

''' Extract column values/index from a DataFrame '''
df["col"].values
df["col"].index

''' Concatenate two DataFrames along the same index '''
df_both = pd.concat([df_one, df_two], axis=1)

''' Sum all values in a df column that have value x '''
sum_cols_equal_to_x = df[df['col'] == x].sum()['col']

''' Slice a df with a Series to extract a the rows which match a list of values '''
series = pd.Series([list, of, items, to, view]
df[df['col'].isin(series)]

''' Unique elements'''
# array
np.unique(array)
# list
set(lst)
# DataFrame column
df['col'].unique()
# DataFrame (distinct rows)
df.drop_duplicates()
# DataFrames (distinct entries in a specific column)
df.drop_duplicates(subset='col', keep='first')

# List conda environments
conda info --envs
<commit_msg>Rotate axis labels<commit_after>''' Command Line Arguments '''
import argparse

parser = argparse.ArgumentParser(description='CLI')
parser.add_argument('--input', '-i', type=str, required=True)
args = parser.parse_args()
print("Input:", args.input)

''' Read a gzipped csv file '''
import gzip
import csv

with gzip.open('file.csv.gz', 'rb') as fp:
  reader = csv.reader(fp)
  for row in reader:
    print(row)

''' Vector multiplication (dot product) '''
import numpy as np

w = np.array([2,2,2,2])
x = np.array([1,2,3,4])
y = np.dot(w,x)

''' Generator MxN random numbers '''
import numpy as np

np.random.random((M,N))

''' Time a function '''
import time

tic = time.time()
# A function!
toc = time.time()
print(toc - tic)

''' Remove list from list '''
new_list = [i for i in old_list if i not in list_of_items_to_remove]

''' Read csv header only '''
import csv

with open(fpath, 'r') as fp:
    reader = csv.DictReader(fp)
    fieldnames = reader.fieldnames

''' Unpack a dictionary '''
# 2.x
d = {}
for k,v in d.iteritems():
	print(k,v)
# 3.x
d = {}
for k,v in d.items():
    print(k,v)

''' List files in directory '''
import os
print os.listdir(<directory_path>)

''' Initialize empty dictionary '''
keys = ['k1', 'k2', 'k3']
d = dict.fromkeys(keys)
# Initialize values to empty lists
d = dict.fromkeys(keys, [])

''' Extract column values/index from a DataFrame '''
df["col"].values
df["col"].index

''' Concatenate two DataFrames/Series along the same index '''
df_both = pd.concat([df_one, df_two], axis=1)

''' Sum all values in a df column that have value x '''
sum_cols_equal_to_x = df[df['col'] == x].sum()['col']

''' Slice a df with a Series to extract a the rows which match a list of values '''
series = pd.Series([list, of, items, to, view]
df[df['col'].isin(series)]

''' Unique elements'''
# array
np.unique(array)
# list
set(lst)
# DataFrame column
df['col'].unique()
# DataFrame (distinct rows)
df.drop_duplicates()
# DataFrames (distinct entries in a specific column)
df.drop_duplicates(subset='col', keep='first')

# List conda environments
conda info --envs

# Rotate axis labels
plt.xticks(rotation=45)
# Set axis limits
ax = plt.gca()
ax.set_xlim([0,1])
<|endoftext|>


<commit_before>from Tkinter import *
from ttk import *
import enchant
from enchant.checker import SpellChecker

class tkSpellCheck:
    def __init__(self, textwidget, lang='en_US'):
        self.textwidget = textwidget
        self.swords = enchant.Dict(lang)        # S(uggested) Words, obviously
        self.checker = SpellChecker(lang)
        self.sp_err = {}

    def __call__(self, mode=False):
        if mode == 'init':
            self.fullcheck()
        if mode == 'inline':
            self.inlinecheck()
        if not mode:
            self.destroy_tags()

    def destroy_tags(self):
        if self.sp_err:
            for tag in self.sp_err.keys():
                mrk = self.sp_err[tag][1]
                self.textwidget.tag_delete(tag)
                self.textwidget.mark_unset(mrk)
            self.sp_err = {}  # Reset the dictionary for next time

    def fullcheck(self):
        self.destroy_tags()
        text = self.__alltxt()
        self.checker.set_text(text)
        i = 1
        curpos = 1.0

        for err in self.checker:
            tag = '{0}-{1}'.format(err.word, i)
            mrk = 'mrk-{0}'.format(tag)
            first_ltr = self.textwidget.search(err.word, curpos, END)
            last_ltr = first_ltr + ('+%dchars' % len(err.word))

            self.textwidget.tag_config(tag, foreground="red", underline=True)
            self.textwidget.tag_add(tag, first_ltr, last_ltr)
            self.textwidget.mark_set(mrk, first_ltr)
            self.textwidget.tag_bind(tag, '<Button-3>', lambda event,
                                     tag=tag: self.show_suggestions(event, tag))
            i += 1
            curpos = last_ltr
            self.sp_err[tag] = (err.word, mrk)

    def show_suggestions(self, event, tag):
        err = self.sp_err[tag][0]
        suggested_words = self.__suggest(err)
        context_menu = self.__build_cnxtmnu(tag, suggested_words)
        print event.x_root, event.y_root
        context_menu.tk_popup(event.x_root, event.y_root)

    def __alltxt(self):
        return self.textwidget.get(1.0, END)

    def __suggest(self, err):
        return self.swords.suggest(err)

    def __build_cnxtmnu(self, tag, suggested):
        err = self.sp_err[tag][0]
        cnxtmnu = Menu(self.textwidget, tearoff=False)
        for word in suggested:
            cnxtmnu.add_command(label=word, command=lambda tag=tag,
                                  word=word: self.__replace(tag,word))
        cnxtmnu.add_separator()
        cnxtmnu.add_command(label='Add to Dictionary', command=lambda
                            err=err: self.__usr_def(err))
        return cnxtmnu

    def __replace(self, tag, word):
        mrk = self.sp_err[tag][1]
        err = self.sp_err[tag][0]
        first_ltr = self.textwidget.index(mrk)
        last_ltr = first_ltr + ('+%dchars' % len(err))
        self.textwidget.delete(first_ltr, last_ltr)
        self.textwidget.insert(first_ltr, word)
        self.textwidget.tag_delete(tag)
        self.textwidget.mark_unset(mrk)

    def __usr_def(self, err):
        '''TODO: Add a personal dictionary file '''
        pass

if __name__ == '__main__':
    main()
<commit_msg>all functions written(i think?)<commit_after>from Tkinter import *
from ttk import *
import enchant
from enchant.checker import SpellChecker

class tkSpellCheck:
    def __init__(self, textwidget, lang='en_US'):
        self.textwidget = textwidget
        self.suggwords = enchant.Dict(lang)
        self.checker = SpellChecker(lang)
        self.spellerrs = {}

    def __call__(self, mode=False):
        if mode == 'fullcheck':
            self.fullcheck()
        if mode == 'inline':
            self.inlinecheck()
        if not mode:
            self.remove_tags()

    def fullcheck(self):
        self.__clearalltags()
        text = self.__alltxt()
        self.__settags(text, startpos='1.0', endpos='END')

    def inlinecheck(self):
        startpos = self.textwidget.index('insert linestart')
        endpos = self.textwidget.index('insert lineend')
        text = self.__linetxt(startpos, endpos)
        self.__clearlntags(currln=startpos)
        self.__settags(text, startpos=startpos, endpos=endpos)

    def __settags(self, text, startpos, endpos):
        i = 1
        curpos = startpos
        self.checker.set_text(text)
        for err in self.checker:
            currln = self.textwidget.index('insert linestart')
            tag = '{0}-{1}.{2}'.format(err.word, currln, i)
            mrk = 'mrk-{0}'.format(tag)
            firstltr = self.textwidget.search(err.word, curpos, endpos)
            lastltr = firstltr + ('+%dchars' % len(err.word))

            self.textwidget.tag_config(tag, foreground="red", underline=True)
            self.textwidget.tag_add(tag, firstltr, lastltr)
            self.textwidget.mark_set(mrk, firstltr)
            self.textwidget.tag_bind(tag, '<Button-3>', lambda event,
                                     tag=tag: self.__suggestedwords(event, tag))
            i += 1
            curpos = last_ltr
            self.spellerrs[tag] = (err.word, mrk, currln)

    def __suggestedwords(self, event, tag):
        err = self.spellerrs[tag][0]
        suggwords = self.__suggest(err)
        context_menu = self.__contextmenu(tag, suggwords)
        context_menu.tk_popup(event.x_root, event.y_root)

    def __clearalltags(self):
        if self.spellerrs:
            for tag in self.spellerrs.keys():
                self.__unsettag(self, tag)
            self.spellerrs = {}  # Reset the dictionary

    def __clearlntags(self, currln):
        for tag, val in self.spellerrs.items():
            if val[2] == currln:
                self.__unsettag(self, tag)
                del self.spellerrs[tag]

    def __unsettag(self, tag):
        mrk = self.spellerrs[tag][1]
        self.textwidget.tag_delete(tag)
        self.textwidget.mark_unset(mrk)

    def __alltxt(self):
        return self.textwidget.get(1.0, END)

    def __linetxt(self, startpos, endpos):
        return self.textwidget.get(startpos, endpos)

    def __suggest(self, err):
        return self.suggwords.suggest(err)

    def __contextmenu(self, tag, suggested):
        err = self.spellerrs[tag][0]
        contextmenu = Menu(self.textwidget, tearoff=False)
        for word in suggested:
            contextmenu.add_command(label=word, command=lambda tag=tag,
                                    word=word: self.__replace(tag, word))
        contextmenu.add_separator()
        contextmenu.add_command(label='Add to Dictionary', command=lambda
                                err=err: self.__usrdict(err))
        return contextmenu

    def __replace(self, tag, word):
        mrk = self.spellerrs[tag][1]
        err = self.spellerrs[tag][0]
        firstltr = self.textwidget.index(mrk)
        lastltr = firstltr + ('+%dchars' % len(err))
        self.textwidget.delete(firstltr, lastltr)
        self.textwidget.insert(firstltr, word)
        self.textwidget.tag_delete(tag)
        self.textwidget.mark_unset(mrk)
        del self.spellerrs[tag]

    def __usrdict(self, err):
        '''TODO: Add a personal dictionary file '''
        pass

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import tmt
import time
import gc3libs
import gc3libs.workflow
from tmt.visi.stk import Stk
from tmt.visi.stk2png import Stk2png

import logging
gc3libs.configure_logger(level=logging.DEBUG)
# gc3libs.configure_logger(level=logging.INFO)

class Visi(object):
    '''
    Class for visi interface.
    '''

    def __init__(self, args):
        self.args = args
        self.args.stk_folder = os.path.abspath(args.stk_folder)

    def joblist(self):
        '''
        Create a list of jobs in YAML format for parallel computing.
        '''
        project = Stk(self.args.stk_folder, self.args.wildcards,
                      config=self.args.config)

        print '. Creating output directories'
        project.create_output_dirs(self.args.split_output)

        print '. Creating joblist'
        project.create_joblist(batch_size=self.args.batch_size,
                               rename=self.args.rename)

        print '. Writing joblist to file'
        project.write_joblist()

    def run(self):
        '''
        Run unpacking of stk files with optional renaming.
        '''
        project = Stk(self.args.stk_folder, self.args.wildcards,
                      config=self.args.config)

        if self.args.job:

            job_ix = self.args.job-1  # job ids are one-based!

            print '. Reading joblist from file'
            joblist = project.read_joblist()

            print '. Processing job #%d' % self.args.job
            batch = joblist[job_ix]
            process = Stk2png(batch['stk_files'], batch['nd_file'],
                              self.args.config)
            print '.. Unpack .stk files and convert them to .png images'
            process.unpack_images(output_dir=batch['output_dir'],
                                  output_files=batch['png_files'],
                                  keep_z=self.args.zstacks)
        else:

            print '. Creating output directories'
            project.create_output_dirs(self.args.output_folder_name,
                                       self.args.split_output)

            print '. Creating joblist'
            joblist = project.create_joblist(batch_size=1,
                                             rename=self.args.rename)

            for batch in joblist:
                print '. Processing job #%d' % batch['job_id']
                process = Stk2png(batch['stk_files'], batch['nd_file'],
                                  self.args.config)

                print '.. Unpack .stk files and convert them to .png images'
                process.unpack_images(output_dir=batch['output_dir'],
                                      output_files=batch['png_files'],
                                      keep_z=self.args.zstacks)

    def submit(self):
        '''
        Run jobs in parallel via gc3pie.
        '''
        project = Stk(self.args.stk_folder, '*', config=self.args.config)
        joblist = project.read_joblist()

        # Prepare for STDOUT log
        log_dir = os.path.join(project.experiment_dir, 'log')
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)

        # Create an `Engine` instance for running jobs in parallel
        e = gc3libs.create_engine()
        # Put all output files in the same directory
        e.retrieve_overwrites = True
        # Create parallel task collection
        # jobs = gc3libs.workflow.ParallelTaskCollection()
        jobs = gc3libs.workflow.SequentialTaskCollection(tasks=None)
        for batch in joblist:

            if batch['job_id'] > 1:
                continue

            timestamp = tmt.cluster.create_timestamp()
            log_file = os.path.join('log',
                                    'visi_%s_%.5d_%s.txt'
                                    % (project.experiment,
                                       batch['job_id'], timestamp))

            if self.args.config_file:
                command = [
                    'visi', 'run', '--job', str(batch['job_id']),
                    '--visi_config', self.args.config_file,
                    self.args.stk_folder
                ]
            else:
                command = [
                    'visi', 'run', '--job', str(batch['job_id']),
                    self.args.stk_folder
                ]

            app = gc3libs.Application(
                arguments=command,
                inputs=[batch['nd_file']] + batch['stk_files'],
                outputs=batch['png_files'],
                output_dir=batch['output_dir'],
                jobname='visi_%s_%.5d' % (project.experiment, batch['job_id']),
                # stdout=log_file
            )
            jobs.add(app)
        e.add(jobs)

        print 'submit jobs'
        # Periodically check the status of the submitted jobs
        while jobs.execution.state != gc3libs.Run.State.TERMINATED:
            print "Jobs in status %s " % jobs.execution.state
            # `Engine.progress()` will do the GC3Pie magic:
            # submit new jobs, update status of submitted jobs, get
            # results of terminating jobs etc...
            e.progress()

            # Wait a few seconds...
            time.sleep(10)

        print 'Job is now terminated.'

        # sequential task collection for "pipelines" of tasks with dependencies

    @staticmethod
    def process_cli_commands(args, subparser):
        cli = Visi(args)
        if subparser.prog == 'visi run':
            cli.run()
        elif subparser.prog == 'visi joblist':
            cli.joblist()
        elif subparser.prog == 'visi submit':
            cli.submit()
        else:
            subparser.print_help()
<commit_msg>Change to parallel mode<commit_after>import os
import tmt
import time
import gc3libs
import gc3libs.workflow
from tmt.visi.stk import Stk
from tmt.visi.stk2png import Stk2png

import logging
gc3libs.configure_logger(level=logging.DEBUG)
# gc3libs.configure_logger(level=logging.INFO)

class Visi(object):
    '''
    Class for visi interface.
    '''

    def __init__(self, args):
        self.args = args
        self.args.stk_folder = os.path.abspath(args.stk_folder)

    def joblist(self):
        '''
        Create a list of jobs in YAML format for parallel computing.
        '''
        project = Stk(self.args.stk_folder, self.args.wildcards,
                      config=self.args.config)

        print '. Creating output directories'
        project.create_output_dirs(self.args.split_output)

        print '. Creating joblist'
        project.create_joblist(batch_size=self.args.batch_size,
                               rename=self.args.rename)

        print '. Writing joblist to file'
        project.write_joblist()

    def run(self):
        '''
        Run unpacking of stk files with optional renaming.
        '''
        project = Stk(self.args.stk_folder, self.args.wildcards,
                      config=self.args.config)

        if self.args.job:

            job_ix = self.args.job-1  # job ids are one-based!

            print '. Reading joblist from file'
            joblist = project.read_joblist()

            print '. Processing job #%d' % self.args.job
            batch = joblist[job_ix]
            process = Stk2png(batch['stk_files'], batch['nd_file'],
                              self.args.config)
            print '.. Unpack .stk files and convert them to .png images'
            process.unpack_images(output_dir=batch['output_dir'],
                                  output_files=batch['png_files'],
                                  keep_z=self.args.zstacks)
        else:

            print '. Creating output directories'
            project.create_output_dirs(self.args.output_folder_name,
                                       self.args.split_output)

            print '. Creating joblist'
            joblist = project.create_joblist(batch_size=1,
                                             rename=self.args.rename)

            for batch in joblist:
                print '. Processing job #%d' % batch['job_id']
                process = Stk2png(batch['stk_files'], batch['nd_file'],
                                  self.args.config)

                print '.. Unpack .stk files and convert them to .png images'
                process.unpack_images(output_dir=batch['output_dir'],
                                      output_files=batch['png_files'],
                                      keep_z=self.args.zstacks)

    def submit(self):
        '''
        Run jobs in parallel via gc3pie.
        '''
        project = Stk(self.args.stk_folder, '*', config=self.args.config)
        joblist = project.read_joblist()

        # Prepare for STDOUT log
        log_dir = os.path.join(project.experiment_dir, 'log')
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)

        # Create an `Engine` instance for running jobs in parallel
        e = gc3libs.create_engine()
        # Put all output files in the same directory
        e.retrieve_overwrites = True
        # Create parallel task collection
        jobs = gc3libs.workflow.ParallelTaskCollection()
        # jobs = gc3libs.workflow.SequentialTaskCollection(tasks=None)
        for batch in joblist:

            if batch['job_id'] > 1:
                continue

            timestamp = tmt.cluster.create_timestamp()
            log_file = os.path.join('log',
                                    'visi_%s_%.5d_%s.txt'
                                    % (project.experiment,
                                       batch['job_id'], timestamp))

            if self.args.config_file:
                command = [
                    'visi', 'run', '--job', str(batch['job_id']),
                    '--visi_config', self.args.config_file,
                    self.args.stk_folder
                ]
            else:
                command = [
                    'visi', 'run', '--job', str(batch['job_id']),
                    self.args.stk_folder
                ]

            app = gc3libs.Application(
                arguments=command,
                inputs=[batch['nd_file']] + batch['stk_files'],
                outputs=batch['png_files'],
                output_dir=batch['output_dir'],
                jobname='visi_%s_%.5d' % (project.experiment, batch['job_id']),
                stdout=log_file
            )
            jobs.add(app)
        e.add(jobs)

        print 'submit jobs'
        # Periodically check the status of the submitted jobs
        while jobs.execution.state != gc3libs.Run.State.TERMINATED:
            print "Jobs in status %s " % jobs.execution.state
            # `Engine.progress()` will do the GC3Pie magic:
            # submit new jobs, update status of submitted jobs, get
            # results of terminating jobs etc...
            e.progress()

            # Wait a few seconds...
            time.sleep(10)

        print 'Job is now terminated.'

        # sequential task collection for "pipelines" of tasks with dependencies

    @staticmethod
    def process_cli_commands(args, subparser):
        cli = Visi(args)
        if subparser.prog == 'visi run':
            cli.run()
        elif subparser.prog == 'visi joblist':
            cli.joblist()
        elif subparser.prog == 'visi submit':
            cli.submit()
        else:
            subparser.print_help()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Pythonization of the :term:`tmux(1)` server.

tmuxp.server
~~~~~~~~~~~~

"""
from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import logging

from .util import tmux_cmd, TmuxRelationalObject
from .session import Session
from . import formats, exc

logger = logging.getLogger(__name__)

class Server(TmuxRelationalObject):

    """The :term:`tmux(1)` server.

    - :attr:`Server._sessions` [:class:`Session`, ...]

      - :attr:`Session._windows` [:class:`Window`, ...]

        - :attr:`Window._panes` [:class:`Pane`, ...]

          - :class:`Pane`

    When instantiated, provides the ``t`` global. stores information on live,
    running tmux server.

    """

    #: ``[-L socket-name]``
    socket_name = None
    #: ``[-S socket-path]``
    socket_path = None
    #: ``[-f file]``
    config_file = None
    #: ``-2`` or ``-8``
    colors = None
    #: unique child ID key
    childIdAttribute = 'session_id'

    def __init__(
        self,
        socket_name=None,
        socket_path=None,
        config_file=None,
        colors=None,
        **kwargs
    ):
        self._windows = []
        self._panes = []

        if socket_name:
            self.socket_name = socket_name

        if socket_path:
            self.socket_path = socket_path

        if config_file:
            self.config_file = config_file

        if colors:
            self.colors = colors

    def cmd(self, *args, **kwargs):
        """Return :class:`util.tmux_cmd` send tmux commands with sockets, colors.

        :rtype: :class:`util.tmux_cmd`

        :versionchanged: 0.8
            Renamed from ``.tmux`` to ``.cmd``.

        """

        args = list(args)
        if self.socket_name:
            args.insert(0, '-L{0}'.format(self.socket_name))
        if self.socket_path:
            args.insert(0, '-S{0}'.format(self.socket_path))
        if self.config_file:
            args.insert(0, '-f{0}'.format(self.config_file))
        if self.colors:
            if self.colors == 256:
                args.insert(0, '-2')
            elif self.colors == 88:
                args.insert(0, '-8')
            else:
                raise ValueError('Server.colors must equal 88 or 256')

        return tmux_cmd(*args, **kwargs)

    def _list_sessions(self):
        """Return list of sessions in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-sessions`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: :py:obj:`list` of :py:obj:`dict`

        """

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % f for f in sformats]

        tmux_args = (
            '-F%s' % '\t'.join(tmux_formats),   # output
        )

        proc = self.cmd(
            'list-sessions',
            *tmux_args
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)
        else:
            session_info = proc.stdout[0]

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % format for format in sformats]
        sessions = proc.stdout

        # combine format keys with values returned from ``tmux list-windows``
        sessions = [dict(zip(
            sformats, session.split('\t'))) for session in sessions]

        # clear up empty dict
        sessions = [
            dict((k, v) for k, v in session.items() if v)
            for session in sessions
        ]

        return sessions

    @property
    def _sessions(self):
        """Property / alias to return :meth:`~._list_sessions`."""

        return self._list_sessions()

    def list_sessions(self):
        """Return list of :class:`Session` from the ``tmux(1)`` session.

        :rtype: :py:obj:`list` of :class:`Session`

        """
        return [
            Session(server=self, **s) for s in self._sessions
        ]

    @property
    def sessions(self):
        """Property / alias to return :meth:`~.list_sessions`."""
        return self.list_sessions()
    #: Alias of :attr:`sessions`.
    children = sessions

    def _list_windows(self):
        """Return list of windows in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-windows`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: list

        """

        wformats = ['session_name', 'session_id'] + formats.WINDOW_FORMATS
        tmux_formats = ['#{%s}' % format for format in wformats]

        proc = self.cmd(
            'list-windows',                     # ``tmux list-windows``
            '-a',
            '-F%s' % '\t'.join(tmux_formats),   # output
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        windows = proc.stdout

        wformats = ['session_name', 'session_id'] + formats.WINDOW_FORMATS

        # combine format keys with values returned from ``tmux list-windows``
        windows = [dict(zip(
            wformats, window.split('\t'))) for window in windows]

        # clear up empty dict
        windows = [
            dict((k, v) for k, v in window.items() if v) for window in windows
        ]

        # tmux < 1.8 doesn't have window_id, use window_name
        for w in windows:
            if not 'window_id' in w:
                w['window_id'] = w['window_name']

        if self._windows:
            self._windows[:] = []

        self._windows.extend(windows)

        return self._windows

    def _update_windows(self):
        """Update internal window data and return ``self`` for chainability.

        :rtype: :class:`Server`

        """
        self._list_windows()
        return self

    def _list_panes(self):
        """Return list of panes in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-panes`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: list

        """

        pformats = [
            'session_name', 'session_id',
            'window_index', 'window_id',
            'window_name'
        ] + formats.PANE_FORMATS
        tmux_formats = ['#{%s}\t' % f for f in pformats]

        proc = self.cmd(
            'list-panes',
            '-a',
            '-F%s' % ''.join(tmux_formats),     # output
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        panes = proc.stdout

        pformats = [
            'session_name', 'session_id',
            'window_index', 'window_id', 'window_name'
        ] + formats.PANE_FORMATS

        # combine format keys with values returned from ``tmux list-panes``
        panes = [dict(zip(
            pformats, window.split('\t'))) for window in panes]

        # clear up empty dict
        panes = [
            dict((k, v) for k, v in window.items() if v) for window in panes
        ]

        if self._panes:
            self._panes[:] = []

        self._panes.extend(panes)

        return self._panes

    def _update_panes(self):
        """Update internal pane data and return ``self`` for chainability.

        :rtype: :class:`Server`

        """
        self._list_panes()
        return self

    def attached_sessions(self):
        """Return active :class:`Session` objects.

        This will not work where multiple tmux sessions are attached.

        :rtype: :py:obj:`list` of :class:`Session`

        """

        sessions = self._sessions
        attached_sessions = list()

        for session in sessions:
            if 'session_attached' in session:
                # for now session_active is a unicode
                if session.get('session_attached') == '1':
                    logger.debug('session %s attached', session.get(
                        'session_name'))
                    attached_sessions.append(session)
                else:
                    continue

        return [
                   Session(server=self, **s) for s in attached_sessions
               ] or None

    def has_session(self, target_session):
        """Return True if session exists. ``$ tmux has-session``.

        :param: target_session: str of session name.
        :rtype: bool

        """

        proc = self.cmd('has-session', '-t%s' % target_session)

        if proc.stdout == []:
            return True
        if any(
            x in proc.stdout for x in
            ['failed to connect to server', 'error connecting to']
        ):
            return False
        elif 'no server running' in proc.stdout:  # tmux 2.0
            return False
        elif 'can\'t find session' in proc.stdout:  # tmux 2.1
            return False
        elif 'session not found' in proc.stdout:
            return False
        else:
            return True

    def kill_server(self):
        """``$ tmux kill-server``."""
        self.cmd('kill-server')

    def kill_session(self, target_session=None):
        """Kill the tmux session with ``$ tmux kill-session``, return ``self``.

        :param: target_session: str. note this accepts ``fnmatch(3)``. 'asdf'
            will kill 'asdfasd'.

        :rtype: :class:`Server`

        """
        proc = self.cmd('kill-session', '-t%s' % target_session)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        return self

    def switch_client(self, target_session):
        """``$ tmux switch-client``.

        :param: target_session: str. name of the session. fnmatch(3) works.

        """

        proc = self.cmd('switch-client', '-t%s' % target_session)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

    def attach_session(self, target_session=None):
        """``$ tmux attach-session`` aka alias: ``$ tmux attach``.

        :param: target_session: str. name of the session. fnmatch(3) works.

        """
        tmux_args = tuple()
        if target_session:
            tmux_args += ('-t%s' % target_session,)

        proc = self.cmd('attach-session', *tmux_args)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

    def new_session(self,
                    session_name=None,
                    kill_session=False,
                    attach=False,
                    *args,
                    **kwargs):
        """Return :class:`Session` from  ``$ tmux new-session``.

        Uses ``-P`` flag to print session info, ``-F`` for return formatting
        returns new Session object.

        ``$ tmux new-session -d`` will create the session in the background
        ``$ tmux new-session -Ad`` will move to the session name if it already
        exists. todo: make an option to handle this.

        :param session_name: session name::

            $ tmux new-session -s <session_name>
        :type session_name: string

        :param detach: create session background::

            $ tmux new-session -d
        :type detach: bool

        :param attach_if_exists: if the session_name exists, attach it.
                                 if False, this method will raise a
                                 :exc:`tmuxp.exc.TmuxSessionExists` exception
        :type attach_if_exists: bool

        :param kill_session: Kill current session if ``$ tmux has-session``
                             Useful for testing workspaces.
        :type kill_session: bool
        :rtype: :class:`Session`

        """

        if self.has_session(session_name):
            if kill_session:
                self.cmd('kill-session', '-t%s' % session_name)
                logger.info('session %s exists. killed it.' % session_name)
            else:
                raise exc.TmuxSessionExists(
                    'Session named %s exists' % session_name
                )

        logger.debug('creating session %s' % session_name)

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % f for f in sformats]

        env = os.environ.get('TMUX')

        if env:
            del os.environ['TMUX']

        tmux_args = (
            '-s%s' % session_name,
            '-P', '-F%s' % '\t'.join(tmux_formats),   # output
        )

        if not attach:
            tmux_args += ('-d',)

        proc = self.cmd(
            'new-session',
            *tmux_args
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        session = proc.stdout[0]

        if env:
            os.environ['TMUX'] = env

        # combine format keys with values returned from ``tmux list-windows``
        session = dict(zip(sformats, session.split('\t')))

        # clear up empty dict
        session = dict((k, v) for k, v in session.items() if v)

        session = Session(server=self, **session)

        return session
<commit_msg>Update new_session documentation<commit_after># -*- coding: utf-8 -*-
"""Pythonization of the :term:`tmux(1)` server.

tmuxp.server
~~~~~~~~~~~~

"""
from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

import os
import logging

from .util import tmux_cmd, TmuxRelationalObject
from .session import Session
from . import formats, exc

logger = logging.getLogger(__name__)

class Server(TmuxRelationalObject):

    """The :term:`tmux(1)` server.

    - :attr:`Server._sessions` [:class:`Session`, ...]

      - :attr:`Session._windows` [:class:`Window`, ...]

        - :attr:`Window._panes` [:class:`Pane`, ...]

          - :class:`Pane`

    When instantiated, provides the ``t`` global. stores information on live,
    running tmux server.

    """

    #: ``[-L socket-name]``
    socket_name = None
    #: ``[-S socket-path]``
    socket_path = None
    #: ``[-f file]``
    config_file = None
    #: ``-2`` or ``-8``
    colors = None
    #: unique child ID key
    childIdAttribute = 'session_id'

    def __init__(
        self,
        socket_name=None,
        socket_path=None,
        config_file=None,
        colors=None,
        **kwargs
    ):
        self._windows = []
        self._panes = []

        if socket_name:
            self.socket_name = socket_name

        if socket_path:
            self.socket_path = socket_path

        if config_file:
            self.config_file = config_file

        if colors:
            self.colors = colors

    def cmd(self, *args, **kwargs):
        """Return :class:`util.tmux_cmd` send tmux commands with sockets, colors.

        :rtype: :class:`util.tmux_cmd`

        :versionchanged: 0.8
            Renamed from ``.tmux`` to ``.cmd``.

        """

        args = list(args)
        if self.socket_name:
            args.insert(0, '-L{0}'.format(self.socket_name))
        if self.socket_path:
            args.insert(0, '-S{0}'.format(self.socket_path))
        if self.config_file:
            args.insert(0, '-f{0}'.format(self.config_file))
        if self.colors:
            if self.colors == 256:
                args.insert(0, '-2')
            elif self.colors == 88:
                args.insert(0, '-8')
            else:
                raise ValueError('Server.colors must equal 88 or 256')

        return tmux_cmd(*args, **kwargs)

    def _list_sessions(self):
        """Return list of sessions in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-sessions`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: :py:obj:`list` of :py:obj:`dict`

        """

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % f for f in sformats]

        tmux_args = (
            '-F%s' % '\t'.join(tmux_formats),   # output
        )

        proc = self.cmd(
            'list-sessions',
            *tmux_args
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)
        else:
            session_info = proc.stdout[0]

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % format for format in sformats]
        sessions = proc.stdout

        # combine format keys with values returned from ``tmux list-windows``
        sessions = [dict(zip(
            sformats, session.split('\t'))) for session in sessions]

        # clear up empty dict
        sessions = [
            dict((k, v) for k, v in session.items() if v)
            for session in sessions
        ]

        return sessions

    @property
    def _sessions(self):
        """Property / alias to return :meth:`~._list_sessions`."""

        return self._list_sessions()

    def list_sessions(self):
        """Return list of :class:`Session` from the ``tmux(1)`` session.

        :rtype: :py:obj:`list` of :class:`Session`

        """
        return [
            Session(server=self, **s) for s in self._sessions
        ]

    @property
    def sessions(self):
        """Property / alias to return :meth:`~.list_sessions`."""
        return self.list_sessions()
    #: Alias of :attr:`sessions`.
    children = sessions

    def _list_windows(self):
        """Return list of windows in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-windows`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: list

        """

        wformats = ['session_name', 'session_id'] + formats.WINDOW_FORMATS
        tmux_formats = ['#{%s}' % format for format in wformats]

        proc = self.cmd(
            'list-windows',                     # ``tmux list-windows``
            '-a',
            '-F%s' % '\t'.join(tmux_formats),   # output
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        windows = proc.stdout

        wformats = ['session_name', 'session_id'] + formats.WINDOW_FORMATS

        # combine format keys with values returned from ``tmux list-windows``
        windows = [dict(zip(
            wformats, window.split('\t'))) for window in windows]

        # clear up empty dict
        windows = [
            dict((k, v) for k, v in window.items() if v) for window in windows
        ]

        # tmux < 1.8 doesn't have window_id, use window_name
        for w in windows:
            if not 'window_id' in w:
                w['window_id'] = w['window_name']

        if self._windows:
            self._windows[:] = []

        self._windows.extend(windows)

        return self._windows

    def _update_windows(self):
        """Update internal window data and return ``self`` for chainability.

        :rtype: :class:`Server`

        """
        self._list_windows()
        return self

    def _list_panes(self):
        """Return list of panes in :py:obj:`dict` form.

        Retrieved from ``$ tmux(1) list-panes`` stdout.

        The :py:obj:`list` is derived from ``stdout`` in :class:`util.tmux_cmd`
        which wraps :py:class:`subprocess.Popen`.

        :rtype: list

        """

        pformats = [
            'session_name', 'session_id',
            'window_index', 'window_id',
            'window_name'
        ] + formats.PANE_FORMATS
        tmux_formats = ['#{%s}\t' % f for f in pformats]

        proc = self.cmd(
            'list-panes',
            '-a',
            '-F%s' % ''.join(tmux_formats),     # output
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        panes = proc.stdout

        pformats = [
            'session_name', 'session_id',
            'window_index', 'window_id', 'window_name'
        ] + formats.PANE_FORMATS

        # combine format keys with values returned from ``tmux list-panes``
        panes = [dict(zip(
            pformats, window.split('\t'))) for window in panes]

        # clear up empty dict
        panes = [
            dict((k, v) for k, v in window.items() if v) for window in panes
        ]

        if self._panes:
            self._panes[:] = []

        self._panes.extend(panes)

        return self._panes

    def _update_panes(self):
        """Update internal pane data and return ``self`` for chainability.

        :rtype: :class:`Server`

        """
        self._list_panes()
        return self

    def attached_sessions(self):
        """Return active :class:`Session` objects.

        This will not work where multiple tmux sessions are attached.

        :rtype: :py:obj:`list` of :class:`Session`

        """

        sessions = self._sessions
        attached_sessions = list()

        for session in sessions:
            if 'session_attached' in session:
                # for now session_active is a unicode
                if session.get('session_attached') == '1':
                    logger.debug('session %s attached', session.get(
                        'session_name'))
                    attached_sessions.append(session)
                else:
                    continue

        return [
                   Session(server=self, **s) for s in attached_sessions
               ] or None

    def has_session(self, target_session):
        """Return True if session exists. ``$ tmux has-session``.

        :param: target_session: str of session name.
        :rtype: bool

        """

        proc = self.cmd('has-session', '-t%s' % target_session)

        if proc.stdout == []:
            return True
        if any(
            x in proc.stdout for x in
            ['failed to connect to server', 'error connecting to']
        ):
            return False
        elif 'no server running' in proc.stdout:  # tmux 2.0
            return False
        elif 'can\'t find session' in proc.stdout:  # tmux 2.1
            return False
        elif 'session not found' in proc.stdout:
            return False
        else:
            return True

    def kill_server(self):
        """``$ tmux kill-server``."""
        self.cmd('kill-server')

    def kill_session(self, target_session=None):
        """Kill the tmux session with ``$ tmux kill-session``, return ``self``.

        :param: target_session: str. note this accepts ``fnmatch(3)``. 'asdf'
            will kill 'asdfasd'.

        :rtype: :class:`Server`

        """
        proc = self.cmd('kill-session', '-t%s' % target_session)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        return self

    def switch_client(self, target_session):
        """``$ tmux switch-client``.

        :param: target_session: str. name of the session. fnmatch(3) works.

        """

        proc = self.cmd('switch-client', '-t%s' % target_session)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

    def attach_session(self, target_session=None):
        """``$ tmux attach-session`` aka alias: ``$ tmux attach``.

        :param: target_session: str. name of the session. fnmatch(3) works.

        """
        tmux_args = tuple()
        if target_session:
            tmux_args += ('-t%s' % target_session,)

        proc = self.cmd('attach-session', *tmux_args)

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

    def new_session(self,
                    session_name=None,
                    kill_session=False,
                    attach=False,
                    *args,
                    **kwargs):
        """Return :class:`Session` from  ``$ tmux new-session``.

        Uses ``-P`` flag to print session info, ``-F`` for return formatting
        returns new Session object.

        ``$ tmux new-session -d`` will create the session in the background
        ``$ tmux new-session -Ad`` will move to the session name if it already
        exists. todo: make an option to handle this.

        :param session_name: session name::

            $ tmux new-session -s <session_name>
        :type session_name: string

        :param attach: create session in the foreground. ``attach=False`` is
        equivalent to::

            $ tmux new-session -d
        :type attach: bool

        :param kill_session: Kill current session if ``$ tmux has-session``
                             Useful for testing workspaces.
        :type kill_session: bool
        :rtype: :class:`Session`

        """

        if self.has_session(session_name):
            if kill_session:
                self.cmd('kill-session', '-t%s' % session_name)
                logger.info('session %s exists. killed it.' % session_name)
            else:
                raise exc.TmuxSessionExists(
                    'Session named %s exists' % session_name
                )

        logger.debug('creating session %s' % session_name)

        sformats = formats.SESSION_FORMATS
        tmux_formats = ['#{%s}' % f for f in sformats]

        env = os.environ.get('TMUX')

        if env:
            del os.environ['TMUX']

        tmux_args = (
            '-s%s' % session_name,
            '-P', '-F%s' % '\t'.join(tmux_formats),   # output
        )

        if not attach:
            tmux_args += ('-d',)

        proc = self.cmd(
            'new-session',
            *tmux_args
        )

        if proc.stderr:
            raise exc.TmuxpException(proc.stderr)

        session = proc.stdout[0]

        if env:
            os.environ['TMUX'] = env

        # combine format keys with values returned from ``tmux list-windows``
        session = dict(zip(sformats, session.split('\t')))

        # clear up empty dict
        session = dict((k, v) for k, v in session.items() if v)

        session = Session(server=self, **session)

        return session
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import datetime

from pyramid.httpexceptions import HTTPUnauthorized

from ines.api import BaseSessionManager
from ines.api import BaseSession
from ines.convert import force_string
from ines.middlewares.logs import LoggingMiddleware

NOW_DATE = datetime.datetime.now

class BaseLogSessionManager(BaseSessionManager):
    __api_name__ = 'logging'
    __middlewares__ = [LoggingMiddleware]

class BaseLogSession(BaseSession):
    __api_name__ = 'logging'

    def log(self, code, message, level='INFO'):
        header = ('-' * 30) + ' ' + level + ' ' + ('-' * 30)
        print header

        arguments = [
            ('Application', self.request.application_name),
            ('Code', code),
            ('URL', self.request.url),
            ('Method', self.request.method),
            ('Date', NOW_DATE()),
            ('Language', self.request.locale_name),
            ('IP address', self.request.ip_address)]

        try:
            authenticated = self.request.authenticated
        except HTTPUnauthorized:
            authenticated = None

        if authenticated:
            arguments.extend([
                ('Session type', self.request.authenticated.session_type),
                ('Session id', self.request.authenticated.session_id)])
        else:
            arguments.append(('Session', 'Without autentication'))

        bigger = max(len(k) for k, v in arguments)
        for key, value in arguments:
            print key, ' ' * (bigger - len(key)), ':', force_string(value)

        print
        try:
            message = force_string(message)
            for line in message.split('\n'):
                print '  %s' % line
        except UnicodeEncodeError:
            pass

        print '-' * len(header)

    def log_debug(self, code, message):
        if self.config.debug:
            return self.log(
                code,
                message,
                level='DEBUG')

    def log_warning(self, code, message):
        return self.log(
            code,
            message,
            level='WARN')

    def log_error(self, code, message):
        return self.log(
            code,
            message,
            level='ERROR')

    def log_critical(self, code, message):
        return self.log(
            code,
            message,
            level='CRITICAL')
<commit_msg>Added extra arguments to log<commit_after># -*- coding: utf-8 -*-

import datetime

from pyramid.httpexceptions import HTTPUnauthorized

from ines.api import BaseSessionManager
from ines.api import BaseSession
from ines.convert import force_string
from ines.middlewares.logs import LoggingMiddleware

NOW_DATE = datetime.datetime.now

class BaseLogSessionManager(BaseSessionManager):
    __api_name__ = 'logging'
    __middlewares__ = [LoggingMiddleware]

class BaseLogSession(BaseSession):
    __api_name__ = 'logging'

    def log(self, code, message, level='INFO', extra=None):
        header = ('-' * 30) + ' ' + level + ' ' + ('-' * 30)
        print header

        arguments = [
            ('Application', self.request.application_name),
            ('Code', code),
            ('URL', self.request.url),
            ('Method', self.request.method),
            ('Date', NOW_DATE()),
            ('Language', self.request.locale_name),
            ('IP address', self.request.ip_address)]

        if extra:
            arguments.extend(extra.items())

        try:
            authenticated = self.request.authenticated
        except HTTPUnauthorized:
            authenticated = None

        if authenticated:
            arguments.extend([
                ('Session type', self.request.authenticated.session_type),
                ('Session id', self.request.authenticated.session_id)])
        else:
            arguments.append(('Session', 'Without autentication'))

        bigger = max(len(k) for k, v in arguments)
        for key, value in arguments:
            print key, ' ' * (bigger - len(key)), ':', force_string(value)

        print
        try:
            message = force_string(message)
            for line in message.split('\n'):
                print '  %s' % line
        except UnicodeEncodeError:
            pass

        print '-' * len(header)

    def log_debug(self, code, message, **kwargs):
        if self.config.debug:
            return self.log(
                code,
                message,
                level='DEBUG',
                **kwargs)

    def log_warning(self, code, message, **kwargs):
        return self.log(
            code,
            message,
            level='WARN',
            **kwargs)

    def log_error(self, code, message, **kwargs):
        return self.log(
            code,
            message,
            level='ERROR',
            **kwargs)

    def log_critical(self, code, message, **kwargs):
        return self.log(
            code,
            message,
            level='CRITICAL',
            **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import json
from subprocess import call, Popen, PIPE

# get role of machine (master or worker)
role = Popen('/usr/share/google/get_metadata_value attributes/dataproc-role', shell=True, stdout=PIPE).communicate()[0]

# initialization actions to perform on master machine only
if role == 'Master':

    # download Anaconda Python 2.7 installation script
    call(['wget', '-P', '/home/anaconda2/', 'https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh'])

    # install Anaconda in /home/anaconda2/
    call(['bash', '/home/anaconda2/Anaconda2-4.3.1-Linux-x86_64.sh', '-b', '-f', '-p', '/home/anaconda2/'])
    os.chmod('/home/anaconda2/', 0777)

    # additional packages to install
    pkgs = [
        'lxml',
        'jupyter-spark',
        'jgscm'
    ]

    # use pip to install packages
    for pkg in pkgs:
        call(['/home/anaconda2/bin/pip', 'install', pkg])

    # create Jupyter kernel spec file
    kernel = {
        'argv': [
            '/home/anaconda2/bin/python',
            '-m',
            'ipykernel',
            '-f',
            '{connection_file}'
        ],
        'display_name': 'Hail',
        'language': 'python',
        'env': {
            'PYTHONHASHSEED': '0',
            'SPARK_HOME': '/usr/lib/spark/',
            'PYTHONPATH': '/usr/lib/spark/python/:/usr/lib/spark/python/lib/py4j-0.10.3-src.zip:/home/hail/pyhail-hail-is-master-{}.zip'.format(hash)
        }
    }

    # write kernel spec file to default Jupyter kernel directory
    os.makedirs('/home/anaconda2/share/jupyter/kernels/hail/')
    with open('/home/anaconda2/share/jupyter/kernels/hail/kernel.json', 'wb') as f:
        json.dump(kernel, f)

    # get latest Hail hash
    hash = Popen(['gsutil', 'cat', 'gs://hail-common/latest-hash.txt'], stdout=PIPE, stderr=PIPE).communicate()[0].strip()

    # Hail jar and zip names
    hail_jar = 'hail-hail-is-master-all-spark2.0.2-{}.jar'.format(hash)
    hail_zip = 'pyhail-hail-is-master-{}.zip'.format(hash)

    # make directory for Hail and Jupyter notebook related files
    os.mkdir('/home/hail/')
    os.chmod('/home/hail/', 0777)

    # copy Hail jar and zip to local directory on master node
    call(['gsutil', 'cp', 'gs://hail-common/{}'.format(hail_jar), '/home/hail/'])
    call(['gsutil', 'cp', 'gs://hail-common/{}'.format(hail_zip), '/home/hail/'])

    # modify default Spark config file to reference Hail jar and zip
    with open('/etc/spark/conf/spark-defaults.conf', 'ab') as f:
        opts = [
            'spark.files=/home/hail/{}'.format(hail_jar),
            'spark.submit.pyFiles=/home/hail/{}'.format(hail_zip),
            'spark.driver.extraClassPath=./{0}:/home/hail/{0}'.format(hail_jar),
            'spark.executor.extraClassPath=./{0}:/home/hail/{0}'.format(hail_jar)
        ]
        f.write('\n'.join(opts))

    # add Spark variable designating Anaconda Python executable as the default on driver
    with open('/etc/spark/conf/spark-env.sh', 'ab') as f:
        f.write('PYSPARK_DRIVER_PYTHON=/home/anaconda2/bin/python' + '\n')

    # create Jupyter configuration file
    call(['mkdir', '-p', '/home/anaconda2/etc/jupyter/'])
    with open('/home/anaconda2/etc/jupyter/jupyter_notebook_config.py', 'wb') as f:
	    opts = [
		    'c.Application.log_level = "DEBUG"',
		    'c.NotebookApp.ip = "127.0.0.1"',
		    'c.NotebookApp.open_browser = False',
		    'c.NotebookApp.port = 8123',
		    'c.NotebookApp.token = ""',
		    'c.NotebookApp.contents_manager_class = "jgscm.GoogleStorageContentManager"'
        ]
	    f.write('\n'.join(opts) + '\n')

    # setup jupyter-spark extension
    call(['/home/anaconda2/bin/jupyter', 'serverextension', 'enable', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'install', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'enable', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'enable', '--user', '--py', 'widgetsnbextension'])

    # create systemd service file for Jupyter notebook server process
    with open('/lib/systemd/system/jupyter.service', 'wb') as f:
    	opts = [
    		'[Unit]',
    		'Description=Jupyter Notebook',
    		'After=hadoop-yarn-resourcemanager.service',
    		'[Service]',
    		'Type=simple',
    		'User=root',
    		'Group=root',
    		'WorkingDirectory=/home/hail/',
            'ExecStart=/home/anaconda2/bin/python /home/anaconda2/bin/jupyter notebook',
    		'Restart=always',
    		'RestartSec=1',
    		'[Install]',
    		'WantedBy=multi-user.target'
    	]
    	f.write('\n'.join(opts) + '\n')

    # add Jupyter service to autorun and start it
    call(['systemctl', 'daemon-reload'])
    call(['systemctl', 'enable', 'jupyter'])
    call(['service', 'jupyter', 'start'])
<commit_msg>fixed order of code in init script<commit_after>#!/usr/bin/env python

import os
import json
from subprocess import call, Popen, PIPE

# get role of machine (master or worker)
role = Popen('/usr/share/google/get_metadata_value attributes/dataproc-role', shell=True, stdout=PIPE).communicate()[0]

# initialization actions to perform on master machine only
if role == 'Master':

    # download Anaconda Python 2.7 installation script
    call(['wget', '-P', '/home/anaconda2/', 'https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh'])

    # install Anaconda in /home/anaconda2/
    call(['bash', '/home/anaconda2/Anaconda2-4.3.1-Linux-x86_64.sh', '-b', '-f', '-p', '/home/anaconda2/'])
    os.chmod('/home/anaconda2/', 0777)

    # additional packages to install
    pkgs = [
        'lxml',
        'jupyter-spark',
        'jgscm'
    ]

    # use pip to install packages
    for pkg in pkgs:
        call(['/home/anaconda2/bin/pip', 'install', pkg])

    # get latest Hail hash
    hash = Popen(['gsutil', 'cat', 'gs://hail-common/latest-hash.txt'], stdout=PIPE, stderr=PIPE).communicate()[0].strip()

    # Hail jar and zip names
    hail_jar = 'hail-hail-is-master-all-spark2.0.2-{}.jar'.format(hash)
    hail_zip = 'pyhail-hail-is-master-{}.zip'.format(hash)

    # make directory for Hail and Jupyter notebook related files
    os.mkdir('/home/hail/')
    os.chmod('/home/hail/', 0777)

    # copy Hail jar and zip to local directory on master node
    call(['gsutil', 'cp', 'gs://hail-common/{}'.format(hail_jar), '/home/hail/'])
    call(['gsutil', 'cp', 'gs://hail-common/{}'.format(hail_zip), '/home/hail/'])

    # create Jupyter kernel spec file
    kernel = {
        'argv': [
            '/home/anaconda2/bin/python',
            '-m',
            'ipykernel',
            '-f',
            '{connection_file}'
        ],
        'display_name': 'Hail',
        'language': 'python',
        'env': {
            'PYTHONHASHSEED': '0',
            'SPARK_HOME': '/usr/lib/spark/',
            'PYTHONPATH': '/usr/lib/spark/python/:/usr/lib/spark/python/lib/py4j-0.10.3-src.zip:/home/hail/pyhail-hail-is-master-{}.zip'.format(hash)
        }
    }

    # write kernel spec file to default Jupyter kernel directory
    os.makedirs('/home/anaconda2/share/jupyter/kernels/hail/')
    with open('/home/anaconda2/share/jupyter/kernels/hail/kernel.json', 'wb') as f:
        json.dump(kernel, f)

    # modify default Spark config file to reference Hail jar and zip
    with open('/etc/spark/conf/spark-defaults.conf', 'ab') as f:
        opts = [
            'spark.files=/home/hail/{}'.format(hail_jar),
            'spark.submit.pyFiles=/home/hail/{}'.format(hail_zip),
            'spark.driver.extraClassPath=./{0}:/home/hail/{0}'.format(hail_jar),
            'spark.executor.extraClassPath=./{0}:/home/hail/{0}'.format(hail_jar)
        ]
        f.write('\n'.join(opts))

    # add Spark variable designating Anaconda Python executable as the default on driver
    with open('/etc/spark/conf/spark-env.sh', 'ab') as f:
        f.write('PYSPARK_DRIVER_PYTHON=/home/anaconda2/bin/python' + '\n')

    # create Jupyter configuration file
    call(['mkdir', '-p', '/home/anaconda2/etc/jupyter/'])
    with open('/home/anaconda2/etc/jupyter/jupyter_notebook_config.py', 'wb') as f:
	    opts = [
		    'c.Application.log_level = "DEBUG"',
		    'c.NotebookApp.ip = "127.0.0.1"',
		    'c.NotebookApp.open_browser = False',
		    'c.NotebookApp.port = 8123',
		    'c.NotebookApp.token = ""',
		    'c.NotebookApp.contents_manager_class = "jgscm.GoogleStorageContentManager"'
        ]
	    f.write('\n'.join(opts) + '\n')

    # setup jupyter-spark extension
    call(['/home/anaconda2/bin/jupyter', 'serverextension', 'enable', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'install', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'enable', '--user', '--py', 'jupyter_spark'])
    call(['/home/anaconda2/bin/jupyter', 'nbextension', 'enable', '--user', '--py', 'widgetsnbextension'])

    # create systemd service file for Jupyter notebook server process
    with open('/lib/systemd/system/jupyter.service', 'wb') as f:
    	opts = [
    		'[Unit]',
    		'Description=Jupyter Notebook',
    		'After=hadoop-yarn-resourcemanager.service',
    		'[Service]',
    		'Type=simple',
    		'User=root',
    		'Group=root',
    		'WorkingDirectory=/home/hail/',
            'ExecStart=/home/anaconda2/bin/python /home/anaconda2/bin/jupyter notebook',
    		'Restart=always',
    		'RestartSec=1',
    		'[Install]',
    		'WantedBy=multi-user.target'
    	]
    	f.write('\n'.join(opts) + '\n')

    # add Jupyter service to autorun and start it
    call(['systemctl', 'daemon-reload'])
    call(['systemctl', 'enable', 'jupyter'])
    call(['service', 'jupyter', 'start'])
<|endoftext|>