<commit_before>
import numpy as np
import random
import itertools
import statsmodels.api as sm
from scipy.stats import norm

class CausalModel(object):

	def __init__(self, Y, D, X):
		self.Y, self.D, self.X = Y, D, X
		self.N, self.k = self.X.shape
		control, treated = (self.D==0), (self.D==1)
		self.Y_c, self.Y_t = self.Y[control], self.Y[treated]
		self.X_c, self.X_t = self.X[control], self.X[treated]
		self.N_c, self.N_t = self.X_c.shape[0], self.X_t.shape[0]

	def __polymatrix(self, X, terms):

		num_of_terms = len(terms)
		X_poly = np.ones((X.shape[0], num_of_terms))
		for i in xrange(num_of_terms):
			for j in terms[i]:
				X_poly[:, i] = X_poly[:, i] * X[:, j]
		return np.hstack((X, X_poly))

	def synthetic(self, poly=0):

		ITT = np.zeros(self.N_t)

		if poly > 1:

			terms = []
			for power in xrange(2, poly+1):
				terms.extend(list(itertools.combinations_with_replacement(range(self.k),
				                                                          power)))
			num_of_terms = len(terms)

			X_control = self.__polymatrix(self.X_c, terms)

			for i in xrange(self.N_t):
				X_treated = self.__polymatrix(self.X_t[i, :].reshape((1, self.k)), terms)
				w = np.linalg.lstsq(X_control.T, X_treated.flatten())[0]
				ITT[i] = self.Y_t[i] - np.dot(w, self.Y_c)

		else:
			# avoids storing the potentially massive N_t-by-N_c weights matrix
			for i in xrange(self.N_t):
				w = np.linalg.lstsq(self.X_c.T, self.X_t[i, ])[0]
				ITT[i] = self.Y_t[i] - np.dot(w, self.Y_c)

		return Results(self, ITT.mean(), ITT)

	def matching(self, replace=False, correct_bias=False):

		match_index = np.zeros(self.N_t, dtype=np.int)

		if replace:
			for i in xrange(self.N_t):
				dX = self.X_c - self.X_t[i]
				match_index[i] = np.argmin((dX**2).sum(axis=1))
		else:
			unmatched = range(self.N_c)
			for i in xrange(self.N_t):
				dX = self.X_c[unmatched] - self.X_t[i]
				match_index[i] = unmatched.pop(np.argmin((dX**2).sum(axis=1)))

		if correct_bias:
			reg = sm.OLS(self.Y_c[match_index],
			             sm.add_constant(self.X_c[match_index])).fit()
			ITT = (self.Y_t - self.Y_c[match_index] -
			       np.dot((self.X_t - self.X_c[match_index]), reg.params[1:]))
		else:
			ITT = self.Y_t - self.Y_c[match_index]

		return Results(self, ITT.mean(), ITT)

	def ols(self):

		D = self.D.reshape((self.N, 1))  # convert D into N-by-1 vector
		dX = self.X - self.X.mean(0)  # demean covariates
		DdX = D * dX
		# construct design matrix
		Z = np.column_stack((D, dX, DdX))

		reg = sm.OLS(self.Y, sm.add_constant(Z)).fit()

		# for derivation of this estimator, see my notes on Treatment Effects
		ITT = reg.params[1] + np.dot(dX[self.D==1], reg.params[-self.k:])

		return Results(self, ITT.mean(), ITT)

class Results(object):

	def __init__(self, model, ATT, ITT):
		self.ATT = ATT
		self.ITT = ITT

	def summary(self):

		print 'Estimated Average Treatment Effect on the Treated:', self.ATT

class parameters(object):

	"""
	Class object that stores the parameter values for use in the baseline model.

	See SimulateData function for model description.

	Args:
		N = Sample size (control + treated units) to generate
		k = Number of covariates
	"""

	def __init__(self, N=500, k=3):  # set initial parameter values
		self.N = N  # sample size (control + treated units)
		self.k = k  # number of covariates

		self.delta = 3
		self.beta = np.ones(k)
		self.theta = np.ones(k)
		self.mu = np.zeros(k)
		self.Sigma = np.identity(k)
		self.Gamma = np.identity(2)

def SimulateData(para=parameters(), nonlinear=False, return_counterfactual=False):

	"""
	Function that generates data according to one of two simple models that
	satisfies the Unconfoundedness assumption.

	The covariates and error terms are generated according to
		X ~ N(mu, Sigma), epsilon ~ N(0, Gamma).
	The counterfactual outcomes are generated by
		Y_0 = X*beta + epsilon_0,
		Y_1 = delta + X*(beta+theta) + epsilon_1,
	if the nonlinear Boolean is False, or by
		Y_0 = sum_of_abs(X) + epsilon_0,
		Y_1 = sum_of_squares(X) + epsilon_1,
	if the nonlinear Boolean is True.

	Selection is done according to the following propensity score function:
		P(D=1|X) = Phi(X*beta),
	where Phi is the standard normal CDF.

	Args:
		para = Model parameter values supplied by parameter class object
		nonlinear = Boolean indicating whether the data generating model should
		            be linear or not
		return_counterfactual = Boolean indicating whether to return vectors of
		                        counterfactual outcomes

	Returns:
		Y = N-dimensional array of observed outcomes
		D = N-dimensional array of treatment indicator; 1=treated, 0=control
		X = N-by-k matrix of covariates
		Y_0 = N-dimensional array of non-treated outcomes
		Y_1 = N-dimensional array of treated outcomes
	"""

	k = len(para.mu)

	X = np.random.multivariate_normal(mean=para.mu, cov=para.Sigma,
	                                  size=para.N)
	epsilon = np.random.multivariate_normal(mean=np.zeros(2), cov=para.Gamma,
	                                        size=para.N)

	Xbeta = np.dot(X, para.beta)

	pscore = norm.cdf(Xbeta)
	# for each p in pscore, generate Bernoulli rv with success probability p
	D = np.array([np.random.binomial(1, p, size=1) for p in pscore]).flatten()

	if nonlinear:
		Y_0 = abs(X).sum(1) + epsilon[:, 0]
		Y_1 = (X**2).sum(1) + epsilon[:, 1]
	else:
		Y_0 = Xbeta + epsilon[:, 0]
		Y_1 = para.delta + np.dot(X, para.beta+para.theta) + epsilon[:, 1]

	Y = (1 - D) * Y_0 + D * Y_1  # compute observed outcome

	if return_counterfactual:
		return Y, D, X, Y_0, Y_1
	else:
		return Y, D, X
<commit_msg>added match by pscore order option<commit_after>
import numpy as np
import random
import itertools
import statsmodels.api as sm
from scipy.stats import norm

class CausalModel(object):

	def __init__(self, Y, D, X):
		self.Y, self.D, self.X = Y, D, X
		self.N, self.k = self.X.shape
		control, treated = (self.D==0), (self.D==1)
		self.Y_c, self.Y_t = self.Y[control], self.Y[treated]
		self.X_c, self.X_t = self.X[control], self.X[treated]
		self.N_c, self.N_t = self.X_c.shape[0], self.X_t.shape[0]

	def __polymatrix(self, X, terms):

		num_of_terms = len(terms)
		X_poly = np.ones((X.shape[0], num_of_terms))
		for i in xrange(num_of_terms):
			for j in terms[i]:
				X_poly[:, i] = X_poly[:, i] * X[:, j]
		return np.hstack((X, X_poly))

	def synthetic(self, poly=0):

		ITT = np.zeros(self.N_t)

		if poly > 1:

			terms = []
			for power in xrange(2, poly+1):
				terms.extend(list(itertools.combinations_with_replacement(range(self.k),
				                                                          power)))
			num_of_terms = len(terms)

			X_control = self.__polymatrix(self.X_c, terms)

			for i in xrange(self.N_t):
				X_treated = self.__polymatrix(self.X_t[i, :].reshape((1, self.k)), terms)
				w = np.linalg.lstsq(X_control.T, X_treated.flatten())[0]
				ITT[i] = self.Y_t[i] - np.dot(w, self.Y_c)

		else:
			# avoids storing the potentially massive N_t-by-N_c weights matrix
			for i in xrange(self.N_t):
				w = np.linalg.lstsq(self.X_c.T, self.X_t[i, ])[0]
				ITT[i] = self.Y_t[i] - np.dot(w, self.Y_c)

		return Results(self, ITT.mean(), ITT)

	def matching(self, replace=False, correct_bias=False, order_by_pscore=False):

		match_index = np.zeros(self.N_t, dtype=np.int)

		if replace:
			for i in xrange(self.N_t):
				dX = self.X_c - self.X_t[i]
				match_index[i] = np.argmin((dX**2).sum(axis=1))
		else:
			unmatched = range(self.N_c)
			if order_by_pscore:
				pscore = sm.Logit(D, X).fit(disp=False).predict()
				order = np.argsort(pscore[D==1])[::-1]
			else:
				order = xrange(self.N_t)
			for i in order:
				dX = self.X_c[unmatched] - self.X_t[i]
				match_index[i] = unmatched.pop(np.argmin((dX**2).sum(axis=1)))

		if correct_bias:
			reg = sm.OLS(self.Y_c[match_index],
			             sm.add_constant(self.X_c[match_index])).fit()
			ITT = (self.Y_t - self.Y_c[match_index] -
			       np.dot((self.X_t - self.X_c[match_index]), reg.params[1:]))
		else:
			ITT = self.Y_t - self.Y_c[match_index]

		return Results(self, ITT.mean(), ITT)

	def ols(self):

		D = self.D.reshape((self.N, 1))  # convert D into N-by-1 vector
		dX = self.X - self.X.mean(0)  # demean covariates
		DdX = D * dX
		# construct design matrix
		Z = np.column_stack((D, dX, DdX))

		reg = sm.OLS(self.Y, sm.add_constant(Z)).fit()

		# for derivation of this estimator, see my notes on Treatment Effects
		ITT = reg.params[1] + np.dot(dX[self.D==1], reg.params[-self.k:])

		return Results(self, ITT.mean(), ITT)

class Results(object):

	def __init__(self, model, ATT, ITT):
		self.ATT = ATT
		self.ITT = ITT

	def summary(self):

		print 'Estimated Average Treatment Effect on the Treated:', self.ATT

class parameters(object):

	"""
	Class object that stores the parameter values for use in the baseline model.

	See SimulateData function for model description.

	Args:
		N = Sample size (control + treated units) to generate
		k = Number of covariates
	"""

	def __init__(self, N=500, k=3):  # set initial parameter values
		self.N = N  # sample size (control + treated units)
		self.k = k  # number of covariates

		self.delta = 3
		self.beta = np.ones(k)
		self.theta = np.ones(k)
		self.mu = np.zeros(k)
		self.Sigma = np.identity(k)
		self.Gamma = np.identity(2)

def SimulateData(para=parameters(), nonlinear=False, return_counterfactual=False):

	"""
	Function that generates data according to one of two simple models that
	satisfies the Unconfoundedness assumption.

	The covariates and error terms are generated according to
		X ~ N(mu, Sigma), epsilon ~ N(0, Gamma).
	The counterfactual outcomes are generated by
		Y_0 = X*beta + epsilon_0,
		Y_1 = delta + X*(beta+theta) + epsilon_1,
	if the nonlinear Boolean is False, or by
		Y_0 = sum_of_abs(X) + epsilon_0,
		Y_1 = sum_of_squares(X) + epsilon_1,
	if the nonlinear Boolean is True.

	Selection is done according to the following propensity score function:
		P(D=1|X) = Phi(X*beta),
	where Phi is the standard normal CDF.

	Args:
		para = Model parameter values supplied by parameter class object
		nonlinear = Boolean indicating whether the data generating model should
		            be linear or not
		return_counterfactual = Boolean indicating whether to return vectors of
		                        counterfactual outcomes

	Returns:
		Y = N-dimensional array of observed outcomes
		D = N-dimensional array of treatment indicator; 1=treated, 0=control
		X = N-by-k matrix of covariates
		Y_0 = N-dimensional array of non-treated outcomes
		Y_1 = N-dimensional array of treated outcomes
	"""

	k = len(para.mu)

	X = np.random.multivariate_normal(mean=para.mu, cov=para.Sigma,
	                                  size=para.N)
	epsilon = np.random.multivariate_normal(mean=np.zeros(2), cov=para.Gamma,
	                                        size=para.N)

	Xbeta = np.dot(X, para.beta)

	pscore = norm.cdf(Xbeta)
	# for each p in pscore, generate Bernoulli rv with success probability p
	D = np.array([np.random.binomial(1, p, size=1) for p in pscore]).flatten()

	if nonlinear:
		Y_0 = abs(X).sum(1) + epsilon[:, 0]
		Y_1 = (X**2).sum(1) + epsilon[:, 1]
	else:
		Y_0 = Xbeta + epsilon[:, 0]
		Y_1 = para.delta + np.dot(X, para.beta+para.theta) + epsilon[:, 1]

	Y = (1 - D) * Y_0 + D * Y_1  # compute observed outcome

	if return_counterfactual:
		return Y, D, X, Y_0, Y_1
	else:
		return Y, D, X
<|endoftext|>


<commit_before>#coding=utf-8

import os
import time
import sys
import shutil
import config
import tornado.httpserver
import tornado.web
import tornado.ioloop
import tornado.options
import tornado.escape
import tornado.template
import misaka as m

from tornado.options import define, options

define("port", default=8888, help="run on the given port", type=int)

from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_by_name

class CatsupRender(m.HtmlRenderer, m.SmartyPants):
    def block_code(self, text, lang):
        if not lang:
            text = tornado.escape.xhtml_escape(text.strip())
            return '\n
%s
\n' % text
        lexer = get_lexer_by_name(lang, stripall=True)
        formatter = HtmlFormatter()
        return highlight(text, lexer, formatter)

    def autolink(self, link, is_email):
        if is_email:
            return '%(link)s' % {'link': link}

        if '.' in link:
            name_extension = link.split('.')[-1].lower()
            if name_extension in ('jpg', 'png', 'git', 'jpeg'):
                return '' % link

        return '%s' % (link, link)

md = m.Markdown(CatsupRender(flags=m.HTML_ESCAPE | m.HTML_USE_XHTML),
    extensions=m.EXT_FENCED_CODE | m.EXT_NO_INTRA_EMPHASIS | m.EXT_AUTOLINK |
        m.EXT_STRIKETHROUGH | m.EXT_SUPERSCRIPT)

def load_post(file_name):
    '''Load a post.return a dict.
    '''
    path = os.path.join(config.posts_path, file_name)
    file = open(path, 'r')
    post = {'file_name': file_name[:-3],
            'tags': [],
            'date': file_name[:10]}
    while True:
        line = file.readline()
        if line.startswith('#'):
            post['title'] = line[1:].strip()
        elif 'tags' in line.lower():
            for tag in line.split(':')[-1].strip().split(','):
                post['tags'].append(tag.strip())
        elif line.startswith('---'):
            content = '\n'.join(file.readlines())
            if isinstance(content, str):
                content = content.decode('utf-8')
            post['content'] = md.render(content)
            post['updated'] = os.stat(path).st_ctime
            updated_xml = time.gmtime(post['updated'])
            post['updated_xml'] = time.strftime('%Y-%m-%dT%H:%M:%SZ',
                updated_xml)
            return post

def load_posts():
    '''load all the posts.return a list.
    Sort with filename.
    '''
    post_files = os.listdir(config.posts_path)
    post_files.sort(reverse=True)
    posts = []
    for file_name in post_files:
        if '.md' in file_name:
            post = load_post(file_name)
            posts.append(post)
    return posts

def get_infos(posts):
    """return the tag list and archive list.
    """
    tags = {}
    archives = {}
    for post in posts:
        for tag in post['tags']:
            if tag in tags:
                tags[tag].append(post)
            else:
                tags[tag] = [post]
        year = post['date'][:4]
        if year in archives:
            archives[year].append(post)
        else:
            archives[year] = [post]

    return sorted(tags.items(), key=lambda x: len(x[1]), reverse=True),\
        sorted(archives.items(), key=lambda x: x[0], reverse=True)

class BaseHandler(tornado.web.RequestHandler):
    def get_error_html(self, *args, **kwargs):
        return self.render_string('404.html')

class MainHandler(BaseHandler):
    def get(self, p=1):
        if p == '1':  # /page_1.html
            self.redirect('/', status=301)
        p = int(p)
        self.render('index.html', posts=self.settings['posts'], p=p)

class ArticleHandler(BaseHandler):
    def get(self, file_name):
        posts_num = len(posts)
        prev = next = None
        for i in range(posts_num):
            post = posts[i]
            if post['file_name'] == file_name:
                if i:  # i>0
                    prev = posts[i - 1]
                if (i + 1) < posts_num:
                    next = posts[i + 1]
                return self.render('article.html', post=post,
                    prev=prev, next=next)
        raise tornado.web.HTTPError(404)

class TagsHandler(BaseHandler):
    def get(self):
        self.render('tags.html')

class TagHandler(BaseHandler):
    def get(self, tag_name):
        prev = next = None
        for i, tag in enumerate(self.settings['tags']):
            if tag[0] == tag_name:
                i += 1
                if i < len(self.settings['tags']):
                    next = self.settings['tags'][i]
                return self.render('tag.html', tag=tag,
                    prev=prev, next=next)
            prev = tag

        raise tornado.web.HTTPError(404)

class ArchivesHandler(BaseHandler):
    def get(self):
        self.render('archives.html')

class ArchiveHandler(BaseHandler):
    def get(self, archive_name):
        prev = next = None
        for i, archive in enumerate(self.settings['archives']):
            if archive[0] == archive_name:
                i += 1
                if i < len(self.settings['archives']):
                    next = self.settings['archives'][i]
                return self.render('archive.html', archive=archive,
                    prev=prev, next=next)
            prev = archive
        raise tornado.web.HTTPError(404)

class LinksHandler(BaseHandler):
    def get(self):
        self.render('links.html')

class FeedHandler(BaseHandler):
    def get(self):
        self.set_header("Content-Type", "application/atom+xml")
        loader = tornado.template.Loader(config.common_template_path,
            autoescape=None)
        p = loader.load("feed.xml").generate(posts=self.settings['posts'],
            handler=config)
        self.write(p)

class SitemapHandler(BaseHandler):
    def get(self):
        self.set_header("Content-Type", "text/plain")
        loader = tornado.template.Loader(config.common_template_path,
            autoescape=None)
        p = loader.load("sitemap.txt").generate(posts=self.settings['posts'],
            handler=config, tags=self.settings['tags'],
            archives=self.settings['archives'])
        self.write(p)

class WebhookHandler(BaseHandler):
    def post(self):
        """Webhook support for GitHub and Bitbucket.
        """
        update_posts()
        if 'tags' in self.settings:
            posts = load_posts()
            tags, archives = get_infos(posts)
            self.settings['posts'] = posts
            self.settings['tags'] = tags
            self.settings['archives'] = archives
        else:
            deploy()

class ErrorHandler(BaseHandler):
    def prepare(self):
        raise tornado.web.HTTPError(404)

posts = load_posts()
tags, archives = get_infos(posts)

def write(file_name, page):
    file_path = os.path.join(deploy_path, file_name)
    open(file_path, 'w').write(page)

def deploy():
    if os.path.exists(config.deploy_path):
        shutil.rmtree(config.deploy_path)

    os.makedirs(config.deploy_path)

    loader = tornado.template.Loader(config.common_template_path,
        autoescape=None)

    print('Generating sitemap')
    page = loader.load("sitemap.txt").generate(posts=posts, handler=config,
        tags=tags, archives=archives)
    write('sitemap.txt', page)

    print('Generating atom')
    page = loader.load("feed.xml").generate(posts=posts, handler=config)
    write('feed.xml', page)

    loader = tornado.template.Loader(config.settings['template_path'],
        autoescape=None)

    print('Start generating index pages')
    generator = loader.load("index.html")
    p = 0
    while posts_num > p * config.settings['post_per_page']:
        p += 1
        print('Start generating page %s' % p)
        page = generator.generate(posts=posts, handler=config,
            p=p)
        write('page_%s.html' % p, page)

    index_1 = os.path.join(deploy_path, 'page_1.html')
    index = os.path.join(deploy_path, 'index.html')
    os.rename(index_1, index)

    print('Start generating articles')
    generator = loader.load("article.html")
    posts.reverse()
    prev = None
    post = posts.pop()
    next = len(posts) and posts.pop() or None
    while post:
        print('Generating %s' % post['file_name'])
        page = generator.generate(post=post, handler=config,
            prev=prev, next=next)
        write('%s.html' % post['file_name'], page)
        prev, post, next = post, next, len(posts) and posts.pop() or None

    print('Start generating tag pages')
    generator = loader.load("tag.html")
    prev = None
    for i, tag in enumerate(tags):
        print('Generating tag %s' % tag[0])
        i += 1
        next = i < len(tags) and tags[i] or None
        page = generator.generate(tag=tag, prev=prev,
            next=next, handler=config)
        write('tag_%s.html' % tag[0], page)
        prev = tag

    print('Start generating archive pages')
    generator = loader.load("archive.html")
    prev = None
    for i, archive in enumerate(archives):
        print('Generating archive %s' % archive[0])
        i += 1
        next = i < len(archives) and archives[i] or None
        page = generator.generate(archive=archive, prev=prev,
            next=next, handler=config)
        write('archive_%s.html' % archive[0], page)
        prev = archive

    print('Start generating other pages')
    for p in ('404', 'tags', 'archives', 'links'):
        page = loader.load("%s.html" % p).generate(handler=config)
        write('%s.html' % p, page)

    print('Copying static files.')
    deploy_static_dir = os.path.join(deploy_path, 'static')
    if os.path.exists(deploy_static_dir):
        shutil.rmtree(deploy_static_dir)
    shutil.copytree(config.settings['static_path'], deploy_static_dir)
    os.chdir(deploy_path)
    os.system('cp static/favicon.ico ./')
    os.system('cp static/robots.txt ./')

    print('Done.')

def update_posts():
    os.chdir(config.posts_path)
    if os.path.isdir(os.path.join(config.posts_path, '.git')):
        os.system('git pull')
    elif os.path.isdir(os.path.join(config.posts_path, '.hg')):
        os.system('hg pull')

if __name__ == '__main__':
    args = sys.argv
    if len(args) < 2:
        print('Useage: catsup.py server/deploy/webhook')
        sys.exit(0)
    cmd = args[1]
    del args[1]
    if cmd == 'server':
        application = tornado.web.Application([
            (r'/', MainHandler),
            (r'/page_(.*?).html', MainHandler),
            (r'/archives.html', ArchivesHandler),
            (r'/archive_(.*?).html', ArchiveHandler),
            (r'/tags.html', TagsHandler),
            (r'/tag_(.*?).html', TagHandler),
            (r'/links.html', LinksHandler),
            (r'/feed.xml', FeedHandler),
            (r'/sitemap.txt', SitemapHandler),
            (r'/webhook', WebhookHandler),
            (r'/(.*).html', ArticleHandler),
            (r'/.*', ErrorHandler),
        ], autoescape=None, posts=posts, tags=tags, archives=archives,
        **config.settings)
    elif cmd == 'deploy':
        deploy_path = config.deploy_path

        posts_num = len(posts)
        config.settings['tags'] = tags
        config.settings['archives'] = archives

        deploy()
    elif cmd == 'webhook':
        application = tornado.web.Application([
            (r'/webhook', WebhookHandler),
        ])
    else:
        print('Unknow Command: %s' % cmd)
        sys.exit(0)

    if 'application' in locals():
        tornado.options.parse_command_line(args)
        http_server = tornado.httpserver.HTTPServer(application, xheaders=True)
        http_server.listen(options.port)
        tornado.ioloop.IOLoop.instance().start()
<commit_msg>use config.deploy_path<commit_after>#coding=utf-8

import os
import time
import sys
import shutil
import config
import tornado.httpserver
import tornado.web
import tornado.ioloop
import tornado.options
import tornado.escape
import tornado.template
import misaka as m

from tornado.options import define, options

define("port", default=8888, help="run on the given port", type=int)

from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import get_lexer_by_name

class CatsupRender(m.HtmlRenderer, m.SmartyPants):
    def block_code(self, text, lang):
        if not lang:
            text = tornado.escape.xhtml_escape(text.strip())
            return '\n
%s
\n' % text
        lexer = get_lexer_by_name(lang, stripall=True)
        formatter = HtmlFormatter()
        return highlight(text, lexer, formatter)

    def autolink(self, link, is_email):
        if is_email:
            return '%(link)s' % {'link': link}

        if '.' in link:
            name_extension = link.split('.')[-1].lower()
            if name_extension in ('jpg', 'png', 'git', 'jpeg'):
                return '' % link

        return '%s' % (link, link)

md = m.Markdown(CatsupRender(flags=m.HTML_ESCAPE | m.HTML_USE_XHTML),
    extensions=m.EXT_FENCED_CODE | m.EXT_NO_INTRA_EMPHASIS | m.EXT_AUTOLINK |
        m.EXT_STRIKETHROUGH | m.EXT_SUPERSCRIPT)

def load_post(file_name):
    '''Load a post.return a dict.
    '''
    path = os.path.join(config.posts_path, file_name)
    file = open(path, 'r')
    post = {'file_name': file_name[:-3],
            'tags': [],
            'date': file_name[:10]}
    while True:
        line = file.readline()
        if line.startswith('#'):
            post['title'] = line[1:].strip()
        elif 'tags' in line.lower():
            for tag in line.split(':')[-1].strip().split(','):
                post['tags'].append(tag.strip())
        elif line.startswith('---'):
            content = '\n'.join(file.readlines())
            if isinstance(content, str):
                content = content.decode('utf-8')
            post['content'] = md.render(content)
            post['updated'] = os.stat(path).st_ctime
            updated_xml = time.gmtime(post['updated'])
            post['updated_xml'] = time.strftime('%Y-%m-%dT%H:%M:%SZ',
                updated_xml)
            return post

def load_posts():
    '''load all the posts.return a list.
    Sort with filename.
    '''
    post_files = os.listdir(config.posts_path)
    post_files.sort(reverse=True)
    posts = []
    for file_name in post_files:
        if '.md' in file_name:
            post = load_post(file_name)
            posts.append(post)
    return posts

def get_infos(posts):
    """return the tag list and archive list.
    """
    tags = {}
    archives = {}
    for post in posts:
        for tag in post['tags']:
            if tag in tags:
                tags[tag].append(post)
            else:
                tags[tag] = [post]
        year = post['date'][:4]
        if year in archives:
            archives[year].append(post)
        else:
            archives[year] = [post]

    return sorted(tags.items(), key=lambda x: len(x[1]), reverse=True),\
        sorted(archives.items(), key=lambda x: x[0], reverse=True)

class BaseHandler(tornado.web.RequestHandler):
    def get_error_html(self, *args, **kwargs):
        return self.render_string('404.html')

class MainHandler(BaseHandler):
    def get(self, p=1):
        if p == '1':  # /page_1.html
            self.redirect('/', status=301)
        p = int(p)
        self.render('index.html', posts=self.settings['posts'], p=p)

class ArticleHandler(BaseHandler):
    def get(self, file_name):
        posts_num = len(posts)
        prev = next = None
        for i in range(posts_num):
            post = posts[i]
            if post['file_name'] == file_name:
                if i:  # i>0
                    prev = posts[i - 1]
                if (i + 1) < posts_num:
                    next = posts[i + 1]
                return self.render('article.html', post=post,
                    prev=prev, next=next)
        raise tornado.web.HTTPError(404)

class TagsHandler(BaseHandler):
    def get(self):
        self.render('tags.html')

class TagHandler(BaseHandler):
    def get(self, tag_name):
        prev = next = None
        for i, tag in enumerate(self.settings['tags']):
            if tag[0] == tag_name:
                i += 1
                if i < len(self.settings['tags']):
                    next = self.settings['tags'][i]
                return self.render('tag.html', tag=tag,
                    prev=prev, next=next)
            prev = tag

        raise tornado.web.HTTPError(404)

class ArchivesHandler(BaseHandler):
    def get(self):
        self.render('archives.html')

class ArchiveHandler(BaseHandler):
    def get(self, archive_name):
        prev = next = None
        for i, archive in enumerate(self.settings['archives']):
            if archive[0] == archive_name:
                i += 1
                if i < len(self.settings['archives']):
                    next = self.settings['archives'][i]
                return self.render('archive.html', archive=archive,
                    prev=prev, next=next)
            prev = archive
        raise tornado.web.HTTPError(404)

class LinksHandler(BaseHandler):
    def get(self):
        self.render('links.html')

class FeedHandler(BaseHandler):
    def get(self):
        self.set_header("Content-Type", "application/atom+xml")
        loader = tornado.template.Loader(config.common_template_path,
            autoescape=None)
        p = loader.load("feed.xml").generate(posts=self.settings['posts'],
            handler=config)
        self.write(p)

class SitemapHandler(BaseHandler):
    def get(self):
        self.set_header("Content-Type", "text/plain")
        loader = tornado.template.Loader(config.common_template_path,
            autoescape=None)
        p = loader.load("sitemap.txt").generate(posts=self.settings['posts'],
            handler=config, tags=self.settings['tags'],
            archives=self.settings['archives'])
        self.write(p)

class WebhookHandler(BaseHandler):
    def post(self):
        """Webhook support for GitHub and Bitbucket.
        """
        update_posts()
        if 'tags' in self.settings:
            posts = load_posts()
            tags, archives = get_infos(posts)
            self.settings['posts'] = posts
            self.settings['tags'] = tags
            self.settings['archives'] = archives
        else:
            deploy()

class ErrorHandler(BaseHandler):
    def prepare(self):
        raise tornado.web.HTTPError(404)

posts = load_posts()
tags, archives = get_infos(posts)

def write(file_name, page):
    file_path = os.path.join(config.deploy_path, file_name)
    open(file_path, 'w').write(page)

def deploy():
    if os.path.exists(config.deploy_path):
        shutil.rmtree(config.deploy_path)

    os.makedirs(config.deploy_path)

    loader = tornado.template.Loader(config.common_template_path,
        autoescape=None)

    print('Generating sitemap')
    page = loader.load("sitemap.txt").generate(posts=posts, handler=config,
        tags=tags, archives=archives)
    write('sitemap.txt', page)

    print('Generating atom')
    page = loader.load("feed.xml").generate(posts=posts, handler=config)
    write('feed.xml', page)

    loader = tornado.template.Loader(config.settings['template_path'],
        autoescape=None)

    print('Start generating index pages')
    generator = loader.load("index.html")
    p = 0
    while posts_num > p * config.settings['post_per_page']:
        p += 1
        print('Start generating page %s' % p)
        page = generator.generate(posts=posts, handler=config,
            p=p)
        write('page_%s.html' % p, page)

    index_1 = os.path.join(config.deploy_path, 'page_1.html')
    index = os.path.join(config.deploy_path, 'index.html')
    os.rename(index_1, index)

    print('Start generating articles')
    generator = loader.load("article.html")
    posts.reverse()
    prev = None
    post = posts.pop()
    next = len(posts) and posts.pop() or None
    while post:
        print('Generating %s' % post['file_name'])
        page = generator.generate(post=post, handler=config,
            prev=prev, next=next)
        write('%s.html' % post['file_name'], page)
        prev, post, next = post, next, len(posts) and posts.pop() or None

    print('Start generating tag pages')
    generator = loader.load("tag.html")
    prev = None
    for i, tag in enumerate(tags):
        print('Generating tag %s' % tag[0])
        i += 1
        next = i < len(tags) and tags[i] or None
        page = generator.generate(tag=tag, prev=prev,
            next=next, handler=config)
        write('tag_%s.html' % tag[0], page)
        prev = tag

    print('Start generating archive pages')
    generator = loader.load("archive.html")
    prev = None
    for i, archive in enumerate(archives):
        print('Generating archive %s' % archive[0])
        i += 1
        next = i < len(archives) and archives[i] or None
        page = generator.generate(archive=archive, prev=prev,
            next=next, handler=config)
        write('archive_%s.html' % archive[0], page)
        prev = archive

    print('Start generating other pages')
    for p in ('404', 'tags', 'archives', 'links'):
        page = loader.load("%s.html" % p).generate(handler=config)
        write('%s.html' % p, page)

    print('Copying static files.')
    deploy_static_dir = os.path.join(config.deploy_path, 'static')
    if os.path.exists(deploy_static_dir):
        shutil.rmtree(deploy_static_dir)
    shutil.copytree(config.settings['static_path'], deploy_static_dir)
    os.chdir(config.deploy_path)
    os.system('cp static/favicon.ico ./')
    os.system('cp static/robots.txt ./')

    print('Done.')

def update_posts():
    os.chdir(config.posts_path)
    if os.path.isdir(os.path.join(config.posts_path, '.git')):
        os.system('git pull')
    elif os.path.isdir(os.path.join(config.posts_path, '.hg')):
        os.system('hg pull')

if __name__ == '__main__':
    args = sys.argv
    if len(args) < 2:
        print('Useage: catsup.py server/deploy/webhook')
        sys.exit(0)
    cmd = args[1]
    del args[1]
    if cmd == 'server':
        application = tornado.web.Application([
            (r'/', MainHandler),
            (r'/page_(.*?).html', MainHandler),
            (r'/archives.html', ArchivesHandler),
            (r'/archive_(.*?).html', ArchiveHandler),
            (r'/tags.html', TagsHandler),
            (r'/tag_(.*?).html', TagHandler),
            (r'/links.html', LinksHandler),
            (r'/feed.xml', FeedHandler),
            (r'/sitemap.txt', SitemapHandler),
            (r'/webhook', WebhookHandler),
            (r'/(.*).html', ArticleHandler),
            (r'/.*', ErrorHandler),
        ], autoescape=None, posts=posts, tags=tags, archives=archives,
        **config.settings)
    elif cmd == 'deploy':

        posts_num = len(posts)
        config.settings['tags'] = tags
        config.settings['archives'] = archives

        deploy()
    elif cmd == 'webhook':
        application = tornado.web.Application([
            (r'/webhook', WebhookHandler),
        ])
    else:
        print('Unknow Command: %s' % cmd)
        sys.exit(0)

    if 'application' in locals():
        tornado.options.parse_command_line(args)
        http_server = tornado.httpserver.HTTPServer(application, xheaders=True)
        http_server.listen(options.port)
        tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import superdesk

from flask import current_app as app
from settings import INGEST_EXPIRY_MINUTES
from datetime import timedelta, timezone, datetime
from werkzeug.exceptions import HTTPException

from superdesk.notification import push_notification
from superdesk.activity import ACTIVITY_EVENT, notify_and_add_activity
from superdesk.io import providers
from superdesk.celery_app import celery
from superdesk.utc import utcnow, get_expiry_date
from superdesk.workflow import set_default_state
from superdesk.errors import ProviderError
from superdesk.stats import stats
from superdesk.upload import url_for_media
from superdesk.media.media_operations import download_file_from_url, process_file
from superdesk.media.renditions import generate_renditions
from superdesk.io.iptc import subject_codes

UPDATE_SCHEDULE_DEFAULT = {'minutes': 5}
LAST_UPDATED = 'last_updated'
LAST_ITEM_UPDATE = 'last_item_update'
STATE_INGESTED = 'ingested'
IDLE_TIME_DEFAULT = {'hours': 0, 'minutes': 0}

logger = logging.getLogger(__name__)

superdesk.workflow_state(STATE_INGESTED)

superdesk.workflow_action(
    name='ingest'
)

def is_valid_type(provider, provider_type_filter=None):
    """Test if given provider has valid type and should be updated.

    :param provider: provider to be updated
    :param provider_type_filter: active provider type filter
    """
    provider_type = provider.get('type')
    if provider_type not in providers:
        return False
    if provider_type_filter and provider_type != provider_type_filter:
        return False
    return True

def is_scheduled(provider):
    """Test if given provider should be scheduled for update.

    :param provider: ingest provider
    """
    now = utcnow()
    last_updated = provider.get(LAST_UPDATED, now - timedelta(days=100))  # if never updated run now
    update_schedule = provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)
    return last_updated + timedelta(**update_schedule) < now

def is_closed(provider):
    """Test if provider is closed.

    :param provider: ingest provider
    """
    return provider.get('is_closed', False)

def filter_expired_items(provider, items):
    def is_not_expired(item):
        expiry = item.get('expiry', item['versioncreated'] + delta)
        if expiry.tzinfo:
            return expiry > utcnow()
        return False

    try:
        delta = timedelta(minutes=provider.get('content_expiry', INGEST_EXPIRY_MINUTES))
        return [item for item in items if is_not_expired(item)]
    except Exception as ex:
        raise ProviderError.providerFilterExpiredContentError(ex, provider)

def get_provider_rule_set(provider):
    if provider.get('rule_set'):
        return superdesk.get_resource_service('rule_sets').find_one(_id=provider['rule_set'], req=None)

def get_task_ttl(provider):
    update_schedule = provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)
    return update_schedule.get('minutes', 0) * 60 + update_schedule.get('hours', 0) * 3600

def get_is_idle(providor):
    last_item = providor.get(LAST_ITEM_UPDATE)
    idle_time = providor.get('idle_time', IDLE_TIME_DEFAULT)
    if isinstance(idle_time['hours'], datetime):
        idle_hours = 0
    else:
        idle_hours = idle_time['hours']
    if isinstance(idle_time['minutes'], datetime):
        idle_minutes = 0
    else:
        idle_minutes = idle_time['minutes']
    # there is an update time and the idle time is none zero
    if last_item and (idle_hours != 0 or idle_minutes != 0):
        if utcnow() > last_item + timedelta(hours=idle_hours, minutes=idle_minutes):
            return True
    return False

def get_task_id(provider):
    return 'update-ingest-{0}-{1}'.format(provider.get('name'), provider.get('_id'))

class UpdateIngest(superdesk.Command):
    """Update ingest providers."""

    option_list = (
        superdesk.Option('--provider', '-p', dest='provider_type'),
    )

    def run(self, provider_type=None):
        for provider in superdesk.get_resource_service('ingest_providers').get(req=None, lookup={}):
            if is_valid_type(provider, provider_type) and is_scheduled(provider) and not is_closed(provider):
                kwargs = {
                    'provider': provider,
                    'rule_set': get_provider_rule_set(provider)
                }
                update_provider.apply_async(
                    expires=get_task_ttl(provider),
                    kwargs=kwargs)

@celery.task(soft_time_limit=1800)
def update_provider(provider, rule_set=None):
    """
    Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
    updates the provider.
    """
    update = {
        LAST_UPDATED: utcnow(),
        # Providing the _etag as system updates to the documents shouldn't override _etag.
        app.config['ETAG']: provider.get(app.config['ETAG'])
    }

    for items in providers[provider.get('type')].update(provider):
        ingest_items(items, provider, rule_set)
        stats.incr('ingest.ingested_items', len(items))
        if items:
            update[LAST_ITEM_UPDATE] = utcnow()

    superdesk.get_resource_service('ingest_providers').update(provider['_id'], update, provider)

    if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
        notify_and_add_activity(
            ACTIVITY_EVENT,
            'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
            user_list=superdesk.get_resource_service('ingest_providers')._get_administrators(),
            name=provider.get('name'),
            last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))

    logger.info('Provider {0} updated'.format(provider['_id']))
    push_notification('ingest:update')

def process_anpa_category(item, provider):
    try:
        anpa_categories = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='categories')
        if anpa_categories:
            for anpa_category in anpa_categories['items']:
                if anpa_category['is_active'] is True \
                        and item['anpa-category']['qcode'].lower() == anpa_category['value'].lower():
                    item['anpa-category'] = {'qcode': item['anpa-category']['qcode'], 'name': anpa_category['name']}
                    break
    except Exception as ex:
        raise ProviderError.anpaError(ex, provider)

def process_iptc_codes(item, provider):
    """
    Ensures that the higher level IPTC codes are present by inserting them if missing, for example
    if given 15039001 (Formula One) make sure that 15039000 (motor racing) and 15000000 (sport) are there as well

    :param item: A story item
    :return: A story item with possible expanded subjects
    """
    try:
        def iptc_already_exists(code):
            for entry in item['subject']:
                if 'qcode' in entry and code == entry['qcode']:
                    return True
            return False

        for subject in item['subject']:
            if 'qcode' in subject and len(subject['qcode']) == 8:

                top_qcode = subject['qcode'][:2] + '000000'
                if not iptc_already_exists(top_qcode):
                    item['subject'].append({'qcode': top_qcode, 'name': subject_codes[top_qcode]})

                mid_qcode = subject['qcode'][:5] + '000'
                if not iptc_already_exists(mid_qcode):
                    item['subject'].append({'qcode': mid_qcode, 'name': subject_codes[mid_qcode]})
    except Exception as ex:
        raise ProviderError.iptcError(ex, provider)

def apply_rule_set(item, provider, rule_set=None):
    """
    Applies rules set on the item to be ingested into the system. If there's no rule set then the item will
    be returned without any change.

    :param item: Item to be ingested
    :param provider: provider object from whom the item was received
    :return: item
    """
    try:
        if rule_set is None and provider.get('rule_set') is not None:
            rule_set = superdesk.get_resource_service('rule_sets').find_one(_id=provider['rule_set'], req=None)

            if rule_set and 'body_html' in item:
                body = item['body_html']

                for rule in rule_set['rules']:
                    body = body.replace(rule['old'], rule['new'])

                item['body_html'] = body

        return item
    except Exception as ex:
        raise ProviderError.ruleError(ex, provider)

def ingest_items(items, provider, rule_set=None):
    all_items = filter_expired_items(provider, items)
    items_dict = {doc['guid']: doc for doc in all_items}

    for item in [doc for doc in all_items if doc.get('type') != 'composite']:
        ingest_item(item, provider, rule_set)

    for item in [doc for doc in all_items if doc.get('type') == 'composite']:
        for ref in [ref for group in item.get('groups', [])
                    for ref in group.get('refs', []) if 'residRef' in ref]:
            ref.setdefault('location', 'ingest')
            itemRendition = items_dict.get(ref['residRef'], {}).get('renditions')
            if itemRendition:
                ref.setdefault('renditions', itemRendition)
        ingest_item(item, provider, rule_set)

def ingest_item(item, provider, rule_set=None):
    item.setdefault('_id', item['guid'])
    providers[provider.get('type')].provider = provider

    item['ingest_provider'] = str(provider['_id'])
    item.setdefault('source', provider.get('source', ''))
    set_default_state(item, STATE_INGESTED)
    item['expiry'] = get_expiry_date(provider.get('content_expiry', INGEST_EXPIRY_MINUTES),
                                     item.get('versioncreated'))

    if 'anpa-category' in item:
        process_anpa_category(item, provider)

    if 'subject' in item:
        process_iptc_codes(item, provider)

    apply_rule_set(item, provider, rule_set)

    ingest_service = superdesk.get_resource_service('ingest')

    if item.get('ingest_provider_sequence') is None:
        ingest_service.set_ingest_provider_sequence(item, provider)

    rend = item.get('renditions', {})
    if rend:
        baseImageRend = rend.get('baseImage') or next(iter(rend.values()))
        if baseImageRend:
            href = providers[provider.get('type')].prepare_href(baseImageRend['href'])
            update_renditions(item, href)

    old_item = ingest_service.find_one(_id=item['guid'], req=None)

    if old_item:
        ingest_service.put(item['guid'], item)
    else:
        try:
            ingest_service.post([item])
        except HTTPException as e:
            logger.error("Exception while persisting item in ingest collection", e)
            ingest_service.put(item['guid'], item)

def update_renditions(item, href):
    inserted = []
    try:
        content, filename, content_type = download_file_from_url(href)
        file_type, ext = content_type.split('/')
        metadata = process_file(content, file_type)
        file_guid = app.media.put(content, filename, content_type, metadata)
        inserted.append(file_guid)

        rendition_spec = app.config.get('RENDITIONS', {}).get('picture', {})
        renditions = generate_renditions(content, file_guid, inserted, file_type,
                                         content_type, rendition_spec, url_for_media)
        item['renditions'] = renditions
        item['mimetype'] = content_type
        item['filemeta'] = metadata
    except Exception as io:
        logger.exception(io)
        for file_id in inserted:
            app.media.delete(file_id)
        raise

superdesk.command('ingest:update', UpdateIngest())
<commit_msg>Fix for expired content ingestion<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import superdesk

from flask import current_app as app
from settings import INGEST_EXPIRY_MINUTES
from datetime import timedelta, timezone, datetime
from werkzeug.exceptions import HTTPException

from superdesk.notification import push_notification
from superdesk.activity import ACTIVITY_EVENT, notify_and_add_activity
from superdesk.io import providers
from superdesk.celery_app import celery
from superdesk.utc import utcnow, get_expiry_date
from superdesk.workflow import set_default_state
from superdesk.errors import ProviderError
from superdesk.stats import stats
from superdesk.upload import url_for_media
from superdesk.media.media_operations import download_file_from_url, process_file
from superdesk.media.renditions import generate_renditions
from superdesk.io.iptc import subject_codes

UPDATE_SCHEDULE_DEFAULT = {'minutes': 5}
LAST_UPDATED = 'last_updated'
LAST_ITEM_UPDATE = 'last_item_update'
STATE_INGESTED = 'ingested'
IDLE_TIME_DEFAULT = {'hours': 0, 'minutes': 0}

logger = logging.getLogger(__name__)

superdesk.workflow_state(STATE_INGESTED)

superdesk.workflow_action(
    name='ingest'
)

def is_valid_type(provider, provider_type_filter=None):
    """Test if given provider has valid type and should be updated.

    :param provider: provider to be updated
    :param provider_type_filter: active provider type filter
    """
    provider_type = provider.get('type')
    if provider_type not in providers:
        return False
    if provider_type_filter and provider_type != provider_type_filter:
        return False
    return True

def is_scheduled(provider):
    """Test if given provider should be scheduled for update.

    :param provider: ingest provider
    """
    now = utcnow()
    last_updated = provider.get(LAST_UPDATED, now - timedelta(days=100))  # if never updated run now
    update_schedule = provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)
    return last_updated + timedelta(**update_schedule) < now

def is_closed(provider):
    """Test if provider is closed.

    :param provider: ingest provider
    """
    return provider.get('is_closed', False)

def filter_expired_items(provider, items):
    def is_not_expired(item):
        if item.get('expiry') or item.get('versioncreated'):
            expiry = item.get('expiry', item['versioncreated'] + delta)
            if expiry.tzinfo:
                return expiry > utcnow()
        return False

    try:
        delta = timedelta(minutes=provider.get('content_expiry', INGEST_EXPIRY_MINUTES))
        return [item for item in items if is_not_expired(item)]
    except Exception as ex:
        raise ProviderError.providerFilterExpiredContentError(ex, provider)

def get_provider_rule_set(provider):
    if provider.get('rule_set'):
        return superdesk.get_resource_service('rule_sets').find_one(_id=provider['rule_set'], req=None)

def get_task_ttl(provider):
    update_schedule = provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)
    return update_schedule.get('minutes', 0) * 60 + update_schedule.get('hours', 0) * 3600

def get_is_idle(providor):
    last_item = providor.get(LAST_ITEM_UPDATE)
    idle_time = providor.get('idle_time', IDLE_TIME_DEFAULT)
    if isinstance(idle_time['hours'], datetime):
        idle_hours = 0
    else:
        idle_hours = idle_time['hours']
    if isinstance(idle_time['minutes'], datetime):
        idle_minutes = 0
    else:
        idle_minutes = idle_time['minutes']
    # there is an update time and the idle time is none zero
    if last_item and (idle_hours != 0 or idle_minutes != 0):
        if utcnow() > last_item + timedelta(hours=idle_hours, minutes=idle_minutes):
            return True
    return False

def get_task_id(provider):
    return 'update-ingest-{0}-{1}'.format(provider.get('name'), provider.get('_id'))

class UpdateIngest(superdesk.Command):
    """Update ingest providers."""

    option_list = (
        superdesk.Option('--provider', '-p', dest='provider_type'),
    )

    def run(self, provider_type=None):
        for provider in superdesk.get_resource_service('ingest_providers').get(req=None, lookup={}):
            if is_valid_type(provider, provider_type) and is_scheduled(provider) and not is_closed(provider):
                kwargs = {
                    'provider': provider,
                    'rule_set': get_provider_rule_set(provider)
                }
                update_provider.apply_async(
                    expires=get_task_ttl(provider),
                    kwargs=kwargs)

@celery.task(soft_time_limit=1800)
def update_provider(provider, rule_set=None):
    """
    Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
    updates the provider.
    """
    update = {
        LAST_UPDATED: utcnow(),
        # Providing the _etag as system updates to the documents shouldn't override _etag.
        app.config['ETAG']: provider.get(app.config['ETAG'])
    }

    for items in providers[provider.get('type')].update(provider):
        ingest_items(items, provider, rule_set)
        stats.incr('ingest.ingested_items', len(items))
        if items:
            update[LAST_ITEM_UPDATE] = utcnow()

    superdesk.get_resource_service('ingest_providers').update(provider['_id'], update, provider)

    if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
        notify_and_add_activity(
            ACTIVITY_EVENT,
            'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
            user_list=superdesk.get_resource_service('ingest_providers')._get_administrators(),
            name=provider.get('name'),
            last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))

    logger.info('Provider {0} updated'.format(provider['_id']))
    push_notification('ingest:update')

def process_anpa_category(item, provider):
    try:
        anpa_categories = superdesk.get_resource_service('vocabularies').find_one(req=None, _id='categories')
        if anpa_categories:
            for anpa_category in anpa_categories['items']:
                if anpa_category['is_active'] is True \
                        and item['anpa-category']['qcode'].lower() == anpa_category['value'].lower():
                    item['anpa-category'] = {'qcode': item['anpa-category']['qcode'], 'name': anpa_category['name']}
                    break
    except Exception as ex:
        raise ProviderError.anpaError(ex, provider)

def process_iptc_codes(item, provider):
    """
    Ensures that the higher level IPTC codes are present by inserting them if missing, for example
    if given 15039001 (Formula One) make sure that 15039000 (motor racing) and 15000000 (sport) are there as well

    :param item: A story item
    :return: A story item with possible expanded subjects
    """
    try:
        def iptc_already_exists(code):
            for entry in item['subject']:
                if 'qcode' in entry and code == entry['qcode']:
                    return True
            return False

        for subject in item['subject']:
            if 'qcode' in subject and len(subject['qcode']) == 8:

                top_qcode = subject['qcode'][:2] + '000000'
                if not iptc_already_exists(top_qcode):
                    item['subject'].append({'qcode': top_qcode, 'name': subject_codes[top_qcode]})

                mid_qcode = subject['qcode'][:5] + '000'
                if not iptc_already_exists(mid_qcode):
                    item['subject'].append({'qcode': mid_qcode, 'name': subject_codes[mid_qcode]})
    except Exception as ex:
        raise ProviderError.iptcError(ex, provider)

def apply_rule_set(item, provider, rule_set=None):
    """
    Applies rules set on the item to be ingested into the system. If there's no rule set then the item will
    be returned without any change.

    :param item: Item to be ingested
    :param provider: provider object from whom the item was received
    :return: item
    """
    try:
        if rule_set is None and provider.get('rule_set') is not None:
            rule_set = superdesk.get_resource_service('rule_sets').find_one(_id=provider['rule_set'], req=None)

            if rule_set and 'body_html' in item:
                body = item['body_html']

                for rule in rule_set['rules']:
                    body = body.replace(rule['old'], rule['new'])

                item['body_html'] = body

        return item
    except Exception as ex:
        raise ProviderError.ruleError(ex, provider)

def ingest_items(items, provider, rule_set=None):
    all_items = filter_expired_items(provider, items)
    items_dict = {doc['guid']: doc for doc in all_items}

    for item in [doc for doc in all_items if doc.get('type') != 'composite']:
        ingest_item(item, provider, rule_set)

    for item in [doc for doc in all_items if doc.get('type') == 'composite']:
        for ref in [ref for group in item.get('groups', [])
                    for ref in group.get('refs', []) if 'residRef' in ref]:
            ref.setdefault('location', 'ingest')
            itemRendition = items_dict.get(ref['residRef'], {}).get('renditions')
            if itemRendition:
                ref.setdefault('renditions', itemRendition)
        ingest_item(item, provider, rule_set)

def ingest_item(item, provider, rule_set=None):
    item.setdefault('_id', item['guid'])
    providers[provider.get('type')].provider = provider

    item['ingest_provider'] = str(provider['_id'])
    item.setdefault('source', provider.get('source', ''))
    set_default_state(item, STATE_INGESTED)
    item['expiry'] = get_expiry_date(provider.get('content_expiry', INGEST_EXPIRY_MINUTES),
                                     item.get('versioncreated'))

    if 'anpa-category' in item:
        process_anpa_category(item, provider)

    if 'subject' in item:
        process_iptc_codes(item, provider)

    apply_rule_set(item, provider, rule_set)

    ingest_service = superdesk.get_resource_service('ingest')

    if item.get('ingest_provider_sequence') is None:
        ingest_service.set_ingest_provider_sequence(item, provider)

    rend = item.get('renditions', {})
    if rend:
        baseImageRend = rend.get('baseImage') or next(iter(rend.values()))
        if baseImageRend:
            href = providers[provider.get('type')].prepare_href(baseImageRend['href'])
            update_renditions(item, href)

    old_item = ingest_service.find_one(_id=item['guid'], req=None)

    if old_item:
        ingest_service.put(item['guid'], item)
    else:
        try:
            ingest_service.post([item])
        except HTTPException as e:
            logger.error("Exception while persisting item in ingest collection", e)
            ingest_service.put(item['guid'], item)

def update_renditions(item, href):
    inserted = []
    try:
        content, filename, content_type = download_file_from_url(href)
        file_type, ext = content_type.split('/')
        metadata = process_file(content, file_type)
        file_guid = app.media.put(content, filename, content_type, metadata)
        inserted.append(file_guid)

        rendition_spec = app.config.get('RENDITIONS', {}).get('picture', {})
        renditions = generate_renditions(content, file_guid, inserted, file_type,
                                         content_type, rendition_spec, url_for_media)
        item['renditions'] = renditions
        item['mimetype'] = content_type
        item['filemeta'] = metadata
    except Exception as io:
        logger.exception(io)
        for file_id in inserted:
            app.media.delete(file_id)
        raise

superdesk.command('ingest:update', UpdateIngest())
<|endoftext|>


<commit_before>import jinja2
from os.path import join

class Template:
    def __init__(self, template_dir):
        self.env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))

    def render(self, template_file, template_data):
        template_obj = self.env.get_template(template_file)
        return template_obj.render(template_data)

<commit_msg>moved template to cluster package<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

"""Detects problems in CoreCommerce product data.

Requires a [website] section in config file for login info.

The [lint_product] section of the config file contains product value
checks.  The key is the product value (column) to check and the value
is a regular expression which the product value must match.  You can
specify multiple checks for a product value by putting them on
multiple lines.  The regex can be followed by a Python "if"
expression.  For example:

    [lint_product]
    Teaser: \S.{9,59}  ; 10-59 chars
    Discontinued Item: Y|N  ; Y or N
    Available:
        Y|N
        N if item["Discontinued Item"] == "Y"
    # UPC must be blank
    UPC:
    Cost: [0-9]+\.[0-9]{2}  ; positive float
    Price: [0-9]+\.[0-9]{2}  ; positive float
    # SKU must be 5 digits.
    # SKU must start with 8 if the category is Cat or Dog.
    SKU:
        [1-9][0-9]{4}
        [8][0-9]{4} if item["Category"] in ("Cat", "Dog")

The [lint_category] section is similar and is used for category value
checks.
"""

import ConfigParser
import cctools
import optparse
import re
import sys

def parse_checks(config, section):
    """Parse value check definitions in config file."""
    checks = list()
    for name, value in config.items(section):
        for check in value.split("\n"):
            check = check.strip()
            if check != "":
                checks.append((name, check))
    return checks

def product_display_name(product):
    """Construct a display name for a product."""
    display_name = "%s %s" % (product["SKU"], product["Product Name"])
    return display_name.strip()

def check_skus(products):
    """Check SKUs for uniqueness."""
    skus = dict()
    for product in products:
        display_name = product_display_name(product)
        sku = product["SKU"]
        if sku != "":
            if sku in skus:
                print "%s '%s': SKU already used by '%s'" % (
                    "Product",
                    display_name,
                    skus[sku]
                )
            else:
                skus[sku] = display_name

def check_item(item_checks, item_type_name, item, item_name):
    """Check category or product for problems."""

    for key, check in item_checks:
        check_parts = check.split(None, 2)
        if len(check_parts) == 1:
            pass
        elif len(check_parts) == 3 and check_parts[1].lower() == "if":
            predicate = check_parts[2]
            if not eval(predicate, {"__builtins__": {}}, {"item": item}):
                continue
        else:
            print "ERROR: Unknown check syntax '%s'" % check
            sys.exit(1)
        pattern = "^" + check_parts[0] + "$"  # entire value must match
        if not re.match(pattern, item[key]):
            print "%s '%s': Invalid '%s' of '%s' (does not match %s)" % (
                item_type_name,
                item_name,
                key,
                item[key],
                pattern
            )

def main():
    """main"""
    option_parser = optparse.OptionParser(
        usage="usage: %prog [options]\n" +
        "  Detects problems in data exported from CoreCommerce."
    )
    option_parser.add_option(
        "--config",
        action="store",
        metavar="FILE",
        default="cctools.cfg",
        help="configuration filename (default=%default)"
    )
    option_parser.add_option(
        "--no-clean",
        action="store_false",
        dest="clean",
        default=True,
        help="do not clean data before checking"
    )
    option_parser.add_option(
        "--cache-ttl",
        action="store",
        type=int,
        metavar="SEC",
        default=3600,
        help="cache TTL in seconds (default=%default)"
    )
    option_parser.add_option(
        "--verbose",
        action="store_true",
        default=False,
        help="display progress messages"
    )

    # Parse command line arguments.
    (options, args) = option_parser.parse_args()
    if len(args) != 0:
        option_parser.error("invalid argument")

    # Read config file.
    config = ConfigParser.RawConfigParser()
    config.optionxform = str  # preserve case of option names
    config.readfp(open(options.config))
    category_checks = parse_checks(config, "lint_category")
    product_checks = parse_checks(config, "lint_product")

    # Create a connection to CoreCommerce.
    cc_browser = cctools.CCBrowser(
        config.get("website", "host"),
        config.get("website", "site"),
        config.get("website", "username"),
        config.get("website", "password"),
        verbose=options.verbose,
        clean=options.clean,
        cache_ttl=options.cache_ttl
    )

    # Check category list.
    categories = cc_browser.get_categories()
    for category in categories:
        check_item(
            category_checks,
            "Category",
            category,
            category["Category Name"]
        )

    # Check products list.
    products = cc_browser.get_products()
    check_skus(products)
    for product in products:
        check_item(
            product_checks,
            "Product",
            product,
            product_display_name(product)
        )

    if options.verbose:
        sys.stderr.write("Checks complete\n")
    return 0

if __name__ == "__main__":
    main()
<commit_msg>cclint.py: add personalization checks<commit_after>#!/usr/bin/env python

"""Detects problems in CoreCommerce product data.

Requires a [website] section in config file for login info.

The [lint_product] section of the config file contains product value
checks.  The key is the product value (column) to check and the value
is a regular expression which the product value must match.  You can
specify multiple checks for a product value by putting them on
multiple lines.  The regex can be followed by a Python "if"
expression.  For example:

    [lint_product]
    Teaser: \S.{9,59}  ; 10-59 chars
    Discontinued Item: Y|N  ; Y or N
    Available:
        Y|N
        N if item["Discontinued Item"] == "Y"
    # UPC must be blank
    UPC:
    Cost: [0-9]+\.[0-9]{2}  ; positive float
    Price: [0-9]+\.[0-9]{2}  ; positive float
    # SKU must be 5 digits.
    # SKU must start with 8 if the category is Cat or Dog.
    SKU:
        [1-9][0-9]{4}
        [8][0-9]{4} if item["Category"] in ("Cat", "Dog")

The [lint_category] section is similar and is used for category value
checks.

The [lint_personalization] section is similar and is used for
personalization value checks.
"""

import ConfigParser
import cctools
import optparse
import re
import sys

def parse_checks(config, section):
    """Parse value check definitions in config file."""
    checks = list()
    for name, value in config.items(section):
        for check in value.split("\n"):
            check = check.strip()
            if check != "":
                checks.append((name, check))
    return checks

def product_display_name(product):
    """Construct a display name for a product."""
    display_name = "%s %s" % (product["SKU"], product["Product Name"])
    return display_name.strip()

def personalization_display_name(personalization):
    """Construct a display name for a personalization."""
    display_name = "%s %s %s" % (
        personalization["Product SKU"],
        personalization["Product Name"],
        personalization["Question|Answer"]
    )
    return display_name.strip()

def check_skus(products):
    """Check SKUs for uniqueness."""
    skus = dict()
    for product in products:
        display_name = product_display_name(product)
        sku = product["SKU"]
        if sku != "":
            if sku in skus:
                print "%s '%s': SKU already used by '%s'" % (
                    "Product",
                    display_name,
                    skus[sku]
                )
            else:
                skus[sku] = display_name

def check_item(item_checks, item_type_name, item, item_name):
    """Check category or product for problems."""

    for key, check in item_checks:
        check_parts = check.split(None, 2)
        if len(check_parts) == 1:
            pass
        elif len(check_parts) == 3 and check_parts[1].lower() == "if":
            predicate = check_parts[2]
            if not eval(predicate, {"__builtins__": {}}, {"item": item}):
                continue
        else:
            print "ERROR: Unknown check syntax '%s'" % check
            sys.exit(1)
        pattern = "^" + check_parts[0] + "$"  # entire value must match
        if not re.match(pattern, item[key]):
            print "%s '%s': Invalid '%s' of '%s' (does not match %s)" % (
                item_type_name,
                item_name,
                key,
                item[key],
                pattern
            )

def main():
    """main"""
    option_parser = optparse.OptionParser(
        usage="usage: %prog [options]\n" +
        "  Detects problems in data exported from CoreCommerce."
    )
    option_parser.add_option(
        "--config",
        action="store",
        metavar="FILE",
        default="cctools.cfg",
        help="configuration filename (default=%default)"
    )
    option_parser.add_option(
        "--no-clean",
        action="store_false",
        dest="clean",
        default=True,
        help="do not clean data before checking"
    )
    option_parser.add_option(
        "--cache-ttl",
        action="store",
        type=int,
        metavar="SEC",
        default=3600,
        help="cache TTL in seconds (default=%default)"
    )
    option_parser.add_option(
        "--verbose",
        action="store_true",
        default=False,
        help="display progress messages"
    )

    # Parse command line arguments.
    (options, args) = option_parser.parse_args()
    if len(args) != 0:
        option_parser.error("invalid argument")

    # Read config file.
    config = ConfigParser.RawConfigParser()
    config.optionxform = str  # preserve case of option names
    config.readfp(open(options.config))
    category_checks = parse_checks(config, "lint_category")
    product_checks = parse_checks(config, "lint_product")
    personalization_checks = parse_checks(config, "lint_personalization")

    # Create a connection to CoreCommerce.
    cc_browser = cctools.CCBrowser(
        config.get("website", "host"),
        config.get("website", "site"),
        config.get("website", "username"),
        config.get("website", "password"),
        verbose=options.verbose,
        clean=options.clean,
        cache_ttl=options.cache_ttl
    )

    # Check category list.
    categories = cc_browser.get_categories()
    for category in categories:
        check_item(
            category_checks,
            "Category",
            category,
            category["Category Name"]
        )

    # Check products list.
    products = cc_browser.get_products()
    check_skus(products)
    for product in products:
        check_item(
            product_checks,
            "Product",
            product,
            product_display_name(product)
        )

    # Check personalizations list.
    personalizations = cc_browser.get_personalizations()
    for personalization in personalizations:
        check_item(
            personalization_checks,
            "Personalization",
            personalization,
            personalization_display_name(personalization)
        )

    if options.verbose:
        sys.stderr.write("Checks complete\n")
    return 0

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from periphery import I2C
import time

class CCS811(object):

	def __init__(self, device_bus=1, device_address=0x5B):
		self.device_bus = device_bus
		self.device_address = device_address
		self.bus = I2C("/dev/i2c-1")

	def start_app(self):
		my_ccs811.write_byte(0xF4)
		print(my_ccs811.read_byte(0x00))
		my_ccs811.write_byte_data(0x01, 0x10)

	def transfer(self, msgs):
		try:
			self.bus.transfer(self.device_address, msgs)
		except periphery.i2c.I2CError as error:
			print(str(error))
		return msgs

	def close(self):
		self.bus.close()

	def read_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.transfer(msgs)
		msgs = [I2C.Message([address], read=True)]
		ret_msg = self.transfer(msgs)

		return (ret_msg[0].data[0])

	def write_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.transfer(msgs)

	def reset(self):
		msgs = [I2C.Message([0xFF, 0x11, 0xE5, 0x72, 0x8A], read=False)]
		self.transfer(msgs)

	def write_byte_data(self, address, data):
		msgs = [I2C.Message([address, data], read=False)]
		self.transfer(msgs)

	def read_bytes(self, count):
		msgs = [I2C.Message([0] * count, read=True)]
		ret_msg = self.transfer(msgs)
		return ret_msg[0].data

if __name__ == "__main__":
	my_ccs811 = CCS811()
	my_ccs811.reset()

	time.sleep(1)

	print(my_ccs811.read_byte(0x00))
	print(my_ccs811.read_byte(0x20))

	my_ccs811.start_app()
	time.sleep(1)

	my_ccs811.read_byte(0x00)

	while True:
		if my_ccs811.read_byte(0x00) == 152:
			time.sleep(0.0625)
			my_ccs811.write_byte(0x02)
			time.sleep(0.0625)
			data = my_ccs811.read_bytes(8)
			print(data)
			eco2 = data[0] << 8 | data[1]
			voc = data[2] << 8 | data[3]
			print(eco2, voc)
		else:
			if my_ccs811.read_byte(0x00) & 0x01:
				#my_ccs811.close()
				#my_ccs811 = CCS811()
				#my_ccs811.reset()
				time.sleep(1)
				#my_ccs811.start_app()

			print(my_ccs811.read_byte(0x00))
		time.sleep(1)

<commit_msg>Exceptions<commit_after>from periphery import I2C
import time

class CCS811(object):

	def __init__(self, device_bus=1, device_address=0x5B):
		self.device_bus = device_bus
		self.device_address = device_address
		self.bus = I2C("/dev/i2c-1")

	def start_app(self):
		my_ccs811.write_byte(0xF4)
		print(my_ccs811.read_byte(0x00))
		my_ccs811.write_byte_data(0x01, 0x10)

	def transfer(self, msgs):
		try:
			self.bus.transfer(self.device_address, msgs)
		except periphery.i2c.I2CError as error:
			print(str(error))
		return msgs

	def close(self):
		self.bus.close()

	def read_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.transfer(msgs)
		msgs = [I2C.Message([address], read=True)]
		ret_msg = self.transfer(msgs)

		return (ret_msg[0].data[0])

	def write_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.transfer(msgs)

	def reset(self):
		msgs = [I2C.Message([0xFF, 0x11, 0xE5, 0x72, 0x8A], read=False)]
		self.transfer(msgs)

	def write_byte_data(self, address, data):
		msgs = [I2C.Message([address, data], read=False)]
		self.transfer(msgs)

	def read_bytes(self, count):
		msgs = [I2C.Message([0] * count, read=True)]
		ret_msg = self.transfer(msgs)
		return ret_msg[0].data

if __name__ == "__main__":
	my_ccs811 = CCS811()
	my_ccs811.reset()

	time.sleep(1)

	print(my_ccs811.read_byte(0x00))
	print(my_ccs811.read_byte(0x20))

	my_ccs811.start_app()
	time.sleep(1)

	my_ccs811.read_byte(0x00)

	while True:
		if my_ccs811.read_byte(0x00) == 152:
			time.sleep(0.0625)
			my_ccs811.write_byte(0x02)
			time.sleep(0.0625)
			data = my_ccs811.read_bytes(8)
			print(data)
			eco2 = data[0] << 8 | data[1]
			voc = data[2] << 8 | data[3]
			print(eco2, voc)
		else:
			if my_ccs811.read_byte(0x00) & 0x01:
				my_ccs811.close()
				my_ccs811 = CCS811()
				my_ccs811.reset()
				time.sleep(1)
				my_ccs811.start_app()

			print(my_ccs811.read_byte(0x00))
		time.sleep(1)

<|endoftext|>


<commit_before>from periphery import I2C
import time

class CCS811(object):

	def __init__(self, device_bus=1, device_address=0x5B):
		self.device_bus = device_bus
		self.device_address = device_address
		self.bus = I2C("/dev/i2c-1")

	def read_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.bus.transfer(self.device_address, msgs)
		time.sleep(0.0625)
		msgs = [I2C.Message([address, 0 , 0 , 0, 0], read=True)]
		self.bus.transfer(self.device_address, msgs)

		return (msgs[0].data[0])

	def write_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def reset(self):
		msgs = [I2C.Message([0xFF, 0x11, 0xE5, 0x72, 0x8A], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def write_byte_data(self, address, data):
		msgs = [I2C.Message([address, data], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def read_bytes(self, count):
		msgs = [I2C.Message([0] * count, read=True)]
		self.bus.transfer(self.device_address, msgs)
		return (msgs[0].data)

if __name__ == "__main__":
	my_ccs811 = CCS811()
	my_ccs811.reset()

	time.sleep(1)

	print(my_ccs811.read_byte(0x00))
	print(my_ccs811.read_byte(0x20))

	my_ccs811.write_byte(0xF4)
	print(my_ccs811.read_byte(0x00))

	my_ccs811.write_byte_data(0x01, 0x10)
	print(my_ccs811.read_byte(0x00))

	time.sleep(1)

	my_ccs811.read_byte(0x00)

	while True:
		if my_ccs811.read_byte(0x00) == 152:
			my_ccs811.write_byte(0x02)
			time.sleep(0.0625)
			data = my_ccs811.read_bytes(8)
			print(data)
			eco2 = data[0] << 8 | data[1]
			voc = data[2] << 8 | data[3]
			print(eco2, voc)
		else:
			if my_ccs811.read_byte(0x00) & 0x01:
				my_ccs811.reset()
			print(my_ccs811.read_byte(0x00))
		time.sleep(1)

<commit_msg>Changing delays()<commit_after>from periphery import I2C
import time

class CCS811(object):

	def __init__(self, device_bus=1, device_address=0x5B):
		self.device_bus = device_bus
		self.device_address = device_address
		self.bus = I2C("/dev/i2c-1")

	def read_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.bus.transfer(self.device_address, msgs)
		time.sleep(0.0625)
		msgs = [I2C.Message([address], read=True)]
		self.bus.transfer(self.device_address, msgs)

		return (msgs[0].data[0])

	def write_byte(self, address):
		msgs = [I2C.Message([address], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def reset(self):
		msgs = [I2C.Message([0xFF, 0x11, 0xE5, 0x72, 0x8A], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def write_byte_data(self, address, data):
		msgs = [I2C.Message([address, data], read=False)]
		self.bus.transfer(self.device_address, msgs)

	def read_bytes(self, count):
		msgs = [I2C.Message([0] * count, read=True)]
		self.bus.transfer(self.device_address, msgs)
		return (msgs[0].data)

if __name__ == "__main__":
	my_ccs811 = CCS811()
	my_ccs811.reset()

	time.sleep(1)

	print(my_ccs811.read_byte(0x00))
	print(my_ccs811.read_byte(0x20))

	my_ccs811.write_byte(0xF4)
	print(my_ccs811.read_byte(0x00))

	my_ccs811.write_byte_data(0x01, 0x10)
	print(my_ccs811.read_byte(0x00))

	time.sleep(1)

	my_ccs811.read_byte(0x00)

	while True:
		if my_ccs811.read_byte(0x00) == 152:
			my_ccs811.write_byte(0x02)
			time.sleep(0.0625)
			data = my_ccs811.read_bytes(8)
			print(data)
			eco2 = data[0] << 8 | data[1]
			voc = data[2] << 8 | data[3]
			print(eco2, voc)
		else:
			if my_ccs811.read_byte(0x00) & 0x01:
				my_ccs811.reset()
			print(my_ccs811.read_byte(0x00))
		time.sleep(1)

<|endoftext|>


<commit_before>#!/usr/bin/env python2.5

'''
Manipulate DJB's Constant Databases. These are 2 level disk-based hash tables
that efficiently handle thousands of keys, while remaining space-efficient.

    http://cr.yp.to/cdb.html

Note the Reader class reads the entire CDB into memory. When using Writer,
consider using Python's hash() instead of djb_hash() for a tidy speedup,
however readers must be similarly configured.
'''

from _struct import Struct
from itertools import chain

def djb_hash(s):
    '''Return the value of DJB's hash function for the given 8-bit string.'''
    h = 5381
    for c in s:
        h = (((h << 5) + h) ^ ord(c)) & 0xffffffff
    return h # for small strings, masking here is faster.

read_2_le4 = Struct('<LL').unpack
write_2_le4 = Struct('<LL').pack

class Reader(object):
    '''A dictionary-like object for reading data stored in a Constant
    Database.'''

    __slots__ = ('data', 'table_start', 'length', 'hashfn')

    def __init__(self, sequence, hashfn=djb_hash):
        '''Create an instance using the given string-like sequence, optionally
        specifying a non-default hash function.'''
        if len(sequence) < 2048:
            raise IOError('CDB too small')
        self.data = sequence

        self.hashfn = hashfn
        self.table_start = None
        self.length = None

    def _get_value(self, pos, key):
        '''Decode the item record stored at the given position, verify its key
        matches the given key, and return its value part on match, otherwise
        return None.'''
        klen, dlen = read_2_le4(self.data[pos:pos+8])
        pos += 8

        if self.data[pos:pos+klen] == key:
            pos += klen
            return self.data[pos:pos+dlen]

    def iteritems(self):
        '''Like dict.iteritems(). Items are returned in insertion order.'''
        if self.table_start is None:
            self.table_start = min(read_2_le4(self.data[i:i+8])[0]
                                   for i in xrange(0, 2048, 8))

        pos = 2048
        while pos < self.table_start:
            klen, dlen = read_2_le4(self.data[pos:pos+8])
            pos += 8

            key = self.data[pos:pos+klen]
            pos += klen

            data = self.data[pos:pos+dlen]
            pos += dlen

            yield key, data

    def items(self):
        '''Like dict.items().'''
        return list(self.iteritems())

    def iterkeys(self):
        '''Like dict.iterkeys().'''
        return (p[0] for p in self.iteritems())
    __iter__ = iterkeys

    def itervalues(self):
        '''Like dict.itervalues().'''
        return (p[1] for p in self.iteritems())

    def keys(self):
        '''Like dict.keys().'''
        return list(p[0] for p in self.iteritems())

    def values(self):
        '''Like dict.values().'''
        return list(p[1] for p in self.iteritems())

    def __getitem__(self, key):
        '''Like dict.__getitem__().'''
        value = self.get(key)
        if value is None:
            raise KeyError(key)
        return value

    def has_key(self, k):
        '''Return True if the given key exists in the database.'''
        return self.get(k, None) is not None
    __contains__ = has_key

    def __len__(self):
        '''Return the number of records in the database.'''
        if self.length is None:
            # TODO(dmw): can't really rely on load factor being 0.5
            self.length = sum(read_2_le4(self.data[i:i+8])[1] / 2
                              for i in xrange(0, 2048, 8))
        return self.length

    def gets(self, key):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted.'''
        # Truncate to 32 bits and remove sign.
        h = self.hashfn(key) & 0xffffffff
        idx = (h << 3) & 2047
        start, nslots = read_2_le4(self.data[idx:idx+8])

        if nslots:
            end = start + (nslots << 3)
            slot_off = start + (((h >> 8) % nslots) << 3)

            for pos in chain(xrange(slot_off, end, 8),
                             xrange(start, slot_off, 8)):
                rec_h, rec_pos = read_2_le4(self.data[pos:pos+8])

                if not rec_h:
                    break
                elif rec_h == h:
                    value = self._get_value(rec_pos, key)
                    if value is not None:
                        yield value

    def get(self, key, default=None):
        '''Return the first value for the given key in the database, or if it
        doesn't exist, return None or the given default.'''
        # Avoid exception catch when handling default case; much faster.
        return chain(self.gets(key), (default,)).next()

    def getint(self, key, default=None, base=0):
        '''Return the first value for the given key converted to an integer, or
        if it doesn't exist, return None or the given default.'''
        value = self.get(key, default)
        if value is not default:
            return int(value, base)
        return value

    def getints(self, key, base=0):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted, after converting them to integers.'''
        return (int(v, base) for v in self.gets(key))

    def getstring(self, key, default=None, encoding='utf-8'):
        '''Return the first value for the given key decoded as a UTF-8 string
        or the encoding specified, or if it doesn't exist, return None or the
        given default.'''
        value = self.get(key, default)
        if value is not default:
            return value.decode(encoding)
        return value

    def getstrings(self, key, encoding='utf-8'):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted, after decoding them as UTF-8 strings, or the
        encoding specified.'''
        return (v.decode(encoding) for v in self.gets(key))

class Writer(object):
    '''Object for building new Constant Databases, and writing them to a
    seekable file-like object.'''

    def __init__(self, fp, hashfn=djb_hash):
        '''Initialize a new instance that writes to the given file-like object
        using the given hash function, or DJB's function if unspecified.'''
        self.fp = fp
        self.hashfn = hashfn

        fp.write('\x00' * 2048)
        self._unordered = [[] for i in xrange(256)]

    def put(self, key, value=''):
        '''Write a string key/value pair to the output file.'''
        assert type(key) is str and type(value) is str

        pos = self.fp.tell()
        self.fp.write(write_2_le4(len(key), len(value)))
        self.fp.write(key)
        self.fp.write(value)

        h = self.hashfn(key) & 0xffffffff
        self._unordered[h & 0xff].append((h, pos))

    def puts(self, key, values):
        '''Write more than one value for the same key to the output file.
        Equivalent to calling put() in a loop.'''
        for value in values:
            self.put(key, value)

    def putint(self, key, value):
        '''Write an integer as a base-10 string associated with the given key
        to the output file.'''
        self.put(key, str(value))

    def putints(self, key, values):
        '''Write zero or more integers for the same key to the output file.
        Equivalent to calling putint() in a loop.'''
        self.puts(key, (str(value) for value in values))

    def putstring(self, key, value, encoding='utf-8'):
        '''Write a unicode string associated with the given key to the output
        file after encoding it as UTF-8 or the given encoding.'''
        self.put(key, unicode.encode(value, encoding))

    def putstrings(self, key, values, encoding='utf-8'):
        '''Write zero or more unicode strings to the output file. Equivalent to
        calling putstring() in a loop.'''
        self.puts(key, (unicode.encode(value, encoding) for value in values))

    def finalize(self):
        '''Write the final hash tables to the output file, and write out its
        index. The output file remains open upon return.'''
        index = []
        for tbl in self._unordered:
            length = len(tbl) << 1
            ordered = [(0, 0)] * length
            for pair in tbl:
                where = (pair[0] >> 8) % length
                for i in chain(xrange(where, length), xrange(0, where)):
                    if not ordered[i][0]:
                        ordered[i] = pair
                        break

            index.append((self.fp.tell(), length))
            for pair in ordered:
                self.fp.write(write_2_le4(*pair))

        self.fp.seek(0)
        for pair in index:
            self.fp.write(write_2_le4(*pair))
        self.fp = None # prevent double finalize()
<commit_msg>Stupid variable name.<commit_after>#!/usr/bin/env python2.5

'''
Manipulate DJB's Constant Databases. These are 2 level disk-based hash tables
that efficiently handle thousands of keys, while remaining space-efficient.

    http://cr.yp.to/cdb.html

Note the Reader class reads the entire CDB into memory. When using Writer,
consider using Python's hash() instead of djb_hash() for a tidy speedup,
however readers must be similarly configured.
'''

from _struct import Struct
from itertools import chain

def djb_hash(s):
    '''Return the value of DJB's hash function for the given 8-bit string.'''
    h = 5381
    for c in s:
        h = (((h << 5) + h) ^ ord(c)) & 0xffffffff
    return h # for small strings, masking here is faster.

read_2_le4 = Struct('<LL').unpack
write_2_le4 = Struct('<LL').pack

class Reader(object):
    '''A dictionary-like object for reading data stored in a Constant
    Database.'''

    __slots__ = ('data', 'table_start', 'length', 'hashfn')

    def __init__(self, data, hashfn=djb_hash):
        '''Create an instance using the given string-like sequence, optionally
        specifying a non-default hash function.'''
        if len(data) < 2048:
            raise IOError('CDB too small')
        self.data = data

        self.hashfn = hashfn
        self.table_start = None
        self.length = None

    def _get_value(self, pos, key):
        '''Decode the item record stored at the given position, verify its key
        matches the given key, and return its value part on match, otherwise
        return None.'''
        klen, dlen = read_2_le4(self.data[pos:pos+8])
        pos += 8

        if self.data[pos:pos+klen] == key:
            pos += klen
            return self.data[pos:pos+dlen]

    def iteritems(self):
        '''Like dict.iteritems(). Items are returned in insertion order.'''
        if self.table_start is None:
            self.table_start = min(read_2_le4(self.data[i:i+8])[0]
                                   for i in xrange(0, 2048, 8))

        pos = 2048
        while pos < self.table_start:
            klen, dlen = read_2_le4(self.data[pos:pos+8])
            pos += 8

            key = self.data[pos:pos+klen]
            pos += klen

            data = self.data[pos:pos+dlen]
            pos += dlen

            yield key, data

    def items(self):
        '''Like dict.items().'''
        return list(self.iteritems())

    def iterkeys(self):
        '''Like dict.iterkeys().'''
        return (p[0] for p in self.iteritems())
    __iter__ = iterkeys

    def itervalues(self):
        '''Like dict.itervalues().'''
        return (p[1] for p in self.iteritems())

    def keys(self):
        '''Like dict.keys().'''
        return list(p[0] for p in self.iteritems())

    def values(self):
        '''Like dict.values().'''
        return list(p[1] for p in self.iteritems())

    def __getitem__(self, key):
        '''Like dict.__getitem__().'''
        value = self.get(key)
        if value is None:
            raise KeyError(key)
        return value

    def has_key(self, k):
        '''Return True if the given key exists in the database.'''
        return self.get(k, None) is not None
    __contains__ = has_key

    def __len__(self):
        '''Return the number of records in the database.'''
        if self.length is None:
            # TODO(dmw): can't really rely on load factor being 0.5
            self.length = sum(read_2_le4(self.data[i:i+8])[1] / 2
                              for i in xrange(0, 2048, 8))
        return self.length

    def gets(self, key):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted.'''
        # Truncate to 32 bits and remove sign.
        h = self.hashfn(key) & 0xffffffff
        idx = (h << 3) & 2047
        start, nslots = read_2_le4(self.data[idx:idx+8])

        if nslots:
            end = start + (nslots << 3)
            slot_off = start + (((h >> 8) % nslots) << 3)

            for pos in chain(xrange(slot_off, end, 8),
                             xrange(start, slot_off, 8)):
                rec_h, rec_pos = read_2_le4(self.data[pos:pos+8])

                if not rec_h:
                    break
                elif rec_h == h:
                    value = self._get_value(rec_pos, key)
                    if value is not None:
                        yield value

    def get(self, key, default=None):
        '''Return the first value for the given key in the database, or if it
        doesn't exist, return None or the given default.'''
        # Avoid exception catch when handling default case; much faster.
        return chain(self.gets(key), (default,)).next()

    def getint(self, key, default=None, base=0):
        '''Return the first value for the given key converted to an integer, or
        if it doesn't exist, return None or the given default.'''
        value = self.get(key, default)
        if value is not default:
            return int(value, base)
        return value

    def getints(self, key, base=0):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted, after converting them to integers.'''
        return (int(v, base) for v in self.gets(key))

    def getstring(self, key, default=None, encoding='utf-8'):
        '''Return the first value for the given key decoded as a UTF-8 string
        or the encoding specified, or if it doesn't exist, return None or the
        given default.'''
        value = self.get(key, default)
        if value is not default:
            return value.decode(encoding)
        return value

    def getstrings(self, key, encoding='utf-8'):
        '''Yield all values for the given key in the database, in the order in
        which they were inserted, after decoding them as UTF-8 strings, or the
        encoding specified.'''
        return (v.decode(encoding) for v in self.gets(key))

class Writer(object):
    '''Object for building new Constant Databases, and writing them to a
    seekable file-like object.'''

    def __init__(self, fp, hashfn=djb_hash):
        '''Initialize a new instance that writes to the given file-like object
        using the given hash function, or DJB's function if unspecified.'''
        self.fp = fp
        self.hashfn = hashfn

        fp.write('\x00' * 2048)
        self._unordered = [[] for i in xrange(256)]

    def put(self, key, value=''):
        '''Write a string key/value pair to the output file.'''
        assert type(key) is str and type(value) is str

        pos = self.fp.tell()
        self.fp.write(write_2_le4(len(key), len(value)))
        self.fp.write(key)
        self.fp.write(value)

        h = self.hashfn(key) & 0xffffffff
        self._unordered[h & 0xff].append((h, pos))

    def puts(self, key, values):
        '''Write more than one value for the same key to the output file.
        Equivalent to calling put() in a loop.'''
        for value in values:
            self.put(key, value)

    def putint(self, key, value):
        '''Write an integer as a base-10 string associated with the given key
        to the output file.'''
        self.put(key, str(value))

    def putints(self, key, values):
        '''Write zero or more integers for the same key to the output file.
        Equivalent to calling putint() in a loop.'''
        self.puts(key, (str(value) for value in values))

    def putstring(self, key, value, encoding='utf-8'):
        '''Write a unicode string associated with the given key to the output
        file after encoding it as UTF-8 or the given encoding.'''
        self.put(key, unicode.encode(value, encoding))

    def putstrings(self, key, values, encoding='utf-8'):
        '''Write zero or more unicode strings to the output file. Equivalent to
        calling putstring() in a loop.'''
        self.puts(key, (unicode.encode(value, encoding) for value in values))

    def finalize(self):
        '''Write the final hash tables to the output file, and write out its
        index. The output file remains open upon return.'''
        index = []
        for tbl in self._unordered:
            length = len(tbl) << 1
            ordered = [(0, 0)] * length
            for pair in tbl:
                where = (pair[0] >> 8) % length
                for i in chain(xrange(where, length), xrange(0, where)):
                    if not ordered[i][0]:
                        ordered[i] = pair
                        break

            index.append((self.fp.tell(), length))
            for pair in ordered:
                self.fp.write(write_2_le4(*pair))

        self.fp.seek(0)
        for pair in index:
            self.fp.write(write_2_le4(*pair))
        self.fp = None # prevent double finalize()
<|endoftext|>