<commit_before>
import datetime
import dateutil
import functools
import futures
import logging
from tornado import web, escape, gen
from tornado.ioloop import IOLoop

import base
import models
import util

class ThisWeekHandler(base.BaseHandler):
    @web.asynchronous
    def get(self, uid):
        user = self.get_current_user()
        data_user = models.User.objects(id=uid).first()
        if not data_user.public and (not user or user.email != data_user.email):
            self.write_error(403)
            return
        date = models._current_monday()
        this_week_runs = models.Run.this_week_runs(data_user)

        expected_dates = set()
        for x in range(7):
            expected_dates.add(date)
            date += dateutil.relativedelta.relativedelta(days=1)

        runs = []
        dates = set()
        for r in this_week_runs:
            if r.date not in dates:
                runs.append({'x': r.date.strftime("%c"), 'y': r.distance})
            else:
                # find and add data
                for r2 in runs:
                    if r2['x'] == r.date.strftime("%c"):
                        r2['y'] += float(r.distance)
            dates.add(r.date)

        # for days without runs yet, add 0 mileage
        for d in expected_dates - dates:
            runs.append({'x': d.strftime("%c"), 'y': 0.0})

        data = {
                'xScale': 'ordinal',
                'yScale': 'linear',
                'main': [
                    {
                        'data': runs
                    }
                ]
        }

        self.finish(data)

class WeeklyMileageHandler(base.BaseHandler):
    @web.asynchronous
    def get(self, uid):
        user = self.get_current_user()
        since = self.get_argument('since', '')
        window_weeks = self.get_argument('window_weeks', '')

        data_user = models.User.objects(id=uid).first()
        if not data_user.public and (not user or user.email != data_user.email):
            self.write_error(403)
            return

        if window_weeks:
            window_weeks = dateutil.relativedelta.relativedelta(weeks=int(window_weeks))

        try:
            since = int(since)
        except ValueError:
            since = None

        if since:
            since = datetime.date(since, 1, 1)
            weeks = models.Week.objects(user=data_user, date__gte=since)
        else:
            if window_weeks:
                weeks = models.Week.objects(user=data_user, date__gte=datetime.date.today() - window_weeks)
            else:
                weeks = models.Week.objects(user=data_user)

        weeks = [week for week in weeks]

        weeks = sorted(weeks, key=lambda w: w.date)
        last_date = weeks[0].date
        offset = dateutil.relativedelta.relativedelta(days=7)
        for x in range(1, len(weeks)):
            if last_date + offset != weeks[x].date:
                while last_date + offset != weeks[x].date:
                    w = models.Week(distance=0, time=0, date=last_date+offset)
                    weeks.append(w)
                    last_date += offset
            last_date += offset
        weeks = sorted(weeks, key=lambda w: w.date)

        # handle the beginning of the year
        year = datetime.date.today().year
        if since and weeks[0].date != datetime.date(year, 1, 1):
            # manually build a partial week
            runs = models.Run.objects(user=data_user, date__lt=weeks[0].date, date__gte=datetime.date(year, 1, 1))
            week = models.Week(user=data_user)
            week.date = datetime.date(year, 1, 1)
            for run in runs:
                week.distance += run.distance
                week.time += run.time
            weeks.insert(0, week)

        weeks = [ {'x': w.date.strftime('%m-%d-%Y'), 'y': w.distance} for w in weeks]

        # if the user only has 1 week display the last week too
        if len(weeks) == 1:
            last_monday = dateutil.parser.parse(weeks[0]['x']) - dateutil.relativedelta.relativedelta(days=7)
            weeks.append({'x': last_monday.strftime('%m-%d-%Y'), 'y': 0})

        data = {
                'xScale': 'time',
                'yScale': 'linear',
                'main': [
                    {
                        'data': weeks
                    }
                ]
        }

        # 0 fill some data if there is none so that the graph shows
        if since and not weeks:
            weeks.append({'x': since.strftime('%x'), 'y':0})
            weeks.append({'x': (since + dateutil.relativedelta.relativedelta(days=7)).strftime('%m-%d-%Y'), 'y':0})
            data['yMin'] = 0
            data['yMax'] = 100

        self.finish(data)

class WeekdayRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        data_user = models.User.objects(id=uid).first()

        def runs_by_day(user):
            run_map = '''
                function() {
                    // remember, we need 0 to be Monday, so let's adjust
                    //var day = this.date.getDay() - 1;
                    //if(day < 0) { day = 6; }
                    emit(this.date.getDay(), 1);
                };
            '''
            run_reduce = '''
                function(day, counts) {
                    return Array.sum(counts);
                };
            '''

            runs = models.Run.objects(user=user)
            mrd = runs.map_reduce(run_map, run_reduce, 'inline')
            return mrd

        map_reduce_document = (yield gen.Task(
            self.execute_thread, runs_by_day, data_user)).result()

        runs_per_weekday = [0] * 7
        for i in map_reduce_document:
            runs_per_weekday[int(i.key)] = int(i.value)

        """
        data = {
            'xScale': 'ordinal',
            'yScale': 'linear',
            'yMin': 0,
            'main': [
                { 'data': [{'x':i, 'y':runs_per_weekday[i]} for i in range(len(runs_per_weekday))]}
            ]
        }
        """
        data = { 'data':
                [
                    {'x':i, 'y':runs_per_weekday[i]} for i in range(
                        len(runs_per_weekday))]}

        self.finish(data)

class DailyRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        user = models.User.objects(id=uid).first()

        def runs_per_day(user):
            plus_one_day = datetime.timedelta(days=1)
            today = datetime.date.today()
            if today.month == 2 and today.day == 29:
                d = datetime.date(today.year-1, today.month, today.day-1)
            else:
                d = datetime.date(today.year-1, today.month, today.day)
            runs = models.Run.objects(user=user,date__gt=d)
            data = {str(r.date).split(' ')[0]: [r.date.isocalendar()[1], 1] for r in runs}
            while d <= today:
                if str(d) not in data.keys():
                    data[str(d)] = [d.isocalendar()[1], 0]
                d += plus_one_day
            ret_data = [[k, v[0], v[1]] for k,v in data.iteritems()]
            ret_data = sorted(ret_data, key=lambda i: i[0])
            return ret_data

        runs = (yield gen.Task(
            self.execute_thread, runs_per_day, user)).result()

        self.finish(str(runs).replace("'", '"'))

class MonthRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        user = models.User.objects(id=uid).first()

        def runs_in_month(user):
            plus_one_day = datetime.timedelta(days=1)
            today = datetime.date.today()
            d = today - (31 * plus_one_day)
            runs = models.Run.objects(user=user,date__gte=d)
            data = {str(r.date).split(' ')[0]: [r.date.isocalendar()[1], 1] for r in runs}
            while d <= today:
                if str(d) not in data.keys():
                    data[str(d)] = [d.isocalendar()[1], 0]
                d += plus_one_day
            ret_data = [[k, v[0], v[1]] for k,v in data.iteritems()]
            ret_data = sorted(ret_data, key=lambda i: i[0])
            return ret_data

        runs = (yield gen.Task(
            self.execute_thread, runs_in_month, user)).result()

        self.finish(str(runs).replace("'", '"'))

<commit_msg>Fixed year view as well, was not getting first day info<commit_after>
import datetime
import dateutil
import functools
import futures
import logging
from tornado import web, escape, gen
from tornado.ioloop import IOLoop

import base
import models
import util

class ThisWeekHandler(base.BaseHandler):
    @web.asynchronous
    def get(self, uid):
        user = self.get_current_user()
        data_user = models.User.objects(id=uid).first()
        if not data_user.public and (not user or user.email != data_user.email):
            self.write_error(403)
            return
        date = models._current_monday()
        this_week_runs = models.Run.this_week_runs(data_user)

        expected_dates = set()
        for x in range(7):
            expected_dates.add(date)
            date += dateutil.relativedelta.relativedelta(days=1)

        runs = []
        dates = set()
        for r in this_week_runs:
            if r.date not in dates:
                runs.append({'x': r.date.strftime("%c"), 'y': r.distance})
            else:
                # find and add data
                for r2 in runs:
                    if r2['x'] == r.date.strftime("%c"):
                        r2['y'] += float(r.distance)
            dates.add(r.date)

        # for days without runs yet, add 0 mileage
        for d in expected_dates - dates:
            runs.append({'x': d.strftime("%c"), 'y': 0.0})

        data = {
                'xScale': 'ordinal',
                'yScale': 'linear',
                'main': [
                    {
                        'data': runs
                    }
                ]
        }

        self.finish(data)

class WeeklyMileageHandler(base.BaseHandler):
    @web.asynchronous
    def get(self, uid):
        user = self.get_current_user()
        since = self.get_argument('since', '')
        window_weeks = self.get_argument('window_weeks', '')

        data_user = models.User.objects(id=uid).first()
        if not data_user.public and (not user or user.email != data_user.email):
            self.write_error(403)
            return

        if window_weeks:
            window_weeks = dateutil.relativedelta.relativedelta(weeks=int(window_weeks))

        try:
            since = int(since)
        except ValueError:
            since = None

        if since:
            since = datetime.date(since, 1, 1)
            weeks = models.Week.objects(user=data_user, date__gte=since)
        else:
            if window_weeks:
                weeks = models.Week.objects(user=data_user, date__gte=datetime.date.today() - window_weeks)
            else:
                weeks = models.Week.objects(user=data_user)

        weeks = [week for week in weeks]

        weeks = sorted(weeks, key=lambda w: w.date)
        last_date = weeks[0].date
        offset = dateutil.relativedelta.relativedelta(days=7)
        for x in range(1, len(weeks)):
            if last_date + offset != weeks[x].date:
                while last_date + offset != weeks[x].date:
                    w = models.Week(distance=0, time=0, date=last_date+offset)
                    weeks.append(w)
                    last_date += offset
            last_date += offset
        weeks = sorted(weeks, key=lambda w: w.date)

        # handle the beginning of the year
        year = datetime.date.today().year
        if since and weeks[0].date != datetime.date(year, 1, 1):
            # manually build a partial week
            runs = models.Run.objects(user=data_user, date__lt=weeks[0].date, date__gte=datetime.date(year, 1, 1))
            week = models.Week(user=data_user)
            week.date = datetime.date(year, 1, 1)
            for run in runs:
                week.distance += run.distance
                week.time += run.time
            weeks.insert(0, week)

        weeks = [ {'x': w.date.strftime('%m-%d-%Y'), 'y': w.distance} for w in weeks]

        # if the user only has 1 week display the last week too
        if len(weeks) == 1:
            last_monday = dateutil.parser.parse(weeks[0]['x']) - dateutil.relativedelta.relativedelta(days=7)
            weeks.append({'x': last_monday.strftime('%m-%d-%Y'), 'y': 0})

        data = {
                'xScale': 'time',
                'yScale': 'linear',
                'main': [
                    {
                        'data': weeks
                    }
                ]
        }

        # 0 fill some data if there is none so that the graph shows
        if since and not weeks:
            weeks.append({'x': since.strftime('%x'), 'y':0})
            weeks.append({'x': (since + dateutil.relativedelta.relativedelta(days=7)).strftime('%m-%d-%Y'), 'y':0})
            data['yMin'] = 0
            data['yMax'] = 100

        self.finish(data)

class WeekdayRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        data_user = models.User.objects(id=uid).first()

        def runs_by_day(user):
            run_map = '''
                function() {
                    // remember, we need 0 to be Monday, so let's adjust
                    //var day = this.date.getDay() - 1;
                    //if(day < 0) { day = 6; }
                    emit(this.date.getDay(), 1);
                };
            '''
            run_reduce = '''
                function(day, counts) {
                    return Array.sum(counts);
                };
            '''

            runs = models.Run.objects(user=user)
            mrd = runs.map_reduce(run_map, run_reduce, 'inline')
            return mrd

        map_reduce_document = (yield gen.Task(
            self.execute_thread, runs_by_day, data_user)).result()

        runs_per_weekday = [0] * 7
        for i in map_reduce_document:
            runs_per_weekday[int(i.key)] = int(i.value)

        """
        data = {
            'xScale': 'ordinal',
            'yScale': 'linear',
            'yMin': 0,
            'main': [
                { 'data': [{'x':i, 'y':runs_per_weekday[i]} for i in range(len(runs_per_weekday))]}
            ]
        }
        """
        data = { 'data':
                [
                    {'x':i, 'y':runs_per_weekday[i]} for i in range(
                        len(runs_per_weekday))]}

        self.finish(data)

class DailyRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        user = models.User.objects(id=uid).first()

        def runs_per_day(user):
            plus_one_day = datetime.timedelta(days=1)
            today = datetime.date.today()
            if today.month == 2 and today.day == 29:
                d = datetime.date(today.year-1, today.month, today.day-1)
            else:
                d = datetime.date(today.year-1, today.month, today.day)
            runs = models.Run.objects(user=user,date__gte=d)
            data = {str(r.date).split(' ')[0]: [r.date.isocalendar()[1], 1] for r in runs}
            while d <= today:
                if str(d) not in data.keys():
                    data[str(d)] = [d.isocalendar()[1], 0]
                d += plus_one_day
            ret_data = [[k, v[0], v[1]] for k,v in data.iteritems()]
            ret_data = sorted(ret_data, key=lambda i: i[0])
            return ret_data

        runs = (yield gen.Task(
            self.execute_thread, runs_per_day, user)).result()

        self.finish(str(runs).replace("'", '"'))

class MonthRunsHandler(base.BaseHandler):
    @web.asynchronous
    @gen.engine
    def get(self, uid):
        user = models.User.objects(id=uid).first()

        def runs_in_month(user):
            plus_one_day = datetime.timedelta(days=1)
            today = datetime.date.today()
            d = today - (31 * plus_one_day)
            runs = models.Run.objects(user=user,date__gte=d)
            data = {str(r.date).split(' ')[0]: [r.date.isocalendar()[1], 1] for r in runs}
            while d <= today:
                if str(d) not in data.keys():
                    data[str(d)] = [d.isocalendar()[1], 0]
                d += plus_one_day
            ret_data = [[k, v[0], v[1]] for k,v in data.iteritems()]
            ret_data = sorted(ret_data, key=lambda i: i[0])
            return ret_data

        runs = (yield gen.Task(
            self.execute_thread, runs_in_month, user)).result()

        self.finish(str(runs).replace("'", '"'))

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import subprocess
import sys

def get_env(var="EP"):
    return os.environ.get(var)

def get_keychain_pass(account=None, server=None):
    if not sys.platform == 'darwin':
        return get_env()

    home = os.environ.get('HOME')
    params = {
        'security': '/usr/bin/security',
        'command': 'find-internet-password',
        'account': account,
        'server': server,
        'keychain': home + '/Library/Keychains/login.keychain',
    }
    command = "sudo -u ksmiley %(security)s -v %(command)s -g -a %(account)s -s %(server)s %(keychain)s" % params
    output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
    outtext = [l for l in output.splitlines()
               if l.startswith('password: ')][0]

    return re.match(r'password: "(.*)"', outtext).group(1)

<commit_msg>Fix offlineimap helpers<commit_after>#!/usr/bin/env python

import os
import re
import subprocess
import sys

def get_env(var="EP"):
    return os.environ.get(var)

def get_keychain_pass(account=None, server=None):
    if not sys.platform == 'darwin':
        return get_env()

    home = os.environ.get('HOME')
    user = os.environ.get('USER')

    params = {
        'security': '/usr/bin/security',
        'command': 'find-internet-password',
        'account': account,
        'server': server,
        'keychain': home + '/Library/Keychains/login.keychain',
        'user': user,
    }
    command = "sudo -u %(user)s %(security)s -v %(command)s -g -a %(account)s -s %(server)s %(keychain)s" % params
    output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT)
    outtext = [l for l in output.splitlines()
               if l.startswith('password: ')][0]

    return re.match(r'password: "(.*)"', outtext).group(1)

<|endoftext|>


<commit_before>
# coding: utf-8

"""This module contains all the page handlers and caching mechanisms."""
import json
import logging
import time
import random
from datetime import datetime, timedelta
from httplib import HTTPException
from xml.dom.minidom import getDOMImplementation

from google.appengine.api import memcache, users
from google.appengine.ext import ndb

import tf2api
import tf2search

import config
from handler import Handler

def updatecache():
    t0 = time.time()

    tf2info = tf2search.gettf2info(config.apikey, config.backpackkey,
                                   config.blueprintsfile)
    itemsdict = tf2search.getitemsdict(tf2info)

    newitems = [itemsdict[index] for index in
                tf2info.newstoreprices]

    nametoindexmap = {}
    itemnames = []
    itemindexes = set()

    sitemap = Sitemap()
    sitemap.add(config.homepage)

    for name, item in tf2info.itemsbyname.items():
        index = item['defindex']
        itemdict = itemsdict[index]

        if tf2search.isvalidresult(itemdict):
            nametoindexmap[name] = index
            itemnames.append(name)
            itemindexes.add(index)

            path = '{0}/item/{1}'.format(config.homepage, index)
            sitemap.add(path)

    memcache.set_multi({'itemsdict': itemsdict,
                        'itemsets': tf2info.itemsets,
                        'bundles': tf2info.bundles,
                        'nametoindexmap': nametoindexmap,
                        'itemnames': itemnames,
                        'itemindexes': itemindexes,
                        'newitems': newitems,
                        'sitemap': sitemap.toxml()})
    t1 = time.time()

    memcache.set('lastupdated', t1)
    logging.debug('Updated Cache. Time taken: {} seconds'.format(t1 - t0))

def getfromcache(key):
    value = memcache.get(key)

    if not value:
        logging.debug("Could not find key '{}'. Updating cache.".format(key))
        updatecache()
        value = memcache.get(key)

    return value

def getsteamid(openiduser):
    return openiduser.nickname().split('/')[-1] if openiduser else None

def getuser(steamid, create=False):
    user = User.get_by_id(steamid)

    if user:
        needsupdate = (datetime.now() - user.lastupdate) > timedelta(minutes=3)
    else:
        if not create:
            return None
        user = User(id=steamid)
        needsupdate = True

    if needsupdate:
        try:
            steamuser = tf2api.getplayersummary(config.apikey, steamid)

        except HTTPException:
            # Postpone update if this is not a new user
            if not create:
                return user

        user.name = steamuser['personaname']
        user.url = steamuser['profileurl'].split('/')[-2]
        user.avatar = steamuser['avatar']
        user.state = 'Online' if steamuser['personastate'] != 0 else 'Offline'

        if 'gameid' in steamuser:
            user.state = 'In-Game'

        user.put()

    return user

class TF2Handler(Handler):
    """Homepage handler"""
    def get(self):
        if self.request.host.endswith('appspot.com'):
            return self.redirect(config.homepage, True)

        t0 = getfromcache('lastupdated')
        lastupdated = int(time.time() - t0) / 60

        user = None

        openidurl = 'http://steamcommunity.com/openid'
        openiduser = users.get_current_user()

        if openiduser:
            steamid = getsteamid(openiduser)
            user = getuser(steamid, True)

            self.response.set_cookie('steam_id', steamid)
        else:
            self.response.delete_cookie('steam_id')

        self.render('tf2.html',
                    homepage=config.homepage,
                    user=user,
                    loginurl=users.create_login_url('/', None, openidurl),
                    tags=tf2api.getalltags(),
                    newitems=random.sample(getfromcache('newitems'), 5),
                    lastupdated=lastupdated)

class TF2SearchHandler(Handler):
    def get(self):
        query = self.request.get('q')

        if query:
            if query == 'random':
                itemindexes = getfromcache('itemindexes')
                return self.redirect(
                    # random.choice does not support sets
                    '/item/{}'.format(random.choice(list(itemindexes))))

            nametoindexmap = getfromcache('nametoindexmap')

            if query in nametoindexmap:
                return self.redirect(
                    '/item/{}'.format(nametoindexmap[query]))

            itemsdict = getfromcache('itemsdict')
            itemsets = getfromcache('itemsets')
            bundles = getfromcache('bundles')

            t0 = time.time()
            results = tf2search.search(query, itemsdict, nametoindexmap,
                                       itemsets, bundles)
            t1 = time.time()

            self.render('tf2results.html',
                        query=query,
                        mainitems=results['mainitems'],
                        otheritems=sorted(results['otheritems'].items(),
                                          key=lambda k: len(k[0]),
                                          reverse=True),
                        itemsets=itemsets,
                        resultslength=results['length'],
                        time=round(t1 - t0, 3))
        else:
            self.redirect('/')

class TF2SuggestHandler(Handler):
    """OpenSearch suggestions handler"""
    def get(self):
        query = self.request.get('q')

        itemnames = getfromcache('itemnames')
        if query:
            suggestions = []
            for name in itemnames:
                foldedname = tf2search.foldaccents(name)
                if query in foldedname or query in foldedname.lower():
                    suggestions.append(name)
        else:
            suggestions = itemnames

        self.response.headers['Content-Type'] = ('application/json;'
                                                 'charset=UTF-8')
        self.write(json.dumps([query, suggestions]))

class TF2UserHandler(Handler):
    def get(self, steamid):
        usersteamid = getsteamid(users.get_current_user())
        # Avoid a Steam API call if the user is visiting his own page
        if usersteamid:
            user = getuser(usersteamid)
            # User is visiting his own page
            if steamid in (usersteamid, user.url):
                steamid = usersteamid

        if steamid != usersteamid:
            steamid = (tf2api.resolvevanityurl(config.apikey, steamid) or
                       steamid)
            user = getuser(steamid)

        if user:
            itemsdict = getfromcache('itemsdict')
            items = []

            wishlist = user.wishlist

            for i, item in enumerate(wishlist):
                itemdict = itemsdict[item['index']].copy()
                item.update({'i': i})
                itemdict.update(item)
                items.append(itemdict)

        else:
            return self.redirect('/')

        self.render('tf2user.html',
                    user=user,
                    wishlist=user.wishlist,
                    items=items)

class TF2WishlistHandler(Handler):
    def post(self, option):
        openiduser = users.get_current_user()

        if not openiduser:
            return self.redirect('/')

        steamid = getsteamid(openiduser)
        user = getuser(steamid)

        if option == 'add':
            index = int(self.request.get('index'))
            quality = int(self.request.get('quality'))

            if (index in getfromcache('itemindexes') and
                    quality in tf2api.getallqualities()):

                user.wishlist.append({'index': index,
                                      'quality': quality})
                user.put()

                self.write('Added')

        elif option == 'remove':
            i = int(self.request.get('i'))

            del user.wishlist[i]
            user.put()

            self.write('Removed')

class TF2ItemHandler(Handler):
    def get(self, defindex, is_json):
        itemsdict = getfromcache('itemsdict')
        index = int(defindex)

        if index in itemsdict:
            itemdict = itemsdict[index]
        else:
            return self.redirect('/')

        if is_json:
            self.response.headers['Content-Type'] = ('application/json;'
                                                     'charset=UTF-8')
            self.write(json.dumps(itemdict, indent=2))
        else:
            desc_list = []

            if itemdict['description']:
                desc_list.append(itemdict['description'].replace('\n', ' '))

            desc_list.append(', '.join(itemdict['classes'])
                             if itemdict['classes'] else 'All Classes')

            if itemdict['tags']:
                desc_list.append(', '.join(itemdict['tags']).title())

            self.render('tf2item.html',
                        item=itemdict,
                        description=' | '.join(desc_list))

class TF2SitemapHandler(Handler):
    def get(self):
        self.response.headers['Content-Type'] = 'application/xml;charset=UTF-8'
        self.write(getfromcache('sitemap'))

class CacheHandler(Handler):
    def get(self, option):
        if option == 'update':
            updatecache()
            self.write('Cache Updated')
        elif option == 'flush':
            memcache.flush_all()
            self.write('Cache Flushed')

class User(ndb.Model):
    name = ndb.StringProperty('n')
    url = ndb.StringProperty('u')
    avatar = ndb.StringProperty('a')
    state = ndb.StringProperty('s')
    wishlist = ndb.PickleProperty('w', default=[])

    lastupdate = ndb.DateTimeProperty('t', auto_now=True)

class Sitemap:
    """A class that is used to create XML sitemaps"""
    def __init__(self):
        impl = getDOMImplementation()
        self.doc = impl.createDocument(None, 'urlset', None)

        self.urlset = self.doc.documentElement
        self.urlset.setAttribute('xmlns',
                                 'http://www.sitemaps.org/schemas/sitemap/0.9')

    def add(self, path):
        """Add a url to the sitemap"""
        url = self.doc.createElement('url')
        loc = url.appendChild(self.doc.createElement('loc'))
        loc.appendChild(self.doc.createTextNode(path))

        return self.urlset.appendChild(url)

    def toxml(self):
        """Return a pretty XML version of the sitemap"""
        return self.doc.toprettyxml(encoding='UTF-8')
<commit_msg>Return proper HTTP response<commit_after>
# coding: utf-8

"""This module contains all the page handlers and caching mechanisms."""
import json
import logging
import time
import random
from datetime import datetime, timedelta
from httplib import HTTPException
from xml.dom.minidom import getDOMImplementation

from google.appengine.api import memcache, users
from google.appengine.ext import ndb

import tf2api
import tf2search

import config
from handler import Handler

def updatecache():
    t0 = time.time()

    tf2info = tf2search.gettf2info(config.apikey, config.backpackkey,
                                   config.blueprintsfile)
    itemsdict = tf2search.getitemsdict(tf2info)

    newitems = [itemsdict[index] for index in
                tf2info.newstoreprices]

    nametoindexmap = {}
    itemnames = []
    itemindexes = set()

    sitemap = Sitemap()
    sitemap.add(config.homepage)

    for name, item in tf2info.itemsbyname.items():
        index = item['defindex']
        itemdict = itemsdict[index]

        if tf2search.isvalidresult(itemdict):
            nametoindexmap[name] = index
            itemnames.append(name)
            itemindexes.add(index)

            path = '{0}/item/{1}'.format(config.homepage, index)
            sitemap.add(path)

    memcache.set_multi({'itemsdict': itemsdict,
                        'itemsets': tf2info.itemsets,
                        'bundles': tf2info.bundles,
                        'nametoindexmap': nametoindexmap,
                        'itemnames': itemnames,
                        'itemindexes': itemindexes,
                        'newitems': newitems,
                        'sitemap': sitemap.toxml()})
    t1 = time.time()

    memcache.set('lastupdated', t1)
    logging.debug('Updated Cache. Time taken: {} seconds'.format(t1 - t0))

def getfromcache(key):
    value = memcache.get(key)

    if not value:
        logging.debug("Could not find key '{}'. Updating cache.".format(key))
        updatecache()
        value = memcache.get(key)

    return value

def getsteamid(openiduser):
    return openiduser.nickname().split('/')[-1] if openiduser else None

def getuser(steamid, create=False):
    user = User.get_by_id(steamid)

    if user:
        needsupdate = (datetime.now() - user.lastupdate) > timedelta(minutes=3)
    else:
        if not create:
            return None
        user = User(id=steamid)
        needsupdate = True

    if needsupdate:
        try:
            steamuser = tf2api.getplayersummary(config.apikey, steamid)

        except HTTPException:
            # Postpone update if this is not a new user
            if not create:
                return user

        user.name = steamuser['personaname']
        user.url = steamuser['profileurl'].split('/')[-2]
        user.avatar = steamuser['avatar']
        user.state = 'Online' if steamuser['personastate'] != 0 else 'Offline'

        if 'gameid' in steamuser:
            user.state = 'In-Game'

        user.put()

    return user

class TF2Handler(Handler):
    """Homepage handler"""
    def get(self):
        if self.request.host.endswith('appspot.com'):
            return self.redirect(config.homepage, True)

        t0 = getfromcache('lastupdated')
        lastupdated = int(time.time() - t0) / 60

        user = None

        openidurl = 'http://steamcommunity.com/openid'
        openiduser = users.get_current_user()

        if openiduser:
            steamid = getsteamid(openiduser)
            user = getuser(steamid, True)

            self.response.set_cookie('steam_id', steamid)
        else:
            self.response.delete_cookie('steam_id')

        self.render('tf2.html',
                    homepage=config.homepage,
                    user=user,
                    loginurl=users.create_login_url('/', None, openidurl),
                    tags=tf2api.getalltags(),
                    newitems=random.sample(getfromcache('newitems'), 5),
                    lastupdated=lastupdated)

class TF2SearchHandler(Handler):
    def get(self):
        query = self.request.get('q')

        if query:
            if query == 'random':
                itemindexes = getfromcache('itemindexes')
                return self.redirect(
                    # random.choice does not support sets
                    '/item/{}'.format(random.choice(list(itemindexes))))

            nametoindexmap = getfromcache('nametoindexmap')

            if query in nametoindexmap:
                return self.redirect(
                    '/item/{}'.format(nametoindexmap[query]))

            itemsdict = getfromcache('itemsdict')
            itemsets = getfromcache('itemsets')
            bundles = getfromcache('bundles')

            t0 = time.time()
            results = tf2search.search(query, itemsdict, nametoindexmap,
                                       itemsets, bundles)
            t1 = time.time()

            self.render('tf2results.html',
                        query=query,
                        mainitems=results['mainitems'],
                        otheritems=sorted(results['otheritems'].items(),
                                          key=lambda k: len(k[0]),
                                          reverse=True),
                        itemsets=itemsets,
                        resultslength=results['length'],
                        time=round(t1 - t0, 3))
        else:
            self.redirect('/')

class TF2SuggestHandler(Handler):
    """OpenSearch suggestions handler"""
    def get(self):
        query = self.request.get('q')

        itemnames = getfromcache('itemnames')
        if query:
            suggestions = []
            for name in itemnames:
                foldedname = tf2search.foldaccents(name)
                if query in foldedname or query in foldedname.lower():
                    suggestions.append(name)
        else:
            suggestions = itemnames

        self.response.headers['Content-Type'] = ('application/json;'
                                                 'charset=UTF-8')
        self.write(json.dumps([query, suggestions]))

class TF2UserHandler(Handler):
    def get(self, steamid):
        usersteamid = getsteamid(users.get_current_user())
        # Avoid a Steam API call if the user is visiting his own page
        if usersteamid:
            user = getuser(usersteamid)
            # User is visiting his own page
            if steamid in (usersteamid, user.url):
                steamid = usersteamid

        if steamid != usersteamid:
            steamid = (tf2api.resolvevanityurl(config.apikey, steamid) or
                       steamid)
            user = getuser(steamid)

        if user:
            itemsdict = getfromcache('itemsdict')
            items = []

            wishlist = user.wishlist

            for i, item in enumerate(wishlist):
                itemdict = itemsdict[item['index']].copy()
                item.update({'i': i})
                itemdict.update(item)
                items.append(itemdict)

        else:
            return self.redirect('/')

        self.render('tf2user.html',
                    user=user,
                    wishlist=user.wishlist,
                    items=items)

class TF2WishlistHandler(Handler):
    def post(self, option):
        openiduser = users.get_current_user()

        if not openiduser:
            self.response.headers['WWW-Authenticate'] = (
                'OpenID identifier="http://steamcommunity.com/openid"')
            return self.response.set_status(401)

        steamid = getsteamid(openiduser)
        user = getuser(steamid)

        if option == 'add':
            index = int(self.request.get('index'))
            quality = int(self.request.get('quality'))

            if (index in getfromcache('itemindexes') and
                    quality in tf2api.getallqualities()):

                user.wishlist.append({'index': index,
                                      'quality': quality})
                user.put()

                self.write('Added')

        elif option == 'remove':
            i = int(self.request.get('i'))

            del user.wishlist[i]
            user.put()

            self.write('Removed')

class TF2ItemHandler(Handler):
    def get(self, defindex, is_json):
        itemsdict = getfromcache('itemsdict')
        index = int(defindex)

        if index in itemsdict:
            itemdict = itemsdict[index]
        else:
            return self.redirect('/')

        if is_json:
            self.response.headers['Content-Type'] = ('application/json;'
                                                     'charset=UTF-8')
            self.write(json.dumps(itemdict, indent=2))
        else:
            desc_list = []

            if itemdict['description']:
                desc_list.append(itemdict['description'].replace('\n', ' '))

            desc_list.append(', '.join(itemdict['classes'])
                             if itemdict['classes'] else 'All Classes')

            if itemdict['tags']:
                desc_list.append(', '.join(itemdict['tags']).title())

            self.render('tf2item.html',
                        item=itemdict,
                        description=' | '.join(desc_list))

class TF2SitemapHandler(Handler):
    def get(self):
        self.response.headers['Content-Type'] = 'application/xml;charset=UTF-8'
        self.write(getfromcache('sitemap'))

class CacheHandler(Handler):
    def get(self, option):
        if option == 'update':
            updatecache()
            self.write('Cache Updated')
        elif option == 'flush':
            memcache.flush_all()
            self.write('Cache Flushed')

class User(ndb.Model):
    name = ndb.StringProperty('n')
    url = ndb.StringProperty('u')
    avatar = ndb.StringProperty('a')
    state = ndb.StringProperty('s')
    wishlist = ndb.PickleProperty('w', default=[])

    lastupdate = ndb.DateTimeProperty('t', auto_now=True)

class Sitemap:
    """A class that is used to create XML sitemaps"""
    def __init__(self):
        impl = getDOMImplementation()
        self.doc = impl.createDocument(None, 'urlset', None)

        self.urlset = self.doc.documentElement
        self.urlset.setAttribute('xmlns',
                                 'http://www.sitemaps.org/schemas/sitemap/0.9')

    def add(self, path):
        """Add a url to the sitemap"""
        url = self.doc.createElement('url')
        loc = url.appendChild(self.doc.createElement('loc'))
        loc.appendChild(self.doc.createTextNode(path))

        return self.urlset.appendChild(url)

    def toxml(self):
        """Return a pretty XML version of the sitemap"""
        return self.doc.toprettyxml(encoding='UTF-8')
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

def opensfm_to_upright(coords, width, height, orientation,
                       new_width=None, new_height=None):
    """
    Transform opensfm coordinates to upright coordinates, correcting for EXIF orientation.

    :param coords: Points in opensfm coordinate system.
    :param width: Width of original image in pixels unadjusted for orientation.
    :param height: Height of original image in pixels unadjusted for orientation.
    :param orientation: Orientation of original image.

    :return: Points in upright coordinate system.

    >>> sfm = np.array([[-0.5, -0.375],
    ...                 [-0.5,  0.375],
    ...                 [ 0.5, -0.375],
    ...                 [ 0.5,  0.375]])
    >>> opensfm_to_upright(sfm, 320, 240, 1)
    array([[  0.,   0.],
           [  0., 240.],
           [320.,   0.],
           [320., 240.]])
    """

    R_T = {
        1: np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        3: np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]),
        6: np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]),
        8: np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]),
    }

    w = float(width)
    h = float(height)
    s = max(w, h)

    H_inv = np.array([[s / w,     0, 0.5],
                      [    0, s / h, 0.5],
                      [    0,     0,   1]])

    T_inv = np.dot(R_T[orientation], H_inv)

    p = np.dot(coords, T_inv[:2, :2].T) + T_inv[:2, 2].reshape(1, 2)

    upright_width = width if orientation < 6 else height
    upright_height = height if orientation < 6 else width
    if new_width is not None:
        upright_width = new_width
    if new_height is not None:
        upright_height = new_height
    p[:, 0] = upright_width * p[:, 0]
    p[:, 1] = upright_height * p[:, 1]

    return p
<commit_msg>fix: stupid doctest spaces<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

def opensfm_to_upright(coords, width, height, orientation,
                       new_width=None, new_height=None):
    """
    Transform opensfm coordinates to upright coordinates, correcting for EXIF orientation.

    :param coords: Points in opensfm coordinate system.
    :param width: Width of original image in pixels unadjusted for orientation.
    :param height: Height of original image in pixels unadjusted for orientation.
    :param orientation: Orientation of original image.

    :return: Points in upright coordinate system.

    >>> sfm = np.array([[-0.5, -0.375],
    ...                 [-0.5,  0.375],
    ...                 [ 0.5, -0.375],
    ...                 [ 0.5,  0.375]])
    >>> opensfm_to_upright(sfm, 320, 240, 1)
    array([[   0.,    0.],
           [   0.,  240.],
           [ 320.,    0.],
           [ 320.,  240.]])
    """

    R_T = {
        1: np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        3: np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]),
        6: np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]),
        8: np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]),
    }

    w = float(width)
    h = float(height)
    s = max(w, h)

    H_inv = np.array([[s / w,     0, 0.5],
                      [    0, s / h, 0.5],
                      [    0,     0,   1]])

    T_inv = np.dot(R_T[orientation], H_inv)

    p = np.dot(coords, T_inv[:2, :2].T) + T_inv[:2, 2].reshape(1, 2)

    upright_width = width if orientation < 6 else height
    upright_height = height if orientation < 6 else width
    if new_width is not None:
        upright_width = new_width
    if new_height is not None:
        upright_height = new_height
    p[:, 0] = upright_width * p[:, 0]
    p[:, 1] = upright_height * p[:, 1]

    return p
<|endoftext|>


<commit_before>import hashlib
import json

class Issue:

    def __init__(self, contract, function, pc, title, _type="Informational", description="", debug=""):

        self.title = title
        self.contract = contract
        self.function = function
        self.pc = pc
        self.description = description
        self.type = _type
        self.debug = debug
        self.filename = None
        self.code = None
        self.lineno = None

    def as_dict(self):

        issue = {'title': self.title, 'description':self.description, 'function': self.function, 'type': self.type, 'address': self.pc, 'debug': self.debug}

        if self.filename and self.lineno:
            issue['filename'] = self.filename
            issue['lineno'] = self.lineno

        if self.code:
            issue['code'] = self.code

        return issue

class Report:

    def __init__(self):
        self.issues = {}
        pass

    def append_issue(self, issue):
        m = hashlib.md5()
        m.update((issue.contract + str(issue.pc) + issue.title).encode('utf-8'))
        self.issues[m.digest()] = issue

    def as_text(self):
        text = ""

        for key, issue in self.issues.items():

            text += "==== " + issue.title + " ====\n"
            text += "Type: " + issue.type + "\n"

            if len(issue.contract):
                text += "Contract: " + issue.contract + "\n"
            else:
                text += "Contract: Unknown\n"

            text += "Function name: " + issue.function + "\n"
            text += "PC address: " + str(issue.pc) + "\n"

            text += issue.description + "\n--------------------\n"

            if issue.filename and issue.lineno:
                 text += "In file: " + issue.filename + ":" + str(issue.lineno)

            if issue.code:
                text += "\n\n" + issue.code + "\n\n--------------------\n"

            text+="\n"

        return text

    def as_json(self):
        issues = []

        for key, issue in self.issues.items():

            issues.append(issue.as_dict())

        return json.dumps(issues)

    def as_markdown(self):
        text = "# Analysis Results\n"

        for key, issue in self.issues.items():

            text += "## " + issue.title + "\n"
            text += "- Type: " + issue.type + "\n"

            if len(issue.contract):
                text += "- Contract: " + issue.contract + "\n"
            else:
                text += "- Contract: Unknown\n"

            text += "- Function name: " + issue.function + "\n"
            text += "- PC address: " + str(issue.pc) + "\n\n"

            text += "### Description\n" + issue.description + "\n"

            if issue.filename and issue.lineno:
                text += "\nIn *%s:*\n%d" % (issue.filename, issue.lineno)

            if issue.code:
                text += "\n```\n" + issue.code + "\n```\n"

        return text
<commit_msg>Fix markdown formatting<commit_after>import hashlib
import json

class Issue:

    def __init__(self, contract, function, pc, title, _type="Informational", description="", debug=""):

        self.title = title
        self.contract = contract
        self.function = function
        self.pc = pc
        self.description = description
        self.type = _type
        self.debug = debug
        self.filename = None
        self.code = None
        self.lineno = None

    def as_dict(self):

        issue = {'title': self.title, 'description':self.description, 'function': self.function, 'type': self.type, 'address': self.pc, 'debug': self.debug}

        if self.filename and self.lineno:
            issue['filename'] = self.filename
            issue['lineno'] = self.lineno

        if self.code:
            issue['code'] = self.code

        return issue

class Report:

    def __init__(self):
        self.issues = {}
        pass

    def append_issue(self, issue):
        m = hashlib.md5()
        m.update((issue.contract + str(issue.pc) + issue.title).encode('utf-8'))
        self.issues[m.digest()] = issue

    def as_text(self):
        text = ""

        for key, issue in self.issues.items():

            text += "==== " + issue.title + " ====\n"
            text += "Type: " + issue.type + "\n"

            if len(issue.contract):
                text += "Contract: " + issue.contract + "\n"
            else:
                text += "Contract: Unknown\n"

            text += "Function name: " + issue.function + "\n"
            text += "PC address: " + str(issue.pc) + "\n"

            text += issue.description + "\n--------------------\n"

            if issue.filename and issue.lineno:
                 text += "In file: " + issue.filename + ":" + str(issue.lineno)

            if issue.code:
                text += "\n\n" + issue.code + "\n\n--------------------\n"

            text+="\n"

        return text

    def as_json(self):
        issues = []

        for key, issue in self.issues.items():

            issues.append(issue.as_dict())

        return json.dumps(issues)

    def as_markdown(self):
        text = "# Analysis Results\n"

        for key, issue in self.issues.items():

            text += "## " + issue.title + "\n"
            text += "- Type: " + issue.type + "\n"

            if len(issue.contract):
                text += "- Contract: " + issue.contract + "\n"
            else:
                text += "- Contract: Unknown\n"

            text += "- Function name: " + issue.function + "\n"
            text += "- PC address: " + str(issue.pc) + "\n\n"

            text += "### Description\n" + issue.description + "\n"

            if issue.filename and issue.lineno:
                text += "\nIn *%s:*%d\n" % (issue.filename, issue.lineno)

            if issue.code:
                text += "\n```\n" + issue.code + "\n```\n"

        return text
<|endoftext|>


<commit_before>import hmac
import hashlib
import os

from flask import Flask, request
from rauth import OAuth1Session

from githubx import Githubx, file_description
from models import Post

ENV_KEYS = ('GH_USER', 'GH_SECRET', 'CIO_KEY', 'CIO_SECRET')

app = Flask(__name__)
app.debug = True

def load_env_conf(keys=ENV_KEYS):
    """A hack, since I'm using Heroku env files + foreman."""
    global app
    for key in keys:
        app.config[key] = os.environ[key]

load_env_conf()

cio_requests = OAuth1Session(os.environ['CIO_KEY'], os.environ['CIO_SECRET'])

githubx = Githubx(
    app.config['GH_USER'],
    app.config['GH_SECRET'],
    user_agent='github.com/simon-weber/the-listserve-archive')

@app.route('/cio/webhook', methods=['POST'])
def receive_mail():
    """POSTed to by context.IO when a new post is received."""
    app.logger.debug("received new post")
    app.logger.debug(request.form)
    app.logger.debug(request.json())

    if not verify_webhook_post(request.json()):
        return "invalid"

    commit_post_data(request.json())

    return "ok"

@app.route('/cio/webhookfailure', methods=['GET', 'POST'])
def handle_webhook_failure():
    """Context.IO might get or post at this for different kinds of failure."""
    app.logger.error("context.IO reports WebHook failure!")
    app.logger.debug(request.args)
    app.logger.debug(request.json())

    return "ok"

def verify_webhook_post(request_json):
    """Return True if the request is from context.IO.

    http://context.io/docs/2.0/accounts/webhooks."""

    sig = hmac.new(app.config['CIO_SECRET'],
                   msg=str(request_json['timestamp']) + request_json['token'],
                   digestmod=hashlib.sha256).hexdigest()

    return sig == request_json['signature']

def commit_post_data(webhook_request_json, branch='gh-pages'):
    """Commit a .html file in _posts/, a .json in data/, and append to the
    cumulative .json in data/."""

    webhook = webhook_request_json  # convenience

    account_id = webhook['account_id']
    msg_id = webhook['message_data']['message_id']

    msg = cio_requests.get(
        "https://api.context.io/2.0/accounts/{aid}/messages/{mid}".format(
            aid=account_id,
            mid=msg_id),
        params={'include_body': 1,
                'body_type': 'text/plain'}
    )

    # context.io responded with a list once?
    app.logger.debug('message response:')
    app.logger.debug(msg.json())

    post = Post.from_cio_message(msg.json())

    path_content_pairs = files_to_create(post)

    githubx.commit(
        repo='the-listserve-archive',
        file_descriptions=[file_description(*pair) for pair in path_content_pairs],
        commit_message="add post (%s)" % post.datestr(),
        branch=branch)

def files_to_create(post):
    """Return a list of (filepath, contents) pairs.

    The first item in the list will be for _posts."""

    post_fname, jekyll_html = post.to_jekyll_html()

    date_path = os.path.join(*post.datestr().split('-'))

    json_fname = date_path + '.json'
    jekyll_json = post.to_jekyll_json()

    multipost_fname = date_path + '.html'
    jekyll_multipost = post.to_jekyll_multipost()

    return [
        (os.path.join('_posts', post_fname),
         jekyll_html),
        (json_fname,
         jekyll_json),
        (multipost_fname,
         jekyll_multipost),
    ]

if __name__ == '__main__':
    port = int(os.environ.get('PORT', 5000))
    app.run(host='0.0.0.0', port=port)
<commit_msg>fix request handling<commit_after>import hmac
import hashlib
import os

from flask import Flask, request
from rauth import OAuth1Session

from githubx import Githubx, file_description
from models import Post

ENV_KEYS = ('GH_USER', 'GH_SECRET', 'CIO_KEY', 'CIO_SECRET')

app = Flask(__name__)
app.debug = True

def load_env_conf(keys=ENV_KEYS):
    """A hack, since I'm using Heroku env files + foreman."""
    global app
    for key in keys:
        app.config[key] = os.environ[key]

load_env_conf()

cio_requests = OAuth1Session(os.environ['CIO_KEY'], os.environ['CIO_SECRET'])

githubx = Githubx(
    app.config['GH_USER'],
    app.config['GH_SECRET'],
    user_agent='github.com/simon-weber/the-listserve-archive')

@app.route('/cio/webhook', methods=['POST'])
def receive_mail():
    """POSTed to by context.IO when a new post is received."""
    app.logger.debug("received new post")
    app.logger.debug(request.form)
    app.logger.debug(request.json)

    if not verify_webhook_post(request.json):
        return "invalid"

    commit_post_data(request.json)

    return "ok"

@app.route('/cio/webhookfailure', methods=['GET', 'POST'])
def handle_webhook_failure():
    """Context.IO might get or post at this for different kinds of failure."""
    app.logger.error("context.IO reports WebHook failure!")
    app.logger.debug(request.args)
    app.logger.debug(request.json)

    return "ok"

def verify_webhook_post(request_json):
    """Return True if the request is from context.IO.

    http://context.io/docs/2.0/accounts/webhooks."""

    sig = hmac.new(app.config['CIO_SECRET'],
                   msg=str(request_json['timestamp']) + request_json['token'],
                   digestmod=hashlib.sha256).hexdigest()

    return sig == request_json['signature']

def commit_post_data(webhook_request_json, branch='gh-pages'):
    """Commit a .html file in _posts/, a .json in data/, and append to the
    cumulative .json in data/."""

    webhook = webhook_request_json  # convenience

    account_id = webhook['account_id']
    msg_id = webhook['message_data']['message_id']

    msg = cio_requests.get(
        "https://api.context.io/2.0/accounts/{aid}/messages/{mid}".format(
            aid=account_id,
            mid=msg_id),
        params={'include_body': 1,
                'body_type': 'text/plain'}
    )

    # context.io responded with a list once?
    app.logger.debug('message response:')
    app.logger.debug(msg.json())

    post = Post.from_cio_message(msg.json())

    path_content_pairs = files_to_create(post)

    githubx.commit(
        repo='the-listserve-archive',
        file_descriptions=[file_description(*pair) for pair in path_content_pairs],
        commit_message="add post (%s)" % post.datestr(),
        branch=branch)

def files_to_create(post):
    """Return a list of (filepath, contents) pairs.

    The first item in the list will be for _posts."""

    post_fname, jekyll_html = post.to_jekyll_html()

    date_path = os.path.join(*post.datestr().split('-'))

    json_fname = date_path + '.json'
    jekyll_json = post.to_jekyll_json()

    multipost_fname = date_path + '.html'
    jekyll_multipost = post.to_jekyll_multipost()

    return [
        (os.path.join('_posts', post_fname),
         jekyll_html),
        (json_fname,
         jekyll_json),
        (multipost_fname,
         jekyll_multipost),
    ]

if __name__ == '__main__':
    port = int(os.environ.get('PORT', 5000))
    app.run(host='0.0.0.0', port=port)
<|endoftext|>


<commit_before>""" Class for loading in modules to OpsDroid """

import logging
import sys
import os
import subprocess
import pip
import shutil
import yaml
import importlib
from opsdroid.const import DEFAULT_GIT_URL, MODULES_DIRECTORY, DEFAULT_MODULE_BRANCH

class Loader:
    def __init__(self, opsdroid):
        self.opsdroid = opsdroid
        logging.debug("Loaded loader")

    def load_config_file(self, config_path):
        """ Load a yaml config file from path """
        if not os.path.isfile(config_path):
            self.opsdroid.critical("Config file " + config_path + " not found", 1)

        try:
            with open(config_path, 'r') as stream:
                return(yaml.load(stream))
        except yaml.YAMLError as e:
            self.opsdroid.critical(e, 1)
        except FileNotFoundError as e:
            self.opsdroid.critical(e, 1)

    def load_config(self, config):
        """ Load all module types based on config """
        logging.debug("Loading modules from config")

        if 'databases' in config.keys():
            # TODO: Implement database modules
            self._load_modules('database', config['databases'])
        else:
            logging.warn("No databases in configuration")

        if 'skills' in config.keys():
            self._setup_modules(
                self._load_modules('skill', config['skills'])
            )
        else:
            self.opsdroid.critical("No skills in configuration, at least 1 required", 1)

        if 'connectors' in config.keys():
            self.opsdroid.start_connectors(self._load_modules('connector', config['connectors']))
        else:
            self.opsdroid.critical("No connectors in configuration, at least 1 required", 1)

    def _load_modules(self, modules_type, modules):
        """ Load modules """
        logging.debug("Loading " + modules_type + " modules")
        loaded_modules = []

        if not os.path.isdir(MODULES_DIRECTORY):
            os.makedirs(MODULES_DIRECTORY)

        for module_name in modules.keys():

            module_path = self._build_module_path(modules_type, module_name)
            install_path = MODULES_DIRECTORY + "/" + modules_type + "/" + module_name

            if "branch" not in modules[module_name]:
                modules[module_name]["branch"] = DEFAULT_MODULE_BRANCH

            # Remove module for reinstall if no-cache set
            if "no-cache" in modules[module_name] \
                        and modules[module_name]["no-cache"] == True \
                        and os.path.isdir(install_path):
                logging.debug("'no-cache' set, removing module " + module_path)
                shutil.rmtree(install_path)

            # Install module
            self._install_module(module_name, modules_type, modules[module_name], install_path)

            # Import module
            try:
                logging.debug("Module path: " + module_path)
                module = importlib.import_module(module_path + "." + module_name)
                logging.debug("Loading " + modules_type + ": " + module_name)
                loaded_modules.append({"module": module, "config": modules[module_name]})
            except ImportError as e:
                logging.error("Failed to load " + modules_type + " " + module_name)
                logging.error(e)

        return loaded_modules

    def _setup_modules(self, modules):
        """ Call the setup function on the passed in modules """
        for module in modules:
            module["module"].setup(self.opsdroid)

    def _install_module(self, module_name, module_type, module_config, install_path):
        """ Install a module """
        logging.debug("Installing " + module_name)

        if os.path.isdir(install_path):
            # TODO Allow for updating or reinstalling of modules
            logging.debug("Module " + module_name + " already installed, skipping")
        else:
            if module_config != None and "repo" in module_config:
                git_url = module_config["repo"]
            else:
                git_url = DEFAULT_GIT_URL + module_type + "-" + module_name + ".git"

            if any(x in git_url for x in ["http", "https", "ssh"]):
                # TODO Test if url or ssh path exists
                # TODO Handle github authentication
                self._git_clone(git_url, install_path, module_config["branch"])
            else:
                if os.path.isdir(git_url):
                    self._git_clone(git_url, install_path, module_config["branch"])
                else:
                    logging.debug("Could not find local git repo " + git_url)

            if os.path.isdir(install_path):
                logging.debug("Installed " + module_name + " to " + install_path)
            else:
                logging.debug("Install of " + module_name + " failed ")

            # Install module dependancies
            if os.path.isfile(install_path + "/requirements.txt"):
                pip.main(["install", "-r", install_path + "/requirements.txt"])

    def _build_module_path(self, mod_type, mod_name):
        """ Generate the module path from name and type """
        return MODULES_DIRECTORY + "." + mod_type + "." + mod_name

    def _git_clone(self, git_url, install_path, branch):
        """ Clone a git repo to a location and wait for finish """
        p = subprocess.Popen(["git", "clone", "-b", branch, git_url, install_path], shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.wait()
<commit_msg>Set module config to empty dict if none provided<commit_after>""" Class for loading in modules to OpsDroid """

import logging
import sys
import os
import subprocess
import pip
import shutil
import yaml
import importlib
from opsdroid.const import DEFAULT_GIT_URL, MODULES_DIRECTORY, DEFAULT_MODULE_BRANCH

class Loader:
    def __init__(self, opsdroid):
        self.opsdroid = opsdroid
        logging.debug("Loaded loader")

    def load_config_file(self, config_path):
        """ Load a yaml config file from path """
        if not os.path.isfile(config_path):
            self.opsdroid.critical("Config file " + config_path + " not found", 1)

        try:
            with open(config_path, 'r') as stream:
                return(yaml.load(stream))
        except yaml.YAMLError as e:
            self.opsdroid.critical(e, 1)
        except FileNotFoundError as e:
            self.opsdroid.critical(e, 1)

    def load_config(self, config):
        """ Load all module types based on config """
        logging.debug("Loading modules from config")

        if 'databases' in config.keys():
            # TODO: Implement database modules
            self._load_modules('database', config['databases'])
        else:
            logging.warn("No databases in configuration")

        if 'skills' in config.keys():
            self._setup_modules(
                self._load_modules('skill', config['skills'])
            )
        else:
            self.opsdroid.critical("No skills in configuration, at least 1 required", 1)

        if 'connectors' in config.keys():
            self.opsdroid.start_connectors(self._load_modules('connector', config['connectors']))
        else:
            self.opsdroid.critical("No connectors in configuration, at least 1 required", 1)

    def _load_modules(self, modules_type, modules):
        """ Load modules """
        logging.debug("Loading " + modules_type + " modules")
        loaded_modules = []

        if not os.path.isdir(MODULES_DIRECTORY):
            os.makedirs(MODULES_DIRECTORY)

        for module_name in modules.keys():

            if modules[module_name] == None:
                modules[module_name] = {}

            module_path = self._build_module_path(modules_type, module_name)
            install_path = MODULES_DIRECTORY + "/" + modules_type + "/" + module_name

            if "branch" not in modules[module_name]:
                modules[module_name]["branch"] = DEFAULT_MODULE_BRANCH

            # Remove module for reinstall if no-cache set
            if "no-cache" in modules[module_name] \
                        and modules[module_name]["no-cache"] == True \
                        and os.path.isdir(install_path):
                logging.debug("'no-cache' set, removing module " + module_path)
                shutil.rmtree(install_path)

            # Install module
            self._install_module(module_name, modules_type, modules[module_name], install_path)

            # Import module
            try:
                logging.debug("Module path: " + module_path)
                module = importlib.import_module(module_path + "." + module_name)
                logging.debug("Loading " + modules_type + ": " + module_name)
                loaded_modules.append({"module": module, "config": modules[module_name]})
            except ImportError as e:
                logging.error("Failed to load " + modules_type + " " + module_name)
                logging.error(e)

        return loaded_modules

    def _setup_modules(self, modules):
        """ Call the setup function on the passed in modules """
        for module in modules:
            module["module"].setup(self.opsdroid)

    def _install_module(self, module_name, module_type, module_config, install_path):
        """ Install a module """
        logging.debug("Installing " + module_name)

        if os.path.isdir(install_path):
            # TODO Allow for updating or reinstalling of modules
            logging.debug("Module " + module_name + " already installed, skipping")
        else:
            if module_config != None and "repo" in module_config:
                git_url = module_config["repo"]
            else:
                git_url = DEFAULT_GIT_URL + module_type + "-" + module_name + ".git"

            if any(x in git_url for x in ["http", "https", "ssh"]):
                # TODO Test if url or ssh path exists
                # TODO Handle github authentication
                self._git_clone(git_url, install_path, module_config["branch"])
            else:
                if os.path.isdir(git_url):
                    self._git_clone(git_url, install_path, module_config["branch"])
                else:
                    logging.debug("Could not find local git repo " + git_url)

            if os.path.isdir(install_path):
                logging.debug("Installed " + module_name + " to " + install_path)
            else:
                logging.debug("Install of " + module_name + " failed ")

            # Install module dependancies
            if os.path.isfile(install_path + "/requirements.txt"):
                pip.main(["install", "-r", install_path + "/requirements.txt"])

    def _build_module_path(self, mod_type, mod_name):
        """ Generate the module path from name and type """
        return MODULES_DIRECTORY + "." + mod_type + "." + mod_name

    def _git_clone(self, git_url, install_path, branch):
        """ Clone a git repo to a location and wait for finish """
        p = subprocess.Popen(["git", "clone", "-b", branch, git_url, install_path], shell=False,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p.wait()
<|endoftext|>


<commit_before>
from itertools import (groupby, starmap)
from nac.basisSet.basisNormalization import compute_normalization_sphericals
from nac.common import (
    Matrix, Tensor3D, Vector, retrieve_hdf5_data, search_data_in_hdf5,
    store_arrays_in_hdf5, triang2mtx)
from nac.integrals.overlapIntegral import calcMtxOverlapP
from nac.integrals.nonAdiabaticCoupling import calculate_spherical_overlap
from nac.integrals.spherical_Cartesian_cgf import calc_transf_matrix
from os.path import join
from scipy import sparse
from typing import (Dict, List, Tuple)
import hashlib
import h5py
import logging
import numpy as np

# Get logger
logger = logging.getLogger(__name__)

def photo_excitation_rate(
        tensor_overlaps: Tensor3D, time_dependent_coeffs: Matrix,
        map_index_pyxaid_hdf5: Matrix, dt_au: float) -> Tuple:
    """
    Calculate the Electron transfer rate, using both adiabatic and nonadiabatic
    components, using equation number 8 from:
    J. AM. CHEM. SOC. 2005, 127, 7941-7951. Ab Initio Nonadiabatic Molecular
    Dynamics of the Ultrafast Electron Injection acrossthe Alizarin-TiO2
    Interface.
    The derivatives are calculated numerically using 3 points.

    :param tensor_overlaps: Overlap matrices at time t - dt, t and t + dt.
    :param time_dependent_coeffs: Time-dependentent coefficients
    at time t - dt, t and t + dt.
    :param map_index_pyxaid_hdf5: Index relation between the Excitations
    in PYXAID and the corresponding molecular orbitals store in the HDF5.
    :param dt_au: Delta time integration used in the dynamics.
    :returns: tuple containing both nonadiabatic and adiabatic components
    """
    # Rearrange the overlap matrix in the PYXAID order
    matrix_overlap_pyxaid_order = retrieve_overlap_elements(
        map_index_pyxaid_hdf5, tensor_overlaps[1])

    # Density of the fragment
    density = np.dot(matrix_overlap_pyxaid_order ** 2,
                     time_dependent_coeffs[1])

    # NonAdiabatic component
    coeff_derivatives = np.apply_along_axis(
        lambda v: (v[0] - v[2]) / (2 * dt_au), 0, time_dependent_coeffs)

    nonadiabatic = np.dot(coeff_derivatives,
                          matrix_overlap_pyxaid_order ** 2)

    # Adiabatic component
    overlap_derv = np.apply_along_axis(
        lambda v: (v[0] - v[2]) / (2 * dt_au), 0, tensor_overlaps)

    overlap_derv_pyxaid_order = retrieve_overlap_elements(
        map_index_pyxaid_hdf5, overlap_derv)

    adiabatic = np.dot(time_dependent_coeffs[1],
                       overlap_derv_pyxaid_order ** 2)

    return density, nonadiabatic, adiabatic

def retrieve_overlap_elements(
        map_index_pyxaid_hdf5: Matrix, arr: Matrix) -> Matrix:
    """
    Read the overlap elements in the PYXAID STATES order
    """
    # Resulring matrix
    overlaps = np.empty(map_index_pyxaid_hdf5.shape[0])

    # indices of the i -> j transitions used by PYXAID
    for k, (i, j) in enumerate(np.rollaxis(map_index_pyxaid_hdf5, axis=0)):
        s_ii = arr[i, i]
        s_jj = arr[j, j]
        s_ij = arr[i, j]
        overlaps[k] = s_ii * s_jj - (s_ij ** 2)

    return overlaps

def compute_overlaps_ET(
        project_name: str, molecules: List, basis_name: str,
        path_hdf5: str, dictCGFs: Dict, mo_paths_hdf5: List,
        fragment_indices: List, enumerate_from: int,
        package_name: int) -> List:
    """
    Given a list of molecular fragments compute the Overlaps in the
    molecular orbital basis for the different fragments and for all the
    molecular frames.
    """
    # Preprocess the indices of the atomic fragments
    fragment_indices = sanitize_fragment_indices(fragment_indices)

    # Matrix containing the lower and upper range for the CGFs of each atom
    # in order.
    indices_range_CGFs_spherical = create_indices_range_CGFs_spherical(
        molecules[0], dictCGFs)

    # Norms of the spherical CGfs
    dict_global_norms = compute_normalization_sphericals(dictCGFs)

    fragment_overlaps = []
    for k, vector_indices in enumerate(fragment_indices):
        logger.info("Computing Overlaps for molecular fragment: {}".format(k))
        # Extract atoms belonging to the fragment
        frames_fragment_atoms = [[mol[i] for i in vector_indices]
                                 for mol in molecules]
        # compute the transformation matrix from cartesian to spherical
        sparse_trans_mtx = compute_fragment_trans_mtx(
            path_hdf5, basis_name, frames_fragment_atoms[0], dict_global_norms,
            package_name)

        # create a Hash for the fragment
        fragment_hash = hashlib.md5(vector_indices.tostring()).hexdigest()
        logger.info("The overlaps for the molecular fragment number {} are going \
        to be stored in the hdf5 using the following hash: {}".format(k, fragment_hash))

        # Compute the indices of the MOs corresponding to the atoms in the
        # fragments
        indices_fragment_mos = compute_indices_fragments_mos(
            vector_indices, indices_range_CGFs_spherical)

        # Overlap matrix for the given fragment
        overlaps = compute_frames_fragment_overlap(
            project_name, frames_fragment_atoms, path_hdf5, mo_paths_hdf5,
            indices_fragment_mos, dictCGFs, sparse_trans_mtx, enumerate_from,
            fragment_hash)
        fragment_overlaps.append(overlaps)

    return fragment_overlaps

def compute_frames_fragment_overlap(
        project_name: str, frames_fragment_atoms: List, path_hdf5: str,
        mo_paths_hdf5: List, indices_fragment_mos: Vector, dictCGFs: Dict,
        sparse_trans_mtx: Matrix, enumerate_from: int,
        fragment_hash: int) -> List:
    """
    Compute all the overlap matrices for the frames of a molecular dynamics
    for a given fragment.
    """
    # Path to store the fragment overlap matrices in the HDF5
    range_points = range(
        enumerate_from, enumerate_from + len(frames_fragment_atoms))
    paths = [join(project_name, 'point_{}'.format(i),
                  'fragment_overlap_hash_{}'.format(fragment_hash))
             for i in range_points]

    return [compute_fragment_overlap(
        path_hdf5, p, fragment, path_mos, indices_fragment_mos, dictCGFs,
        sparse_trans_mtx)
        for p, fragment, path_mos in
        zip(paths, frames_fragment_atoms, mo_paths_hdf5)]

def compute_fragment_overlap(
        path_hdf5: str, path_overlap: str, fragment_atoms: List, path_mos: str,
        indices_fragment_mos: Vector, dictCGFs: Dict,
        trans_mtx: Matrix) -> str:
    """
    Compute the overlap matrix only for those atoms included in the fragment
    """
    if not search_data_in_hdf5(path_hdf5, path_overlap):
        # Compute the overlap in the atomic basis
        dim_cart = trans_mtx.shape[1]
        overlap_AO = triang2mtx(
            calcMtxOverlapP(fragment_atoms, dictCGFs), dim_cart)
        # Read all the molecular orbital coefficients
        coefficients = retrieve_hdf5_data(path_hdf5, path_mos[1])
        # Number of Orbitals stored in the HDF5
        dim_y = coefficients.shape[1]
        # Extract the coefficients belonging to the fragment
        dim_x = indices_fragment_mos.size

        # Extract the MO Coefficients belonging to the fragment
        x_range = np.repeat(indices_fragment_mos, dim_y)
        y_range = np.tile(np.arange(dim_y), dim_x)
        fragment_coefficients = coefficients[x_range, y_range].reshape(dim_x, dim_y)

        # Compute the overlap in spherical coordinates
        overlap_spherical = calculate_spherical_overlap(
            trans_mtx, overlap_AO, fragment_coefficients, fragment_coefficients)
        store_arrays_in_hdf5(path_hdf5, path_overlap, overlap_spherical)
    else:
        logger.info("overlap:{} already on HDF5".format(path_overlap))

    return path_overlap

def create_indices_range_CGFs_spherical(
        molecule: List, dictCGFs: Dict) -> Matrix:
    """
    Creates a matrix containing the lower and upper(exclusive) indices
    of the CGFs for each atoms in order.
    """
    # Compute how many CGFs are in Spherical coordinates

    lens_CGFs_spherical = compute_lens_CGFs_sphericals(molecule, dictCGFs)
    ranges = np.empty((len(molecule), 2), dtype=np.int32)

    lower = 0
    for i, at in enumerate(molecule):
        upper = lower + lens_CGFs_spherical[at.symbol]
        ranges[i] = lower, upper
        lower = upper

    return ranges

def compute_lens_CGFs_sphericals(molecule: List, dictCGFs: Dict) -> Dict:
    """
    Calculate how many CGFs in sphericals are there per atom
    """
    # Number of CGFs per angular momenta in Cartesian coordinates
    CGFs_cartesians = {'S': 1, 'P': 3, 'D': 6, 'F': 10}

    # Number of CGFs per angular momenta in spherical coordinates
    CGFs_sphericals = {'S': 1, 'P': 3, 'D': 5, 'F': 7}

    # Unique labels
    labels = set(at.symbol for at in molecule)

    # Compute number of spherical CGFs per atoms
    lens_CGFs_spherical = {
        l: int(
            sum(len(list(vals)) * CGFs_sphericals[g] / CGFs_cartesians[g]
                for g, vals in groupby(dictCGFs[l], lambda cgf: cgf.orbType[0])))
        for l in labels}

    return lens_CGFs_spherical

def compute_fragment_trans_mtx(
        path_hdf5: str, basis_name: str, fragment_atoms: List,
        dict_global_norms: Dict, package_name: str) -> Matrix:
    """
    Calculate the cartesian to spherical transformation matrix for
    the given fragment
    """
    with h5py.File(path_hdf5, 'r') as f5:
        trans_mtx = calc_transf_matrix(
            f5, fragment_atoms, dict_global_norms, basis_name, package_name)

    return sparse.csr_matrix(trans_mtx)

def compute_indices_fragments_mos(
        vector_indices: Vector, indices_range_CGFs: Matrix) -> Vector:
    """
    Compute the indices of the CGFs of the fragments to extract from
    the coefficients stored in the HDF5.
    """
    # extract the ranges of CGFs for the fragments
    fragment_ranges = indices_range_CGFs[vector_indices]

    return np.concatenate(list(starmap(np.arange, fragment_ranges)))

def sanitize_fragment_indices(fragment_indices: List) -> Matrix:
    """
    transform to numpy array and start the indices from 0.

    :param fragment_indices: indices of the atoms belonging to the different
    molecular fragments.
    """
    # Convert the indices to a numpy array
    if isinstance(fragment_indices[0], list):
        fragment_indices = [np.array(xs, dtype=np.int) for xs in fragment_indices]
    else:
        fragment_indices = [np.array(fragment_indices, dtype=np.int)]

    # Shift the index 1 position to start from 0
    fragment_indices = [xs - 1 for xs in fragment_indices]

    return fragment_indices
<commit_msg>fixed issued callling  the transf_mtx in the ET comput<commit_after>
from itertools import (groupby, starmap)
from nac.basisSet.basisNormalization import compute_normalization_sphericals
from nac.common import (
    Matrix, Tensor3D, Vector, retrieve_hdf5_data, search_data_in_hdf5,
    store_arrays_in_hdf5, triang2mtx)
from nac.integrals.overlapIntegral import calcMtxOverlapP
from nac.integrals.nonAdiabaticCoupling import calculate_spherical_overlap
from nac.integrals.spherical_Cartesian_cgf import calc_transf_matrix
from os.path import join
from scipy import sparse
from typing import (Dict, List, Tuple)
import hashlib
import h5py
import logging
import numpy as np

# Get logger
logger = logging.getLogger(__name__)

def photo_excitation_rate(
        tensor_overlaps: Tensor3D, time_dependent_coeffs: Matrix,
        map_index_pyxaid_hdf5: Matrix, dt_au: float) -> Tuple:
    """
    Calculate the Electron transfer rate, using both adiabatic and nonadiabatic
    components, using equation number 8 from:
    J. AM. CHEM. SOC. 2005, 127, 7941-7951. Ab Initio Nonadiabatic Molecular
    Dynamics of the Ultrafast Electron Injection acrossthe Alizarin-TiO2
    Interface.
    The derivatives are calculated numerically using 3 points.

    :param tensor_overlaps: Overlap matrices at time t - dt, t and t + dt.
    :param time_dependent_coeffs: Time-dependentent coefficients
    at time t - dt, t and t + dt.
    :param map_index_pyxaid_hdf5: Index relation between the Excitations
    in PYXAID and the corresponding molecular orbitals store in the HDF5.
    :param dt_au: Delta time integration used in the dynamics.
    :returns: tuple containing both nonadiabatic and adiabatic components
    """
    # Rearrange the overlap matrix in the PYXAID order
    matrix_overlap_pyxaid_order = retrieve_overlap_elements(
        map_index_pyxaid_hdf5, tensor_overlaps[1])

    # Density of the fragment
    density = np.dot(matrix_overlap_pyxaid_order ** 2,
                     time_dependent_coeffs[1])

    # NonAdiabatic component
    coeff_derivatives = np.apply_along_axis(
        lambda v: (v[0] - v[2]) / (2 * dt_au), 0, time_dependent_coeffs)

    nonadiabatic = np.dot(coeff_derivatives,
                          matrix_overlap_pyxaid_order ** 2)

    # Adiabatic component
    overlap_derv = np.apply_along_axis(
        lambda v: (v[0] - v[2]) / (2 * dt_au), 0, tensor_overlaps)

    overlap_derv_pyxaid_order = retrieve_overlap_elements(
        map_index_pyxaid_hdf5, overlap_derv)

    adiabatic = np.dot(time_dependent_coeffs[1],
                       overlap_derv_pyxaid_order ** 2)

    return density, nonadiabatic, adiabatic

def retrieve_overlap_elements(
        map_index_pyxaid_hdf5: Matrix, arr: Matrix) -> Matrix:
    """
    Read the overlap elements in the PYXAID STATES order
    """
    # Resulring matrix
    overlaps = np.empty(map_index_pyxaid_hdf5.shape[0])

    # indices of the i -> j transitions used by PYXAID
    for k, (i, j) in enumerate(np.rollaxis(map_index_pyxaid_hdf5, axis=0)):
        s_ii = arr[i, i]
        s_jj = arr[j, j]
        s_ij = arr[i, j]
        overlaps[k] = s_ii * s_jj - (s_ij ** 2)

    return overlaps

def compute_overlaps_ET(
        project_name: str, molecules: List, basis_name: str,
        path_hdf5: str, dictCGFs: Dict, mo_paths_hdf5: List,
        fragment_indices: List, enumerate_from: int,
        package_name: int) -> List:
    """
    Given a list of molecular fragments compute the Overlaps in the
    molecular orbital basis for the different fragments and for all the
    molecular frames.
    """
    # Preprocess the indices of the atomic fragments
    fragment_indices = sanitize_fragment_indices(fragment_indices)

    # Matrix containing the lower and upper range for the CGFs of each atom
    # in order.
    indices_range_CGFs_spherical = create_indices_range_CGFs_spherical(
        molecules[0], dictCGFs)

    # Norms of the spherical CGfs
    dict_global_norms = compute_normalization_sphericals(dictCGFs)

    fragment_overlaps = []
    for k, vector_indices in enumerate(fragment_indices):
        logger.info("Computing Overlaps for molecular fragment: {}".format(k))
        # Extract atoms belonging to the fragment
        frames_fragment_atoms = [[mol[i] for i in vector_indices]
                                 for mol in molecules]
        # compute the transformation matrix from cartesian to spherical
        sparse_trans_mtx = compute_fragment_trans_mtx(
            path_hdf5, basis_name, frames_fragment_atoms[0], dict_global_norms,
            package_name)

        # create a Hash for the fragment
        fragment_hash = hashlib.md5(vector_indices.tostring()).hexdigest()
        logger.info("The overlaps for the molecular fragment number {} are going \
        to be stored in the hdf5 using the following hash: {}".format(k, fragment_hash))

        # Compute the indices of the MOs corresponding to the atoms in the
        # fragments
        indices_fragment_mos = compute_indices_fragments_mos(
            vector_indices, indices_range_CGFs_spherical)

        # Overlap matrix for the given fragment
        overlaps = compute_frames_fragment_overlap(
            project_name, frames_fragment_atoms, path_hdf5, mo_paths_hdf5,
            indices_fragment_mos, dictCGFs, sparse_trans_mtx, enumerate_from,
            fragment_hash)
        fragment_overlaps.append(overlaps)

    return fragment_overlaps

def compute_frames_fragment_overlap(
        project_name: str, frames_fragment_atoms: List, path_hdf5: str,
        mo_paths_hdf5: List, indices_fragment_mos: Vector, dictCGFs: Dict,
        sparse_trans_mtx: Matrix, enumerate_from: int,
        fragment_hash: int) -> List:
    """
    Compute all the overlap matrices for the frames of a molecular dynamics
    for a given fragment.
    """
    # Path to store the fragment overlap matrices in the HDF5
    range_points = range(
        enumerate_from, enumerate_from + len(frames_fragment_atoms))
    paths = [join(project_name, 'point_{}'.format(i),
                  'fragment_overlap_hash_{}'.format(fragment_hash))
             for i in range_points]

    return [compute_fragment_overlap(
        path_hdf5, p, fragment, path_mos, indices_fragment_mos, dictCGFs,
        sparse_trans_mtx)
        for p, fragment, path_mos in
        zip(paths, frames_fragment_atoms, mo_paths_hdf5)]

def compute_fragment_overlap(
        path_hdf5: str, path_overlap: str, fragment_atoms: List, path_mos: str,
        indices_fragment_mos: Vector, dictCGFs: Dict,
        trans_mtx: Matrix) -> str:
    """
    Compute the overlap matrix only for those atoms included in the fragment
    """
    if not search_data_in_hdf5(path_hdf5, path_overlap):
        # Compute the overlap in the atomic basis
        dim_cart = trans_mtx.shape[1]
        overlap_AO = triang2mtx(
            calcMtxOverlapP(fragment_atoms, dictCGFs), dim_cart)
        # Read all the molecular orbital coefficients
        coefficients = retrieve_hdf5_data(path_hdf5, path_mos[1])
        # Number of Orbitals stored in the HDF5
        dim_y = coefficients.shape[1]
        # Extract the coefficients belonging to the fragment
        dim_x = indices_fragment_mos.size

        # Extract the MO Coefficients belonging to the fragment
        x_range = np.repeat(indices_fragment_mos, dim_y)
        y_range = np.tile(np.arange(dim_y), dim_x)
        fragment_coefficients = coefficients[x_range, y_range].reshape(dim_x, dim_y)

        # Compute the overlap in spherical coordinates
        overlap_spherical = calculate_spherical_overlap(
            trans_mtx, overlap_AO, fragment_coefficients, fragment_coefficients)
        store_arrays_in_hdf5(path_hdf5, path_overlap, overlap_spherical)
    else:
        logger.info("overlap:{} already on HDF5".format(path_overlap))

    return path_overlap

def create_indices_range_CGFs_spherical(
        molecule: List, dictCGFs: Dict) -> Matrix:
    """
    Creates a matrix containing the lower and upper(exclusive) indices
    of the CGFs for each atoms in order.
    """
    # Compute how many CGFs are in Spherical coordinates

    lens_CGFs_spherical = compute_lens_CGFs_sphericals(molecule, dictCGFs)
    ranges = np.empty((len(molecule), 2), dtype=np.int32)

    lower = 0
    for i, at in enumerate(molecule):
        upper = lower + lens_CGFs_spherical[at.symbol]
        ranges[i] = lower, upper
        lower = upper

    return ranges

def compute_lens_CGFs_sphericals(molecule: List, dictCGFs: Dict) -> Dict:
    """
    Calculate how many CGFs in sphericals are there per atom
    """
    # Number of CGFs per angular momenta in Cartesian coordinates
    CGFs_cartesians = {'S': 1, 'P': 3, 'D': 6, 'F': 10}

    # Number of CGFs per angular momenta in spherical coordinates
    CGFs_sphericals = {'S': 1, 'P': 3, 'D': 5, 'F': 7}

    # Unique labels
    labels = set(at.symbol for at in molecule)

    # Compute number of spherical CGFs per atoms
    lens_CGFs_spherical = {
        l: int(
            sum(len(list(vals)) * CGFs_sphericals[g] / CGFs_cartesians[g]
                for g, vals in groupby(dictCGFs[l], lambda cgf: cgf.orbType[0])))
        for l in labels}

    return lens_CGFs_spherical

def compute_fragment_trans_mtx(
        path_hdf5: str, basis_name: str, fragment_atoms: List,
        dict_global_norms: Dict, package_name: str) -> Matrix:
    """
    Calculate the cartesian to spherical transformation matrix for
    the given fragment
    """
    with h5py.File(path_hdf5, 'r') as f5:
        trans_mtx = calc_transf_matrix(
            f5, fragment_atoms, basis_name, dict_global_norms, package_name)

    return sparse.csr_matrix(trans_mtx)

def compute_indices_fragments_mos(
        vector_indices: Vector, indices_range_CGFs: Matrix) -> Vector:
    """
    Compute the indices of the CGFs of the fragments to extract from
    the coefficients stored in the HDF5.
    """
    # extract the ranges of CGFs for the fragments
    fragment_ranges = indices_range_CGFs[vector_indices]

    return np.concatenate(list(starmap(np.arange, fragment_ranges)))

def sanitize_fragment_indices(fragment_indices: List) -> Matrix:
    """
    transform to numpy array and start the indices from 0.

    :param fragment_indices: indices of the atoms belonging to the different
    molecular fragments.
    """
    # Convert the indices to a numpy array
    if isinstance(fragment_indices[0], list):
        fragment_indices = [np.array(xs, dtype=np.int) for xs in fragment_indices]
    else:
        fragment_indices = [np.array(fragment_indices, dtype=np.int)]

    # Shift the index 1 position to start from 0
    fragment_indices = [xs - 1 for xs in fragment_indices]

    return fragment_indices
<|endoftext|>