<commit_before>import fcntl
import os
import pickle
import stat
import sys

from epucontrol.api.exceptions import *
import epucontrol.main.ec_args as ec_args

class Persistence:
    def __init__(self, params, common):
        self.p = params
        self.c = common
        self.pdir = None
        self.lockfilepath = None

    def validate(self):
        pdir = self.p.get_conf_or_none("persistence", "persistencedir")
        if not pdir:
            raise InvalidConfig("There is no persistence->persistencedir configuration")

        if not os.path.isabs(pdir):
            pdir = self.c.resolve_var_dir(pdir)

        if not os.path.exists(pdir):
            try:
                os.mkdir(pdir)
                self.c.log.debug("created persistence directory: %s" % pdir)
            except:
                exception_type = sys.exc_type
                try:
                    exceptname = exception_type.__name__
                except AttributeError:
                    exceptname = exception_type
                errstr = "problem creating persistence dir '%s': %s: %s" % (pdir, str(exceptname), str(sys.exc_value))

                fullerrstr = "persistence directory does not have valid permissions and cannot be made to have valid permissions: '%s'" % pdir
                fullerrstr += errstr
                self.c.log.error(fullerrstr)
                raise IncompatibleEnvironment(fullerrstr)

        if os.access(pdir, os.W_OK | os.X_OK | os.R_OK):
            self.c.log.debug("persistence directory is rwx-able: %s" % pdir)
        else:
            try:
                os.chmod(pdir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
            except:
                exception_type = sys.exc_type
                try:
                    exceptname = exception_type.__name__
                except AttributeError:
                    exceptname = exception_type
                errstr = " - problem changing persistence directory permissions '%s': %s: %s" % (pdir, str(exceptname), str(sys.exc_value))

                fullerrstr = "persistence directory does not have valid permissions and cannot be made to have valid permissions: '%s'" % pdir
                fullerrstr += errstr
                raise IncompatibleEnvironment(fullerrstr)

            self.c.log.debug("persistence directory was made to be rwx-able: %s" % pdir)

        lockfilepath = os.path.join(pdir, "persistence.lock")
        if not os.path.exists(lockfilepath):
            open(lockfilepath, 'w').close()

        self.lockfilepath = lockfilepath
        self.pdir = pdir

    def new_vm(self, run_name, vm):
        """Adds VM to a run_vms list if it exists for "run_name".  If list
        does not exist, it will be created."""
        return self.run_with_flock(self._new_vm, run_name, vm)

    def new_vm_maybe(self, run_name, vm):
        """Adds VM to a run_vms list if it exists for "run_name".  If list
        does not exist, it will be created.  If VM instance ID is present,
        it won't be added."""
        return self.run_with_flock(self._new_vm_maybe, run_name, vm)

    def _new_vm(self, run_name, vm):
        """Run this under a lock so that the list is not messed up"""
        run_vms = self.get_run_vms_or_none(run_name)
        if not run_vms:
            run_vms = []
        run_vms.append(vm)
        self.store_run_vms(run_name, run_vms)

    def _new_vm_maybe(self, run_name, vm):
        """Run this under a lock so that the list is not messed up"""
        run_vms = self.get_run_vms_or_none(run_name)
        if not run_vms:
            run_vms = []

        found = False
        for avm in run_vms:
            if avm.instanceid == vm.instanceid:
                # late binding hostnames are possible
                if not avm.hostname and vm.hostname:
                    avm.hostname = vm.hostname
                    strparams = (avm.instanceid, avm.hostname)
                    self.c.log.debug("found hostname for '%s': %s" % strparams)
                found = True

        if found:
            self.store_run_vms(run_name, run_vms)
            return False # not new
        else:
            run_vms.append(vm)
            self.store_run_vms(run_name, run_vms)
            return True

    def run_with_flock(self, f, *args, **kw):
        """Run function with persistence's filesystem-based lock.

        Would be nice to use a decorator but want to find/create lockfile
        during class instantiation, need to figure that out later.

        Anyhow, we will likely switch to SQLite at some point.
        """

        lockfile = None
        try:
            lockfile = open(self.lockfilepath, "r")
            fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX)
            return f(*args, **kw)
        finally:
            if lockfile:
                lockfile.close()

    def store_run_vms(self, run_name, run_vms):
        if not self.pdir:
            raise ProgrammingError("cannot persist anything without setup/validation")

        if not run_vms:
            raise ProgrammingError("no run_vms")

        if not run_name:
            raise ProgrammingError("no run_name")

        pobject = self._derive_runset_filepath(run_name)

        f = None
        try:
            f = open(pobject, 'w')
            pickle.dump(run_vms, f)
        finally:
            if f:
                f.close()

    def get_run_vms_or_none(self, run_name):
        """Get list of VMs for a run name or return None.

        If you mutate this list, you should lock first (we will switch
        to SQLite at some point)."""

        if not self.pdir:
            raise ProgrammingError("cannot persist anything without setup/validation")

        pobject = self._derive_runset_filepath(run_name)
        if not os.path.exists(pobject):
            return None

        f = None
        try:
            f = open(pobject, 'r')
            x = pickle.load(f)
            return x
        finally:
            if f:
                f.close()

    def _derive_runset_filepath(self, run_name):
        if not run_name:
            raise ProgrammingError("no run_name")
        return os.path.join(self.pdir, run_name + "-run_vms_list")
        <commit_msg>was pickling too aggressively which is very noticeable for high number of VM instances<commit_after>import fcntl
import os
import pickle
import stat
import sys

from epucontrol.api.exceptions import *
import epucontrol.main.ec_args as ec_args

class Persistence:
    def __init__(self, params, common):
        self.p = params
        self.c = common
        self.pdir = None
        self.lockfilepath = None

    def validate(self):
        pdir = self.p.get_conf_or_none("persistence", "persistencedir")
        if not pdir:
            raise InvalidConfig("There is no persistence->persistencedir configuration")

        if not os.path.isabs(pdir):
            pdir = self.c.resolve_var_dir(pdir)

        if not os.path.exists(pdir):
            try:
                os.mkdir(pdir)
                self.c.log.debug("created persistence directory: %s" % pdir)
            except:
                exception_type = sys.exc_type
                try:
                    exceptname = exception_type.__name__
                except AttributeError:
                    exceptname = exception_type
                errstr = "problem creating persistence dir '%s': %s: %s" % (pdir, str(exceptname), str(sys.exc_value))

                fullerrstr = "persistence directory does not have valid permissions and cannot be made to have valid permissions: '%s'" % pdir
                fullerrstr += errstr
                self.c.log.error(fullerrstr)
                raise IncompatibleEnvironment(fullerrstr)

        if os.access(pdir, os.W_OK | os.X_OK | os.R_OK):
            self.c.log.debug("persistence directory is rwx-able: %s" % pdir)
        else:
            try:
                os.chmod(pdir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
            except:
                exception_type = sys.exc_type
                try:
                    exceptname = exception_type.__name__
                except AttributeError:
                    exceptname = exception_type
                errstr = " - problem changing persistence directory permissions '%s': %s: %s" % (pdir, str(exceptname), str(sys.exc_value))

                fullerrstr = "persistence directory does not have valid permissions and cannot be made to have valid permissions: '%s'" % pdir
                fullerrstr += errstr
                raise IncompatibleEnvironment(fullerrstr)

            self.c.log.debug("persistence directory was made to be rwx-able: %s" % pdir)

        lockfilepath = os.path.join(pdir, "persistence.lock")
        if not os.path.exists(lockfilepath):
            open(lockfilepath, 'w').close()

        self.lockfilepath = lockfilepath
        self.pdir = pdir

    def new_vm(self, run_name, vm):
        """Adds VM to a run_vms list if it exists for "run_name".  If list
        does not exist, it will be created."""
        return self.run_with_flock(self._new_vm, run_name, vm)

    def new_vm_maybe(self, run_name, vm):
        """Adds VM to a run_vms list if it exists for "run_name".  If list
        does not exist, it will be created.  If VM instance ID is present,
        it won't be added."""
        return self.run_with_flock(self._new_vm_maybe, run_name, vm)

    def _new_vm(self, run_name, vm):
        """Run this under a lock so that the list is not messed up"""
        run_vms = self.get_run_vms_or_none(run_name)
        if not run_vms:
            run_vms = []
        run_vms.append(vm)
        self.store_run_vms(run_name, run_vms)

    def _new_vm_maybe(self, run_name, vm):
        """Run this under a lock so that the list is not messed up"""
        run_vms = self.get_run_vms_or_none(run_name)
        if not run_vms:
            run_vms = []

        found = False
        newhostname = False
        for avm in run_vms:
            if avm.instanceid == vm.instanceid:
                # late binding hostnames are possible
                if not avm.hostname and vm.hostname:
                    avm.hostname = vm.hostname
                    strparams = (avm.instanceid, avm.hostname)
                    self.c.log.debug("found hostname for '%s': %s" % strparams)
                    newhostname = True
                found = True

        if newhostname:
            self.store_run_vms(run_name, run_vms)

        if found:
            return False # not new
        else:
            run_vms.append(vm)
            self.store_run_vms(run_name, run_vms)
            return True # new one

    def run_with_flock(self, f, *args, **kw):
        """Run function with persistence's filesystem-based lock.

        Would be nice to use a decorator but want to find/create lockfile
        during class instantiation, need to figure that out later.

        Anyhow, we will likely switch to SQLite at some point.
        """

        lockfile = None
        try:
            lockfile = open(self.lockfilepath, "r")
            fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX)
            return f(*args, **kw)
        finally:
            if lockfile:
                lockfile.close()

    def store_run_vms(self, run_name, run_vms):
        if not self.pdir:
            raise ProgrammingError("cannot persist anything without setup/validation")

        if not run_vms:
            raise ProgrammingError("no run_vms")

        if not run_name:
            raise ProgrammingError("no run_name")

        pobject = self._derive_runset_filepath(run_name)

        f = None
        try:
            f = open(pobject, 'w')
            pickle.dump(run_vms, f)
        finally:
            if f:
                f.close()

    def get_run_vms_or_none(self, run_name):
        """Get list of VMs for a run name or return None.

        If you mutate this list, you should lock first (we will switch
        to SQLite at some point)."""

        if not self.pdir:
            raise ProgrammingError("cannot persist anything without setup/validation")

        pobject = self._derive_runset_filepath(run_name)
        if not os.path.exists(pobject):
            return None

        f = None
        try:
            f = open(pobject, 'r')
            x = pickle.load(f)
            return x
        finally:
            if f:
                f.close()

    def _derive_runset_filepath(self, run_name):
        if not run_name:
            raise ProgrammingError("no run_name")
        return os.path.join(self.pdir, run_name + "-run_vms_list")
        <|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import json

from django.http import HttpResponse, JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from jsonview.decorators import json_view
from jsonview.exceptions import BadRequest
from papers.baremodels import BareName
from papers.baremodels import BarePaper
from papers.bibtex import format_paper_citation_dict
from papers.errors import MetadataSourceException
from papers.models import Paper
from papers.name import parse_comma_name
from papers.utils import tolerant_datestamp_to_datetime
from papers.views import PaperSearchView, ResearcherView
from ratelimit.decorators import ratelimit

def api_paper_common(request, paper):
    if 'format' in request.GET and request.GET['format'] == 'bibtex':
        response = HttpResponse(paper.bibtex(), content_type='application/x-bibtex')
        response['Content-Disposition'] = 'attachement; filename={}.bib'.format(paper.slug)
        return response
    else:
        return JsonResponse({
            'status': 'ok',
            'paper': paper.json()
        })

@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_pk(request, pk):
    p = Paper.objects.filter(pk=pk).first()
    if p is None:
        return JsonResponse({
            'error': 404,
            'message': 'The paper you requested could not be found.',
        }, status=404)
    return api_paper_common(request, p)

@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_doi(request, doi):
    p = None
    try:
        p = Paper.get_by_doi(doi)
        if not p:
            p = Paper.create_by_doi(doi)
    except MetadataSourceException:
        pass
    if p is None:
        return JsonResponse({
            'error': 404,
            'message': 'The paper you requested could not be found.',
        }, status=404)

    return api_paper_common(request, p)

class PaperSearchAPI(PaperSearchView):
    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(PaperSearchAPI, self).dispatch(*args, **kwargs)

    def render_to_response(self, context, **kwargs):
        if 'format' in self.request.GET and self.request.GET['format'] == 'bibtex':
            bibtex = format_paper_citation_dict(
                [
                    r.object.citation_dict()
                    for r in sorted(
                        context['object_list'],
                        key=lambda x: (x.object.pubdate, x.object.title)
                    )
                ],
                indent='  '
            )
            return HttpResponse(bibtex, content_type='application/x-bibtex')
        else:
            stats = context['search_stats'].pie_data()
            papers = [
                result.object.json()
                for result in context['object_list']
            ]
            response = {
                'messages': context['messages'],
                'stats': stats,
                'nb_results': context['nb_results'],
                'papers': papers,
            }
            return JsonResponse(response)

class ResearcherAPI(ResearcherView):
    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(ResearcherAPI, self).dispatch(*args, **kwargs)

    def render_to_response(self, context, **kwargs):
        if 'format' in self.request.GET and self.request.GET['format'] == 'bibtex':
            bibtex = format_paper_citation_dict(
                [
                    r.object.citation_dict()
                    for r in sorted(
                        context['object_list'],
                        key=lambda x: (x.object.pubdate, x.object.title)
                    )
                ],
                indent='  '
            )
            return HttpResponse(bibtex, content_type='application/x-bibtex')
        else:
            # TODO: Export the full researcher object (not just the papers) as
            # JSON?
            stats = context['search_stats'].pie_data()
            papers = [
                result.object.json()
                for result in context['object_list']
            ]
            response = {
                'messages': context['messages'],
                'stats': stats,
                'nb_results': context['nb_results'],
                'papers': papers,
            }
            return JsonResponse(response)

@json_view
@csrf_exempt
@require_POST
@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_query(request):
    try:
        fields = json.loads(request.body.decode('utf-8'))
    except (ValueError, UnicodeDecodeError):
        raise BadRequest('Invalid JSON payload')

    doi = fields.get('doi')
    if doi:
        p = None
        try:
            p = Paper.get_by_doi(doi)
            if not p:
                p = Paper.create_by_doi(doi)
        except MetadataSourceException:
            pass
        if p is None:
            raise BadRequest('Could not find a paper with this DOI')
        return {'status': 'ok', 'paper': p.json()}

    title = fields.get('title')
    if not isinstance(title,  str) or not title or len(title) > 512:
        raise BadRequest(
            'Invalid title, has to be a non-empty string shorter than 512 characters')

    date = fields.get('date')
    if not isinstance(date, str):
        raise BadRequest('A date is required')
    try:
        date = tolerant_datestamp_to_datetime(date)
    except ValueError as e:
        raise BadRequest(str(e))

    authors = fields.get('authors')
    if not isinstance(authors, list):
        raise BadRequest('A list of authors is expected')

    parsed_authors = []
    for a in authors:
        author = None
        if not isinstance(a, dict):
            raise BadRequest('Invalid author')

        if 'first' in a and 'last' in a:
            if not isinstance(a['first'], str) or not isinstance(a['last'], str) or not a['last']:
                raise BadRequest('Invalid (first,last) name provided')
            else:
                author = (a['first'], a['last'])
        elif 'plain' in a:
            if not isinstance(a['plain'], str) or not a['plain']:
                raise BadRequest('Invalid plain name provided')
            else:
                author = parse_comma_name(a['plain'])

        if author is None:
            raise BadRequest('Invalid author')

        parsed_authors.append(BareName.create(author[0], author[1]))

    if not authors:
        raise BadRequest('No authors provided')

    try:
        # Validate the metadata against our data model,
        # and compute the fingerprint to look up the paper in the DB.
        # This does NOT create a paper in the database - we do not want
        # to create papers for every search query we get!
        p = BarePaper.create(title, parsed_authors, date)
    except ValueError as e:
        raise BadRequest('Invalid paper: {}'.format(e))

    try:
        model_paper = Paper.objects.get(fingerprint=p.fingerprint)
        return {'status': 'ok', 'paper': model_paper.json()}
    except Paper.DoesNotExist:
        return {'status': 'not found'}, 404
<commit_msg>Context does not have messages if user is not logged in<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import json

from django.http import HttpResponse, JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from jsonview.decorators import json_view
from jsonview.exceptions import BadRequest
from papers.baremodels import BareName
from papers.baremodels import BarePaper
from papers.bibtex import format_paper_citation_dict
from papers.errors import MetadataSourceException
from papers.models import Paper
from papers.name import parse_comma_name
from papers.utils import tolerant_datestamp_to_datetime
from papers.views import PaperSearchView, ResearcherView
from ratelimit.decorators import ratelimit

def api_paper_common(request, paper):
    if 'format' in request.GET and request.GET['format'] == 'bibtex':
        response = HttpResponse(paper.bibtex(), content_type='application/x-bibtex')
        response['Content-Disposition'] = 'attachement; filename={}.bib'.format(paper.slug)
        return response
    else:
        return JsonResponse({
            'status': 'ok',
            'paper': paper.json()
        })

@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_pk(request, pk):
    p = Paper.objects.filter(pk=pk).first()
    if p is None:
        return JsonResponse({
            'error': 404,
            'message': 'The paper you requested could not be found.',
        }, status=404)
    return api_paper_common(request, p)

@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_doi(request, doi):
    p = None
    try:
        p = Paper.get_by_doi(doi)
        if not p:
            p = Paper.create_by_doi(doi)
    except MetadataSourceException:
        pass
    if p is None:
        return JsonResponse({
            'error': 404,
            'message': 'The paper you requested could not be found.',
        }, status=404)

    return api_paper_common(request, p)

class PaperSearchAPI(PaperSearchView):
    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(PaperSearchAPI, self).dispatch(*args, **kwargs)

    def render_to_response(self, context, **kwargs):
        if 'format' in self.request.GET and self.request.GET['format'] == 'bibtex':
            bibtex = format_paper_citation_dict(
                [
                    r.object.citation_dict()
                    for r in sorted(
                        context['object_list'],
                        key=lambda x: (x.object.pubdate, x.object.title)
                    )
                ],
                indent='  '
            )
            return HttpResponse(bibtex, content_type='application/x-bibtex')
        else:
            stats = context['search_stats'].pie_data()
            papers = [
                result.object.json()
                for result in context['object_list']
            ]
            response = {
                'messages': context.get('messages'),
                'stats': stats,
                'nb_results': context['nb_results'],
                'papers': papers,
            }
            return JsonResponse(response)

class ResearcherAPI(ResearcherView):
    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(ResearcherAPI, self).dispatch(*args, **kwargs)

    def render_to_response(self, context, **kwargs):
        if 'format' in self.request.GET and self.request.GET['format'] == 'bibtex':
            bibtex = format_paper_citation_dict(
                [
                    r.object.citation_dict()
                    for r in sorted(
                        context['object_list'],
                        key=lambda x: (x.object.pubdate, x.object.title)
                    )
                ],
                indent='  '
            )
            return HttpResponse(bibtex, content_type='application/x-bibtex')
        else:
            # TODO: Export the full researcher object (not just the papers) as
            # JSON?
            stats = context['search_stats'].pie_data()
            papers = [
                result.object.json()
                for result in context['object_list']
            ]
            response = {
                'messages': context['messages'],
                'stats': stats,
                'nb_results': context['nb_results'],
                'papers': papers,
            }
            return JsonResponse(response)

@json_view
@csrf_exempt
@require_POST
@ratelimit(key='ip',rate='300/m', block=True)
def api_paper_query(request):
    try:
        fields = json.loads(request.body.decode('utf-8'))
    except (ValueError, UnicodeDecodeError):
        raise BadRequest('Invalid JSON payload')

    doi = fields.get('doi')
    if doi:
        p = None
        try:
            p = Paper.get_by_doi(doi)
            if not p:
                p = Paper.create_by_doi(doi)
        except MetadataSourceException:
            pass
        if p is None:
            raise BadRequest('Could not find a paper with this DOI')
        return {'status': 'ok', 'paper': p.json()}

    title = fields.get('title')
    if not isinstance(title,  str) or not title or len(title) > 512:
        raise BadRequest(
            'Invalid title, has to be a non-empty string shorter than 512 characters')

    date = fields.get('date')
    if not isinstance(date, str):
        raise BadRequest('A date is required')
    try:
        date = tolerant_datestamp_to_datetime(date)
    except ValueError as e:
        raise BadRequest(str(e))

    authors = fields.get('authors')
    if not isinstance(authors, list):
        raise BadRequest('A list of authors is expected')

    parsed_authors = []
    for a in authors:
        author = None
        if not isinstance(a, dict):
            raise BadRequest('Invalid author')

        if 'first' in a and 'last' in a:
            if not isinstance(a['first'], str) or not isinstance(a['last'], str) or not a['last']:
                raise BadRequest('Invalid (first,last) name provided')
            else:
                author = (a['first'], a['last'])
        elif 'plain' in a:
            if not isinstance(a['plain'], str) or not a['plain']:
                raise BadRequest('Invalid plain name provided')
            else:
                author = parse_comma_name(a['plain'])

        if author is None:
            raise BadRequest('Invalid author')

        parsed_authors.append(BareName.create(author[0], author[1]))

    if not authors:
        raise BadRequest('No authors provided')

    try:
        # Validate the metadata against our data model,
        # and compute the fingerprint to look up the paper in the DB.
        # This does NOT create a paper in the database - we do not want
        # to create papers for every search query we get!
        p = BarePaper.create(title, parsed_authors, date)
    except ValueError as e:
        raise BadRequest('Invalid paper: {}'.format(e))

    try:
        model_paper = Paper.objects.get(fingerprint=p.fingerprint)
        return {'status': 'ok', 'paper': model_paper.json()}
    except Paper.DoesNotExist:
        return {'status': 'not found'}, 404
<|endoftext|>


<commit_before>import time
import Adafruit_Nokia_LCD as LCD
import Adafruit_GPIO.SPI as SPI

import Image
import ImageDraw
import ImageFont
import subprocess
import json
import os

bigfontsize = 22

DC = 23
RST = 24
SPI_PORT = 0
SPI_DEVICE = 0

basedir = os.path.dirname(os.path.realpath(__file__))
subprocess.call([os.path.join(basedir, 'wunderground-update.sh')])

disp = LCD.PCD8544(DC, RST, spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE, max_speed_hz=4000000))

disp.begin(contrast=40)

image = Image.new('1', (LCD.LCDWIDTH, LCD.LCDHEIGHT))
draw = ImageDraw.Draw(image)
draw.rectangle((0,0,LCD.LCDWIDTH,LCD.LCDHEIGHT), outline=255, fill=255)

big_font   = ImageFont.truetype(os.path.join('/usr/share/fonts/truetype/freefont/', 'FreeSansBold.ttf'), bigfontsize)
small_font = ImageFont.load_default()

with open('wunderground-conditions-data.json') as conditions_data_file:
    conditions_data = json.load(conditions_data_file)

with open('wunderground-forecast-data.json') as forecast_data_file:
    forecast_data = json.load(forecast_data_file)

temp_num = float(conditions_data[u'current_observation'][u'temp_c'])
temp_cur = "%2.0f" % temp_num
feelslike_num = float(conditions_data[u'current_observation'][u'feelslike_c'])
feelslike_str = "/ %2.0f" % feelslike_num
condition_now = str(conditions_data[u'current_observation'][u'weather'])
condition_today = str(forecast_data[u'forecast'][u'simpleforecast'][u'forecastday'][0][u'conditions'])
epoch = int(conditions_data[u'current_observation'][u'observation_epoch'])
utime = time.strftime('%H:%M', time.localtime(epoch))

print(temp_cur)
print(feelslike_str)
print(condition_now)
print(condition_today)

draw.text((0,0), temp_cur, font=big_font)
draw.text((36,0), feelslike_str, font=big_font)
draw.text((0,20), condition_now, font=small_font)
draw.text((0,30), condition_today, font=small_font)
draw.text((45,39), utime, font=small_font)

disp.image(image)
disp.display()

print("Done.")

<commit_msg>Use abslute path for cron to work<commit_after>import time
import Adafruit_Nokia_LCD as LCD
import Adafruit_GPIO.SPI as SPI

import Image
import ImageDraw
import ImageFont
import subprocess
import json
import os

bigfontsize = 22

DC = 23
RST = 24
SPI_PORT = 0
SPI_DEVICE = 0

basedir = os.path.dirname(os.path.realpath(__file__))
subprocess.call([os.path.join(basedir, 'wunderground-update.sh')])

disp = LCD.PCD8544(DC, RST, spi=SPI.SpiDev(SPI_PORT, SPI_DEVICE, max_speed_hz=4000000))

disp.begin(contrast=40)

image = Image.new('1', (LCD.LCDWIDTH, LCD.LCDHEIGHT))
draw = ImageDraw.Draw(image)
draw.rectangle((0,0,LCD.LCDWIDTH,LCD.LCDHEIGHT), outline=255, fill=255)

big_font   = ImageFont.truetype(os.path.join('/usr/share/fonts/truetype/freefont/', 'FreeSansBold.ttf'), bigfontsize)
small_font = ImageFont.load_default()

with open(os.path.join(basedir, 'wunderground-conditions-data.json')) as conditions_data_file:
    conditions_data = json.load(conditions_data_file)

with open(os.path.join(basedir, 'wunderground-forecast-data.json')) as forecast_data_file:
    forecast_data = json.load(forecast_data_file)

temp_num = float(conditions_data[u'current_observation'][u'temp_c'])
temp_cur = "%2.0f" % temp_num
feelslike_num = float(conditions_data[u'current_observation'][u'feelslike_c'])
feelslike_str = "/ %2.0f" % feelslike_num
condition_now = str(conditions_data[u'current_observation'][u'weather'])
condition_today = str(forecast_data[u'forecast'][u'simpleforecast'][u'forecastday'][0][u'conditions'])
epoch = int(conditions_data[u'current_observation'][u'observation_epoch'])
utime = time.strftime('%H:%M', time.localtime(epoch))

print(temp_cur)
print(feelslike_str)
print(condition_now)
print(condition_today)

draw.text((0,0), temp_cur, font=big_font)
draw.text((36,0), feelslike_str, font=big_font)
draw.text((0,20), condition_now, font=small_font)
draw.text((0,30), condition_today, font=small_font)
draw.text((45,39), utime, font=small_font)

disp.image(image)
disp.display()

print("Done.")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Run Python processes in background

:copyright: Copyright (c) 2016 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkconfig
from pykern import pkio
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp
import os
import re
import signal
import subprocess
import sys

def run_program(cmd, output='mpi_run.out', env=None):
    """Execute python script with mpi.

    Args:
        cmd (list): cmd to run
        output (str): where to write stdout and stderr
        env (dict): what to pass as env
    """
    from sirepo import simulation_db
    try:
        cmd = [
            'mpiexec',
            '--bind-to',
            'none',
            '-n',
            str(cfg.cores),

        ] + cmd
        pksubprocess.check_call_with_signals(
            cmd,
            msg=pkdp,
            output=str(output),
            env=env,
        )
    except Exception as e:
        simulation_db.write_result({'state': 'error', 'error': str(e)})
        raise

def run_script(script):
    """Execute python script with mpi.

    Args:
        script (str): python text
    """
    abort = '''

from mpi4py import MPI
if MPI.COMM_WORLD.Get_rank():
    import signal
    signal.signal(signal.SIGTERM, lambda x, y: MPI.COMM_WORLD.Abort(1))

'''
    n = re.sub(r'^from __future.*', abort, script, count=1, flags=re.MULTILINE)
    script = abort + script if n == script else n
    fn = 'mpi_run.py'
    pkio.write_text(fn, script)
    p = None
    return run_program([sys.executable or 'python', fn])

cfg = pkconfig.init(
    cores=(1, int, 'cores to use per run'),
    slaves=(1, int, 'DEPRECATED: set $SIREPO_MPI_CORES'),
)
cfg.cores = max(cfg.cores, cfg.slaves)
<commit_msg>use pkdlog instead of pkdp for msg<commit_after># -*- coding: utf-8 -*-
"""Run Python processes in background

:copyright: Copyright (c) 2016 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkconfig
from pykern import pkio
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
import os
import re
import signal
import subprocess
import sys

def run_program(cmd, output='mpi_run.out', env=None):
    """Execute python script with mpi.

    Args:
        cmd (list): cmd to run
        output (str): where to write stdout and stderr
        env (dict): what to pass as env
    """
    from sirepo import simulation_db
    try:
        cmd = [
            'mpiexec',
            '--bind-to',
            'none',
            '-n',
            str(cfg.cores),

        ] + cmd
        pksubprocess.check_call_with_signals(
            cmd,
            msg=pkdlog,
            output=str(output),
            env=env,
        )
    except Exception as e:
        simulation_db.write_result({'state': 'error', 'error': str(e)})
        raise

def run_script(script):
    """Execute python script with mpi.

    Args:
        script (str): python text
    """
    abort = '''

from mpi4py import MPI
if MPI.COMM_WORLD.Get_rank():
    import signal
    signal.signal(signal.SIGTERM, lambda x, y: MPI.COMM_WORLD.Abort(1))

'''
    n = re.sub(r'^from __future.*', abort, script, count=1, flags=re.MULTILINE)
    script = abort + script if n == script else n
    fn = 'mpi_run.py'
    pkio.write_text(fn, script)
    p = None
    return run_program([sys.executable or 'python', fn])

cfg = pkconfig.init(
    cores=(1, int, 'cores to use per run'),
    slaves=(1, int, 'DEPRECATED: set $SIREPO_MPI_CORES'),
)
cfg.cores = max(cfg.cores, cfg.slaves)
<|endoftext|>


<commit_before>from twython import Twython
import markovify
import os

# Set these values
APP_KEY = os.environ['CONSUMER_KEY']
APP_SECRET = os.environ['CONSUMER_SECRET']

twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)

ACCESS_TOKEN = twitter.obtain_access_token()

twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)

longstring = ''''''

#goes through results and print out the actual tweet
results = twitter.search(q='AltRightStarWars', count=300)
for result in results['statuses']:
    longstring += result['text']

text_model = markovify.Text(longstring)

for i in range(1, 10):
    print text_model.make_short_sentence(140)
<commit_msg>Got trending hashTags<commit_after>from twython import Twython
import markovify
import os

# Set these values
APP_KEY = os.environ['CONSUMER_KEY']
APP_SECRET = os.environ['CONSUMER_SECRET']

twitter = Twython(APP_KEY, APP_SECRET, oauth_version=2)

ACCESS_TOKEN = twitter.obtain_access_token()

twitter = Twython(APP_KEY, access_token=ACCESS_TOKEN)

toronto = twitter.get_place_trends(id=2487956)
trend_array = []
if toronto:
   for trend in toronto[0].get('trends', []):
      trend_array.append(trend['name'])

#print trend_array
longstring = ''''''

search = trend_array[0]
search = search[1:]

#print search

#goes through results and print out the actual tweet
results = twitter.search(q=search, count=300)
for result in results['statuses']:
    longstring += result['text']

text_model = markovify.Text(longstring)

twitter.update_status(status=text_model.make_short_sentence(140))

<|endoftext|>


<commit_before>import errno
import os
from os.path import join
from tempfile import TemporaryDirectory
import unittest

from hwt.pyUtils.fileHelpers import find_files

class FileUtilsTC(unittest.TestCase):
    def mkFile(self, d, fileName):
        fileName = join(d, fileName)
        if not os.path.exists(os.path.dirname(fileName)):
            try:
                os.makedirs(os.path.dirname(fileName))
            except OSError as exc:
                # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise

        open(fileName, 'w').close()
        return fileName

    def assertFound(self, directory, pattern, fileNames, recursive=True):
        f = set(find_files(directory, pattern, recursive=recursive))
        self.assertEqual(f, set(fileNames))

    def test_basic(self):
        with TemporaryDirectory() as d:
            self.assertFound(d, "*", {})

            a = self.mkFile(d, "a.a")
            self.assertFound(d, "*", {a})
            self.assertFound(d, "*.a", {a})
            self.assertFound(d, "*.*", {a})
            self.assertFound(d, "a.*", {a})
            self.assertFound(d, "b.*", {})
            self.assertFound(d, "*.b", {})
            self.assertFound(d, "", {})

            a2 = self.mkFile(d, "a2.a")
            self.assertFound(d, "*", {a, a2})
            self.assertFound(d, "*.a", {a, a2})
            self.assertFound(d, "*.*", {a, a2})
            self.assertFound(d, "a.*", {a})

            a3 = self.mkFile(d, join("a", "b", "c", "d.a"))
            self.assertFound(d, "*", {a, a2, a3})
            self.assertFound(d, "*.a", {a, a2, a3})
            self.assertFound(d, "*.*", {a, a2, a3})
            self.assertFound(d, "a.*", {a})

if __name__ == "__main__":
    suite = unittest.TestSuite()
    # suite.addTest(FrameTemplateTC('test_sWithStartPadding'))
    suite.addTest(unittest.makeSuite(FileUtilsTC))
    runner = unittest.TextTestRunner(verbosity=3)
    runner.run(suite)
<commit_msg>tests for find_files recursive = False<commit_after>import errno
import os
from os.path import join
from tempfile import TemporaryDirectory
import unittest

from hwt.pyUtils.fileHelpers import find_files

class FileUtilsTC(unittest.TestCase):
    def mkFile(self, d, fileName):
        fileName = join(d, fileName)
        if not os.path.exists(os.path.dirname(fileName)):
            try:
                os.makedirs(os.path.dirname(fileName))
            except OSError as exc:
                # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise

        open(fileName, 'w').close()
        return fileName

    def assertFound(self, directory, pattern, fileNames, recursive=True):
        f = set(find_files(directory, pattern, recursive=recursive))
        self.assertEqual(f, set(fileNames))

    def test_basic(self):
        with TemporaryDirectory() as d:
            self.assertFound(d, "*", {})

            a = self.mkFile(d, "a.a")
            self.assertFound(d, "*", {a})
            self.assertFound(d, "*.a", {a})
            self.assertFound(d, "*.*", {a})
            self.assertFound(d, "a.*", {a})
            self.assertFound(d, "b.*", {})
            self.assertFound(d, "*.b", {})
            self.assertFound(d, "", {})

            a2 = self.mkFile(d, "a2.a")
            self.assertFound(d, "*", {a, a2})
            self.assertFound(d, "*.a", {a, a2})
            self.assertFound(d, "*.*", {a, a2})
            self.assertFound(d, "a.*", {a})

            a3 = self.mkFile(d, join("a", "b", "c", "d.a"))
            self.assertFound(d, "*", {a, a2, a3})
            self.assertFound(d, "*.a", {a, a2, a3})
            self.assertFound(d, "*.*", {a, a2, a3})
            self.assertFound(d, "a.*", {a})
            self.assertFound(d, "*", {a, a2}, recursive=False)

if __name__ == "__main__":
    suite = unittest.TestSuite()
    # suite.addTest(FrameTemplateTC('test_sWithStartPadding'))
    suite.addTest(unittest.makeSuite(FileUtilsTC))
    runner = unittest.TextTestRunner(verbosity=3)
    runner.run(suite)
<|endoftext|>


<commit_before>import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))

import matplotlib.pyplot as plt
import numpy as np
from threading import RLock # re-entrant lock
import copy

from base.config import *
from base.SO3 import rodrigues, SO3
from visualization.visualization_camera import VisualizationCamera

class PlotterSharedData:
    """
    Shared data between the visualization context and the application context.
    """
    def __init__(self):
        self._lock = RLock()
        self._stopped = False
        self._time = 0.0
        self._text = ""
        self._true_orientation = None
        self._estimated_orientation = None

    def acquire_lock(self):
        self._lock.acquire()

    def release_lock(self):
        self._lock.release()

    @property
    def time(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._time

    @time.setter
    def time(self, t):
        with self._lock:
            # POD's don't need the extra copy
            self._time = t * 1.0

    @property
    def text(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._text

    @text.setter
    def text(self, t):
        with self._lock:
            # POD's don't need the extra copy
            self._text = str(t)

    @property
    def true_orientation(self):
        with self._lock:
            result = copy.deepcopy(self._true_orientation)
            return result

    @true_orientation.setter
    def true_orientation(self, R):
        with self._lock:
            self._true_orientation = copy.deepcopy(R)

    def stop(self):
        with self._lock:
            self._stopped = True

    @property
    def stopped(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._stopped

    @property
    def estimated_orientation(self):
        with self._lock:
            result = copy.deepcopy(self._estimated_orientation)
            return result

    @estimated_orientation.setter
    def estimated_orientation(self, R):
        with self._lock:
            self._estimated_orientation = copy.deepcopy(R)

class Plotter:
    """
    A collection of visualization utilities.
    """
    def __init__(self, shared_data, view_point=None):
        """
        :param shared_data: of class PlotterSharedData
        """
        self._shared_data = shared_data

        if (view_point is None):
            view_point = VIEW_POINT

        translation = np.array(view_point, dtype=np.float)

        if (np.linalg.norm(translation) < TOLERANCE):
            raise ValueError("Invalid view point: {}".format(view_point))

        z_camera_in_camera = np.array([0, 0, 1], dtype=np.float)
        z_camera_in_world = -np.array(view_point, dtype=np.float)
        rotation = SO3.from_two_directions(z_camera_in_camera, z_camera_in_world)

        self._camera = VisualizationCamera(rotation, translation)

        # generate 3 axes
        self._offset = np.array([-1, -1, 0], dtype=np.float)
        self._origin = np.array([0, 0, 0], dtype=np.float)
        self._axis_x = np.array([1, 0, 0], dtype=np.float)
        self._axis_y = np.array([0, 1, 0], dtype=np.float)
        self._axis_z = np.array([0, 0, 1], dtype=np.float)

    @property
    def stopped(self):
        return self._shared_data.stopped

    def _get_frame_axes(self, R_from_body_to_world, scale=1.0):
        """
        calculate the axes of the reference frame which is transformed through R
        """
        axis_x = R_from_body_to_world * self._axis_x * scale + self._offset
        axis_y = R_from_body_to_world * self._axis_y * scale + self._offset
        axis_z = R_from_body_to_world * self._axis_z * scale + self._offset
        origin = R_from_body_to_world * self._origin * scale + self._offset

        points = [axis_x, axis_y, axis_z, origin]
        points = self._camera.project_points(points)

        body_axis_x_1 = [points[3][0], points[0][0]]
        body_axis_x_2 = [points[3][1], points[0][1]]
        body_axis_y_1 = [points[3][0], points[1][0]]
        body_axis_y_2 = [points[3][1], points[1][1]]
        body_axis_z_1 = [points[3][0], points[2][0]]
        body_axis_z_2 = [points[3][1], points[2][1]]

        return (body_axis_x_1, body_axis_x_2,
                body_axis_y_1, body_axis_y_2,
                body_axis_z_1, body_axis_z_2)

    def draw(self):
        """
        Visualize shared data
        """
        true_body_frame_scale = 0.75
        body_frame_scale = 0.5
        body_axis_width = 10
        world_axis_width = 5

        plt.clf()

        # true body ref frame
        R_from_body_to_world = self._shared_data.true_orientation
        if (R_from_body_to_world is not None):
            body_axes = self._get_frame_axes(R_from_body_to_world, true_body_frame_scale)
            plt.plot(body_axes[0], body_axes[1], 'r--', linewidth=body_axis_width)
            plt.plot(body_axes[2], body_axes[3], 'g--', linewidth=body_axis_width)
            plt.plot(body_axes[4], body_axes[5], 'b--', linewidth=body_axis_width)

        # estimated body ref frame
        R_from_body_to_world = self._shared_data.estimated_orientation
        if (R_from_body_to_world is not None):
            body_axes = self._get_frame_axes(R_from_body_to_world, body_frame_scale)
            plt.plot(body_axes[0], body_axes[1], 'r-', linewidth=body_axis_width)
            plt.plot(body_axes[2], body_axes[3], 'g-', linewidth=body_axis_width)
            plt.plot(body_axes[4], body_axes[5], 'b-', linewidth=body_axis_width)

        # world ref frame
        R_from_body_to_world = SO3()
        body_axes = self._get_frame_axes(R_from_body_to_world)
        plt.plot(body_axes[0], body_axes[1], 'r--', linewidth=world_axis_width)
        plt.plot(body_axes[2], body_axes[3], 'g--', linewidth=world_axis_width)
        plt.plot(body_axes[4], body_axes[5], 'b--', linewidth=world_axis_width)

        # axes
        plt.ylim((-VIEW_RANGE, VIEW_RANGE))
        plt.xlim((-VIEW_RANGE, VIEW_RANGE))
        plt.axes().set_aspect('equal')

        # all the plots are overlayed, so long as draw() has not been caleed.
        plt.title("time = %.2f s" % (self._shared_data.time))
        plt.text(-VIEW_RANGE, -VIEW_RANGE, self._shared_data.text, fontsize=20, color='r')
        plt.draw()
        plt.show(block=False)

if (__name__ == '__main__'):
    pass
<commit_msg>Fixed a visualization issue.<commit_after>import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))

import matplotlib.pyplot as plt
import numpy as np
from threading import RLock # re-entrant lock
import copy

from base.config import *
from base.SO3 import rodrigues, SO3
from visualization.visualization_camera import VisualizationCamera

class PlotterSharedData:
    """
    Shared data between the visualization context and the application context.
    """
    def __init__(self):
        self._lock = RLock()
        self._stopped = False
        self._time = 0.0
        self._text = ""
        self._true_orientation = None
        self._estimated_orientation = None

    def acquire_lock(self):
        self._lock.acquire()

    def release_lock(self):
        self._lock.release()

    @property
    def time(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._time

    @time.setter
    def time(self, t):
        with self._lock:
            # POD's don't need the extra copy
            self._time = t * 1.0

    @property
    def text(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._text

    @text.setter
    def text(self, t):
        with self._lock:
            # POD's don't need the extra copy
            self._text = str(t)

    @property
    def true_orientation(self):
        with self._lock:
            result = copy.deepcopy(self._true_orientation)
            return result

    @true_orientation.setter
    def true_orientation(self, R):
        with self._lock:
            self._true_orientation = copy.deepcopy(R)

    def stop(self):
        with self._lock:
            self._stopped = True

    @property
    def stopped(self):
        with self._lock:
            # POD's don't need the extra copy
            return self._stopped

    @property
    def estimated_orientation(self):
        with self._lock:
            result = copy.deepcopy(self._estimated_orientation)
            return result

    @estimated_orientation.setter
    def estimated_orientation(self, R):
        with self._lock:
            self._estimated_orientation = copy.deepcopy(R)

class Plotter:
    """
    A collection of visualization utilities.
    """
    def __init__(self, shared_data, view_point=None):
        """
        :param shared_data: of class PlotterSharedData
        """
        self._shared_data = shared_data

        if (view_point is None):
            view_point = VIEW_POINT

        translation = np.array(view_point, dtype=np.float)

        if (np.linalg.norm(translation) < TOLERANCE):
            raise ValueError("Invalid view point: {}".format(view_point))

        z_camera_in_camera = np.array([0, 0, 1], dtype=np.float)
        z_camera_in_world = -np.array(view_point, dtype=np.float)
        rotation = SO3.from_two_directions(z_camera_in_camera, z_camera_in_world)

        self._camera = VisualizationCamera(rotation, translation)

        # generate 3 axes
        self._offset = np.array([-0.5, -0.5, 0], dtype=np.float)
        self._origin = np.array([0, 0, 0], dtype=np.float)
        self._axis_x = np.array([1, 0, 0], dtype=np.float)
        self._axis_y = np.array([0, 1, 0], dtype=np.float)
        self._axis_z = np.array([0, 0, 1], dtype=np.float)

    @property
    def stopped(self):
        return self._shared_data.stopped

    def _get_frame_points(self, R_from_body_to_world, scale=1.0):
        """
        Calculate the canvas coordinate of the reference frame, which
        is characterized by R_from_body_to_world
        """
        # calculate new end points in world ref frame.
        axis_x = R_from_body_to_world * self._axis_x * scale + self._offset
        axis_y = R_from_body_to_world * self._axis_y * scale + self._offset
        axis_z = R_from_body_to_world * self._axis_z * scale + self._offset
        origin = R_from_body_to_world * self._origin * scale + self._offset

        points = [axis_x, axis_y, axis_z, origin]
        points = self._camera.project_points(points)

        # calculate canvas coordinates
        # NOTE: for the camera image, x-axis points to the right and
        #       y-axis points downward; for the canvas, x-axis points
        #       to the right and y-axis points up.
        body_axis_x_1 = [ points[3][0],  points[0][0]]
        body_axis_x_2 = [-points[3][1], -points[0][1]]
        body_axis_y_1 = [ points[3][0],  points[1][0]]
        body_axis_y_2 = [-points[3][1], -points[1][1]]
        body_axis_z_1 = [ points[3][0],  points[2][0]]
        body_axis_z_2 = [-points[3][1], -points[2][1]]

        points = (body_axis_x_1, body_axis_x_2,
                  body_axis_y_1, body_axis_y_2,
                  body_axis_z_1, body_axis_z_2)
        return points

    def draw(self):
        """
        Visualize shared data
        """
        true_body_frame_scale = 0.75
        body_frame_scale = 0.5
        body_axis_width = 10
        world_axis_width = 5

        plt.clf()

        # true body ref frame
        R_from_body_to_world = self._shared_data.true_orientation
        if (R_from_body_to_world is not None):
            points = self._get_frame_points(R_from_body_to_world, true_body_frame_scale)
            plt.plot(points[0], points[1], 'r--', linewidth=body_axis_width)
            plt.plot(points[2], points[3], 'g--', linewidth=body_axis_width)
            plt.plot(points[4], points[5], 'b--', linewidth=body_axis_width)

        # estimated body ref frame
        R_from_body_to_world = self._shared_data.estimated_orientation
        if (R_from_body_to_world is not None):
            points = self._get_frame_points(R_from_body_to_world, body_frame_scale)
            plt.plot(points[0], points[1], 'r-', linewidth=body_axis_width)
            plt.plot(points[2], points[3], 'g-', linewidth=body_axis_width)
            plt.plot(points[4], points[5], 'b-', linewidth=body_axis_width)

        # world ref frame
        R_from_body_to_world = SO3()
        points = self._get_frame_points(R_from_body_to_world)
        plt.plot(points[0], points[1], 'r--', linewidth=world_axis_width)
        plt.plot(points[2], points[3], 'g--', linewidth=world_axis_width)
        plt.plot(points[4], points[5], 'b--', linewidth=world_axis_width)

        # axes
        plt.ylim((-VIEW_RANGE, VIEW_RANGE))
        plt.xlim((-VIEW_RANGE, VIEW_RANGE))
        plt.axes().set_aspect('equal')

        # all the plots are overlayed, so long as draw() has not been caleed.
        plt.title("time = %.2f s" % (self._shared_data.time))
        plt.text(-VIEW_RANGE, -VIEW_RANGE, self._shared_data.text, fontsize=20, color='r')
        plt.draw()
        plt.show(block=False)

if (__name__ == '__main__'):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Run multiple simulations varying a single parameter.
"""

import foampy
from foampy.dictionaries import replace_single_line_value
import numpy as np
from subprocess import call
import os
import pandas as pd
from modules import processing as pr

def zero_tsr_fluc():
    """Set TSR fluctuation amplitude to zero."""
    replace_single_line_value("fvOptions", "tsrAmplitude", 0.0)

def set_tsr(val):
    """Set mean tip speed ratio."""
    print("Setting TSR to", tsr)
    replace_single_line_value("fvOptions", "tipSpeedRatio", val)

def log_perf(param="tsr", append=True):
    """Log performance to file."""
    if not os.path.isdir("processed"):
        os.mkdir("processed")
    fpath = "processed/{}_sweep.csv".format(param)
    if append and os.path.isfile(fpath)
        df = pd.read_csv(fpath)
    else:
        df = pd.DataFrame(columns=["tsr", "cp", "cd"])
    df = df.append(pr.calc_perf(t1=3.0), ignore_index=True)
    df.to_csv("fpath", index=False)

def tsr_sweep(start=0.4, stop=3.4, step=0.5, append=False):
    """Run over multiple TSRs. `stop` will be included."""
    if not append and os.path.isfile("processed/tsr_sweep.csv"):
        os.remove("processed/tsr_sweep.csv")
    tsrs = np.arange(start, stop + 0.5*step, step)
    zero_tsr_fluc()
    cp = []
    cd = []
    for tsr in tsrs:
        set_tsr(tsr)
        if tsr == tsrs[0]:
            call("./Allclean")
            call("./Allrun")
        else:
            call("pimpleFoam | tee log.pimpleFoam", shell=True)
        os.rename("log.pimpleFoam", "log.pimpleFoam." + str(tsr))
        log_perf(append=True)
    # Checkout original fvOptions
    call["git", "checkout", "system/fvOptions"]

if __name__ == "__main__":
    tsr_sweep(0.4, 3.4, 0.5, append=False)
<commit_msg>Fix some bugs in paramsweep.py<commit_after>#!/usr/bin/env python
"""
Run multiple simulations varying a single parameter.
"""

import foampy
from foampy.dictionaries import replace_value
import numpy as np
from subprocess import call
import os
import pandas as pd
from modules import processing as pr

def zero_tsr_fluc():
    """Set TSR fluctuation amplitude to zero."""
    replace_value("system/fvOptions", "tsrAmplitude", 0.0)

def set_tsr(val):
    """Set mean tip speed ratio."""
    print("Setting TSR to", val)
    replace_value("system/fvOptions", "tipSpeedRatio", val)

def log_perf(param="tsr", append=True):
    """Log performance to file."""
    if not os.path.isdir("processed"):
        os.mkdir("processed")
    fpath = "processed/{}_sweep.csv".format(param)
    if append and os.path.isfile(fpath):
        df = pd.read_csv(fpath)
    else:
        df = pd.DataFrame(columns=["tsr", "cp", "cd"])
    df = df.append(pr.calc_perf(t1=3.0), ignore_index=True)
    df.to_csv("fpath", index=False)

def tsr_sweep(start=0.4, stop=3.4, step=0.5, append=False):
    """Run over multiple TSRs. `stop` will be included."""
    if not append and os.path.isfile("processed/tsr_sweep.csv"):
        os.remove("processed/tsr_sweep.csv")
    tsrs = np.arange(start, stop + 0.5*step, step)
    zero_tsr_fluc()
    cp = []
    cd = []
    for tsr in tsrs:
        set_tsr(tsr)
        if tsr == tsrs[0]:
            call("./Allclean")
            call("./Allrun")
        else:
            call("pimpleFoam | tee log.pimpleFoam", shell=True)
        os.rename("log.pimpleFoam", "log.pimpleFoam." + str(tsr))
        log_perf(append=True)
    # Checkout original fvOptions
    call["git", "checkout", "system/fvOptions"]

if __name__ == "__main__":
    tsr_sweep(0.4, 3.4, 0.5, append=False)
<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Input ops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

def parse_sequence_example(serialized, image_feature, caption_feature):
  """Parses a tensorflow.SequenceExample into an image and caption.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.
    image_feature: Name of SequenceExample context feature containing image
      data.
    caption_feature: Name of SequenceExample feature list containing integer
      captions.

  Returns:
    encoded_image: A scalar string Tensor containing a JPEG encoded image.
    caption: A 1-D uint64 Tensor with dynamically specified length.
  """
  context, sequence = tf.parse_single_sequence_example(
      serialized,
      context_features={
          image_feature: tf.FixedLenFeature([], dtype=tf.string)
      },
      sequence_features={
          caption_feature: tf.FixedLenSequenceFeature([], dtype=tf.int64),
      })

  encoded_image = context[image_feature]
  caption = sequence[caption_feature]
  return encoded_image, caption

def prefetch_input_data(reader,
                        file_pattern,
                        is_training,
                        batch_size,
                        values_per_shard,
                        input_queue_capacity_factor=16,
                        num_reader_threads=1,
                        shard_queue_name="filename_queue",
                        value_queue_name="input_queue"):
  """Prefetches string values from disk into an input queue.

  In training the capacity of the queue is important because a larger queue
  means better mixing of training examples between shards. The minimum number of
  values kept in the queue is values_per_shard * input_queue_capacity_factor,
  where input_queue_memory factor should be chosen to trade-off better mixing
  with memory usage.

  Args:
    reader: Instance of tf.ReaderBase.
    file_pattern: Comma-separated list of file patterns (e.g.
        /tmp/train_data-?????-of-00100).
    is_training: Boolean; whether prefetching for training or eval.
    batch_size: Model batch size used to determine queue capacity.
    values_per_shard: Approximate number of values per shard.
    input_queue_capacity_factor: Minimum number of values to keep in the queue
      in multiples of values_per_shard. See comments above.
    num_reader_threads: Number of reader threads to fill the queue.
    shard_queue_name: Name for the shards filename queue.
    value_queue_name: Name for the values input queue.

  Returns:
    A Queue containing prefetched string values.
  """
  data_files = []
  for pattern in file_pattern.split(","):
    data_files.extend(tf.gfile.Glob(pattern))
  if not data_files:
    tf.logging.fatal("Found no input files matching %s", file_pattern)
  else:
    tf.logging.info("Prefetching values from %d files matching %s",
                    len(data_files), file_pattern)

  if is_training:
    filename_queue = tf.train.string_input_producer(
        data_files, shuffle=True, capacity=16, name=shard_queue_name)
    min_queue_examples = values_per_shard * input_queue_capacity_factor
    capacity = min_queue_examples + 100 * batch_size
    values_queue = tf.RandomShuffleQueue(
        capacity=capacity,
        min_after_dequeue=min_queue_examples,
        dtypes=[tf.string],
        name="random_" + value_queue_name)
  else:
    filename_queue = tf.train.string_input_producer(
        data_files, shuffle=False, capacity=1, name=shard_queue_name)
    capacity = values_per_shard + 3 * batch_size
    values_queue = tf.FIFOQueue(
        capacity=capacity, dtypes=[tf.string], name="fifo_" + value_queue_name)

  enqueue_ops = []
  for _ in range(num_reader_threads):
    _, value = reader.read(filename_queue)
    enqueue_ops.append(values_queue.enqueue([value]))
  tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(
      values_queue, enqueue_ops))
  tf.summary.scalar(
      "queue/%s/fraction_of_%d_full" % (values_queue.name, capacity),
      tf.cast(values_queue.size(), tf.float32) * (1. / capacity))

  return values_queue

def batch_with_dynamic_pad(images_and_captions,
                           batch_size,
                           queue_capacity,
                           add_summaries=True):
  """Batches input images and captions.

  This function splits the caption into an input sequence and a target sequence,
  where the target sequence is the input sequence right-shifted by 1. Input and
  target sequences are batched and padded up to the maximum length of sequences
  in the batch. A mask is created to distinguish real words from padding words.

  Example:
    Actual captions in the batch ('-' denotes padded character):
      [
        [ 1 2 5 4 5 ],
        [ 1 2 3 4 - ],
        [ 1 2 3 - - ],
      ]

    input_seqs:
      [
        [ 1 2 3 4 ],
        [ 1 2 3 - ],
        [ 1 2 - - ],
      ]

    target_seqs:
      [
        [ 2 3 4 5 ],
        [ 2 3 4 - ],
        [ 2 3 - - ],
      ]

    mask:
      [
        [ 1 1 1 1 ],
        [ 1 1 1 0 ],
        [ 1 1 0 0 ],
      ]

  Args:
    images_and_captions: A list of pairs [image, caption], where image is a
      Tensor of shape [height, width, channels] and caption is a 1-D Tensor of
      any length. Each pair will be processed and added to the queue in a
      separate thread.
    batch_size: Batch size.
    queue_capacity: Queue capacity.
    add_summaries: If true, add caption length summaries.

  Returns:
    images: A Tensor of shape [batch_size, height, width, channels].
    input_seqs: An int32 Tensor of shape [batch_size, padded_length].
    target_seqs: An int32 Tensor of shape [batch_size, padded_length].
    mask: An int32 0/1 Tensor of shape [batch_size, padded_length].
  """
  enqueue_list = []
  for image, caption in images_and_captions:
    caption_length = tf.shape(caption)[0]
    input_length = tf.expand_dims(tf.subtract(caption_length, 1), 0)

    input_seq = tf.slice(caption, [0], input_length)
    target_seq = tf.slice(caption, [1], input_length)
    indicator = tf.ones(input_length, dtype=tf.int32)
    enqueue_list.append([image, input_seq, target_seq, indicator])

  images, input_seqs, target_seqs, mask = tf.train.batch_join(
      enqueue_list,
      batch_size=batch_size,
      capacity=queue_capacity,
      dynamic_pad=True,
      name="batch_and_pad")

  if add_summaries:
    lengths = tf.add(tf.reduce_sum(mask, 1), 1)
    tf.summary.scalar("caption_length/batch_min", tf.reduce_min(lengths))
    tf.summary.scalar("caption_length/batch_max", tf.reduce_max(lengths))
    tf.summary.scalar("caption_length/batch_mean", tf.reduce_mean(lengths))

  return images, input_seqs, target_seqs, mask
<commit_msg>Fix wrong explanation<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Input ops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

def parse_sequence_example(serialized, image_feature, caption_feature):
  """Parses a tensorflow.SequenceExample into an image and caption.

  Args:
    serialized: A scalar string Tensor; a single serialized SequenceExample.
    image_feature: Name of SequenceExample context feature containing image
      data.
    caption_feature: Name of SequenceExample feature list containing integer
      captions.

  Returns:
    encoded_image: A scalar string Tensor containing a JPEG encoded image.
    caption: A 1-D uint64 Tensor with dynamically specified length.
  """
  context, sequence = tf.parse_single_sequence_example(
      serialized,
      context_features={
          image_feature: tf.FixedLenFeature([], dtype=tf.string)
      },
      sequence_features={
          caption_feature: tf.FixedLenSequenceFeature([], dtype=tf.int64),
      })

  encoded_image = context[image_feature]
  caption = sequence[caption_feature]
  return encoded_image, caption

def prefetch_input_data(reader,
                        file_pattern,
                        is_training,
                        batch_size,
                        values_per_shard,
                        input_queue_capacity_factor=16,
                        num_reader_threads=1,
                        shard_queue_name="filename_queue",
                        value_queue_name="input_queue"):
  """Prefetches string values from disk into an input queue.

  In training the capacity of the queue is important because a larger queue
  means better mixing of training examples between shards. The minimum number of
  values kept in the queue is values_per_shard * input_queue_capacity_factor,
  where input_queue_memory factor should be chosen to trade-off better mixing
  with memory usage.

  Args:
    reader: Instance of tf.ReaderBase.
    file_pattern: Comma-separated list of file patterns (e.g.
        /tmp/train_data-?????-of-00100).
    is_training: Boolean; whether prefetching for training or eval.
    batch_size: Model batch size used to determine queue capacity.
    values_per_shard: Approximate number of values per shard.
    input_queue_capacity_factor: Minimum number of values to keep in the queue
      in multiples of values_per_shard. See comments above.
    num_reader_threads: Number of reader threads to fill the queue.
    shard_queue_name: Name for the shards filename queue.
    value_queue_name: Name for the values input queue.

  Returns:
    A Queue containing prefetched string values.
  """
  data_files = []
  for pattern in file_pattern.split(","):
    data_files.extend(tf.gfile.Glob(pattern))
  if not data_files:
    tf.logging.fatal("Found no input files matching %s", file_pattern)
  else:
    tf.logging.info("Prefetching values from %d files matching %s",
                    len(data_files), file_pattern)

  if is_training:
    filename_queue = tf.train.string_input_producer(
        data_files, shuffle=True, capacity=16, name=shard_queue_name)
    min_queue_examples = values_per_shard * input_queue_capacity_factor
    capacity = min_queue_examples + 100 * batch_size
    values_queue = tf.RandomShuffleQueue(
        capacity=capacity,
        min_after_dequeue=min_queue_examples,
        dtypes=[tf.string],
        name="random_" + value_queue_name)
  else:
    filename_queue = tf.train.string_input_producer(
        data_files, shuffle=False, capacity=1, name=shard_queue_name)
    capacity = values_per_shard + 3 * batch_size
    values_queue = tf.FIFOQueue(
        capacity=capacity, dtypes=[tf.string], name="fifo_" + value_queue_name)

  enqueue_ops = []
  for _ in range(num_reader_threads):
    _, value = reader.read(filename_queue)
    enqueue_ops.append(values_queue.enqueue([value]))
  tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(
      values_queue, enqueue_ops))
  tf.summary.scalar(
      "queue/%s/fraction_of_%d_full" % (values_queue.name, capacity),
      tf.cast(values_queue.size(), tf.float32) * (1. / capacity))

  return values_queue

def batch_with_dynamic_pad(images_and_captions,
                           batch_size,
                           queue_capacity,
                           add_summaries=True):
  """Batches input images and captions.

  This function splits the caption into an input sequence and a target sequence,
  where the target sequence is the input sequence right-shifted by 1. Input and
  target sequences are batched and padded up to the maximum length of sequences
  in the batch. A mask is created to distinguish real words from padding words.

  Example:
    Actual captions in the batch ('-' denotes padded character):
      [
        [ 1 2 3 4 5 ],
        [ 1 2 3 4 - ],
        [ 1 2 3 - - ],
      ]

    input_seqs:
      [
        [ 1 2 3 4 ],
        [ 1 2 3 - ],
        [ 1 2 - - ],
      ]

    target_seqs:
      [
        [ 2 3 4 5 ],
        [ 2 3 4 - ],
        [ 2 3 - - ],
      ]

    mask:
      [
        [ 1 1 1 1 ],
        [ 1 1 1 0 ],
        [ 1 1 0 0 ],
      ]

  Args:
    images_and_captions: A list of pairs [image, caption], where image is a
      Tensor of shape [height, width, channels] and caption is a 1-D Tensor of
      any length. Each pair will be processed and added to the queue in a
      separate thread.
    batch_size: Batch size.
    queue_capacity: Queue capacity.
    add_summaries: If true, add caption length summaries.

  Returns:
    images: A Tensor of shape [batch_size, height, width, channels].
    input_seqs: An int32 Tensor of shape [batch_size, padded_length].
    target_seqs: An int32 Tensor of shape [batch_size, padded_length].
    mask: An int32 0/1 Tensor of shape [batch_size, padded_length].
  """
  enqueue_list = []
  for image, caption in images_and_captions:
    caption_length = tf.shape(caption)[0]
    input_length = tf.expand_dims(tf.subtract(caption_length, 1), 0)

    input_seq = tf.slice(caption, [0], input_length)
    target_seq = tf.slice(caption, [1], input_length)
    indicator = tf.ones(input_length, dtype=tf.int32)
    enqueue_list.append([image, input_seq, target_seq, indicator])

  images, input_seqs, target_seqs, mask = tf.train.batch_join(
      enqueue_list,
      batch_size=batch_size,
      capacity=queue_capacity,
      dynamic_pad=True,
      name="batch_and_pad")

  if add_summaries:
    lengths = tf.add(tf.reduce_sum(mask, 1), 1)
    tf.summary.scalar("caption_length/batch_min", tf.reduce_min(lengths))
    tf.summary.scalar("caption_length/batch_max", tf.reduce_max(lengths))
    tf.summary.scalar("caption_length/batch_mean", tf.reduce_mean(lengths))

  return images, input_seqs, target_seqs, mask
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
#
# Markov chain manipulation
# Copyright (C) 2016 greenify
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict
from itertools import chain
from . import utils

__author__ = "greenify"
__copyright__ = "Copyright 2016, greenify"

class Markov:
    """ A simple markov chain implementation

        Attributes:
            corpus: given corpus (a corpus_entry needs to be a tuple or array)
            order: maximal order to look back for a given state (default 1)
            tokenize: function how to split an element of the corpus (e.g sentences into words)
    """

    def __init__(self, corpus, order=1, tokenize=lambda x: x):
        """ Initializes the Markov Chain with a given corpus and order """
        assert order >= 1, "Invalid Markov chain order"
        assert order <= 20, "Markov chain order too high"
        assert corpus is not None, "Corpus is empty"
        self.order = order
        self._start_symbol = '^'
        self._end_symbol = '$'
        self.tokenize = tokenize
        self._compute_transitions(corpus, self.order)

    def _compute_transitions(self, corpus, order=1):
        """ Computes the transition probabilities of a corpus
        Args:
            corpus: the given corpus (a corpus_entry needs to be iterable)
            order: the maximal Markov chain order
        """
        self.transitions = defaultdict(lambda: defaultdict(int))

        for corpus_entry in corpus:
            tokens = self.tokenize(corpus_entry)

            last_tokens = utils.prefilled_buffer(
                self._start_symbol, length=self.order)
            # count the occurrences of "present | past"
            for token_value in chain(tokens, self._end_symbol):
                for suffix in utils.get_suffixes(last_tokens):
                    self.transitions[suffix][token_value] += 1

                last_tokens.append(token_value)

        self._compute_relative_probs(self.transitions)

    def _compute_relative_probs(self, prob_dict):
        """ computes the relative probabilities for every state """
        for transition_counts in prob_dict.values():
            summed_occurences = sum(transition_counts.values())
            if summed_occurences > 0:
                for token in transition_counts.keys():
                    transition_counts[token] = transition_counts[
                        token] * 1.0 / summed_occurences

    def _text_generator(self, next_token=None, emit=lambda x, _, __: x):
        """ loops from the start state to the end state and records the emissions
        Tokens are joint to sentences by looking ahead for the next token type

        emit: by default the markovian emit (see HMM for different emission forms)
        """
        assert next_token is not None
        last_tokens = utils.prefilled_buffer(
            self._start_symbol, length=self.order)
        if hasattr(self, "order_emissions"):
            order_emissions = self.order_emissions
        else:
            order_emissions = self.order
        last_emissions = utils.prefilled_buffer(
            self._start_symbol, length=order_emissions)
        generated_tokens = []
        while last_tokens[-1] != self._end_symbol:
            new_token = next_token(last_tokens)
            emission = emit(new_token, last_tokens, last_emissions)
            last_emissions.append(emission)
            generated_tokens.append(emission)
            last_tokens.append(new_token)
        text = generated_tokens[:-1]
        return text

    def generate_text(self):
        """ Generates sentences from a given corpus
        Returns:
            Properly formatted string of generated sentences
        """
        return self._text_generator(next_token=self._generate_next_token)

    def _generate_next_token_helper(self, past_states, transitions):
        """ generates next token based previous states """
        key = tuple(past_states)
        assert key in transitions, "%s" % str(key)
        return utils.weighted_choice(transitions[key].items())

    def _generate_next_token(self, past_states):
        """ generates next token based previous words """
        return self._generate_next_token_helper(past_states, self.transitions)
<commit_msg>add max_length to text generation<commit_after>#!/usr/bin/env python
# encoding: utf-8
#
# Markov chain manipulation
#
# The MIT License (MIT)
#
# Copyright (c) 2016 greenify
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from collections import defaultdict
from itertools import chain
from . import utils

__author__ = "greenify"
__copyright__ = "Copyright 2016, greenify"

class Markov:
    """ A simple markov chain implementation

        Attributes:
            corpus: given corpus (a corpus_entry needs to be a tuple or array)
            order: maximal order to look back for a given state (default 1)
            tokenize: function how to split an element of the corpus (e.g sentences into words)
    """

    def __init__(self, corpus, order=1, tokenize=lambda x: x):
        """ Initializes the Markov Chain with a given corpus and order """
        assert order >= 1, "Invalid Markov chain order"
        assert order <= 20, "Markov chain order too high"
        assert corpus is not None, "Corpus is empty"
        self.order = order
        self._start_symbol = '^'
        self._end_symbol = '$'
        self.tokenize = tokenize
        self._compute_transitions(corpus, self.order)

    def _compute_transitions(self, corpus, order=1):
        """ Computes the transition probabilities of a corpus
        Args:
            corpus: the given corpus (a corpus_entry needs to be iterable)
            order: the maximal Markov chain order
        """
        self.transitions = defaultdict(lambda: defaultdict(int))

        for corpus_entry in corpus:
            tokens = self.tokenize(corpus_entry)

            last_tokens = utils.prefilled_buffer(
                self._start_symbol, length=self.order)
            # count the occurrences of "present | past"
            for token_value in chain(tokens, self._end_symbol):
                for suffix in utils.get_suffixes(last_tokens):
                    self.transitions[suffix][token_value] += 1

                last_tokens.append(token_value)

        self._compute_relative_probs(self.transitions)

    def _compute_relative_probs(self, prob_dict):
        """ computes the relative probabilities for every state """
        for transition_counts in prob_dict.values():
            summed_occurences = sum(transition_counts.values())
            if summed_occurences > 0:
                for token in transition_counts.keys():
                    transition_counts[token] = transition_counts[
                        token] * 1.0 / summed_occurences

    def _text_generator(self, next_token=None, emit=lambda x, _, __: x, max_length=None):
        """ loops from the start state to the end state and records the emissions
        Tokens are joint to sentences by looking ahead for the next token type

        emit: by default the markovian emit (see HMM for different emission forms)
        max_length: maximum number of emissions
        """
        assert next_token is not None
        last_tokens = utils.prefilled_buffer(
            self._start_symbol, length=self.order)
        if hasattr(self, "order_emissions"):
            order_emissions = self.order_emissions
        else:
            order_emissions = self.order
        last_emissions = utils.prefilled_buffer(
            self._start_symbol, length=order_emissions)
        generated_tokens = []
        while last_tokens[-1] != self._end_symbol:
            new_token = next_token(last_tokens)
            emission = emit(new_token, last_tokens, last_emissions)
            last_emissions.append(emission)
            generated_tokens.append(emission)
            last_tokens.append(new_token)
            # cut-off for max. emissions
            # + 1 for the start state
            if max_length is not None and max_length + 1 <= len(generated_tokens):
                break
        text = generated_tokens[:-1]
        return text

    def generate_text(self, max_length=None):
        """ Generates sentences from a given corpus
        max_length: maximum number of emissions
        Returns:
            Properly formatted string of generated sentences
        """
        return self._text_generator(next_token=self._generate_next_token, max_length=max_length)

    def _generate_next_token_helper(self, past_states, transitions):
        """ generates next token based previous states """
        key = tuple(past_states)
        assert key in transitions, "%s" % str(key)
        return utils.weighted_choice(transitions[key].items())

    def _generate_next_token(self, past_states):
        """ generates next token based previous words """
        return self._generate_next_token_helper(past_states, self.transitions)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2017 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino <ecino@compassion.ch>
#
#    The licence is in the file __manifest__.py
#
##############################################################################
from odoo import models, api, fields, _
from odoo.tools import safe_eval
from odoo.addons.queue_job.job import job

class GenerateCommunicationWizard(models.TransientModel):
    _name = 'partner.communication.generate.wizard'

    state = fields.Selection([
        ('edit', 'edit'),
        ('preview', 'preview'),
        ('generation', 'generation')
    ], default='edit')
    selection_domain = fields.Char(default="[]")
    partner_ids = fields.Many2many(
        'res.partner', string='Recipients',
        default=lambda s: s._default_partners()
    )
    force_language = fields.Selection('_lang_select')
    model_id = fields.Many2one(
        'partner.communication.config', 'Template',
        domain=[('model', '=', 'res.partner')]
    )
    send_mode = fields.Selection('_send_mode_select', default='physical')
    customize_template = fields.Boolean()
    subject = fields.Char()
    body_html = fields.Html()
    report_id = fields.Many2one(
        'ir.actions.report.xml', 'Letter template',
        domain=[('model', '=', 'partner.communication.job')]
    )
    language_added_in_domain = fields.Boolean()
    preview = fields.Html(readonly=True)
    communication_ids = fields.Many2many(
        'partner.communication.job', 'partner_communication_generation_rel',
        string='Communications')
    progress = fields.Float(compute='_compute_progress')

    ##########################################################################
    #                             FIELDS METHODS                             #
    ##########################################################################
    @api.model
    def _default_partners(self):
        partners = self.env['res.partner'].browse(self.env.context.get(
            'active_ids', []))
        return partners.ids

    @api.model
    def _lang_select(self):
        languages = self.env['res.lang'].search([])
        return [(language.code, language.name) for language in languages]

    @api.model
    def _send_mode_select(self):
        return self.env['partner.communication.job'].send_mode_select()

    @api.multi
    def _compute_progress(self):
        for wizard in self:
            wizard.progress = float(len(wizard.communication_ids) * 100) / (
                len(wizard.partner_ids) or 1)

    ##########################################################################
    #                             VIEW CALLBACKS                             #
    ##########################################################################
    @api.onchange('selection_domain', 'force_language')
    def onchange_domain(self):
        if self.force_language and not self.language_added_in_domain:
            domain = self.selection_domain or '[]'
            domain = domain[:-1] + ", ('lang', '=', '{}')]".format(
                self.force_language)
            self.selection_domain = domain.replace('[, ', '[')
            self.language_added_in_domain = True
        if self.selection_domain:
            self.partner_ids = self.env['res.partner'].search(
                safe_eval(self.selection_domain))
        if not self.force_language:
            self.language_added_in_domain = False

    @api.onchange('model_id')
    def onchange_model_id(self):
        if self.model_id:
            self.body_html = self.model_id.email_template_id.body_html
            self.subject = self.model_id.email_template_id.subject
            send_mode = self.model_id.send_mode.replace('auto_', '')
            if send_mode in [m[0] for m in self._send_mode_select()]:
                self.send_mode = send_mode

    @api.multi
    def get_preview(self):
        comm_model = 'partner.communication.job'
        config = self.model_id or self.env.ref(
            'partner_communication.default_communication')
        partner = self.partner_ids[0]
        template = config.email_template_id.with_context(lang=partner.lang)
        model = template.model or comm_model
        res_preview = self.env[comm_model].create({
            'partner_id': partner.id,
            'config_id': config.id,
            'object_ids': self.env.context.get('object_ids', partner.id),
        })
        self.preview = template.render_template(
            self.body_html, model, res_preview.ids)[res_preview.id]
        res_preview.unlink()
        self.state = 'preview'
        return self.reload()

    @api.multi
    def edit(self):
        self.state = 'edit'
        return self.reload()

    @api.multi
    def generate(self):
        self.state = 'generation'
        if len(self.partner_ids) > 5:
            self.with_delay().generate_communications()
            return self.reload()
        else:
            self.generate_communications(async_mode=False)
            return self.close()

    @api.multi
    def reload(self):
        if not self.exists():
            return True
        return {
            'type': 'ir.actions.act_window',
            'name': _('Generate Communications'),
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'view_type': 'form',
            'context': self._context,
            'target': 'new',
        }

    @api.multi
    def close(self):
        return {
            'type': 'ir.actions.act_window',
            'name': _('Communications'),
            'res_model': 'partner.communication.job',
            'domain': [('id', 'in', self.communication_ids.ids)],
            'view_mode': 'tree,form',
            'view_type': 'form',
            'context': self._context,
        }

    @job
    def generate_communications(self, async_mode=True):
        """ Create the communication records """
        default = self.env.ref('partner_communication.default_communication')
        model = self.model_id or default
        for partner in self.partner_ids:
            vals = {
                'partner_id': partner.id,
                'object_ids': partner.id,
                'config_id': model.id,
                'auto_send': False,
                'send_mode': self.send_mode,
                'report_id': self.report_id.id or model.report_id.id,
            }
            if async_mode:
                self.with_delay().create_communication(vals)
            else:
                self.create_communication(vals)
        return True

    @job(default_channel='root.partner_communication')
    def create_communication(self, vals):
        """ Generate partner communication """
        communication = self.env['partner.communication.job'].create(vals)
        if self.customize_template or not self.model_id:
            default = self.env.ref(
                'partner_communication.default_communication')
            model = self.model_id or default
            template = model.email_template_id.with_context(
                lang=self.force_language or self.env.context.lang)
            new_subject = template.render_template(
                self.subject, 'partner.communication.job',
                communication.ids)
            new_text = template.render_template(
                self.body_html, 'partner.communication.job',
                communication.ids)
            communication.body_html = new_text[communication.id]
            communication.subject = new_subject[communication.id]

        self.communication_ids += communication
<commit_msg>CO-1301 FIX generation communication preview language<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2017 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino <ecino@compassion.ch>
#
#    The licence is in the file __manifest__.py
#
##############################################################################
from odoo import models, api, fields, _
from odoo.tools import safe_eval
from odoo.addons.queue_job.job import job

class GenerateCommunicationWizard(models.TransientModel):
    _name = 'partner.communication.generate.wizard'

    state = fields.Selection([
        ('edit', 'edit'),
        ('preview', 'preview'),
        ('generation', 'generation')
    ], default='edit')
    selection_domain = fields.Char(default="[]")
    partner_ids = fields.Many2many(
        'res.partner', string='Recipients',
        default=lambda s: s._default_partners()
    )
    force_language = fields.Selection('_lang_select')
    model_id = fields.Many2one(
        'partner.communication.config', 'Template',
        domain=[('model', '=', 'res.partner')]
    )
    send_mode = fields.Selection('_send_mode_select', default='physical')
    customize_template = fields.Boolean()
    subject = fields.Char()
    body_html = fields.Html()
    report_id = fields.Many2one(
        'ir.actions.report.xml', 'Letter template',
        domain=[('model', '=', 'partner.communication.job')]
    )
    language_added_in_domain = fields.Boolean()
    preview = fields.Html(readonly=True)
    communication_ids = fields.Many2many(
        'partner.communication.job', 'partner_communication_generation_rel',
        string='Communications')
    progress = fields.Float(compute='_compute_progress')

    ##########################################################################
    #                             FIELDS METHODS                             #
    ##########################################################################
    @api.model
    def _default_partners(self):
        partners = self.env['res.partner'].browse(self.env.context.get(
            'active_ids', []))
        return partners.ids

    @api.model
    def _lang_select(self):
        languages = self.env['res.lang'].search([])
        return [(language.code, language.name) for language in languages]

    @api.model
    def _send_mode_select(self):
        return self.env['partner.communication.job'].send_mode_select()

    @api.multi
    def _compute_progress(self):
        for wizard in self:
            wizard.progress = float(len(wizard.communication_ids) * 100) / (
                len(wizard.partner_ids) or 1)

    ##########################################################################
    #                             VIEW CALLBACKS                             #
    ##########################################################################
    @api.onchange('selection_domain', 'force_language')
    def onchange_domain(self):
        if self.force_language and not self.language_added_in_domain:
            domain = self.selection_domain or '[]'
            domain = domain[:-1] + ", ('lang', '=', '{}')]".format(
                self.force_language)
            self.selection_domain = domain.replace('[, ', '[')
            self.language_added_in_domain = True
        if self.selection_domain:
            self.partner_ids = self.env['res.partner'].search(
                safe_eval(self.selection_domain))
        if not self.force_language:
            self.language_added_in_domain = False

    @api.onchange('model_id')
    def onchange_model_id(self):
        if self.model_id:
            self.body_html = self.model_id.email_template_id.body_html
            self.subject = self.model_id.email_template_id.subject
            send_mode = self.model_id.send_mode.replace('auto_', '')
            if send_mode in [m[0] for m in self._send_mode_select()]:
                self.send_mode = send_mode

    @api.multi
    def get_preview(self):
        comm_model = 'partner.communication.job'
        config = self.model_id or self.env.ref(
            'partner_communication.default_communication')
        partner = self.partner_ids[0]
        template = config.email_template_id.with_context(lang=partner.lang)
        model = template.model or comm_model
        res_preview = self.env[comm_model].create({
            'partner_id': partner.id,
            'config_id': config.id,
            'object_ids': self.env.context.get('object_ids', partner.id),
        })
        self.preview = template.render_template(
            template.body_html, model, res_preview.ids)[res_preview.id]
        res_preview.unlink()
        self.state = 'preview'
        return self.reload()

    @api.multi
    def edit(self):
        self.state = 'edit'
        return self.reload()

    @api.multi
    def generate(self):
        self.state = 'generation'
        if len(self.partner_ids) > 5:
            self.with_delay().generate_communications()
            return self.reload()
        else:
            self.generate_communications(async_mode=False)
            return self.close()

    @api.multi
    def reload(self):
        if not self.exists():
            return True
        return {
            'type': 'ir.actions.act_window',
            'name': _('Generate Communications'),
            'res_model': self._name,
            'res_id': self.id,
            'view_mode': 'form',
            'view_type': 'form',
            'context': self._context,
            'target': 'new',
        }

    @api.multi
    def close(self):
        return {
            'type': 'ir.actions.act_window',
            'name': _('Communications'),
            'res_model': 'partner.communication.job',
            'domain': [('id', 'in', self.communication_ids.ids)],
            'view_mode': 'tree,form',
            'view_type': 'form',
            'context': self._context,
        }

    @job
    def generate_communications(self, async_mode=True):
        """ Create the communication records """
        default = self.env.ref('partner_communication.default_communication')
        model = self.model_id or default
        for partner in self.partner_ids:
            vals = {
                'partner_id': partner.id,
                'object_ids': partner.id,
                'config_id': model.id,
                'auto_send': False,
                'send_mode': self.send_mode,
                'report_id': self.report_id.id or model.report_id.id,
            }
            if async_mode:
                self.with_delay().create_communication(vals)
            else:
                self.create_communication(vals)
        return True

    @job(default_channel='root.partner_communication')
    def create_communication(self, vals):
        """ Generate partner communication """
        communication = self.env['partner.communication.job'].create(vals)
        if self.customize_template or not self.model_id:
            default = self.env.ref(
                'partner_communication.default_communication')
            model = self.model_id or default
            template = model.email_template_id.with_context(
                lang=self.force_language or self.env.context.lang)
            new_subject = template.render_template(
                self.subject, 'partner.communication.job',
                communication.ids)
            new_text = template.render_template(
                self.body_html, 'partner.communication.job',
                communication.ids)
            communication.body_html = new_text[communication.id]
            communication.subject = new_subject[communication.id]

        self.communication_ids += communication
<|endoftext|>