<commit_before>from portality import models
from portality.core import app
import esprit
import re

def rewrite_keywords(journal):
    bib = journal.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
<commit_msg>naming<commit_after>from portality import models
from portality.core import app
import esprit
import re

def rewrite_keywords(journal_like):
    bib = journal_like.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
<|endoftext|>


<commit_before># unit tests for the patch
import mesh.patch as patch
import numpy as np

from nose.tools.nontrivial import with_setup
from numpy.testing import assert_array_equal
from nose.tools import assert_equal

def setup_func():
    print "setup"

def teardown_func():
    pass

# Grid2d tests

@with_setup(setup_func, teardown_func)
def test_dx_dy():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_equal(g.dx, 0.25)
    assert_equal(g.dy, 0.25)

def test_grid_coords():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_array_equal(g.x[g.ilo:g.ihi+1], np.array([0.125, 0.375, 0.625, 0.875]))
    assert_array_equal(g.y[g.jlo:g.jhi+1], np.array([0.125, 0.375, 0.625, 0.875, 1.125, 1.375]))

def test_grid_2d_coords():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_array_equal(g.x, g.x2d[:,g.jc])
    assert_array_equal(g.y, g.y2d[g.ic,:])

def test_course_like():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)
    c = g.coarse_like(2)

    assert_equal(c.nx, 2)
    assert_equal(c.ny, 3)

# CellCenterData2d tests
def test_bcs():

    myg = patch.Grid2d(4,4, ng = 2, xmax=1.0, ymax=1.0)
    myd = patch.CellCenterData2d(myg, dtype=np.int)

    bco = patch.BCObject(xlb="outflow", xrb="outflow",
                         ylb="outflow", yrb="outflow")
    myd.register_var("outflow", bco)

    bcp = patch.BCObject(xlb="periodic", xrb="periodic",
                         ylb="periodic", yrb="periodic")
    myd.register_var("periodic", bcp)

    bcre = patch.BCObject(xlb="reflect-even", xrb="reflect-even",
                          ylb="reflect-even", yrb="reflect-even")
    myd.register_var("reflect-even", bcre)

    bcro = patch.BCObject(xlb="reflect-odd", xrb="reflect-odd",
                          ylb="reflect-odd", yrb="reflect-odd")
    myd.register_var("reflect-odd", bcro)

    myd.create()

    a = myd.get_var("outflow")
    a.v()[:,:] = np.fromfunction(lambda i, j: i+10*j+1, (4,4), dtype=int)

    b = myd.get_var("periodic")
    c = myd.get_var("reflect-even")
    d = myd.get_var("reflect-odd")

    b.d[:,:] = a.d[:,:]
    c.d[:,:] = a.d[:,:]
    d.d[:,:] = a.d[:,:]

    myd.fill_BC("outflow")
    # left ghost
    assert_array_equal(a.d[myg.ilo-1,myg.jlo:myg.jhi+1], np.array([ 1, 11, 21, 31]))
    # right ghost
    assert_array_equal(a.d[myg.ihi+1,myg.jlo:myg.jhi+1], np.array([ 4, 14, 24, 34]))
    # bottom ghost
    assert_array_equal(a.d[myg.ilo:myg.ihi+1,myg.jlo-1], np.array([ 1, 2, 3, 4]))
    # top ghost
    assert_array_equal(a.d[myg.ilo:myg.ihi+1,myg.jhi+1], np.array([31, 32, 33, 34]))

    myd.fill_BC("periodic")
    # x-boundaries
    assert_array_equal(b.d[myg.ilo-1,myg.jlo:myg.jhi+1],
                       b.d[myg.ihi,myg.jlo:myg.jhi+1])
    assert_array_equal(b.d[myg.ilo,myg.jlo:myg.jhi+1],
                       b.d[myg.ihi+1,myg.jlo:myg.jhi+1])
    # y-boundaries
    assert_array_equal(b.d[myg.ilo:myg.ihi+1,myg.jlo-1],
                       b.d[myg.ilo:myg.ihi+1,myg.jhi])
    assert_array_equal(b.d[myg.ilo:myg.ihi+1,myg.jlo],
                       b.d[myg.ilo:myg.ihi+1,myg.jhi+1])

    myd.fill_BC("reflect-even")
    # left -- we'll check 2 ghost cells here -- now we use flipud here
    # because our 'x' is the row index
    # left
    assert_array_equal(c.d[myg.ilo:myg.ilo+2,myg.jlo:myg.ihi+1],
                       np.flipud(c.d[myg.ilo-2:myg.ilo,myg.jlo:myg.jhi+1]))
    # right
    assert_array_equal(c.d[myg.ihi-1:myg.ihi+1,myg.jlo:myg.jhi+1],
                       np.flipud(c.d[myg.ihi+1:myg.ihi+3,myg.jlo:myg.jhi+1]))

    # bottom
    assert_array_equal(c.d[myg.ilo:myg.ihi+1,myg.jlo:myg.jlo+2],
                       np.fliplr(c.d[myg.ilo:myg.ihi+1,myg.jlo-2:myg.jlo]))
    # top
    assert_array_equal(c.d[myg.ilo:myg.ihi+1,myg.jhi-1:myg.jhi+1],
                       np.fliplr(c.d[myg.ilo:myg.ihi+1,myg.jhi+1:myg.jhi+3]))

    myd.fill_BC("reflect-odd")
    # left -- we'll check 2 ghost cells here -- now we use flipud here
    # because our 'x' is the row index
    # left
    assert_array_equal(d.d[myg.ilo:myg.ilo+2,myg.jlo:myg.ihi+1],
                       -np.flipud(d.d[myg.ilo-2:myg.ilo,myg.jlo:myg.jhi+1]))
    # right
    assert_array_equal(d.d[myg.ihi-1:myg.ihi+1,myg.jlo:myg.jhi+1],
                       -np.flipud(d.d[myg.ihi+1:myg.ihi+3,myg.jlo:myg.jhi+1]))

    # bottom
    assert_array_equal(d.d[myg.ilo:myg.ihi+1,myg.jlo:myg.jlo+2],
                       -np.fliplr(d.d[myg.ilo:myg.ihi+1,myg.jlo-2:myg.jlo]))
    # top
    assert_array_equal(d.d[myg.ilo:myg.ihi+1,myg.jhi-1:myg.jhi+1],
                       -np.fliplr(d.d[myg.ilo:myg.ihi+1,myg.jhi+1:myg.jhi+3]))

<commit_msg>more tests<commit_after># unit tests for the patch
import mesh.patch as patch
import numpy as np

from nose.tools.nontrivial import with_setup
from numpy.testing import assert_array_equal
from nose.tools import assert_equal

def setup_func():
    print "setup"

def teardown_func():
    pass

# utilities
def test_buf_split():
    assert_array_equal(patch._buf_split(2), [2, 2, 2, 2])
    assert_array_equal(patch._buf_split((2, 3)), [2, 3, 2, 3])

# Grid2d tests

@with_setup(setup_func, teardown_func)
def test_dx_dy():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_equal(g.dx, 0.25)
    assert_equal(g.dy, 0.25)

def test_grid_coords():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_array_equal(g.x[g.ilo:g.ihi+1], np.array([0.125, 0.375, 0.625, 0.875]))
    assert_array_equal(g.y[g.jlo:g.jhi+1], np.array([0.125, 0.375, 0.625, 0.875, 1.125, 1.375]))

def test_grid_2d_coords():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)

    assert_array_equal(g.x, g.x2d[:,g.jc])
    assert_array_equal(g.y, g.y2d[g.ic,:])

def test_course_like():
    g = patch.Grid2d(4, 6, ng=2, ymax=1.5)
    c = g.coarse_like(2)

    assert_equal(c.nx, 2)
    assert_equal(c.ny, 3)

# CellCenterData2d tests
def test_bcs():

    myg = patch.Grid2d(4,4, ng = 2, xmax=1.0, ymax=1.0)
    myd = patch.CellCenterData2d(myg, dtype=np.int)

    bco = patch.BCObject(xlb="outflow", xrb="outflow",
                         ylb="outflow", yrb="outflow")
    myd.register_var("outflow", bco)

    bcp = patch.BCObject(xlb="periodic", xrb="periodic",
                         ylb="periodic", yrb="periodic")
    myd.register_var("periodic", bcp)

    bcre = patch.BCObject(xlb="reflect-even", xrb="reflect-even",
                          ylb="reflect-even", yrb="reflect-even")
    myd.register_var("reflect-even", bcre)

    bcro = patch.BCObject(xlb="reflect-odd", xrb="reflect-odd",
                          ylb="reflect-odd", yrb="reflect-odd")
    myd.register_var("reflect-odd", bcro)

    myd.create()

    a = myd.get_var("outflow")
    a.v()[:,:] = np.fromfunction(lambda i, j: i+10*j+1, (4,4), dtype=int)

    b = myd.get_var("periodic")
    c = myd.get_var("reflect-even")
    d = myd.get_var("reflect-odd")

    b.d[:,:] = a.d[:,:]
    c.d[:,:] = a.d[:,:]
    d.d[:,:] = a.d[:,:]

    myd.fill_BC("outflow")
    # left ghost
    assert_array_equal(a.d[myg.ilo-1,myg.jlo:myg.jhi+1], np.array([ 1, 11, 21, 31]))
    # right ghost
    assert_array_equal(a.d[myg.ihi+1,myg.jlo:myg.jhi+1], np.array([ 4, 14, 24, 34]))
    # bottom ghost
    assert_array_equal(a.d[myg.ilo:myg.ihi+1,myg.jlo-1], np.array([ 1, 2, 3, 4]))
    # top ghost
    assert_array_equal(a.d[myg.ilo:myg.ihi+1,myg.jhi+1], np.array([31, 32, 33, 34]))

    myd.fill_BC("periodic")
    # x-boundaries
    assert_array_equal(b.d[myg.ilo-1,myg.jlo:myg.jhi+1],
                       b.d[myg.ihi,myg.jlo:myg.jhi+1])
    assert_array_equal(b.d[myg.ilo,myg.jlo:myg.jhi+1],
                       b.d[myg.ihi+1,myg.jlo:myg.jhi+1])
    # y-boundaries
    assert_array_equal(b.d[myg.ilo:myg.ihi+1,myg.jlo-1],
                       b.d[myg.ilo:myg.ihi+1,myg.jhi])
    assert_array_equal(b.d[myg.ilo:myg.ihi+1,myg.jlo],
                       b.d[myg.ilo:myg.ihi+1,myg.jhi+1])

    myd.fill_BC("reflect-even")
    # left -- we'll check 2 ghost cells here -- now we use flipud here
    # because our 'x' is the row index
    # left
    assert_array_equal(c.d[myg.ilo:myg.ilo+2,myg.jlo:myg.ihi+1],
                       np.flipud(c.d[myg.ilo-2:myg.ilo,myg.jlo:myg.jhi+1]))
    # right
    assert_array_equal(c.d[myg.ihi-1:myg.ihi+1,myg.jlo:myg.jhi+1],
                       np.flipud(c.d[myg.ihi+1:myg.ihi+3,myg.jlo:myg.jhi+1]))

    # bottom
    assert_array_equal(c.d[myg.ilo:myg.ihi+1,myg.jlo:myg.jlo+2],
                       np.fliplr(c.d[myg.ilo:myg.ihi+1,myg.jlo-2:myg.jlo]))
    # top
    assert_array_equal(c.d[myg.ilo:myg.ihi+1,myg.jhi-1:myg.jhi+1],
                       np.fliplr(c.d[myg.ilo:myg.ihi+1,myg.jhi+1:myg.jhi+3]))

    myd.fill_BC("reflect-odd")
    # left -- we'll check 2 ghost cells here -- now we use flipud here
    # because our 'x' is the row index
    # left
    assert_array_equal(d.d[myg.ilo:myg.ilo+2,myg.jlo:myg.ihi+1],
                       -np.flipud(d.d[myg.ilo-2:myg.ilo,myg.jlo:myg.jhi+1]))
    # right
    assert_array_equal(d.d[myg.ihi-1:myg.ihi+1,myg.jlo:myg.jhi+1],
                       -np.flipud(d.d[myg.ihi+1:myg.ihi+3,myg.jlo:myg.jhi+1]))

    # bottom
    assert_array_equal(d.d[myg.ilo:myg.ihi+1,myg.jlo:myg.jlo+2],
                       -np.fliplr(d.d[myg.ilo:myg.ihi+1,myg.jlo-2:myg.jlo]))
    # top
    assert_array_equal(d.d[myg.ilo:myg.ihi+1,myg.jhi-1:myg.jhi+1],
                       -np.fliplr(d.d[myg.ilo:myg.ihi+1,myg.jhi+1:myg.jhi+3]))

<|endoftext|>


<commit_before>import os

def get_report_list():
    ''' creates reports to show user '''
    curpath = os.getcwd()
    newpath = curpath + '/reports/'

    report_list = []

    for report in os.listdir(newpath):
        if report.startswith('site_report-') and report.endswith('.json'):
            # print (report)
            report_list.append(report)

    return report_list

if __name__ == "__main__":
    pass<commit_msg>Refactoring slightly and fixing docstrings<commit_after>import os

def get_report_list():
    ''' lists all site reports in the reports directory '''
    report_list = []

    for report in os.listdir(os.getcwd() + '/reports/'):
        if report.startswith('site_report-') and report.endswith('.json'):
            report_list.append(report)

    return report_list
<|endoftext|>


<commit_before>"""Support for SleepIQ Sensor."""
from __future__ import annotations

from asyncsleepiq import SleepIQBed, SleepIQSleeper

from homeassistant.components.sensor import SensorEntity
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator

from .const import DOMAIN, PRESSURE, SLEEP_NUMBER
from .coordinator import SleepIQData
from .entity import SleepIQSleeperEntity

SENSORS = [PRESSURE, SLEEP_NUMBER]

async def async_setup_entry(
    hass: HomeAssistant,
    entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
) -> None:
    """Set up the SleepIQ bed sensors."""
    data: SleepIQData = hass.data[DOMAIN][entry.entry_id]
    async_add_entities(
        SleepIQSensorEntity(data.data_coordinator, bed, sleeper, sensor_type)
        for bed in data.client.beds.values()
        for sleeper in bed.sleepers
        for sensor_type in SENSORS
    )

class SleepIQSensorEntity(SleepIQSleeperEntity, SensorEntity):
    """Representation of an SleepIQ Entity with CoordinatorEntity."""

    _attr_icon = "mdi:bed"

    def __init__(
        self,
        coordinator: DataUpdateCoordinator,
        bed: SleepIQBed,
        sleeper: SleepIQSleeper,
        sensor_type: str,
    ) -> None:
        """Initialize the sensor."""
        self.sensor_type = sensor_type
        super().__init__(coordinator, bed, sleeper, sensor_type)

    @callback
    def _async_update_attrs(self) -> None:
        """Update sensor attributes."""
        self._attr_native_value = getattr(self.sleeper, self.sensor_type)
<commit_msg>Add sensor state class for SleepIQ sensors (#76372)<commit_after>"""Support for SleepIQ Sensor."""
from __future__ import annotations

from asyncsleepiq import SleepIQBed, SleepIQSleeper

from homeassistant.components.sensor import SensorEntity, SensorStateClass
from homeassistant.config_entries import ConfigEntry
from homeassistant.core import HomeAssistant, callback
from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator

from .const import DOMAIN, PRESSURE, SLEEP_NUMBER
from .coordinator import SleepIQData
from .entity import SleepIQSleeperEntity

SENSORS = [PRESSURE, SLEEP_NUMBER]

async def async_setup_entry(
    hass: HomeAssistant,
    entry: ConfigEntry,
    async_add_entities: AddEntitiesCallback,
) -> None:
    """Set up the SleepIQ bed sensors."""
    data: SleepIQData = hass.data[DOMAIN][entry.entry_id]
    async_add_entities(
        SleepIQSensorEntity(data.data_coordinator, bed, sleeper, sensor_type)
        for bed in data.client.beds.values()
        for sleeper in bed.sleepers
        for sensor_type in SENSORS
    )

class SleepIQSensorEntity(SleepIQSleeperEntity, SensorEntity):
    """Representation of an SleepIQ Entity with CoordinatorEntity."""

    _attr_icon = "mdi:bed"

    def __init__(
        self,
        coordinator: DataUpdateCoordinator,
        bed: SleepIQBed,
        sleeper: SleepIQSleeper,
        sensor_type: str,
    ) -> None:
        """Initialize the sensor."""
        self.sensor_type = sensor_type
        self._attr_state_class = SensorStateClass.MEASUREMENT
        super().__init__(coordinator, bed, sleeper, sensor_type)

    @callback
    def _async_update_attrs(self) -> None:
        """Update sensor attributes."""
        self._attr_native_value = getattr(self.sleeper, self.sensor_type)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals, absolute_import

from django.contrib.staticfiles.storage import staticfiles_storage
from django.template.loader import render_to_string

from pipeline.conf import settings
from pipeline.utils import guess_type
from pipeline.packager import Packager, PackageNotFound

from django_jinja import library, utils
lib = library.Library()

@lib.global_function
@utils.safe
def compressed_css(name):
    package = settings.PIPELINE_CSS.get(name, {})
    if package:
        package = {name: package}

    packager = Packager(css_packages=package, js_packages={})

    try:
        package = packager.package_for('css', name)
    except PackageNotFound:
        return ""

    def _render_css(path):
        template_name = package.template_name or "pipeline/css.jinja"

        context = package.extra_context
        context.update({
            'type': guess_type(path, 'text/css'),
            'url': staticfiles_storage.url(path)
        })

        return render_to_string(template_name, context)

    if not settings.DEBUG:
        return _render_css(package.output_filename)

    paths = packager.compile(package.paths)
    tags = [_render_css(path) for path in paths]

    return '\n'.join(tags)

@lib.global_function
@utils.safe
def compressed_js(name):
    package = settings.PIPELINE_JS.get(name, {})
    if package:
        package = {name: package}

    packager = Packager(css_packages={}, js_packages=package)
    try:
        package = packager.package_for('js', name)
    except PackageNotFound:
        return ""

    def _render_js(path):
        template_name = package.template_name or "pipeline/js.jinja"
        context = package.extra_context
        context.update({
            'type': guess_type(path, 'text/javascript'),
            'url': staticfiles_storage.url(path),
        })
        return render_to_string(template_name, context)

    def _render_inline_js(js):
        context = package.extra_context
        context.update({
            'source': js
        })
        return render_to_string("pipeline/inline_js.jinja", context)

    if not settings.DEBUG:
        return _render_js(package.output_filename)

    paths = packager.compile(package.paths)
    templates = packager.pack_templates(package)
    tags = [_render_js(js) for js in paths]

    if templates:
        tags.append(_render_inline(templates))

    return '\n'.join(tags)
<commit_msg>settings.PIPELINE_ENABLED<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals, absolute_import

from django.contrib.staticfiles.storage import staticfiles_storage
from django.template.loader import render_to_string

from pipeline.conf import settings
from pipeline.utils import guess_type
from pipeline.packager import Packager, PackageNotFound

from django_jinja import library, utils
lib = library.Library()

@lib.global_function
@utils.safe
def compressed_css(name):
    package = settings.PIPELINE_CSS.get(name, {})
    if package:
        package = {name: package}

    packager = Packager(css_packages=package, js_packages={})

    try:
        package = packager.package_for('css', name)
    except PackageNotFound:
        return ""

    def _render_css(path):
        template_name = package.template_name or "pipeline/css.jinja"

        context = package.extra_context
        context.update({
            'type': guess_type(path, 'text/css'),
            'url': staticfiles_storage.url(path)
        })

        return render_to_string(template_name, context)

    if settings.PIPELINE_ENABLED:
        return _render_css(package.output_filename)

    paths = packager.compile(package.paths)
    tags = [_render_css(path) for path in paths]

    return '\n'.join(tags)

@lib.global_function
@utils.safe
def compressed_js(name):
    package = settings.PIPELINE_JS.get(name, {})
    if package:
        package = {name: package}

    packager = Packager(css_packages={}, js_packages=package)
    try:
        package = packager.package_for('js', name)
    except PackageNotFound:
        return ""

    def _render_js(path):
        template_name = package.template_name or "pipeline/js.jinja"
        context = package.extra_context
        context.update({
            'type': guess_type(path, 'text/javascript'),
            'url': staticfiles_storage.url(path),
        })
        return render_to_string(template_name, context)

    def _render_inline_js(js):
        context = package.extra_context
        context.update({
            'source': js
        })
        return render_to_string("pipeline/inline_js.jinja", context)

    if settings.PIPELINE_ENABLED:
        return _render_js(package.output_filename)

    paths = packager.compile(package.paths)
    templates = packager.pack_templates(package)
    tags = [_render_js(js) for js in paths]

    if templates:
        tags.append(_render_inline_js(templates))

    return '\n'.join(tags)
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.http import JsonResponse, HttpResponse
import http.client as http
import json

from backend.hashids import Hashids
from v1.models import *

def hash_email(email):
    # TODO: use real hashing..
    return email

def poll(request, poll_id):
    try:
        poll = Poll.objects.get(hash_id=poll_id)
        # PARANOID: possible fuckup - more than one object returned
    except Poll.DoesNotExist:
        return HttpResponse('', status=http.NOT_FOUND)
    try:
        key = request.GET['key']
    except KeyError:
        return HttpResponse('', status=http.BAD_REQUEST)

    try:
        poll.allowed_hashes.get(value=key)
    except VotingHash.DoesNotExist:
        return HttpResponse('', status=http.UNAUTHORIZED)

    return JsonResponse(poll.json_dict())

def create(request):
    if request.method != 'POST':
        return HttpResponse('', status=http.BAD_REQUEST)
    try:
        # PARANOID: is this safe?
        data = json.loads(request.body.decode('utf-8'))
        question = str(data['question'])
        choices = list(map(str, data['choices']))
        recipients = list(map(str, data['recipients']))
    except (TypeError, ValueError, KeyError):
        return HttpResponse('', status=http.BAD_REQUEST)

    ballot = Ballot()
    ballot.question = question
    ballot.save()

    for choice_text in choices:
        choice = Choice()
        choice.value = choice_text
        choice.save()
        ballot.choices.add(choice)

    ballot.save()

    poll = Poll()
    poll.ballot = ballot
    poll.save()

    for recipient_text in recipients:
        recipient = VotingHash()
        recipient.value = hash_email(recipient_text)
        recipient.save()
        poll.allowed_hashes.add(recipient)

    poll.hash_id = Hashids().encode(poll.id)
    poll.save()

    return JsonResponse({'id': poll.hash_id})
<commit_msg>added email hashing<commit_after>from django.shortcuts import render
from django.http import JsonResponse, HttpResponse
import http.client as http
import json
import hashlib

from backend.hashids import Hashids
from v1.models import *

def hash_email(email):
    return hashlib.sha256(email.encode('utf-8')).hexdigest()

def poll(request, poll_id):
    try:
        poll = Poll.objects.get(hash_id=poll_id)
        # PARANOID: possible fuckup - more than one object returned
    except Poll.DoesNotExist:
        return HttpResponse('', status=http.NOT_FOUND)
    try:
        key = request.GET['key']
    except KeyError:
        return HttpResponse('', status=http.BAD_REQUEST)

    try:
        poll.allowed_hashes.get(value=key)
    except VotingHash.DoesNotExist:
        return HttpResponse('', status=http.UNAUTHORIZED)

    return JsonResponse(poll.json_dict())

def create(request):
    if request.method != 'POST':
        return HttpResponse('', status=http.BAD_REQUEST)
    try:
        # PARANOID: is this safe?
        data = json.loads(request.body.decode('utf-8'))
        question = str(data['question'])
        choices = list(map(str, data['choices']))
        recipients = list(map(str, data['recipients']))
    except (TypeError, ValueError, KeyError):
        return HttpResponse('', status=http.BAD_REQUEST)

    ballot = Ballot()
    ballot.question = question
    ballot.save()

    for choice_text in choices:
        choice = Choice()
        choice.value = choice_text
        choice.save()
        ballot.choices.add(choice)

    ballot.save()

    poll = Poll()
    poll.ballot = ballot
    poll.save()

    for recipient_text in recipients:
        recipient = VotingHash()
        recipient.value = hash_email(recipient_text)
        recipient.save()
        poll.allowed_hashes.add(recipient)

    poll.hash_id = Hashids().encode(poll.id)
    poll.save()

    return JsonResponse({'id': poll.hash_id})
<|endoftext|>


<commit_before>from .base import *

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'CHANGEME!!!'

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

# Process all tasks synchronously.
# Helpful for local development and running tests
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_ALWAYS_EAGER = True

try:
    from .local import *
except ImportError:
    pass
<commit_msg>Update TEMPLATE_DEBUG to Django 1.8 version<commit_after>from .base import *

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATES[0]['OPTIONS']['debug'] = True

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'CHANGEME!!!'

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

# Process all tasks synchronously.
# Helpful for local development and running tests
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_ALWAYS_EAGER = True

try:
    from .local import *
except ImportError:
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import sys
from uuid import uuid4

from collections import namedtuple

from django.conf import settings
from django.core.mail import EmailMessage, EmailMultiAlternatives
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import python_2_unicode_compatible
from post_office.fields import CommaSeparatedEmailField

try:
    from django.utils.encoding import smart_text  # For Django >= 1.5
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text

from django.template import Context, Template

from jsonfield import JSONField
from post_office import cache
from .compat import text_type
from .connections import connections
from .settings import context_field_class, get_log_level
from .validators import validate_email_with_name, validate_template_syntax

PRIORITY = namedtuple('PRIORITY', 'low medium high now')._make(range(4))
STATUS = namedtuple('STATUS', 'sent failed queued')._make(range(3))

@python_2_unicode_compatible
class Email(models.Model):
    """
    A model to hold email information.
    """

    PRIORITY_CHOICES = [(PRIORITY.low, _("low")), (PRIORITY.medium, _("medium")),
                        (PRIORITY.high, _("high")), (PRIORITY.now, _("now"))]
    STATUS_CHOICES = [(STATUS.sent, _("sent")), (STATUS.failed, _("failed")),
                      (STATUS.queued, _("queued"))]

    from_email = models.CharField(_("Email From"), max_length=254,
                                  validators=[validate_email_with_name])
    to = CommaSeparatedEmailField(_("Email To"))
    cc = CommaSeparatedEmailField(_("Cc"))
    bcc = CommaSeparatedEmailField(("Bcc"))
    subject = models.CharField(_("Subject"), max_length=255, blank=True)
    message = models.TextField(_("Message"), blank=True)
    html_message = models.TextField(_("HTML Message"), blank=True)
    """
    Emails with 'queued' status will get processed by ``send_queued`` command.
    Status field will then be set to ``failed`` or ``sent`` depending on
    whether it's successfully delivered.
    """
    status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES, db_index=True,
                                              blank=True, null=True)
    priority = models.PositiveSmallIntegerField(choices=PRIORITY_CHOICES,
                                                blank=True, null=True)
    created = models.DateTimeField(auto_now_add=True, db_index=True)
    last_updated = models.DateTimeField(db_index=True, auto_now=True)
    scheduled_time = models.DateTimeField(blank=True, null=True, db_index=True)
    headers = JSONField(blank=True, null=True)
    template = models.ForeignKey('post_office.EmailTemplate', blank=True, null=True)
    context = context_field_class(blank=True, null=True)
    backend_alias = models.CharField(blank=True, default='', max_length=64)

    class Meta:
        app_label = 'post_office'

    def __str__(self):
        return u'%s' % self.to

    def email_message(self, connection=None):
        """
        Returns a django ``EmailMessage`` or ``EmailMultiAlternatives`` object,
        depending on whether html_message is empty.
        """
        subject = smart_text(self.subject)

        if self.template is not None:
            _context = Context(self.context)
            subject = Template(self.template.subject).render(_context)
            message = Template(self.template.content).render(_context)
            html_message = Template(self.template.html_content).render(_context)

        else:
            subject = self.subject
            message = self.message
            html_message = self.html_message

        if html_message:
            msg = EmailMultiAlternatives(
                subject=subject, body=message, from_email=self.from_email,
                to=self.to, bcc=self.bcc, cc=self.cc,
                connection=connection, headers=self.headers)
            msg.attach_alternative(html_message, "text/html")
        else:
            msg = EmailMessage(
                subject=subject, body=message, from_email=self.from_email,
                to=self.to, bcc=self.bcc, cc=self.cc,
                connection=connection, headers=self.headers)

        for attachment in self.attachments.all():
            msg.attach(attachment.name, attachment.file.read())

        return msg

    def dispatch(self, log_level=None, disconnect_after_delivery=True):
        """
        Actually send out the email and log the result
        """

        if log_level is None:
            log_level = get_log_level()

        connection = None
        try:
            connection = connections[self.backend_alias or 'default']
            self.email_message(connection=connection).send()
            status = STATUS.sent
            message = ''
            exception_type = ''
        except:
            status = STATUS.failed
            exception, message, _ = sys.exc_info()
            exception_type = exception.__name__

        if connection and disconnect_after_delivery:
            connection.close()

        self.status = status
        self.save()

        # If log level is 0, log nothing, 1 logs only sending failures
        # and 2 means log both successes and failures
        if log_level == 1:
            if status == STATUS.failed:
                self.logs.create(status=status, message=message,
                                 exception_type=exception_type)
        elif log_level == 2:
            self.logs.create(status=status, message=message,
                             exception_type=exception_type)

        return status

    def save(self, *args, **kwargs):
        self.full_clean()
        return super(Email, self).save(*args, **kwargs)

@python_2_unicode_compatible
class Log(models.Model):
    """
    A model to record sending email sending activities.
    """

    STATUS_CHOICES = [(STATUS.sent, _("sent")), (STATUS.failed, _("failed"))]

    email = models.ForeignKey(Email, editable=False, related_name='logs')
    date = models.DateTimeField(auto_now_add=True)
    status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES)
    exception_type = models.CharField(max_length=255, blank=True)
    message = models.TextField()

    class Meta:
        app_label = 'post_office'

    def __str__(self):
        return text_type(self.date)

@python_2_unicode_compatible
class EmailTemplate(models.Model):
    """
    Model to hold template information from db
    """
    name = models.CharField(max_length=255, help_text=_("e.g: 'welcome_email'"))
    description = models.TextField(blank=True,
        help_text=_("Description of this template."))
    created = models.DateTimeField(auto_now_add=True)
    last_updated = models.DateTimeField(auto_now=True)
    subject = models.CharField(max_length=255, blank=True,
        verbose_name=_("Subject"), validators=[validate_template_syntax])
    content = models.TextField(blank=True,
        verbose_name=_("Content"), validators=[validate_template_syntax])
    html_content = models.TextField(blank=True,
        verbose_name=_("HTML content"), validators=[validate_template_syntax])
    language = models.CharField(max_length=12, choices=settings.LANGUAGES,
        help_text=_("Render template in alternative language"),
        default='', blank=True)
    default_template = models.ForeignKey('self', related_name='translated_templates',
        null=True, default=None)

    class Meta:
        app_label = 'post_office'
        unique_together = ('language', 'default_template')
        verbose_name = _("Email Template")
        verbose_name_plural = _("Email Templates")

    def __str__(self):
        return u'%s %s' % (self.name, self.language)

    def save(self, *args, **kwargs):
        # If template is a translation, use default template's name
        if self.default_template and not self.name:
            self.name = self.default_template.name

        template = super(EmailTemplate, self).save(*args, **kwargs)
        cache.delete(self.name)
        return template

def get_upload_path(instance, filename):
    """Overriding to store the original filename"""
    if not instance.name:
        instance.name = filename  # set original filename

    filename = '{name}.{ext}'.format(name=uuid4().hex,
                                     ext=filename.split('.')[-1])

    return 'post_office_attachments/' + filename

class Attachment(models.Model):
    """
    A model describing an email attachment.
    """
    file = models.FileField(upload_to=get_upload_path)
    name = models.CharField(max_length=255, help_text=_("The original filename"))
    emails = models.ManyToManyField(Email, related_name='attachments')

    class Meta:
        app_label = 'post_office'
<commit_msg>__str__() method for Attachment<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import sys
from uuid import uuid4

from collections import namedtuple

from django.conf import settings
from django.core.mail import EmailMessage, EmailMultiAlternatives
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import python_2_unicode_compatible
from post_office.fields import CommaSeparatedEmailField

try:
    from django.utils.encoding import smart_text  # For Django >= 1.5
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text

from django.template import Context, Template

from jsonfield import JSONField
from post_office import cache
from .compat import text_type
from .connections import connections
from .settings import context_field_class, get_log_level
from .validators import validate_email_with_name, validate_template_syntax

PRIORITY = namedtuple('PRIORITY', 'low medium high now')._make(range(4))
STATUS = namedtuple('STATUS', 'sent failed queued')._make(range(3))

@python_2_unicode_compatible
class Email(models.Model):
    """
    A model to hold email information.
    """

    PRIORITY_CHOICES = [(PRIORITY.low, _("low")), (PRIORITY.medium, _("medium")),
                        (PRIORITY.high, _("high")), (PRIORITY.now, _("now"))]
    STATUS_CHOICES = [(STATUS.sent, _("sent")), (STATUS.failed, _("failed")),
                      (STATUS.queued, _("queued"))]

    from_email = models.CharField(_("Email From"), max_length=254,
                                  validators=[validate_email_with_name])
    to = CommaSeparatedEmailField(_("Email To"))
    cc = CommaSeparatedEmailField(_("Cc"))
    bcc = CommaSeparatedEmailField(("Bcc"))
    subject = models.CharField(_("Subject"), max_length=255, blank=True)
    message = models.TextField(_("Message"), blank=True)
    html_message = models.TextField(_("HTML Message"), blank=True)
    """
    Emails with 'queued' status will get processed by ``send_queued`` command.
    Status field will then be set to ``failed`` or ``sent`` depending on
    whether it's successfully delivered.
    """
    status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES, db_index=True,
                                              blank=True, null=True)
    priority = models.PositiveSmallIntegerField(choices=PRIORITY_CHOICES,
                                                blank=True, null=True)
    created = models.DateTimeField(auto_now_add=True, db_index=True)
    last_updated = models.DateTimeField(db_index=True, auto_now=True)
    scheduled_time = models.DateTimeField(blank=True, null=True, db_index=True)
    headers = JSONField(blank=True, null=True)
    template = models.ForeignKey('post_office.EmailTemplate', blank=True, null=True)
    context = context_field_class(blank=True, null=True)
    backend_alias = models.CharField(blank=True, default='', max_length=64)

    class Meta:
        app_label = 'post_office'

    def __str__(self):
        return u'%s' % self.to

    def email_message(self, connection=None):
        """
        Returns a django ``EmailMessage`` or ``EmailMultiAlternatives`` object,
        depending on whether html_message is empty.
        """
        subject = smart_text(self.subject)

        if self.template is not None:
            _context = Context(self.context)
            subject = Template(self.template.subject).render(_context)
            message = Template(self.template.content).render(_context)
            html_message = Template(self.template.html_content).render(_context)

        else:
            subject = self.subject
            message = self.message
            html_message = self.html_message

        if html_message:
            msg = EmailMultiAlternatives(
                subject=subject, body=message, from_email=self.from_email,
                to=self.to, bcc=self.bcc, cc=self.cc,
                connection=connection, headers=self.headers)
            msg.attach_alternative(html_message, "text/html")
        else:
            msg = EmailMessage(
                subject=subject, body=message, from_email=self.from_email,
                to=self.to, bcc=self.bcc, cc=self.cc,
                connection=connection, headers=self.headers)

        for attachment in self.attachments.all():
            msg.attach(attachment.name, attachment.file.read())

        return msg

    def dispatch(self, log_level=None, disconnect_after_delivery=True):
        """
        Actually send out the email and log the result
        """

        if log_level is None:
            log_level = get_log_level()

        connection = None
        try:
            connection = connections[self.backend_alias or 'default']
            self.email_message(connection=connection).send()
            status = STATUS.sent
            message = ''
            exception_type = ''
        except:
            status = STATUS.failed
            exception, message, _ = sys.exc_info()
            exception_type = exception.__name__

        if connection and disconnect_after_delivery:
            connection.close()

        self.status = status
        self.save()

        # If log level is 0, log nothing, 1 logs only sending failures
        # and 2 means log both successes and failures
        if log_level == 1:
            if status == STATUS.failed:
                self.logs.create(status=status, message=message,
                                 exception_type=exception_type)
        elif log_level == 2:
            self.logs.create(status=status, message=message,
                             exception_type=exception_type)

        return status

    def save(self, *args, **kwargs):
        self.full_clean()
        return super(Email, self).save(*args, **kwargs)

@python_2_unicode_compatible
class Log(models.Model):
    """
    A model to record sending email sending activities.
    """

    STATUS_CHOICES = [(STATUS.sent, _("sent")), (STATUS.failed, _("failed"))]

    email = models.ForeignKey(Email, editable=False, related_name='logs')
    date = models.DateTimeField(auto_now_add=True)
    status = models.PositiveSmallIntegerField(choices=STATUS_CHOICES)
    exception_type = models.CharField(max_length=255, blank=True)
    message = models.TextField()

    class Meta:
        app_label = 'post_office'

    def __str__(self):
        return text_type(self.date)

@python_2_unicode_compatible
class EmailTemplate(models.Model):
    """
    Model to hold template information from db
    """
    name = models.CharField(max_length=255, help_text=_("e.g: 'welcome_email'"))
    description = models.TextField(blank=True,
        help_text=_("Description of this template."))
    created = models.DateTimeField(auto_now_add=True)
    last_updated = models.DateTimeField(auto_now=True)
    subject = models.CharField(max_length=255, blank=True,
        verbose_name=_("Subject"), validators=[validate_template_syntax])
    content = models.TextField(blank=True,
        verbose_name=_("Content"), validators=[validate_template_syntax])
    html_content = models.TextField(blank=True,
        verbose_name=_("HTML content"), validators=[validate_template_syntax])
    language = models.CharField(max_length=12, choices=settings.LANGUAGES,
        help_text=_("Render template in alternative language"),
        default='', blank=True)
    default_template = models.ForeignKey('self', related_name='translated_templates',
        null=True, default=None)

    class Meta:
        app_label = 'post_office'
        unique_together = ('language', 'default_template')
        verbose_name = _("Email Template")
        verbose_name_plural = _("Email Templates")

    def __str__(self):
        return u'%s %s' % (self.name, self.language)

    def save(self, *args, **kwargs):
        # If template is a translation, use default template's name
        if self.default_template and not self.name:
            self.name = self.default_template.name

        template = super(EmailTemplate, self).save(*args, **kwargs)
        cache.delete(self.name)
        return template

def get_upload_path(instance, filename):
    """Overriding to store the original filename"""
    if not instance.name:
        instance.name = filename  # set original filename

    filename = '{name}.{ext}'.format(name=uuid4().hex,
                                     ext=filename.split('.')[-1])

    return 'post_office_attachments/' + filename

@python_2_unicode_compatible
class Attachment(models.Model):
    """
    A model describing an email attachment.
    """
    file = models.FileField(upload_to=get_upload_path)
    name = models.CharField(max_length=255, help_text=_("The original filename"))
    emails = models.ManyToManyField(Email, related_name='attachments')

    class Meta:
        app_label = 'post_office'

    def __str__(self):
        return self.name
<|endoftext|>


<commit_before>import requests
import re
import os
base = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../')
import rethinkdb as r
conn = r.connect( "localhost", 28015).repl()
db = r.db('public')
organization_id = list(db.table('organizations').filter({'name': 'Seattle Police Department'}).run(conn))[0]['id']

def parse_txt_files(txt_files=None):
    import re
    import os
    import dateutil
    if not txt_files:
        txt_files = sorted([f for f in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts')])
    opa_files = []
    for filename in txt_files:
    #for filename in files:
        f = open(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts/'+filename, 'r')

        opa_file = str(f.read()).replace('\n', ' ').replace('  ', ' ')
        if not opa_file:
            continue
        print opa_file
        opa_file_dict = {'organization_id': organization_id}
        regex =  re.search('Complaint Number(?P<num>.*?)Issued', opa_file)

        opa_file_dict['Complaint number'] = regex.group('num').strip(' :').replace(' ', '') if regex else filename[:filename.find('ccs')]
        regex = re.search('Issued Date(?P<date>.*?)Named', opa_file)
        opa_file_dict['Issued date'] = regex.group('date').strip(' :') if regex else None
        regex = re.search('lssued Date(?P<date>.*?)Named', opa_file)
        if regex:
            opa_file_dict['Issued date'] = regex.group('date').strip(' :')

        regex = re.search('OPA Finding(?P<findings>.*?)Final', opa_file)
        opa_file_dict['OPA finding'] = regex.group('findings').strip(' :') if regex else None

        regex = re.search('FINDINGS(?P<findings>.*?)NOTE', opa_file,  re.MULTILINE|re.DOTALL)
        if regex:
            opa_file_dict['Findings'] = regex.group('findings').strip(' :\n').replace('\n', '')
        regex = re.search('Final Discipline\s*(?P<discipline>.*?)INCIDENT', opa_file)
        opa_file_dict['Final discipline'] = regex.group('discipline').strip(' :') if regex else None
        regex = re.search('INCIDENT SYNOPSIS(?P<synopsis>.*?)(Complaint|COMPLAINT)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Incident synopsis'] = regex.group('synopsis').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('COMPLAINT(?P<complaint>.*?)(Complaint|INVESTIGATION)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Complaint'] = regex.group('complaint').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('INVESTIGATION(?P<investigation>.*?)(Complaint|ANALYSIS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Investigation'] = regex.group('investigation').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('ANALYSIS AND CONCLUSION(?P<analysis>.*?)(Complaint|FINDINGS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Analysis and conclusion'] = regex.group('analysis').strip(' :\n').replace('\n', '') if regex else None
        opa_file_dict['txt'] = filename
        opa_file_dict['pdf'] = filename[:-4]+'.pdf'
        try:
            # Get the entire findings section
            f = open('txts/'+filename, 'r')

            opa_file2 = f.read()
            import re
            m = re.search('Issued(.*?)Complaint', opa_file2,  re.MULTILINE|re.DOTALL)
            if m:
                section = m.group()
                lines = section.split('\n')[3:-3]
                structured_version = []
                current = None
                opa_file_dict['is_sustained'] = False
                for i, line in enumerate(lines):
                    if line.startswith('Named Employee'):
                        if current:
                            structured_version.append(current)
                        current = {'heading': line.strip(), 'allegations': []}
                    if line.startswith('Allegation'):
                        allegation_structure = {'heading': line.strip()}
                        allegation_structure['allegation'] = lines[i+2].strip()
                        j = 0
                        while True:
                            j += 1
                            the_line = lines[i + 2 + j]
                            if not the_line:
                                break
                            allegation_structure['allegation'] += ' '+the_line.strip()

                        current['allegations'].append(allegation_structure)
                    if line.startswith('OPA Finding'):
                        current['allegations'][-1]['opa_finding'] = lines[i+2].strip()
                        if current['allegations'][-1]['opa_finding'] == 'Sustained':
                            opa_file_dict['is_sustained'] = True
                    if line.startswith('Final Discipline'):
                        current['final_discipline'] = lines[i+2].strip()
                structured_version.append(current)
            opa_file_dict['Summarized results'] = structured_version
        except Exception, e:
            opa_file_dict['parsing_error'] = str(e)
        opa_files.append(opa_file_dict)
        f.close()
    import sys

    reload(sys)
    sys.setdefaultencoding('utf8')
    print opa_files

    db.table('police_internal_affairs_cases').insert(opa_files, conflict='update').run(conn)
    the_html = '
'
    columns = ['Issued date', 'Complaint number', 'Complaint', 'Incident synopsis', 'Investigation', 'Analysis and conclusion', 'OPA finding', 'Final discipline']
    from dateutil.parser import parse
    import datetime
    mindate = datetime.date(datetime.MINYEAR, 1, 1)
    opa_files = sorted(opa_files, key=lambda x: parse(x['Issued date']).date() if '/' in x['Issued date'] else mindate, reverse=True)

def download():
    import os
    os.system('mkdir %s.crawler_data' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/' % (base))
    html = requests.get('http://www.seattle.gov/opa/closed-case-summaries').text
    os.system('mkdir pdfs')
    files = re.findall('ClosedCaseSummaries/(?P<filename>.*?)\.pdf', html)
    print files
    for f in files:
        if not f+'.pdf' in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs'):
            os.system('wget -O %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/%s.pdf http://www.seattle.gov/Documents/Departments/OPA/ClosedCaseSummaries/%s.pdf' % (base, f, f))
    import os
    files = sorted([f for f in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs')])
    new_files = []
    for filename in files:
        print filename
        if not filename[:-4]+'.txt' in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts'):
            print 'converting'
            os.system('pdf2txt.py -A %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/%s > %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/%s' % (base, filename, base, filename[:-4]+'.txt'))
            new_files.append(filename[:-4]+'.txt')
    parse_txt_files(new_files)

if __name__ == "__main__":
    import sys
    if len(sys.argv) > 1:
        parse_txt_files()
    else:
        download()
<commit_msg>auto commit<commit_after>import requests
import re
import os
base = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../')
import rethinkdb as r
conn = r.connect( "localhost", 28015).repl()
db = r.db('public')
organization_id = list(db.table('organizations').filter({'name': 'Seattle Police Department'}).run(conn))[0]['id']

def parse_txt_files(txt_files=None):
    import re
    import os
    import dateutil
    if not txt_files:
        txt_files = sorted([f for f in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts')])
    opa_files = []
    for filename in txt_files:
    #for filename in files:
        f = open(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts/'+filename, 'r')

        opa_file = str(f.read()).replace('\n', ' ').replace('  ', ' ')
        if not opa_file:
            continue
        print opa_file
        opa_file_dict = {'organization_id': organization_id}
        regex =  re.search('Complaint Number(?P<num>.*?)Issued', opa_file)

        opa_file_dict['Complaint number'] = regex.group('num').strip(' :').replace(' ', '') if regex else filename[:filename.find('ccs')]
        regex = re.search('Issued Date(?P<date>.*?)Named', opa_file)
        opa_file_dict['Issued date'] = regex.group('date').strip(' :') if regex else None
        regex = re.search('lssued Date(?P<date>.*?)Named', opa_file)
        if regex:
            opa_file_dict['Issued date'] = regex.group('date').strip(' :')

        regex = re.search('OPA Finding(?P<findings>.*?)Final', opa_file)
        opa_file_dict['OPA finding'] = regex.group('findings').strip(' :') if regex else None

        regex = re.search('FINDINGS(?P<findings>.*?)NOTE', opa_file,  re.MULTILINE|re.DOTALL)
        if regex:
            opa_file_dict['Findings'] = regex.group('findings').strip(' :\n').replace('\n', '')
        regex = re.search('Final Discipline\s*(?P<discipline>.*?)INCIDENT', opa_file)
        opa_file_dict['Final discipline'] = regex.group('discipline').strip(' :') if regex else None
        regex = re.search('INCIDENT SYNOPSIS(?P<synopsis>.*?)(Complaint|COMPLAINT)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Incident synopsis'] = regex.group('synopsis').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('COMPLAINT(?P<complaint>.*?)(Complaint|INVESTIGATION)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Complaint'] = regex.group('complaint').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('INVESTIGATION(?P<investigation>.*?)(Complaint|ANALYSIS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Investigation'] = regex.group('investigation').strip(' :\n').replace('\n', '') if regex else None
        regex = re.search('ANALYSIS AND CONCLUSION(?P<analysis>.*?)(Complaint|FINDINGS)', opa_file,  re.MULTILINE|re.DOTALL)
        opa_file_dict['Analysis and conclusion'] = regex.group('analysis').strip(' :\n').replace('\n', '') if regex else None
        opa_file_dict['txt'] = filename
        opa_file_dict['pdf'] = filename[:-4]+'.pdf'
        try:
            # Get the entire findings section
            f = open(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts/'+filename, 'r')

            opa_file2 = f.read()
            import re
            m = re.search('Issued(.*?)Complaint', opa_file2,  re.MULTILINE|re.DOTALL)
            if m:
                section = m.group()
                lines = section.split('\n')[3:-3]
                structured_version = []
                current = None
                opa_file_dict['is_sustained'] = False
                for i, line in enumerate(lines):
                    if line.startswith('Named Employee'):
                        if current:
                            structured_version.append(current)
                        current = {'heading': line.strip(), 'allegations': []}
                    if line.startswith('Allegation'):
                        allegation_structure = {'heading': line.strip()}
                        allegation_structure['allegation'] = lines[i+2].strip()
                        j = 0
                        while True:
                            j += 1
                            the_line = lines[i + 2 + j]
                            if not the_line:
                                break
                            allegation_structure['allegation'] += ' '+the_line.strip()

                        current['allegations'].append(allegation_structure)
                    if line.startswith('OPA Finding'):
                        current['allegations'][-1]['opa_finding'] = lines[i+2].strip()
                        if current['allegations'][-1]['opa_finding'] == 'Sustained':
                            opa_file_dict['is_sustained'] = True
                    if line.startswith('Final Discipline'):
                        current['final_discipline'] = lines[i+2].strip()
                structured_version.append(current)
            opa_file_dict['Summarized results'] = structured_version
        except Exception, e:
            opa_file_dict['parsing_error'] = str(e)
        opa_files.append(opa_file_dict)
        f.close()
    import sys

    reload(sys)
    sys.setdefaultencoding('utf8')
    print opa_files

    db.table('police_internal_affairs_cases').insert(opa_files, conflict='update').run(conn)
    the_html = '
'
    columns = ['Issued date', 'Complaint number', 'Complaint', 'Incident synopsis', 'Investigation', 'Analysis and conclusion', 'OPA finding', 'Final discipline']
    from dateutil.parser import parse
    import datetime
    mindate = datetime.date(datetime.MINYEAR, 1, 1)
    opa_files = sorted(opa_files, key=lambda x: parse(x['Issued date']).date() if '/' in x['Issued date'] else mindate, reverse=True)

def download():
    import os
    os.system('mkdir %s.crawler_data' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/' % (base))
    os.system('mkdir %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/' % (base))
    html = requests.get('http://www.seattle.gov/opa/closed-case-summaries').text
    os.system('mkdir pdfs')
    files = re.findall('ClosedCaseSummaries/(?P<filename>.*?)\.pdf', html)
    print files
    for f in files:
        if not f+'.pdf' in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs'):
            os.system('wget -O %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/%s.pdf http://www.seattle.gov/Documents/Departments/OPA/ClosedCaseSummaries/%s.pdf' % (base, f, f))
    import os
    files = sorted([f for f in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs')])
    new_files = []
    for filename in files:
        print filename
        if not filename[:-4]+'.txt' in os.listdir(base+'.crawler_data/seattle_police_internal_affairs_closed_cases/txts'):
            print 'converting'
            os.system('pdf2txt.py -A %s.crawler_data/seattle_police_internal_affairs_closed_cases/pdfs/%s > %s.crawler_data/seattle_police_internal_affairs_closed_cases/txts/%s' % (base, filename, base, filename[:-4]+'.txt'))
            new_files.append(filename[:-4]+'.txt')
    parse_txt_files(new_files)

if __name__ == "__main__":
    import sys
    if len(sys.argv) > 1:
        parse_txt_files()
    else:
        download()
<|endoftext|>


<commit_before>import os
from pathlib import Path
import logging
import asyncio
import re
from opentrons import HERE
from opentrons import server
from opentrons.hardware_control import API, ThreadManager
from opentrons.server.main import build_arg_parser
from argparse import ArgumentParser
from opentrons import __version__
from opentrons.config import (feature_flags as ff, name,
                              robot_configs, IS_ROBOT, ROBOT_FIRMWARE_DIR)
from opentrons.util import logging_config
from opentrons.drivers.smoothie_drivers.driver_3_0 import SmoothieDriver_3_0_0

try:
    from opentrons.hardware_control.socket_server\
        import run as install_hardware_server
except ImportError:
    async def install_hardware_server(sock_path, api):  # type: ignore
        log.warning("Cannot start hardware server: missing dependency")

log = logging.getLogger(__name__)

SMOOTHIE_HEX_RE = re.compile('smoothie-(.*).hex')

def _find_smoothie_file():

    resources = []

    # Search for smoothie files in /usr/lib/firmware first then fall back to
    # value packed in wheel
    if IS_ROBOT:
        resources.extend(ROBOT_FIRMWARE_DIR.iterdir())

    resources_path = Path(HERE) / 'resources'
    resources.extend(resources_path.iterdir())

    for path in resources:
        matches = SMOOTHIE_HEX_RE.search(path.name)
        if matches:
            branch_plus_ref = matches.group(1)
            return path, branch_plus_ref
    raise OSError(f"Could not find smoothie firmware file in {resources_path}")

async def _do_fw_update(new_fw_path, new_fw_ver):
    """ Update the connected smoothie board, with retries

    When the API server boots, it talks to the motor controller board for the
    first time. Sometimes the board is in a bad state - it might have the
    wrong firmware version (i.e. this is the first boot after an update), or it
    might just not be communicating correctly. Sometimes, the motor controller
    not communicating correctly in fact means it needs a firmware update; other
    times, it might mean it just needs to be reset.

    This function is called when the API server boots if either of the above
    cases happens. Its job is to make the motor controller board ready by
    updating its firmware, regardless of the state of the rest of the stack.

    To that end, this function uses the smoothie driver directly (so it can
    ignore the rest of the stack) and has a couple retries with different
    hardware line changes in between (so it can catch all failure modes). If
    this method ultimately fails, it lets the server boot by telling it to
    consider itself virtual.

    After this function has completed, it is always safe to call
    hardware.connect() - it just might be virtual
    """
    explicit_modeset = False
    driver = SmoothieDriver_3_0_0(robot_configs.load())
    for attempts in range(3):
        try:
            await driver.update_firmware(
                new_fw_path,
                explicit_modeset=explicit_modeset)
        except RuntimeError:
            explicit_modeset = True
            continue

        if driver.get_fw_version() == new_fw_ver:
            log.info(f"Smoothie fw update complete in {attempts} tries")
            break
        else:
            log.error(
                "Failed to update smoothie: did not connect after update")
    else:
        log.error("Could not update smoothie, forcing virtual")
        os.environ['ENABLE_VIRTUAL_SMOOTHIE'] = 'true'

async def initialize_robot() -> ThreadManager:
    packed_smoothie_fw_file, packed_smoothie_fw_ver = _find_smoothie_file()
    if os.environ.get("ENABLE_VIRTUAL_SMOOTHIE"):
        log.info("Initialized robot using virtual Smoothie")
        return ThreadManager(API.build_hardware_simulator)

    try:
        hardware = ThreadManager(API.build_hardware_controller)
    except Exception as e:
        # The most common reason for this exception (aside from hardware
        # failures such as a disconnected smoothie) is that the smoothie
        # is in programming mode. If it is, then we still want to update
        # it (so it can boot again), but we don’t have to do the GPIO
        # manipulations that _put_ it in programming mode
        log.exception("Error while connecting to motor driver: {}".format(e))
        fw_version = None
    else:
        fw_version = await hardware.fw_version
        hardware.clean_up()

    log.info(f"Smoothie FW version: {fw_version}")
    if fw_version != packed_smoothie_fw_ver:
        log.info(f"Executing smoothie update: current vers {fw_version},"
                 f" packed vers {packed_smoothie_fw_ver}")
        await _do_fw_update(packed_smoothie_fw_file, packed_smoothie_fw_ver)
    else:
        log.info(f"FW version OK: {packed_smoothie_fw_ver}")
    return ThreadManager(API.build_hardware_controller)

def initialize(
        hardware_server: bool = False,
        hardware_server_socket: str = "/var/run/opentrons-hardware.sock") \
        -> ThreadManager:
    """
    Initialize the Opentrons hardware returning a hardware instance.

    :param hardware_server: Run a jsonrpc server allowing rpc to the  hardware
     controller. Only works on buildroot because extra dependencies are
     required.
    :param hardware_server_socket: Override for the hardware server socket
    """
    robot_conf = robot_configs.load()
    logging_config.log_init(robot_conf.log_level)

    log.info(f"API server version:  {__version__}")
    log.info(f"Robot Name: {name()}")

    loop = asyncio.get_event_loop()
    hardware = loop.run_until_complete(initialize_robot())

    if not ff.disable_home_on_boot():
        log.info("Homing Z axes")
        loop.run_until_complete(hardware.home_z())

    if hardware_server:
        #  TODO: BC 2020-02-25 adapt hardware socket server to ThreadManager
        loop.run_until_complete(
                install_hardware_server(hardware_server_socket,
                                        hardware))  # type: ignore

    return hardware

def run(**kwargs):  # noqa(C901)
    """
    This function was necessary to separate from main() to accommodate for
    server startup path on system 3.0, which is server.main. In the case where
    the api is on system 3.0, server.main will redirect to this function with
    an additional argument of 'patch_old_init'. kwargs are hence used to allow
    the use of different length args
    """
    hardware = initialize(kwargs.get('hardware_server'),
                          kwargs.get('hardware_server_socket'))

    server.run(hardware,
               kwargs.get('hostname'),
               kwargs.get('port'),
               kwargs.get('path'))

def main():
    """ The main entrypoint for the Opentrons robot API server stack.

    This function
    - creates and starts the server for both the RPC routes
      handled by :py:mod:`opentrons.server.rpc` and the HTTP routes handled
      by :py:mod:`opentrons.server.http`
    - initializes the hardware interaction handled by either
      :py:mod:`opentrons.legacy_api` or :py:mod:`opentrons.hardware_control`

    This function does not return until the server is brought down.
    """

    arg_parser = ArgumentParser(
        description="Opentrons robot software",
        parents=[build_arg_parser()])
    arg_parser.add_argument(
        '--hardware-server', action='store_true',
        help='Run a jsonrpc server allowing rpc to the'
        ' hardware controller. Only works on buildroot '
        'because extra dependencies are required.')
    arg_parser.add_argument(
        '--hardware-server-socket', action='store',
        default='/var/run/opentrons-hardware.sock',
        help='Override for the hardware server socket')
    args = arg_parser.parse_args()

    run(**vars(args))
    arg_parser.exit(message="Stopped\n")

if __name__ == "__main__":
    main()
<commit_msg>fix(api): compare smoothie version before hardware controller on boot (#5165)<commit_after>import os
from pathlib import Path
import logging
import asyncio
import re
from opentrons import HERE
from opentrons import server
from opentrons.hardware_control import API, ThreadManager
from opentrons.server.main import build_arg_parser
from argparse import ArgumentParser
from opentrons import __version__
from opentrons.config import (feature_flags as ff, name,
                              robot_configs, IS_ROBOT, ROBOT_FIRMWARE_DIR)
from opentrons.util import logging_config
from opentrons.drivers.smoothie_drivers.driver_3_0 import SmoothieDriver_3_0_0

try:
    from opentrons.hardware_control.socket_server\
        import run as install_hardware_server
except ImportError:
    async def install_hardware_server(sock_path, api):  # type: ignore
        log.warning("Cannot start hardware server: missing dependency")

log = logging.getLogger(__name__)

SMOOTHIE_HEX_RE = re.compile('smoothie-(.*).hex')

def _find_smoothie_file():

    resources = []

    # Search for smoothie files in /usr/lib/firmware first then fall back to
    # value packed in wheel
    if IS_ROBOT:
        resources.extend(ROBOT_FIRMWARE_DIR.iterdir())

    resources_path = Path(HERE) / 'resources'
    resources.extend(resources_path.iterdir())

    for path in resources:
        matches = SMOOTHIE_HEX_RE.search(path.name)
        if matches:
            branch_plus_ref = matches.group(1)
            return path, branch_plus_ref
    raise OSError(f"Could not find smoothie firmware file in {resources_path}")

async def _do_fw_update(driver, new_fw_path, new_fw_ver):
    """ Update the connected smoothie board, with retries

    When the API server boots, it talks to the motor controller board for the
    first time. Sometimes the board is in a bad state - it might have the
    wrong firmware version (i.e. this is the first boot after an update), or it
    might just not be communicating correctly. Sometimes, the motor controller
    not communicating correctly in fact means it needs a firmware update; other
    times, it might mean it just needs to be reset.

    This function is called when the API server boots if either of the above
    cases happens. Its job is to make the motor controller board ready by
    updating its firmware, regardless of the state of the rest of the stack.

    To that end, this function uses the smoothie driver directly (so it can
    ignore the rest of the stack) and has a couple retries with different
    hardware line changes in between (so it can catch all failure modes). If
    this method ultimately fails, it lets the server boot by telling it to
    consider itself virtual.

    After this function has completed, it is always safe to call
    hardware.connect() - it just might be virtual
    """
    explicit_modeset = False
    for attempts in range(3):
        try:
            await driver.update_firmware(
                new_fw_path,
                explicit_modeset=explicit_modeset)
        except RuntimeError:
            explicit_modeset = True
            continue

        if driver.get_fw_version() == new_fw_ver:
            log.info(f"Smoothie fw update complete in {attempts} tries")
            break
        else:
            log.error(
                "Failed to update smoothie: did not connect after update")
    else:
        log.error("Could not update smoothie, forcing virtual")
        os.environ['ENABLE_VIRTUAL_SMOOTHIE'] = 'true'

async def check_for_smoothie_update():
    driver = SmoothieDriver_3_0_0(robot_configs.load())
    driver.connect()
    try:
        fw_version = driver.get_fw_version()
    except Exception as e:
        # The most common reason for this exception (aside from hardware
        # failures such as a disconnected smoothie) is that the smoothie
        # is in programming mode. If it is, then we still want to update
        # it (so it can boot again), but we don’t have to do the GPIO
        # manipulations that _put_ it in programming mode
        log.exception("Error while connecting to motor driver: {}".format(e))
        fw_version = None

    log.info(f"Smoothie FW version: {fw_version}")
    packed_smoothie_fw_file, packed_smoothie_fw_ver = _find_smoothie_file()
    if fw_version != packed_smoothie_fw_ver:
        log.info(f"Executing smoothie update: current vers {fw_version},"
                 f" packed vers {packed_smoothie_fw_ver}")
        await _do_fw_update(driver, packed_smoothie_fw_file,
                            packed_smoothie_fw_ver)
    else:
        log.info(f"FW version OK: {packed_smoothie_fw_ver}")

async def initialize_robot() -> ThreadManager:
    if os.environ.get("ENABLE_VIRTUAL_SMOOTHIE"):
        log.info("Initialized robot using virtual Smoothie")
        return ThreadManager(API.build_hardware_simulator)

    await check_for_smoothie_update()

    hardware = ThreadManager(API.build_hardware_controller)

    if not ff.disable_home_on_boot():
        log.info("Homing Z axes")
        await hardware.home_z()

    return hardware

def initialize(
        hardware_server: bool = False,
        hardware_server_socket: str = "/var/run/opentrons-hardware.sock") \
        -> ThreadManager:
    """
    Initialize the Opentrons hardware returning a hardware instance.

    :param hardware_server: Run a jsonrpc server allowing rpc to the  hardware
     controller. Only works on buildroot because extra dependencies are
     required.
    :param hardware_server_socket: Override for the hardware server socket
    """
    robot_conf = robot_configs.load()
    logging_config.log_init(robot_conf.log_level)

    log.info(f"API server version:  {__version__}")
    log.info(f"Robot Name: {name()}")

    loop = asyncio.get_event_loop()
    hardware = loop.run_until_complete(initialize_robot())

    if hardware_server:
        #  TODO: BC 2020-02-25 adapt hardware socket server to ThreadManager
        loop.run_until_complete(
                install_hardware_server(hardware_server_socket,
                                        hardware))  # type: ignore

    return hardware

def run(**kwargs):  # noqa(C901)
    """
    This function was necessary to separate from main() to accommodate for
    server startup path on system 3.0, which is server.main. In the case where
    the api is on system 3.0, server.main will redirect to this function with
    an additional argument of 'patch_old_init'. kwargs are hence used to allow
    the use of different length args
    """
    hardware = initialize(kwargs.get('hardware_server'),
                          kwargs.get('hardware_server_socket'))

    server.run(hardware,
               kwargs.get('hostname'),
               kwargs.get('port'),
               kwargs.get('path'))

def main():
    """ The main entrypoint for the Opentrons robot API server stack.

    This function
    - creates and starts the server for both the RPC routes
      handled by :py:mod:`opentrons.server.rpc` and the HTTP routes handled
      by :py:mod:`opentrons.server.http`
    - initializes the hardware interaction handled by either
      :py:mod:`opentrons.legacy_api` or :py:mod:`opentrons.hardware_control`

    This function does not return until the server is brought down.
    """

    arg_parser = ArgumentParser(
        description="Opentrons robot software",
        parents=[build_arg_parser()])
    arg_parser.add_argument(
        '--hardware-server', action='store_true',
        help='Run a jsonrpc server allowing rpc to the'
        ' hardware controller. Only works on buildroot '
        'because extra dependencies are required.')
    arg_parser.add_argument(
        '--hardware-server-socket', action='store',
        default='/var/run/opentrons-hardware.sock',
        help='Override for the hardware server socket')
    args = arg_parser.parse_args()

    run(**vars(args))
    arg_parser.exit(message="Stopped\n")

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from collections import defaultdict

import numpy as np
from sklearn import preprocessing
from sklearn import linear_model
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from firebase.firebase import FirebaseApplication

from . import mapper
from . import utils
import DatasetCreation

def clean_weather_string(s):
    return str(s).replace("/", " and ").replace("  ", " ")

def preprocess_weather():
    firebase_clean = mapper.get_clean_firebase()
    response = firebase_clean.get_all()
    weathers = {}
    for airport_code, date_status_dict in response.items():
        if type(date_status_dict) != dict or airport_code == "weathers":
            continue
        for date, event in date_status_dict.items():
            cleaned = clean_weather_string(event["weather"])
            weathers[cleaned] = True

    label_encoder = preprocessing.LabelEncoder()
    weathers = weathers.keys()
    label_encoder.fit(weathers)
    transformed = label_encoder.transform(weathers)
    weather_dict = {}
    for i in range(len(weathers)):
        weather_str = clean_weather_string(weathers[i])
        weather_dict[weather_str] = transformed[i]

    firebase_clean.firebase.put("/metadata", name="weathers", data=weather_dict)
    return weather_dict

def get_weather_array(weather_dict, weather_str):
    cleaned = clean_weather_string(weather_str)
    arr = np.zeros(len(weather_dict))
    if cleaned in weather_dict:
        pos = weather_dict[ cleaned]
        arr[pos] = 1
    else:
        print "ERROR could not clean:", weather_str
    return arr

def get_all_weathers_binarized(clean_data):
    firebase_clean = mapper.get_clean_firebase()
    weathers = firebase_clean.get_metadata()["weathers"]

    weather_series = []
    for code, events in clean_data.items():
        for date, event in events.items():
            weather_series.append(get_weather_array(weathers, event["weather"]))

    return np.array(weather_series)

def get_all_delays_binarized(clean_data):
    delay_series = []
    for code, events in clean_data.items():
        for date, event in events.items():
            delay_series.append(event["delay"])

    return np.array(delay_series)

def get_all_airports_binarized(clean_data):

    transformed = DatasetCreation.writeCities_Airports(clean_data)
    airports_binarized = []
    for code, events in clean_data.items():

        for date, event in events.items():
            airports_binarized.append(DatasetCreation.getAirportBinarizedRepresentation(transformed, code))

    return np.array(airports_binarized)

def get_all_wind_binarized(clean_data):

    windPair = []
    for code, events in clean_data.items():
        for date, event in events.items():
            windPair.append( [float(event["wind_x"]), float(event["wind_y"])] )

    return np.array(windPair)

def merge_binarized(arrays):
    return np.concatenate(arrays, axis=1)

def permute(datapoints, labels):

    p = np.random.permutation(len(labels))

    datapoints = np.array(datapoints)
    labels = np.array(labels)

    return (datapoints, labels)

def decide_model(datapoints, labels):

    classifiers = []
    classifiers.append( RandomForestClassifier(n_estimators=10, min_samples_split=1) )
    classifiers.append( DecisionTreeClassifier(max_depth=None, min_samples_split=1, random_state=0) )
    classifiers.append( linear_model.Perceptron() )
    #classifiers.append( linear_model.SGDClassifier() )
    #classifiers.append( KNeighborsClassifier() )

    best = 0
    model = None
    for clf in classifiers:
        fit = clf.fit(datapoints, labels)
        scores = cross_val_score(fit, datapoints, labels, cv=10, n_jobs=-1)
        res = scores.mean()

        if res > best:
            best = res
            model = fit

    print model, best
    return model

def predict(airport_code):
    firebase_source = mapper.get_source_firebase()
    firebase_clean = mapper.get_clean_firebase()

    all_clean = firebase_clean.get_all()

    airports_binarized = get_all_airports_binarized(all_clean)
    weathers_binarized = get_all_weathers_binarized(all_clean)
    wind_binarized = get_all_wind_binarized(all_clean)
    print "-----", airports_binarized.shape, weathers_binarized.shape, wind_binarized.shape

    features = [airports_binarized, weathers_binarized, wind_binarized]
    merged_binarized = merge_binarized(features)
    print merged_binarized.shape

    delays_binarized = get_all_delays_binarized(all_clean)
    print delays_binarized.shape

    datapoints, labels = permute(merged_binarized, delays_binarized)
    model = decide_model(datapoints, labels)
    #model = linear_model.Perceptron().fit(merged_binarized, delays_binarized)

    airport_status = firebase_source.get_airport(airport_code)
    cleaned_data = utils.get_clean_data(airport_status)
    weathers = firebase_clean.get_metadata()["weathers"]
    weather_binarized = get_weather_array(weathers, cleaned_data["weather"])
    transformed = DatasetCreation.writeCities_Airports(all_clean)
    airport_binarized = DatasetCreation.getAirportBinarizedRepresentation(transformed, airport_code)
    airport_binarized = np.concatenate((airport_binarized, weather_binarized))

    return model.predict([airport_binarized])
<commit_msg>Fixed the problem with the wind ... testing<commit_after>from collections import defaultdict

import numpy as np
from sklearn import preprocessing
from sklearn import linear_model
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.cross_validation import cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from firebase.firebase import FirebaseApplication

from . import mapper
from . import utils
import DatasetCreation

def clean_weather_string(s):
    return str(s).replace("/", " and ").replace("  ", " ")

def preprocess_weather():
    firebase_clean = mapper.get_clean_firebase()
    response = firebase_clean.get_all()
    weathers = {}
    for airport_code, date_status_dict in response.items():
        if type(date_status_dict) != dict or airport_code == "weathers":
            continue
        for date, event in date_status_dict.items():
            cleaned = clean_weather_string(event["weather"])
            weathers[cleaned] = True

    label_encoder = preprocessing.LabelEncoder()
    weathers = weathers.keys()
    label_encoder.fit(weathers)
    transformed = label_encoder.transform(weathers)
    weather_dict = {}
    for i in range(len(weathers)):
        weather_str = clean_weather_string(weathers[i])
        weather_dict[weather_str] = transformed[i]

    firebase_clean.firebase.put("/metadata", name="weathers", data=weather_dict)
    return weather_dict

def get_weather_array(weather_dict, weather_str):
    cleaned = clean_weather_string(weather_str)
    arr = np.zeros(len(weather_dict))
    if cleaned in weather_dict:
        pos = weather_dict[ cleaned]
        arr[pos] = 1
    else:
        print "ERROR could not clean:", weather_str
    return arr

def get_all_weathers_binarized(clean_data):
    firebase_clean = mapper.get_clean_firebase()
    weathers = firebase_clean.get_metadata()["weathers"]

    weather_series = []
    for code, events in clean_data.items():
        for date, event in events.items():
            weather_series.append(get_weather_array(weathers, event["weather"]))

    return np.array(weather_series)

def get_all_delays_binarized(clean_data):
    delay_series = []
    for code, events in clean_data.items():
        for date, event in events.items():
            delay_series.append(event["delay"])

    return np.array(delay_series)

def get_all_airports_binarized(clean_data):

    transformed = DatasetCreation.writeCities_Airports(clean_data)
    airports_binarized = []
    for code, events in clean_data.items():

        for date, event in events.items():
            airports_binarized.append(DatasetCreation.getAirportBinarizedRepresentation(transformed, code))

    return np.array(airports_binarized)

def get_all_wind_binarized(clean_data):

    windPair = []
    for code, events in clean_data.items():
        for date, event in events.items():
            windPair.append( (float(event["wind_x"]), float(event["wind_y"])) )

    return np.array(windPair)

def merge_binarized(arrays):
    return np.concatenate(arrays, axis=1)

def permute(datapoints, labels):

    p = np.random.permutation(len(labels))

    datapoints = np.array(datapoints)
    labels = np.array(labels)

    return (datapoints, labels)

def decide_model(datapoints, labels):

    classifiers = []
    classifiers.append( RandomForestClassifier(n_estimators=10, min_samples_split=1) )
    classifiers.append( DecisionTreeClassifier(max_depth=None, min_samples_split=1, random_state=0) )
    classifiers.append( linear_model.Perceptron() )
    #classifiers.append( linear_model.SGDClassifier() )
    #classifiers.append( KNeighborsClassifier() )

    best = 0
    model = None
    for clf in classifiers:
        fit = clf.fit(datapoints, labels)
        scores = cross_val_score(fit, datapoints, labels, cv=10, n_jobs=-1)
        res = scores.mean()

        if res > best:
            best = res
            model = fit

    print model, best
    return model

def predict(airport_code):
    firebase_source = mapper.get_source_firebase()
    firebase_clean = mapper.get_clean_firebase()

    all_clean = firebase_clean.get_all()

    airports_binarized = get_all_airports_binarized(all_clean)
    weathers_binarized = get_all_weathers_binarized(all_clean)
    wind_binarized = get_all_wind_binarized(all_clean)
    print "-----", airports_binarized.shape, weathers_binarized.shape, wind_binarized.shape

    features = [airports_binarized, weathers_binarized, wind_binarized] # , wind_binarized
    datapoints = merge_binarized(features)

    labels = get_all_delays_binarized(all_clean)

    model = decide_model(datapoints, labels)
    #model = linear_model.Perceptron().fit(merged_binarized, delays_binarized)

    airport_status = firebase_source.get_airport(airport_code)
    cleaned_data = utils.get_clean_data(airport_status)
    weathers = firebase_clean.get_metadata()["weathers"]
    weather_binarized = get_weather_array(weathers, cleaned_data["weather"])
    transformed = DatasetCreation.writeCities_Airports(all_clean)
    airport_binarized = DatasetCreation.getAirportBinarizedRepresentation(transformed, airport_code)
    wind = [ cleaned_data["wind_x"], cleaned_data["wind_y"] ]
    airport_binarized = np.concatenate((airport_binarized, weather_binarized, wind))

    return model.predict([airport_binarized])
<|endoftext|>


<commit_before>#!/usr/bin/python

#####################################################################################
# This was inspired by a guide on ModMyPi (http://bit.ly/mmpsms).                   #
#                                                                                #
# Use a soil moisture sensor and a Raspberry Pi to monitor the soil in a plant pot  #
# and warn the office, via Slack, that it needs watering.                           #
#                                                                                #
# By Wesley Archer (@raspberrycoulis - https://www.raspberrycoulis.co.uk)           #
#####################################################################################

# Import the necessary libraries:
import RPi.GPIO as GPIO
import time
import httplib, urllib
import urllib2
import json

# Slack webhook - get this from https://api.slack.com/custom-integrations/incoming-webhooks
webhook_url = "ADD_HERE"

# This is the function that calls the Slack webhook to notify you:
def postToSlack():
    data = '{"attachments":[{"fallback":"Water plant!","pretext":"The soil is too dry!","color":"#cc0000","fields":[{"title":"The Garden Room plant needs watering!","short":false}]}]}'
    slack = urllib2.Request(webhook_url, data, {'Content-Type': 'application/json'})
    post = urllib2.urlopen(slack)
    post.close()

# This is the function that checks if the soil is moist or dry:
def moisture(channel):
    if GPIO.input(channel):
            #print "Soil is dry!"    # Uncomment to show feedback in console
            postToSlack()
    else:
            #print "Soil is moist"   # Uncomment to show feedback in console
            return "Soil is moist"

# Set the GPIO mode:
GPIO.setmode(GPIO.BCM)

# Define the GPIO pin that the moisture sensor (D0 on the sensor) is connected to:
channel = 17

# Set the GPIO pin above as an input:
GPIO.setup(channel, GPIO.IN)

# Check whether the GPIO pin is high or low. The bouncetime is the minimum time between two callbacks (default=300) in millseconds:
GPIO.add_event_detect(channel, GPIO.BOTH, bouncetime=300)

# This line assigns a function to the GPIO pin so that when the above line tells us there is a change on the pin, run this function:
GPIO.add_event_callback(channel, moisture)

# Run the code in an infinite loop and tells the script to wait 0.1 seconds, this is so the script doesnt hog all of the CPU:
while True:
        time.sleep(0.1)
<commit_msg>Prevent Slack spamming<commit_after>#!/usr/bin/python

#####################################################################################
# This was inspired by a guide on ModMyPi (http://bit.ly/mmpsms).                   #
#                                                                                #
# Use a soil moisture sensor and a Raspberry Pi to monitor the soil in a plant pot  #
# and warn the office, via Slack, that it needs watering.                           #
#                                                                                #
# By Wesley Archer (@raspberrycoulis - https://www.raspberrycoulis.co.uk)           #
#####################################################################################

# Import the necessary libraries:
import RPi.GPIO as GPIO
import time
import httplib, urllib
import urllib2
import json

# Set the GPIO mode:
GPIO.setmode(GPIO.BCM)

# Define the GPIO pin that the moisture sensor (D0 on the sensor) is connected to:
channel = 17

# Set the GPIO pin above as an input and set the internal pull-up resistor down:
GPIO.setup(channel, GPIO.IN,pull_up_down=GPIO.PUD_DOWN)

# Slack webhook - get this from https://api.slack.com/custom-integrations/incoming-webhooks
webhook_url = "ADD_HERE"

# This is the function that calls the Slack webhook to notify you:
def postToSlack():
    data = '{"attachments":[{"fallback":"Water plant!","pretext":"The soil is too dry!","color":"#cc0000","fields":[{"title":"The Garden Room plant needs watering!","short":false}]}]}'
    slack = urllib2.Request(webhook_url, data, {'Content-Type': 'application/json'})
    post = urllib2.urlopen(slack)
    post.close()

# Run the code in an infinite loop. If the soil is dry, a Slack notification is triggered:
while True:
    if GPIO.input(channel)==False:
        #print('Soil is moist')     # Uncomment to print console commands
        time.sleep(900)             # Sleep for 15 minutes (900 seconds)
    else:
        #print('Soil is dry!')      # Uncomment to print console commands
        postToSlack()               # Trigger the Slack webhook notification
        time.sleep(2700)            # Sleep for 45 minutes (2700 seconds)
<|endoftext|>


<commit_before>"""
This command queries the pdl_seguimientos table to calculate some statictics
for `/stats` page. These stat values go to tables for this app.

* projects that are being accumulated in each `comisión`.
* projects that were aproved without 2nd round of votes.
"""
import datetime
import re

from django.core.management.base import BaseCommand, CommandError

from pdl.models import Seguimientos
from stats.models import ComisionCount
from stats.models import Dispensed
from stats.models import WithDictamenButNotVoted

class Command(BaseCommand):
    help = 'Parses the table `pdl_seguimientos` and creates a summary table ' \
           'containing `En Comision` events for each proyecto, only if this ' \
           'is the last event for them.'

    def handle(self, *args, **options):
        queryset = Seguimientos.objects.all()

        # There are 24 `comisiones` in total
        comisiones = set()
        for i in queryset:
            # res = re.match("(en\s+comisión(\s\w+,*)+)", i.evento, re.I)
            res = re.match("(en\s+comisión(\s\w+)+)", i.evento, re.I)
            if res:
                this_comision = re.sub("En\s+comisión\s+", "", res.groups()[0])
                this_comision = re.sub("\s+y\s+.+", "", this_comision)
                comisiones.add(this_comision)

        queryset = Seguimientos.objects.order_by('proyecto_id', '-fecha')
        comisiones_count = {}

        this_proyecto_id = ''
        for i in queryset:
            if i.proyecto_id != this_proyecto_id:
                for comision in comisiones:
                    if comision in i.evento:
                        if comision not in comisiones_count:
                            comisiones_count[comision] = 0
                        comisiones_count[comision] += 1
            this_proyecto_id = i.proyecto_id

        for k, v in comisiones_count.items():
            ComisionCount.objects.update_or_create(
                comision=k, defaults={'count': v},
            )

        # Get projects that did not go to 2da round of votes
        self.get_dispensed_projects()

        # Proyectos con dictamen pero sin votación
        self.get_with_dictamen_but_not_voted()

    def get_dispensed_projects(self):
        total_approved = Seguimientos.objects.filter(evento__icontains='aprobado').count()
        total_dispensed = Seguimientos.objects.filter(evento__icontains='dispensado 2da').count()
        dispensed_by_plenary = Seguimientos.objects.filter(
            evento__icontains='dispensado 2da').filter(evento__icontains='pleno').count()
        dispensed_by_spokesmen = Seguimientos.objects.filter(
            evento__icontains='dispensado 2da').filter(evento__icontains='portavoces').count()
        dispensed_others = total_dispensed - dispensed_by_plenary - dispensed_by_spokesmen

        Dispensed.objects.update_or_create(
            id=1, defaults={
                'total_approved': total_approved,
                'total_dispensed': total_dispensed,
                'dispensed_by_plenary': dispensed_by_plenary,
                'dispensed_by_spokesmen': dispensed_by_spokesmen,
                'dispensed_others': dispensed_others,
            }
        )

    def get_with_dictamen_but_not_voted(self):
        """
        Crea tabla con lista de: proyectos que no figure
          "publicado" || promulgado || votación || en lista de seguimientos,
        pero tenga "dictamen".
        """
        queryset = Seguimientos.objects.all().order_by('proyecto_id').values('proyecto_id', 'evento')
        proyect_ids = self.get_proyect_ids(queryset)

        proyects_with_dictamen_but_not_voted = []
        for proyecto_id in proyect_ids:
            if self.has_dictamen(proyecto_id, queryset) is True and \
                    self.is_voted(proyecto_id, queryset) is False:
                proyects_with_dictamen_but_not_voted.append(WithDictamenButNotVoted(proyect_id=proyecto_id))
        WithDictamenButNotVoted.objects.bulk_create(proyects_with_dictamen_but_not_voted)

    def get_proyect_ids(self, queryset):
        proyect_ids = set()
        for i in queryset:
            proyect_ids.add(i['proyecto_id'])
        return proyect_ids

    def is_voted(self, proyect_id, queryset):
        events = self.get_events(proyect_id, queryset)
        for i in events:
            if 'publicado' in i.lower() or 'promulgado' in i.lower() or 'votación' in i.lower():
                return True
        return False

    def has_dictamen(self, proyect_id, queryset):
        events = self.get_events(proyect_id, queryset)
        for i in events:
            if 'dictamen' in i.lower():
                return True
        return False

    def get_events(self, proyect_id, queryset):
        events = []
        for i in queryset:
            if i['proyecto_id'] == proyect_id:
                events.append(i['evento'])
        return events
<commit_msg>rename variable<commit_after>"""
This command queries the pdl_seguimientos table to calculate some statictics
for `/stats` page. These stat values go to tables for this app.

* projects that are being accumulated in each `comisión`.
* projects that were aproved without 2nd round of votes.
"""
import datetime
import re

from django.core.management.base import BaseCommand, CommandError

from pdl.models import Seguimientos
from stats.models import ComisionCount
from stats.models import Dispensed
from stats.models import WithDictamenButNotVoted

class Command(BaseCommand):
    help = 'Parses the table `pdl_seguimientos` and creates a summary table ' \
           'containing `En Comision` events for each proyecto, only if this ' \
           'is the last event for them.'

    def handle(self, *args, **options):
        queryset = Seguimientos.objects.all()

        # There are 24 `comisiones` in total
        comisiones = set()
        for i in queryset:
            # res = re.match("(en\s+comisión(\s\w+,*)+)", i.evento, re.I)
            res = re.match("(en\s+comisión(\s\w+)+)", i.evento, re.I)
            if res:
                this_comision = re.sub("En\s+comisión\s+", "", res.groups()[0])
                this_comision = re.sub("\s+y\s+.+", "", this_comision)
                comisiones.add(this_comision)

        queryset = Seguimientos.objects.order_by('proyecto_id', '-fecha')
        comisiones_count = {}

        this_proyecto_id = ''
        for i in queryset:
            if i.proyecto_id != this_proyecto_id:
                for comision in comisiones:
                    if comision in i.evento:
                        if comision not in comisiones_count:
                            comisiones_count[comision] = 0
                        comisiones_count[comision] += 1
            this_proyecto_id = i.proyecto_id

        for k, v in comisiones_count.items():
            ComisionCount.objects.update_or_create(
                comision=k, defaults={'count': v},
            )

        # Get projects that did not go to 2da round of votes
        self.get_dispensed_projects()

        # Proyectos con dictamen pero sin votación
        self.get_with_dictamen_but_not_voted()

    def get_dispensed_projects(self):
        total_approved = Seguimientos.objects.filter(evento__icontains='aprobado').count()
        total_dispensed = Seguimientos.objects.filter(evento__icontains='dispensado 2da').count()
        dispensed_by_plenary = Seguimientos.objects.filter(
            evento__icontains='dispensado 2da').filter(evento__icontains='pleno').count()
        dispensed_by_spokesmen = Seguimientos.objects.filter(
            evento__icontains='dispensado 2da').filter(evento__icontains='portavoces').count()
        dispensed_others = total_dispensed - dispensed_by_plenary - dispensed_by_spokesmen

        Dispensed.objects.update_or_create(
            id=1, defaults={
                'total_approved': total_approved,
                'total_dispensed': total_dispensed,
                'dispensed_by_plenary': dispensed_by_plenary,
                'dispensed_by_spokesmen': dispensed_by_spokesmen,
                'dispensed_others': dispensed_others,
            }
        )

    def get_with_dictamen_but_not_voted(self):
        """
        Crea tabla con lista de: proyectos que no figure
          "publicado" || promulgado || votación || en lista de seguimientos,
        pero tenga "dictamen".
        """
        queryset = Seguimientos.objects.all().order_by('proyecto_id').values('proyecto_id', 'evento')
        proyect_ids = self.get_proyect_ids(queryset)

        projects = []
        for proyecto_id in proyect_ids:
            if self.has_dictamen(proyecto_id, queryset) is True and \
                    self.is_voted(proyecto_id, queryset) is False:
                projects.append(WithDictamenButNotVoted(proyect_id=proyecto_id))
        WithDictamenButNotVoted.objects.bulk_create(projects)

    def get_proyect_ids(self, queryset):
        proyect_ids = set()
        for i in queryset:
            proyect_ids.add(i['proyecto_id'])
        return proyect_ids

    def is_voted(self, proyect_id, queryset):
        events = self.get_events(proyect_id, queryset)
        for i in events:
            if 'publicado' in i.lower() or 'promulgado' in i.lower() or 'votación' in i.lower():
                return True
        return False

    def has_dictamen(self, proyect_id, queryset):
        events = self.get_events(proyect_id, queryset)
        for i in events:
            if 'dictamen' in i.lower():
                return True
        return False

    def get_events(self, proyect_id, queryset):
        events = []
        for i in queryset:
            if i['proyecto_id'] == proyect_id:
                events.append(i['evento'])
        return events
<|endoftext|>