<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2013-Today OpenERP SA (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import SUPERUSER_ID
from openerp.addons.web import http
from openerp.addons.web.http import request
from openerp.tools.translate import _
from openerp.addons.website.controllers.main import Website as controllers
controllers = controllers()

import logging
_logger = logging.getLogger(__name__)

from datetime import datetime, timedelta
import time
from dateutil.relativedelta import relativedelta
from openerp import tools
import werkzeug.urls
from openerp.addons.website.models.website import slug

try:
    import GeoIP
except ImportError:
    GeoIP = None
    _logger.warn("Please install GeoIP python module to use events localisation.")

class website_event(http.Controller):
    @http.route(['/event', '/event/page/<int:page>'], type='http', auth="public", website=True, multilang=True)
    def events(self, page=1, **searches):
        cr, uid, context = request.cr, request.uid, request.context
        event_obj = request.registry['event.event']
        type_obj = request.registry['event.type']
        country_obj = request.registry['res.country']

        searches.setdefault('date', 'all')
        searches.setdefault('type', 'all')
        searches.setdefault('country', 'all')

        domain_search = {}

        def sdn(date):
            return date.strftime('%Y-%m-%d 23:59:59')
        def sd(date):
            return date.strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT)
        today = datetime.today()
        dates = [
            ['all', _('Next Events'), [("date_end", ">", sd(today))], 0],
            ['today', _('Today'), [
                ("date_end", ">", sd(today)),
                ("date_begin", "<", sdn(today))],
                0],
            ['week', _('This Week'), [
                ("date_end", ">=", sd(today + relativedelta(days=-today.weekday()))),
                ("date_begin", "<", sdn(today  + relativedelta(days=6-today.weekday())))],
                0],
            ['nextweek', _('Next Week'), [
                ("date_end", ">=", sd(today + relativedelta(days=7-today.weekday()))),
                ("date_begin", "<", sdn(today  + relativedelta(days=13-today.weekday())))],
                0],
            ['month', _('This month'), [
                ("date_end", ">=", sd(today.replace(day=1))),
                ("date_begin", "<", (today.replace(day=1) + relativedelta(months=1)).strftime('%Y-%m-%d 00:00:00'))],
                0],
            ['nextmonth', _('Next month'), [
                ("date_end", ">=", sd(today.replace(day=1) + relativedelta(months=1))),
                ("date_begin", "<", (today.replace(day=1)  + relativedelta(months=2)).strftime('%Y-%m-%d 00:00:00'))],
                0],
            ['old', _('Old Events'), [
                ("date_end", "<", today.strftime('%Y-%m-%d 00:00:00'))],
                0],
        ]

        # search domains
        current_date = None
        current_type = None
        current_country = None
        for date in dates:
            if searches["date"] == date[0]:
                domain_search["date"] = date[2]
                if date[0] != 'all':
                    current_date = date[1]
        if searches["type"] != 'all':
            current_type = type_obj.browse(cr, uid, int(searches['type']), context=context)
            domain_search["type"] = [("type", "=", int(searches["type"]))]
        if searches["country"] != 'all':
            current_country = country_obj.browse(cr, uid, int(searches['country']), context=context)
            domain_search["country"] = [("country_id", "=", int(searches["country"]))]

        def dom_without(without):
            domain = [('state', "in", ['draft','confirm','done'])]
            for key, search in domain_search.items():
                if key != without:
                    domain += search
            return domain

        # count by domains without self search
        for date in dates:
            if date[0] <> 'old':
                date[3] = event_obj.search(
                    request.cr, request.uid, dom_without('date') + date[2],
                    count=True, context=request.context)

        domain = dom_without('type')
        types = event_obj.read_group(
            request.cr, request.uid, domain, ["id", "type"], groupby="type",
            orderby="type", context=request.context)
        type_count = event_obj.search(request.cr, request.uid, domain,
                                      count=True, context=request.context)
        types.insert(0, {
            'type_count': type_count,
            'type': ("all", _("All Categories"))
        })

        domain = dom_without('country')
        countries = event_obj.read_group(
            request.cr, request.uid, domain, ["id", "country_id"],
            groupby="country_id", orderby="country_id", context=request.context)
        country_id_count = event_obj.search(request.cr, request.uid, domain,
                                            count=True, context=request.context)
        countries.insert(0, {
            'country_id_count': country_id_count,
            'country_id': ("all", _("All Countries"))
        })

        step = 5
        event_count = event_obj.search(
            request.cr, request.uid, dom_without("none"), count=True,
            context=request.context)
        pager = request.website.pager(url="/event", total=event_count, page=page, step=step, scope=5)

        order = 'website_published desc, date_begin'
        if searches.get('date','all') == 'old':
            order = 'website_published desc, date_begin desc'
        obj_ids = event_obj.search(
            request.cr, request.uid, dom_without("none"), limit=step,
            offset=pager['offset'], order=order, context=request.context)
        events_ids = event_obj.browse(request.cr, request.uid, obj_ids,
                                      context=request.context)

        values = {
            'current_date': current_date,
            'current_country': current_country,
            'current_type': current_type,
            'event_ids': events_ids,
            'dates': dates,
            'types': types,
            'countries': countries,
            'pager': pager,
            'searches': searches,
            'search_path': "?%s" % werkzeug.url_encode(searches),
        }

        return request.website.render("website_event.index", values)

    @http.route(['/event/<model("event.event"):event>/page/<page:path>'], type='http', auth="public", website=True, multilang=True)
    def event_page(self, event, page, **post):
        values = {
            'event': event,
            'main_object': event
        }
        return request.website.render(page, values)

    @http.route(['/event/<model("event.event"):event>'], type='http', auth="public", website=True, multilang=True)
    def event(self, event, **post):
        if event.menu_id and event.menu_id.child_id:
            target_url = event.menu_id.child_id[0].url
        else:
            target_url = '/event/%s/register' % str(event.id)
        if post.get('enable_editor') == '1':
            target_url += '?enable_editor=1'
        return request.redirect(target_url);

    @http.route(['/event/<model("event.event"):event>/register'], type='http', auth="public", website=True, multilang=True)
    def event_register(self, event, **post):
        values = {
            'event': event,
            'main_object': event,
            'range': range,
        }
        return request.website.render("website_event.event_description_full", values)

    @http.route('/event/add_event', type='http', auth="user", multilang=True, methods=['POST'], website=True)
    def add_event(self, event_name="New Event", **kwargs):
        return self._add_event(event_name, request.context, **kwargs)

    def _add_event(self, event_name=None, context={}, **kwargs):
        if not event_name:
            event_name = _("New Event")
        Event = request.registry.get('event.event')
        date_begin = datetime.today() + timedelta(days=(14))
        vals = {
            'name': event_name,
            'date_begin': date_begin.strftime('%Y-%m-%d'),
            'date_end': (date_begin + timedelta(days=(1))).strftime('%Y-%m-%d'),
        }
        event_id = Event.create(request.cr, request.uid, vals, context=context)
        event = Event.browse(request.cr, request.uid, event_id, context=context)
        return request.redirect("/event/%s/register?enable_editor=1" % slug(event))

    def get_visitors_country(self):
        GI = GeoIP.open('/usr/share/GeoIP/GeoIP.dat', 0)
        return {'country_code': GI.country_code_by_addr(request.httprequest.remote_addr), 'country_name': GI.country_name_by_addr(request.httprequest.remote_addr)}

    def get_formated_date(self, event):
        start_date = datetime.strptime(event.date_begin, tools.DEFAULT_SERVER_DATETIME_FORMAT).date()
        end_date = datetime.strptime(event.date_end, tools.DEFAULT_SERVER_DATETIME_FORMAT).date()
        return ('%s %s%s') % (start_date.strftime("%b"), start_date.strftime("%e"), (end_date != start_date and ("-"+end_date.strftime("%e")) or ""))

    @http.route('/event/get_country_event_list', type='http', auth='public', website=True)
    def get_country_events(self ,**post):
        if not GeoIP:
            return ""
        country_obj = request.registry['res.country']
        event_obj = request.registry['event.event']
        cr, uid, context,event_ids = request.cr, request.uid, request.context,[]
        country_code = self.get_visitors_country()['country_code']
        result = {'events':[],'country':False}
        if country_code:
            country_ids = country_obj.search(request.cr, request.uid, [('code', '=', country_code)], context=request.context)
            event_ids = event_obj.search(request.cr, request.uid, ['|', ('address_id', '=', None),('country_id.code', '=', country_code),('date_begin','>=', time.strftime('%Y-%m-%d 00:00:00')),('state', '=', 'confirm')], order="date_begin", context=request.context)
        if not event_ids:
            event_ids = event_obj.search(request.cr, request.uid, [('date_begin','>=', time.strftime('%Y-%m-%d 00:00:00')),('state', '=', 'confirm')], order="date_begin", context=request.context)
        for event in event_obj.browse(request.cr, request.uid, event_ids, context=request.context)[:6]:
            if country_code and event.country_id.code == country_code:
                result['country'] = country_obj.browse(request.cr, request.uid, country_ids[0], context=request.context)
            result['events'].append({
                 "date": self.get_formated_date(event),
                 "event": event,
                 "url": event.website_url})
        return request.website.render("website_event.country_events_list",result)
<commit_msg>Broken @route arguments for events pages<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2013-Today OpenERP SA (<http://www.openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import SUPERUSER_ID
from openerp.addons.web import http
from openerp.addons.web.http import request
from openerp.tools.translate import _
from openerp.addons.website.controllers.main import Website as controllers
controllers = controllers()

import logging
_logger = logging.getLogger(__name__)

from datetime import datetime, timedelta
import time
from dateutil.relativedelta import relativedelta
from openerp import tools
import werkzeug.urls
from openerp.addons.website.models.website import slug

try:
    import GeoIP
except ImportError:
    GeoIP = None
    _logger.warn("Please install GeoIP python module to use events localisation.")

class website_event(http.Controller):
    @http.route(['/event', '/event/page/<int:page>'], type='http', auth="public", website=True, multilang=True)
    def events(self, page=1, **searches):
        cr, uid, context = request.cr, request.uid, request.context
        event_obj = request.registry['event.event']
        type_obj = request.registry['event.type']
        country_obj = request.registry['res.country']

        searches.setdefault('date', 'all')
        searches.setdefault('type', 'all')
        searches.setdefault('country', 'all')

        domain_search = {}

        def sdn(date):
            return date.strftime('%Y-%m-%d 23:59:59')
        def sd(date):
            return date.strftime(tools.DEFAULT_SERVER_DATETIME_FORMAT)
        today = datetime.today()
        dates = [
            ['all', _('Next Events'), [("date_end", ">", sd(today))], 0],
            ['today', _('Today'), [
                ("date_end", ">", sd(today)),
                ("date_begin", "<", sdn(today))],
                0],
            ['week', _('This Week'), [
                ("date_end", ">=", sd(today + relativedelta(days=-today.weekday()))),
                ("date_begin", "<", sdn(today  + relativedelta(days=6-today.weekday())))],
                0],
            ['nextweek', _('Next Week'), [
                ("date_end", ">=", sd(today + relativedelta(days=7-today.weekday()))),
                ("date_begin", "<", sdn(today  + relativedelta(days=13-today.weekday())))],
                0],
            ['month', _('This month'), [
                ("date_end", ">=", sd(today.replace(day=1))),
                ("date_begin", "<", (today.replace(day=1) + relativedelta(months=1)).strftime('%Y-%m-%d 00:00:00'))],
                0],
            ['nextmonth', _('Next month'), [
                ("date_end", ">=", sd(today.replace(day=1) + relativedelta(months=1))),
                ("date_begin", "<", (today.replace(day=1)  + relativedelta(months=2)).strftime('%Y-%m-%d 00:00:00'))],
                0],
            ['old', _('Old Events'), [
                ("date_end", "<", today.strftime('%Y-%m-%d 00:00:00'))],
                0],
        ]

        # search domains
        current_date = None
        current_type = None
        current_country = None
        for date in dates:
            if searches["date"] == date[0]:
                domain_search["date"] = date[2]
                if date[0] != 'all':
                    current_date = date[1]
        if searches["type"] != 'all':
            current_type = type_obj.browse(cr, uid, int(searches['type']), context=context)
            domain_search["type"] = [("type", "=", int(searches["type"]))]
        if searches["country"] != 'all':
            current_country = country_obj.browse(cr, uid, int(searches['country']), context=context)
            domain_search["country"] = [("country_id", "=", int(searches["country"]))]

        def dom_without(without):
            domain = [('state', "in", ['draft','confirm','done'])]
            for key, search in domain_search.items():
                if key != without:
                    domain += search
            return domain

        # count by domains without self search
        for date in dates:
            if date[0] <> 'old':
                date[3] = event_obj.search(
                    request.cr, request.uid, dom_without('date') + date[2],
                    count=True, context=request.context)

        domain = dom_without('type')
        types = event_obj.read_group(
            request.cr, request.uid, domain, ["id", "type"], groupby="type",
            orderby="type", context=request.context)
        type_count = event_obj.search(request.cr, request.uid, domain,
                                      count=True, context=request.context)
        types.insert(0, {
            'type_count': type_count,
            'type': ("all", _("All Categories"))
        })

        domain = dom_without('country')
        countries = event_obj.read_group(
            request.cr, request.uid, domain, ["id", "country_id"],
            groupby="country_id", orderby="country_id", context=request.context)
        country_id_count = event_obj.search(request.cr, request.uid, domain,
                                            count=True, context=request.context)
        countries.insert(0, {
            'country_id_count': country_id_count,
            'country_id': ("all", _("All Countries"))
        })

        step = 5
        event_count = event_obj.search(
            request.cr, request.uid, dom_without("none"), count=True,
            context=request.context)
        pager = request.website.pager(url="/event", total=event_count, page=page, step=step, scope=5)

        order = 'website_published desc, date_begin'
        if searches.get('date','all') == 'old':
            order = 'website_published desc, date_begin desc'
        obj_ids = event_obj.search(
            request.cr, request.uid, dom_without("none"), limit=step,
            offset=pager['offset'], order=order, context=request.context)
        events_ids = event_obj.browse(request.cr, request.uid, obj_ids,
                                      context=request.context)

        values = {
            'current_date': current_date,
            'current_country': current_country,
            'current_type': current_type,
            'event_ids': events_ids,
            'dates': dates,
            'types': types,
            'countries': countries,
            'pager': pager,
            'searches': searches,
            'search_path': "?%s" % werkzeug.url_encode(searches),
        }

        return request.website.render("website_event.index", values)

    @http.route(['/event/<model("event.event"):event>/page/<path:page>'], type='http', auth="public", website=True, multilang=True)
    def event_page(self, event, page, **post):
        values = {
            'event': event,
            'main_object': event
        }
        return request.website.render(page, values)

    @http.route(['/event/<model("event.event"):event>'], type='http', auth="public", website=True, multilang=True)
    def event(self, event, **post):
        if event.menu_id and event.menu_id.child_id:
            target_url = event.menu_id.child_id[0].url
        else:
            target_url = '/event/%s/register' % str(event.id)
        if post.get('enable_editor') == '1':
            target_url += '?enable_editor=1'
        return request.redirect(target_url);

    @http.route(['/event/<model("event.event"):event>/register'], type='http', auth="public", website=True, multilang=True)
    def event_register(self, event, **post):
        values = {
            'event': event,
            'main_object': event,
            'range': range,
        }
        return request.website.render("website_event.event_description_full", values)

    @http.route('/event/add_event', type='http', auth="user", multilang=True, methods=['POST'], website=True)
    def add_event(self, event_name="New Event", **kwargs):
        return self._add_event(event_name, request.context, **kwargs)

    def _add_event(self, event_name=None, context={}, **kwargs):
        if not event_name:
            event_name = _("New Event")
        Event = request.registry.get('event.event')
        date_begin = datetime.today() + timedelta(days=(14))
        vals = {
            'name': event_name,
            'date_begin': date_begin.strftime('%Y-%m-%d'),
            'date_end': (date_begin + timedelta(days=(1))).strftime('%Y-%m-%d'),
        }
        event_id = Event.create(request.cr, request.uid, vals, context=context)
        event = Event.browse(request.cr, request.uid, event_id, context=context)
        return request.redirect("/event/%s/register?enable_editor=1" % slug(event))

    def get_visitors_country(self):
        GI = GeoIP.open('/usr/share/GeoIP/GeoIP.dat', 0)
        return {'country_code': GI.country_code_by_addr(request.httprequest.remote_addr), 'country_name': GI.country_name_by_addr(request.httprequest.remote_addr)}

    def get_formated_date(self, event):
        start_date = datetime.strptime(event.date_begin, tools.DEFAULT_SERVER_DATETIME_FORMAT).date()
        end_date = datetime.strptime(event.date_end, tools.DEFAULT_SERVER_DATETIME_FORMAT).date()
        return ('%s %s%s') % (start_date.strftime("%b"), start_date.strftime("%e"), (end_date != start_date and ("-"+end_date.strftime("%e")) or ""))

    @http.route('/event/get_country_event_list', type='http', auth='public', website=True)
    def get_country_events(self ,**post):
        if not GeoIP:
            return ""
        country_obj = request.registry['res.country']
        event_obj = request.registry['event.event']
        cr, uid, context,event_ids = request.cr, request.uid, request.context,[]
        country_code = self.get_visitors_country()['country_code']
        result = {'events':[],'country':False}
        if country_code:
            country_ids = country_obj.search(request.cr, request.uid, [('code', '=', country_code)], context=request.context)
            event_ids = event_obj.search(request.cr, request.uid, ['|', ('address_id', '=', None),('country_id.code', '=', country_code),('date_begin','>=', time.strftime('%Y-%m-%d 00:00:00')),('state', '=', 'confirm')], order="date_begin", context=request.context)
        if not event_ids:
            event_ids = event_obj.search(request.cr, request.uid, [('date_begin','>=', time.strftime('%Y-%m-%d 00:00:00')),('state', '=', 'confirm')], order="date_begin", context=request.context)
        for event in event_obj.browse(request.cr, request.uid, event_ids, context=request.context)[:6]:
            if country_code and event.country_id.code == country_code:
                result['country'] = country_obj.browse(request.cr, request.uid, country_ids[0], context=request.context)
            result['events'].append({
                 "date": self.get_formated_date(event),
                 "event": event,
                 "url": event.website_url})
        return request.website.render("website_event.country_events_list",result)
<|endoftext|>


<commit_before>"""
Process here will be to iterate over every item in the SCDB and to locate it in
CourtListener.

Location is done by:
 - Looking in the `supreme_court_db_id` field. During the first run of this
   program we expect this to fail for all items since they will not have this
   field populated yet. During subsequent runs, this field will have hits and
   will provide improved performance.
 - Looking for matching U.S. and docket number.

Once located, we update items:
 - Case name -- will this automatically fix the docket as well?
 - Citations (Lexis, US, L.Ed., etc.)
 - Docket number?
 - supreme_court_db_id
"""
import os
import sys
from lxml.etree import XMLSyntaxError

execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "alert.settings")

from alert.citations.tasks import update_document_by_id
from alert.corpus_importer.dup_helpers import get_html_from_raw_text
from alert.lib.string_diff import get_cosine_similarity
from alert.search.models import Document
import csv
from datetime import date
from django.db.models.query import EmptyQuerySet

DATA_DIR = os.path.dirname(__name__)
SCDB_FILENAME = os.path.join(DATA_DIR, 'SCDB_2014_01_caseCentered_Citation.csv')
SCDB_BEGINS = date(1946, 11, 18)
SCDB_ENDS = date(2014, 6, 19)
START_ROW = 7290

def merge_docs(first_pk, second_pk):
    first = Document.objects.get(pk=first_pk)
    second = Document.objects.get(pk=second_pk)
    first.source = 'LR'
    first.judges = second.judges
    first.html_lawbox = second.html_lawbox
    first.save()
    first.citation.case_name = second.citation.case_name
    first.citation.slug = second.citation.slug
    first.citation.save()
    second.docket.delete()
    second.citation.delete()
    second.delete()
    update_document_by_id(first_pk)

with open(SCDB_FILENAME) as f:
    dialect = csv.Sniffer().sniff(f.read(1024))
    f.seek(0)
    reader = csv.DictReader(f, dialect=dialect)
    for i, d in enumerate(reader):
        # Iterate over every item, looking for matches in various ways.
        if i < START_ROW:
            continue
        print "Row is: %s. ID is: %s" % (i, d['caseId'])

        ds = EmptyQuerySet()
        if len(ds) == 0:
            print "  Checking by caseID...",
            ds = Document.objects.filter(
                supreme_court_db_id=d['caseId'])
            print "%s matches found." % len(ds)
        if d['usCite'].strip():
            # Only do these lookups if there is in fact a usCite value.
            if len(ds) == 0:
                # None found by supreme_court_db_id. Try by citation number.
                print "  Checking by federal_cite_one..",
                ds = Document.objects.filter(
                    citation__federal_cite_one=d['usCite'])
                print "%s matches found." % len(ds)
            if len(ds) == 0:
                print "  Checking by federal_cite_one...",
                ds = Document.objects.filter(
                    citation__federal_cite_two=d['usCite'])
                print "%s matches found." % len(ds)
            if len(ds) == 0:
                print "  Checking by federal_cite_three...",
                ds = Document.objects.filter(
                    citation__federal_cite_three=d['usCite'])
                print "%s matches found." % len(ds)

        if len(ds) == 0:
            print '  No items found.'
        elif len(ds) == 1:
            print '  Exactly one match found.'
        elif len(ds) == 2:
            print '  Two items found.'
            print '    Absolute URLs:\n      %s' % '\n      '.join([
                'https://www.courtlistener.com/opinion/%s/slug/' % d.pk
                for d in ds])

            # Get the cosine similarity
            try:
                _, _, _, body_text_0 = get_html_from_raw_text(ds[0].html)
                _, _, _, body_text_1 = get_html_from_raw_text(ds[1].html_lawbox)
                cos_sim = get_cosine_similarity(body_text_0, body_text_1)
                print '    Cosine similarity is: %s' % cos_sim
            except XMLSyntaxError:
                # hit error on item 1414
                print '    Unable to form XML. Forcing manual review.'
                cos_sim = 0

            if cos_sim > 0.96:
                proceed = True
            else:
                proceed = raw_input("    Should we merge these? (Ctrl+C to "
                                    "quit, or y to merge or anything else to "
                                    "move simply move on): ")
                if 'y' == proceed.lower():
                    proceed = True
                else:
                    proceed = False
            if proceed:
                print '      --> Doing merge.'
                merge_docs(first_pk=ds[0].pk, second_pk=ds[1].pk)
<commit_msg>Moves SCDB code forward.<commit_after>"""
Process here will be to iterate over every item in the SCDB and to locate it in
CourtListener.

Location is done by:
 - Looking in the `supreme_court_db_id` field. During the first run of this
   program we expect this to fail for all items since they will not have this
   field populated yet. During subsequent runs, this field will have hits and
   will provide improved performance.
 - Looking for matching U.S. and docket number.

Once located, we update items:
 - Case name -- will this automatically fix the docket as well?
 - Citations (Lexis, US, L.Ed., etc.)
 - Docket number?
 - supreme_court_db_id
"""
import os
import sys
from lxml.etree import XMLSyntaxError

execfile('/etc/courtlistener')
sys.path.append(INSTALL_ROOT)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "alert.settings")

from alert.citations.tasks import update_document_by_id
from alert.corpus_importer.dup_helpers import get_html_from_raw_text
from alert.lib.string_diff import get_cosine_similarity
from alert.search.models import Document
import csv
from datetime import date
from django.db.models.query import EmptyQuerySet

DATA_DIR = os.path.dirname(__name__)
SCDB_FILENAME = os.path.join(DATA_DIR, 'SCDB_2014_01_caseCentered_Citation.csv')
SCDB_BEGINS = date(1946, 11, 18)
SCDB_ENDS = date(2014, 6, 19)
START_ROW = 7348

def merge_docs(first_pk, second_pk):
    first = Document.objects.get(pk=first_pk)
    second = Document.objects.get(pk=second_pk)
    first.source = 'LR'
    first.judges = second.judges
    first.html_lawbox = second.html_lawbox
    first.save()
    first.citation.case_name = second.citation.case_name
    first.citation.slug = second.citation.slug
    first.citation.save()
    second.docket.delete()
    second.citation.delete()
    second.delete()
    update_document_by_id(first_pk)

with open(SCDB_FILENAME) as f:
    dialect = csv.Sniffer().sniff(f.read(1024))
    f.seek(0)
    reader = csv.DictReader(f, dialect=dialect)
    for i, d in enumerate(reader):
        # Iterate over every item, looking for matches in various ways.
        if i < START_ROW:
            continue
        print "Row is: %s. ID is: %s" % (i, d['caseId'])

        ds = EmptyQuerySet()
        if len(ds) == 0:
            print "  Checking by caseID...",
            ds = Document.objects.filter(
                supreme_court_db_id=d['caseId'])
            print "%s matches found." % len(ds)
        if d['usCite'].strip():
            # Only do these lookups if there is in fact a usCite value.
            if len(ds) == 0:
                # None found by supreme_court_db_id. Try by citation number.
                print "  Checking by federal_cite_one..",
                ds = Document.objects.filter(
                    citation__federal_cite_one=d['usCite'])
                print "%s matches found." % len(ds)
            if len(ds) == 0:
                print "  Checking by federal_cite_one...",
                ds = Document.objects.filter(
                    citation__federal_cite_two=d['usCite'])
                print "%s matches found." % len(ds)
            if len(ds) == 0:
                print "  Checking by federal_cite_three...",
                ds = Document.objects.filter(
                    citation__federal_cite_three=d['usCite'])
                print "%s matches found." % len(ds)

        if len(ds) == 0:
            print '  No items found.'
        elif len(ds) == 1:
            print '  Exactly one match found.'
        elif len(ds) == 2:
            print '  Two items found.'
            print '    Absolute URLs:\n      %s' % '\n      '.join([
                'https://www.courtlistener.com/opinion/%s/slug/' % d.pk
                for d in ds])

            # Get the cosine similarity
            try:
                _, _, _, body_text_0 = get_html_from_raw_text(ds[0].html)
                _, _, _, body_text_1 = get_html_from_raw_text(ds[1].html_lawbox)
                cos_sim = get_cosine_similarity(body_text_0, body_text_1)
                print '    Cosine similarity is: %s' % cos_sim
            except XMLSyntaxError:
                # hit error on item 1414
                print '    Unable to form XML. Forcing manual review.'
                cos_sim = 0

            if cos_sim > 0.96:
                proceed = True
            else:
                proceed = raw_input("    Should we merge these? (Ctrl+C to "
                                    "quit, or y to merge or anything else to "
                                    "move simply move on): ")
                if 'y' == proceed.lower():
                    proceed = True
                else:
                    proceed = False
            if proceed:
                print '      --> Doing merge.'
                merge_docs(first_pk=ds[0].pk, second_pk=ds[1].pk)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import traceback

import numpy as np

from . import core
from .kernelcl import environment
from .compare import (MODELS, randomize_pars, suppress_pd, eval_sasview,
                      eval_opencl, eval_ctypes, make_data, get_demo_pars,
                      columnize, constrain_pars, constrain_new_to_old,
                      make_engine)

def calc_stats(target, value, index):
    resid = abs(value-target)[index]
    relerr = resid/target[index]
    srel = np.argsort(relerr)
    #p90 = int(len(relerr)*0.90)
    p95 = int(len(relerr)*0.95)
    maxrel = np.max(relerr)
    rel95 = relerr[srel[p95]]
    maxabs = np.max(resid[srel[p95:]])
    maxval = np.max(value[srel[p95:]])
    return maxrel,rel95,maxabs,maxval

def print_column_headers(pars, parts):
    stats = list('Max rel err|95% rel err|Max abs err above 90% rel|Max value above 90% rel'.split('|'))
    groups = ['']
    for p in parts:
        groups.append(p)
        groups.extend(['']*(len(stats)-1))
    groups.append("Parameters")
    columns = ['Seed'] + stats*len(parts) +  list(sorted(pars.keys()))
    print(','.join('"%s"'%c for c in groups))
    print(','.join('"%s"'%c for c in columns))

PRECISION = {
    'fast': 1e-3,
    'half': 1e-3,
    'single': 5e-5,
    'double': 5e-14,
    'single!': 5e-5,
    'double!': 5e-14,
    'quad!': 5e-18,
    'sasview': 5e-14,
}
def compare_instance(name, data, index, N=1, mono=True, cutoff=1e-5,
                     base='sasview', comp='double'):
    model_definition = core.load_model_definition(name)
    pars = get_demo_pars(model_definition)
    header = '\n"Model","%s","Count","%d"'%(name, N)
    if not mono: header += ',"Cutoff",%g'%(cutoff,)
    print(header)

    # Some not very clean macros for evaluating the models and checking the
    # results.  They freely use variables from the current scope, even some
    # which have not been defined yet, complete with abuse of mutable lists
    # to allow them to update values in the current scope since nonlocal
    # declarations are not available in python 2.7.
    def try_model(fn, pars):
        try:
            result = fn(**pars)
        except KeyboardInterrupt:
            raise
        except:
            traceback.print_exc()
            print("when comparing %s for %d"%(name, seed))
            if hasattr(data, 'qx_data'):
                result = np.NaN*data.data
            else:
                result = np.NaN*data.x
        return result
    def check_model(pars):
        base_value = try_model(calc_base, pars)
        comp_value = try_model(calc_comp, pars)
        stats = calc_stats(base_value, comp_value, index)
        max_diff[0] = max(max_diff[0], stats[0])
        good[0] = good[0] and (stats[0] < expected)
        return list(stats)

    calc_base = make_engine(model_definition, data, base, cutoff)
    calc_comp = make_engine(model_definition, data, comp, cutoff)
    expected = max(PRECISION[base], PRECISION[comp])

    num_good = 0
    first = True
    max_diff = [0]
    for k in range(N):
        print("%s %d"%(name, k))
        seed = np.random.randint(1e6)
        pars_i = randomize_pars(pars, seed)
        constrain_pars(model_definition, pars_i)
        constrain_new_to_old(model_definition, pars_i)
        if mono:
            pars_i = suppress_pd(pars_i)

        good = [True]
        columns = check_model(pars_i)
        columns += [v for _,v in sorted(pars_i.items())]
        if first:
            labels = [" vs. ".join((calc_base.engine, calc_comp.engine))]
            print_column_headers(pars_i, labels)
            first = False
        if good[0]:
            num_good += 1
        else:
            print(("%d,"%seed)+','.join("%s"%v for v in columns))
    print('"good","%d/%d","max diff",%g'%(num_good, N, max_diff[0]))

def print_usage():
    print("usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono) (single|double|quad)")

def print_models():
    print(columnize(MODELS, indent="  "))

def print_help():
    print_usage()
    print("""\

MODEL is the model name of the model or "all" for all the models
in alphabetical order.

COUNT is the number of randomly generated parameter sets to try. A value
of "10000" is a reasonable check for monodisperse models, or "100" for
polydisperse models.   For a quick check, use "100" and "5" respectively.

NQ is the number of Q values to calculate.  If it starts with "1d", then
it is a 1-dimensional problem, with log spaced Q points from 1e-3 to 1.0.
If it starts with "2d" then it is a 2-dimensional problem, with linearly
spaced points Q points from -1.0 to 1.0 in each dimension. The usual
values are "1d100" for 1-D and "2d32" for 2-D.

CUTOFF is the cutoff value to use for the polydisperse distribution. Weights
below the cutoff will be ignored.  Use "mono" for monodisperse models.  The
choice of polydisperse parameters, and the number of points in the distribution
is set in compare.py defaults for each model.

PRECISION is the floating point precision to use for comparisons.  If two
precisions are given, then compare one to the other, ignoring sasview.

Available models:
""")
    print_models()

def main():
    if len(sys.argv) not in (6,7):
        print_help()
        sys.exit(1)

    model = sys.argv[1]
    if not (model in MODELS) and (model != "all"):
        print('Bad model %s.  Use "all" or one of:')
        print_models()
        sys.exit(1)
    try:
        count = int(sys.argv[2])
        is2D = sys.argv[3].startswith('2d')
        assert sys.argv[3][1] == 'd'
        Nq = int(sys.argv[3][2:])
        mono = sys.argv[4] == 'mono'
        cutoff = float(sys.argv[4]) if not mono else 0
        base = sys.argv[5]
        comp = sys.argv[6] if len(sys.argv) > 6 else "sasview"
    except:
        traceback.print_exc()
        print_usage()
        sys.exit(1)

    data, index = make_data({'qmax':1.0, 'is2d':is2D, 'nq':Nq, 'res':0.,
                              'accuracy': 'Low', 'view':'log'})
    model_list = [model] if model != "all" else MODELS
    for model in model_list:
        compare_instance(model, data, index, N=count, mono=mono,
                         cutoff=cutoff, base=base, comp=comp)

if __name__ == "__main__":
    main()
<commit_msg>fix 2d multi-compare<commit_after>#!/usr/bin/env python
from __future__ import print_function

import sys
import traceback

import numpy as np

from . import core
from . import generate
from .compare import (MODELS, randomize_pars, suppress_pd, make_data,
                      make_engine, get_demo_pars, columnize,
                      constrain_pars, constrain_new_to_old)

def calc_stats(target, value, index):
    resid = abs(value-target)[index]
    relerr = resid/target[index]
    srel = np.argsort(relerr)
    #p90 = int(len(relerr)*0.90)
    p95 = int(len(relerr)*0.95)
    maxrel = np.max(relerr)
    rel95 = relerr[srel[p95]]
    maxabs = np.max(resid[srel[p95:]])
    maxval = np.max(value[srel[p95:]])
    return maxrel,rel95,maxabs,maxval

def print_column_headers(pars, parts):
    stats = list('Max rel err|95% rel err|Max abs err above 90% rel|Max value above 90% rel'.split('|'))
    groups = ['']
    for p in parts:
        groups.append(p)
        groups.extend(['']*(len(stats)-1))
    groups.append("Parameters")
    columns = ['Seed'] + stats*len(parts) +  list(sorted(pars.keys()))
    print(','.join('"%s"'%c for c in groups))
    print(','.join('"%s"'%c for c in columns))

PRECISION = {
    'fast': 1e-3,
    'half': 1e-3,
    'single': 5e-5,
    'double': 5e-14,
    'single!': 5e-5,
    'double!': 5e-14,
    'quad!': 5e-18,
    'sasview': 5e-14,
}
def compare_instance(name, data, index, N=1, mono=True, cutoff=1e-5,
                     base='sasview', comp='double'):
    is2D = hasattr(data, 'qx_data')
    model_definition = core.load_model_definition(name)
    pars = get_demo_pars(model_definition)
    header = ('\n"Model","%s","Count","%d","Dimension","%s"'
              % (name, N, "2D" if is2D else "1D"))
    if not mono: header += ',"Cutoff",%g'%(cutoff,)
    print(header)

    if is2D:
        info = generate.make_info(model_definition)
        partype = info['partype']
        if not partype['orientation'] and not partype['magnetic']:
            print(',"1-D only"')
            return

    # Some not very clean macros for evaluating the models and checking the
    # results.  They freely use variables from the current scope, even some
    # which have not been defined yet, complete with abuse of mutable lists
    # to allow them to update values in the current scope since nonlocal
    # declarations are not available in python 2.7.
    def try_model(fn, pars):
        try:
            result = fn(**pars)
        except KeyboardInterrupt:
            raise
        except:
            traceback.print_exc()
            print("when comparing %s for %d"%(name, seed))
            if hasattr(data, 'qx_data'):
                result = np.NaN*data.data
            else:
                result = np.NaN*data.x
        return result
    def check_model(pars):
        base_value = try_model(calc_base, pars)
        comp_value = try_model(calc_comp, pars)
        stats = calc_stats(base_value, comp_value, index)
        max_diff[0] = max(max_diff[0], stats[0])
        good[0] = good[0] and (stats[0] < expected)
        return list(stats)

    calc_base = make_engine(model_definition, data, base, cutoff)
    calc_comp = make_engine(model_definition, data, comp, cutoff)
    expected = max(PRECISION[base], PRECISION[comp])

    num_good = 0
    first = True
    max_diff = [0]
    for k in range(N):
        print("%s %d"%(name, k), file=sys.stderr)
        seed = np.random.randint(1e6)
        pars_i = randomize_pars(pars, seed)
        constrain_pars(model_definition, pars_i)
        constrain_new_to_old(model_definition, pars_i)
        if mono:
            pars_i = suppress_pd(pars_i)

        good = [True]
        columns = check_model(pars_i)
        columns += [v for _,v in sorted(pars_i.items())]
        if first:
            labels = [" vs. ".join((calc_base.engine, calc_comp.engine))]
            print_column_headers(pars_i, labels)
            first = False
        if good[0]:
            num_good += 1
        else:
            print(("%d,"%seed)+','.join("%s"%v for v in columns))
    print('"good","%d/%d","max diff",%g'%(num_good, N, max_diff[0]))

def print_usage():
    print("usage: compare_many.py MODEL COUNT (1dNQ|2dNQ) (CUTOFF|mono) (single|double|quad)")

def print_models():
    print(columnize(MODELS, indent="  "))

def print_help():
    print_usage()
    print("""\

MODEL is the model name of the model or "all" for all the models
in alphabetical order.

COUNT is the number of randomly generated parameter sets to try. A value
of "10000" is a reasonable check for monodisperse models, or "100" for
polydisperse models.   For a quick check, use "100" and "5" respectively.

NQ is the number of Q values to calculate.  If it starts with "1d", then
it is a 1-dimensional problem, with log spaced Q points from 1e-3 to 1.0.
If it starts with "2d" then it is a 2-dimensional problem, with linearly
spaced points Q points from -1.0 to 1.0 in each dimension. The usual
values are "1d100" for 1-D and "2d32" for 2-D.

CUTOFF is the cutoff value to use for the polydisperse distribution. Weights
below the cutoff will be ignored.  Use "mono" for monodisperse models.  The
choice of polydisperse parameters, and the number of points in the distribution
is set in compare.py defaults for each model.

PRECISION is the floating point precision to use for comparisons.  If two
precisions are given, then compare one to the other, ignoring sasview.

Available models:
""")
    print_models()

def main():
    if len(sys.argv) not in (6,7):
        print_help()
        sys.exit(1)

    model = sys.argv[1]
    if not (model in MODELS) and (model != "all"):
        print('Bad model %s.  Use "all" or one of:')
        print_models()
        sys.exit(1)
    try:
        count = int(sys.argv[2])
        is2D = sys.argv[3].startswith('2d')
        assert sys.argv[3][1] == 'd'
        Nq = int(sys.argv[3][2:])
        mono = sys.argv[4] == 'mono'
        cutoff = float(sys.argv[4]) if not mono else 0
        base = sys.argv[5]
        comp = sys.argv[6] if len(sys.argv) > 6 else "sasview"
    except:
        traceback.print_exc()
        print_usage()
        sys.exit(1)

    data, index = make_data({'qmax':1.0, 'is2d':is2D, 'nq':Nq, 'res':0.,
                              'accuracy': 'Low', 'view':'log'})
    model_list = [model] if model != "all" else MODELS
    for model in model_list:
        compare_instance(model, data, index, N=count, mono=mono,
                         cutoff=cutoff, base=base, comp=comp)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from django.contrib.staticfiles.finders import get_finders
from django.core.files.storage import FileSystemStorage

class SassFileStorage(FileSystemStorage):
    def __init__(self, location=None, base_url=None, *args, **kwargs):
        if location is None:
            location = getattr(settings, 'SASS_PROCESSOR_ROOT', settings.STATIC_ROOT)
        if base_url is None:
            base_url = settings.STATIC_URL
        super(SassFileStorage, self).__init__(location, base_url, *args, **kwargs)

try:
    from storages.backends.s3boto3 import S3Boto3Storage

    class SassS3Boto3Storage(S3Boto3Storage):
        base_url = '{}.s3.amazonaws.com'.format(settings.AWS_STORAGE_BUCKET_NAME)

except (AttributeError, ImportError):
    pass

def find_file(path):
    for finder in get_finders():
        result = finder.find(path)
        if result:
            return result
<commit_msg>Catch exception if s3boto is not installed.<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from django.contrib.staticfiles.finders import get_finders
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import FileSystemStorage

class SassFileStorage(FileSystemStorage):
    def __init__(self, location=None, base_url=None, *args, **kwargs):
        if location is None:
            location = getattr(settings, 'SASS_PROCESSOR_ROOT', settings.STATIC_ROOT)
        if base_url is None:
            base_url = settings.STATIC_URL
        super(SassFileStorage, self).__init__(location, base_url, *args, **kwargs)

try:
    from storages.backends.s3boto3 import S3Boto3Storage

    class SassS3Boto3Storage(S3Boto3Storage):
        base_url = '{}.s3.amazonaws.com'.format(settings.AWS_STORAGE_BUCKET_NAME)

except (AttributeError, ImportError, ImproperlyConfigured):
    pass

def find_file(path):
    for finder in get_finders():
        result = finder.find(path)
        if result:
            return result
<|endoftext|>


<commit_before>import sys
import logging

__version__ = "0.1"

def set_logging(name="sen", level=logging.DEBUG):
    logger = logging.getLogger(name)
    logger.setLevel(level)

    if level == logging.DEBUG:
        handler = logging.FileHandler("debug.log")
    else:
        handler = logging.StreamHandler(sys.stderr)

    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s %(name)-18s %(levelname)-6s %(message)s', '%H:%M:%S')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
<commit_msg>don't log to stderr<commit_after>import sys
import logging

__version__ = "0.1"

def set_logging(name="sen", level=logging.DEBUG):
    logger = logging.getLogger(name)
    logger.setLevel(level)

    if level == logging.DEBUG:
        handler = logging.FileHandler("debug.log")
        handler.setLevel(logging.DEBUG)
    # else:
    #     handler = logging.StreamHandler(sys.stderr)

        formatter = logging.Formatter('%(asctime)s %(name)-18s %(levelname)-6s %(message)s',
                                      '%H:%M:%S')
        handler.setFormatter(formatter)
        logger.addHandler(handler)
<|endoftext|>


<commit_before>from collections import defaultdict
import Shapes
import Colors

animation = list()

class LSFrameGen:

    def __init__(self, rows, cols):
        self.rows = rows
        self.cols = cols
        self.frame = defaultdict(lambda: defaultdict(int))

    # Allows you to edit an existing frame structure, if no colormask is set
    # then the tile will keep its current colormask
    def edit(self,row,col,shape=None,colormask=None):
        if colormask is None:
            try:
                colormask = self.frame[row][col][1]
            except TypeError:
                print("Frame warning: No colormask set for cell at ({:d},{:d})".format(row,col))
        if shape is None:
            try:
                shape = self.frame[row][col][0]
            except TypeError:
                print("Frame warning: No shape set for cell at ({:d},{:d})".format(row,col))
        self.frame[row][col] = [shape, colormask]

    def print(self):
        for row in self.frame:
            print([self.frame[row][col] for col in self.frame[row]])

    def get(self):
        frameOut = list()
        frameOut.append(self.cols)
        for row in self.frame:
            for col in self.frame:
                cell = self.frame[row][col]
         #       frameOut.append(row)
         #       frameOut.append(col)
                frameOut.append(cell[0])
                frameOut.append(cell[1][0])
                frameOut.append(cell[1][1])
                frameOut.append(cell[1][2])
        return(frameOut)  # frameOut is a list consisting of the number of columns in the frame
                          # followed by a repeating pattern of 4 integers, each representing a
                          # subsequent tile's shape, and red, green, and blue colormasks

def main():
    print("TODO: testing lsanimate")

    colormask = Colors.segmentsToRgb([Colors.RED, Colors.RED])

    frame = LSFrameGen(2,2)
    frame.edit(1,1,Shapes.ZERO, colormask)
    frame.edit(1,1,Shapes.SEVEN)
    frame.edit(1,2,Shapes.ONE, colormask)
    frame.edit(2,2,Shapes.ZERO, colormask)
    frame.edit(2,1,Shapes.ONE, colormask)

    frame.print()
    print("")
    print(frame.get())

if __name__ == '__main__':
    main()
<commit_msg>Added lsanimate.py<commit_after>from collections import defaultdict
import Shapes
import Colors

class LSAnimation:

    def __init__(self, rowOffset, colOffset)
        self.offset = (rowOffset, colOffset)
        self._frames = list()

    def addFrame(self, frame):
        self.validateFrame(frame)
        self._frames.append(frame)

    def validateFrame(self, frame):
        print(frame.len())

class LSFrameGen:

    def __init__(self, rows, cols):
        self.rows = rows
        self.cols = cols
        self.frame = defaultdict(lambda: defaultdict(int))

    # Allows you to edit an existing frame structure, if no colormask is set
    # then the tile will keep its current colormask
    def edit(self,row,col,shape=None,colormask=None):
        if colormask is None:
            try:
                colormask = self.frame[row][col][1]
            except TypeError:
                print("Frame warning: No colormask set for cell at ({:d},{:d})".format(row,col))
        if shape is None:
            try:
                shape = self.frame[row][col][0]
            except TypeError:
                print("Frame warning: No shape set for cell at ({:d},{:d})".format(row,col))
        self.frame[row][col] = [shape, colormask]

    def print(self):
        for row in self.frame:
            print([self.frame[row][col] for col in self.frame[row]])

    def get(self):
        frameOut = list()
        frameOut.append(self.cols)
        for row in self.frame:
            for col in self.frame:
                cell = self.frame[row][col]
         #       frameOut.append(row)
         #       frameOut.append(col)
                frameOut.append(cell[0])
                frameOut.append(cell[1][0])
                frameOut.append(cell[1][1])
                frameOut.append(cell[1][2])
        return(frameOut)  # frameOut is a list consisting of the number of columns in the frame
                          # followed by a repeating pattern of 4 integers, each representing a
                          # subsequent tile's shape, and red, green, and blue colormasks

def main():
    print("TODO: testing lsanimate")

    colormask = Colors.segmentsToRgb([Colors.RED, Colors.RED])

    frame = LSFrameGen(2,2)
    frame.edit(1,1,Shapes.ZERO, colormask)
    frame.edit(1,1,Shapes.SEVEN)
    frame.edit(1,2,Shapes.ONE, colormask)
    frame.edit(2,2,Shapes.ZERO, colormask)
    frame.edit(2,1,Shapes.ONE, colormask)

    thisFrame = frame.get()

    ourAnimation = LSAnimation(0,0)

    ourAnimation.addFrame(thisFrame)
    ourAnimation.addFrame(thisFrame)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Plugin for uploading html files of a certain markup, provided by BZK
"""

from __future__ import unicode_literals, absolute_import
from amcat.scripts.article_upload.upload import UploadScript
from amcat.tools.toolkit import readDate
from amcat.models.medium import Medium
from amcat.models.article import Article
from lxml import html
from html2text import html2text
import re
import logging; log = logging.getLogger(__name__)
from amcat.scripts.article_upload.bzk_aliases import BZK_ALIASES as MEDIUM_ALIASES

class BZK(UploadScript):
    def _scrape_unit(self, _file):
        if type(_file) == unicode: #command line
            etree = html.fromstring(_file)
        else: #web interface
            try:
                etree = html.parse(_file).getroot()
            except IOError as e:
                log.exception("Failed html.parse")
                raise TypeError("failed HTML parsing. Are you sure you've inserted the right file?\n{e}".format(**locals()))

        title = etree.cssselect("title")
        if title:
            title_text = title[0].text.lower().strip()
            if "intranet/rss" in title_text or "werkmap" in title_text:
                for article in self.scrape_1(etree, title_text):
                    yield article
        elif etree.cssselect("h1"):
            for article in self.scrape_2(etree):
                yield article
        else:
            for article in self.scrape_3(etree):
                yield article

    def scrape_1(self, _html, t):
        """format of mostly 2013"""
        if "werkmap" in t:
            divs = _html.cssselect("#articleTable div")
        elif "intranet/rss" in t:
            divs = [div for div in _html.cssselect("#sort div") if "sort_" in div.get('id')]

        for div in divs:
            article = Article(metastring = {})
            article.metastring['html'] = div
            article.headline = div.cssselect("#articleTitle")[0].text_content()
            article.text = div.cssselect("#articleIntro")[0]
            articlepage = div.cssselect("#articlePage")
            if articlepage:
                article.pagenr, article.section = self.get_pagenum(articlepage[0].text)

            article.medium = self.get_medium(div.cssselect("#sourceTitle")[0].text)
            date_str = div.cssselect("#articleDate")[0].text
            try:
                article.date = readDate(date_str)
            except ValueError:
                log.error("parsing date \"{date_str}\" failed".format(**locals()))
            else:
                yield article

    def scrape_2(self, _html):
        """New format as of 2014 and a few days before"""

        docdate = readDate(_html.cssselect("h1")[0].text.split("-")[1])

        #split body by

        items = []
        item = []
        tags = set()
        for child in _html.cssselect("body > *"):
            tags.add(child.tag)
            if child.tag == "hr":
                items.append(item)
                item = []
            else:
                item.append(child)

        #first item is the index
        items = items[1:]

        for item in items:
            article = self.parse_item(item)
            if not article.date:
                article.date = docdate
            yield article

    def scrape_3(self, _html):
        """Some ugly MS Word format, as of 2014-03-03"""
        # Partition articles
        part = []
        articles = []
        for tag in _html.cssselect("body > div > *"):
            if tag.cssselect("hr"):
                articles.append(part)
                part = []
            else:
                part.append(tag)
        for tags in articles[1:]:
            article = Article()
            dateline = tags[1].text_content().strip()
            article = self.parse_dateline(dateline, article)
            article.headline = tags[1].text_content().strip()
            html_str = "".join([html.tostring(t) for t in tags[2:]])
            article.text = html2text(html_str)
            article.metastring = {'html': html_str}

            yield article

    def parse_dateline(self, text, article):
        bits = text.split()
        if "-" in bits[-1]:
            article.date = readDate(bits[-1])
            article.medium = self.get_medium(" ".join(bits[:-1]))
        elif bits[-1].isdigit():
            article.date = readDate(" ".join(bits[-3:]))
            article.medium = self.get_medium(" ".join(bits[:-3]))
        else:
            article.medium = self.get_medium(" ".join(bits))
            article.date = None
        return article

    def parse_item(self, item):
        #item: a list of html tags
        article = Article(metastring = {})
        for tag in item:
            if tag.tag in ("p","div"):
                if not (hasattr(article,'text') or article.text):
                    article.text.append(tag)
                else:
                    article.text = [tag]
            elif tag.tag == "h2":
                article.headline = tag.text
            elif tag.tag == "i":
                article = self.parse_dateline(tag.text, article)
        #process html
        article.text = "\n".join([html2text(html.tostring(bit)) for bit in article.text])
        return article

    def get_medium(self, text):
        if not text:
            text = "unknown"
        if text in MEDIUM_ALIASES.keys():
            return Medium.get_or_create(MEDIUM_ALIASES[text])
        else:
            return Medium.get_or_create(text)

    def get_pagenum(self, text):
        p = re.compile("pagina ([0-9]+)([,\-][0-9]+)?([a-zA-Z0-9 ]+)?")
        m = p.search(text.strip())
        pagenum, otherpage, section = m.groups()
        if section:
            section = section.strip()
        return int(pagenum), section

if __name__ == "__main__":
    from amcat.scripts.tools import cli
    cli.run_cli(BZK)

###########################################################################
#                          U N I T   T E S T S                            #
###########################################################################

from amcat.tools import amcattest

class TestBZK(amcattest.AmCATTestCase):
    def setUp(self):
        from django.core.files import File
        import os.path, json
        self.dir = os.path.join(os.path.dirname(__file__), 'test_files', 'bzk')
        self.bzk = BZK(project = amcattest.create_test_project().id,
                  file = File(open(os.path.join(self.dir, 'test.html'))),
                  articleset = amcattest.create_test_set().id)
        self.result = self.bzk.run()

        def test_scrape_unit(self):
            self.assertTrue(self.result)

        def test_scrape_file(self):
            #props to check for:
            # headline, text, pagenr, section, medium, date
            must_props = ('headline', 'text', 'medium', 'date')
            may_props = ('pagenr','section')
            must_props = [[getattr(a,prop) for a in self.result] for prop in must_props]
            may_props = [[getattr(a,prop) for a in self.result] for prop in may_props]

            for proplist in must_props:
                self.assertTrue(all(proplist))
            for proplist in may_props:
                #assuming at least one of the articles has the property. if not, break
                self.assertTrue(any(proplist))

<commit_msg>cherry-picked cf8f25e<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Plugin for uploading html files of a certain markup, provided by BZK
"""

from __future__ import unicode_literals, absolute_import
from amcat.scripts.article_upload.upload import UploadScript
from amcat.tools.toolkit import readDate
from amcat.models.medium import Medium
from amcat.models.article import Article
from lxml import html
from html2text import html2text
import re
import logging; log = logging.getLogger(__name__)
from amcat.scripts.article_upload.bzk_aliases import BZK_ALIASES as MEDIUM_ALIASES

class BZK(UploadScript):
    def _scrape_unit(self, _file):
        if type(_file) == unicode: #command line
            etree = html.fromstring(_file)
        else: #web interface
            try:
                etree = html.parse(_file).getroot()
            except IOError as e:
                log.exception("Failed html.parse")
                raise TypeError("failed HTML parsing. Are you sure you've inserted the right file?\n{e}".format(**locals()))

        title = etree.cssselect("title")
        if title:
            title_text = title[0].text.lower().strip()
            if "intranet/rss" in title_text or "werkmap" in title_text:
                for article in self.scrape_1(etree, title_text):
                    yield article
        elif etree.cssselect("h1"):
            for article in self.scrape_2(etree):
                yield article
        else:
            for article in self.scrape_3(etree):
                yield article

    def scrape_1(self, _html, t):
        """format of mostly 2013"""
        if "werkmap" in t:
            divs = _html.cssselect("#articleTable div")
        elif "intranet/rss" in t:
            divs = [div for div in _html.cssselect("#sort div") if "sort_" in div.get('id')]

        for div in divs:
            article = Article(metastring = {})
            article.metastring['html'] = div
            article.headline = div.cssselect("#articleTitle")[0].text_content()
            article.text = div.cssselect("#articleIntro")[0]
            articlepage = div.cssselect("#articlePage")
            if articlepage:
                article.pagenr, article.section = self.get_pagenum(articlepage[0].text)

            article.medium = self.get_medium(div.cssselect("#sourceTitle")[0].text)
            date_str = div.cssselect("#articleDate")[0].text
            try:
                article.date = readDate(date_str)
            except ValueError:
                log.error("parsing date \"{date_str}\" failed".format(**locals()))
            else:
                yield article

    def scrape_2(self, _html):
        """New format as of 2014 and a few days before"""
        docdate = readDate(_html.cssselect("h1")[0].text.split("-")[1])

        #split body by

        items = []
        item = []
        if len(_html.cssselect("body > *")) == 1:
            tags = _html.cssselect("body > div > *") #extra div wrapper as of 2014-04-08
        else:
            tags = _html.cssselect("body > *")

        for child in tags:
            if child.tag == "hr":
                items.append(item)
                item = []
            else:
                item.append(child)

        #first item is the index
        items = items[1:]
        for item in items:
            article = self.parse_item(item)
            if not article.date:
                article.date = docdate
            yield article

    def scrape_3(self, _html):
        """Some ugly MS Word format, as of 2014-03-03"""
        # Partition articles
        part = []
        articles = []
        for tag in _html.cssselect("body > div > *"):
            if tag.cssselect("hr"):
                articles.append(part)
                part = []
            else:
                part.append(tag)
        for tags in articles[1:]:
            article = Article()
            dateline = tags[1].text_content().strip()
            article = self.parse_dateline(dateline, article)
            article.headline = tags[1].text_content().strip()
            html_str = "".join([html.tostring(t) for t in tags[2:]])
            article.text = html2text(html_str)
            article.metastring = {'html': html_str}

            yield article

    def parse_dateline(self, text, article):
        bits = text.split()
        if "-" in bits[-1]:
            article.date = readDate(bits[-1])
            article.medium = self.get_medium(" ".join(bits[:-1]))
        elif bits[-1].isdigit() and bits[-3].isdigit():
            article.date = readDate(" ".join(bits[-3:]))
            article.medium = self.get_medium(" ".join(bits[:-3]))
        else:
            article.medium = self.get_medium(" ".join(bits))
            article.date = None
        return article

    def parse_item(self, item):
        #item: a list of html tags
        article = Article(metastring = {})
        for tag in item:
            if tag.tag in ("p","div"):
                if not (hasattr(article,'text') or article.text):
                    article.text.append(tag)
                else:
                    article.text = [tag]
            elif tag.tag == "h2":
                article.headline = tag.text
            elif tag.tag == "i":
                article = self.parse_dateline(tag.text_content(), article)
        #process html
        article.text = "\n".join([html2text(html.tostring(bit)) for bit in article.text])
        return article

    def get_medium(self, text):
        if not text:
            text = "unknown"
        if text in MEDIUM_ALIASES.keys():
            return Medium.get_or_create(MEDIUM_ALIASES[text])
        else:
            return Medium.get_or_create(text)

    def get_pagenum(self, text):
        p = re.compile("pagina ([0-9]+)([,\-][0-9]+)?([a-zA-Z0-9 ]+)?")
        m = p.search(text.strip())
        pagenum, otherpage, section = m.groups()
        if section:
            section = section.strip()
        return int(pagenum), section

if __name__ == "__main__":
    from amcat.scripts.tools import cli
    cli.run_cli(BZK)

###########################################################################
#                          U N I T   T E S T S                            #
###########################################################################

from amcat.tools import amcattest

class TestBZK(amcattest.AmCATTestCase):
    def setUp(self):
        from django.core.files import File
        import os.path, json
        self.dir = os.path.join(os.path.dirname(__file__), 'test_files', 'bzk')
        self.bzk = BZK(project = amcattest.create_test_project().id,
                  file = File(open(os.path.join(self.dir, 'test.html'))),
                  articleset = amcattest.create_test_set().id)
        self.result = self.bzk.run()

        def test_scrape_unit(self):
            self.assertTrue(self.result)

        def test_scrape_file(self):
            #props to check for:
            # headline, text, pagenr, section, medium, date
            must_props = ('headline', 'text', 'medium', 'date')
            may_props = ('pagenr','section')
            must_props = [[getattr(a,prop) for a in self.result] for prop in must_props]
            may_props = [[getattr(a,prop) for a in self.result] for prop in may_props]

            for proplist in must_props:
                self.assertTrue(all(proplist))
            for proplist in may_props:
                #assuming at least one of the articles has the property. if not, break
                self.assertTrue(any(proplist))

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Telegram bot to play UNO in group chats
# Copyright (c) 2016 Jannes Höke <uno@jhoeke.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from telegram import ParseMode
from telegram.ext import CommandHandler

from user_setting import UserSetting
from utils import send_async
from shared_vars import dispatcher
from internationalization import _, user_locale

@user_locale
def help_handler(bot, update):
    """Handler for the /help command"""
    help_text = _("Follow these steps:\n\n"
      "1. Add this bot to a group\n"
      "2. In the group, start a new game with /new or join an already"
      " running game with /join\n"
      "3. After at least two players have joined, start the game with"
      " /start\n"
      "4. Type @unobot into your chat box and hit "
      "space, or click the via @unobot text "
      "next to messages. You will see your cards (some greyed out), "
      "any extra options like drawing, and a ? to see the "
      "current game state. The greyed out cards are those you "
      "can not play at the moment. Tap an option to execute "
      "the selected action.\n"
      "Players can join the game at any time. To leave a game, "
      "use /leave. If a player takes more than 90 seconds to play, "
      "you can use /skip to skip that player. Use /notify_me to "
      "receive a private message when a new game is started.\n\n"
      "Language and other settings: /settings\n"
      "Other commands (only game creator):\n"
      "/close - Close lobby\n"
      "/open - Open lobby\n"
      "/kill - Terminate the game\n"
      "/kick - Select a player to kick "
      "by replying to him or her\n"
      "/enable_translations - Translate relevant texts into all "
      "languages spoken in a game\n"
      "/disable_translations - Use English for those texts\n\n"
      "Experimental: Play in multiple groups at the same time. "
      "Press the Current game: ... button and select the "
      "group you want to play a card in.\n"
      "If you enjoy this bot, "
      ""
      "rate me, join the "
      "update channel"
      " and buy an UNO card game.")

    send_async(bot, update.message.chat_id, text=help_text,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def modes(bot, update):
    """Handler for the /help command"""
    modes_explanation = _("This UNO bot has four game modes: Classic, Sanic, Wild and Text.\n\n"
      " 🎻 The Classic mode uses the conventional UNO deck and there is no auto skip.\n"
      " 🚀 The Sanic mode uses the conventional UNO deck and the bot automatically skips a player if he/she takes too long to play its turn\n"
      " 🐉 The Wild mode uses a deck with more special cards, less number variety and no auto skip.\n"
      " ✍ The Text mode uses the conventional UNO deck but instead of stickers it uses the text.\n\n"
      "To change the game mode, the GAME CREATOR has to type the bot nickname and a space, "
      "just like when playing a card, and all gamemode options should appear.")
    send_async(bot, update.message.chat_id, text=modes_explanation,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def source(bot, update):
    """Handler for the /help command"""
    source_text = _("This bot is Free Software and licensed under the AGPL. "
      "The code is available here: \n"
      "https://github.com/jh0ker/mau_mau_bot")
    attributions = _("Attributions:\n"
      'Draw icon by '
      'Faithtoken\n'
      'Pass icon by '
      'Delapouite\n'
      "Originals available on http://game-icons.net\n"
      "Icons edited by ɳick")

    send_async(bot, update.message.chat_id, text=source_text + '\n' +
                                                 attributions,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def news(bot, update):
    """Handler for the /news command"""
    send_async(bot, update.message.chat_id,
               text=_("All news here: https://telegram.me/unobotupdates"),
               disable_web_page_preview=True)

@user_locale
def stats(bot, update):
    user = update.message.from_user
    us = UserSetting.get(id=user.id)
    if not us or not us.stats:
        send_async(bot, update.message.chat_id,
                   text=_("You did not enable statistics. Use /settings in "
                          "a private chat with the bot to enable them."))
    else:
        stats_text = list()

        n = us.games_played
        stats_text.append(
            _("{number} game played",
              "{number} games played",
              n).format(number=n)
        )

        n = us.first_places
        m = round((us.first_places / us.games_played) * 100)
        stats_text.append(
            _("{number} first place ({percent}%)",
              "{number} first places ({percent}%)",
              n).format(number=n, percent=m)
        )

        n = us.cards_played
        stats_text.append(
            _("{number} card played",
              "{number} cards played",
              n).format(number=n)
        )

        send_async(bot, update.message.chat_id,
                   text='\n'.join(stats_text))

def register():
    dispatcher.add_handler(CommandHandler('help', help_handler))
    dispatcher.add_handler(CommandHandler('source', source))
    dispatcher.add_handler(CommandHandler('news', news))
    dispatcher.add_handler(CommandHandler('stats', stats))
    dispatcher.add_handler(CommandHandler('modes', modes))
<commit_msg>We shall not divide by zero<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Telegram bot to play UNO in group chats
# Copyright (c) 2016 Jannes Höke <uno@jhoeke.de>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from telegram import ParseMode
from telegram.ext import CommandHandler

from user_setting import UserSetting
from utils import send_async
from shared_vars import dispatcher
from internationalization import _, user_locale

@user_locale
def help_handler(bot, update):
    """Handler for the /help command"""
    help_text = _("Follow these steps:\n\n"
      "1. Add this bot to a group\n"
      "2. In the group, start a new game with /new or join an already"
      " running game with /join\n"
      "3. After at least two players have joined, start the game with"
      " /start\n"
      "4. Type @unobot into your chat box and hit "
      "space, or click the via @unobot text "
      "next to messages. You will see your cards (some greyed out), "
      "any extra options like drawing, and a ? to see the "
      "current game state. The greyed out cards are those you "
      "can not play at the moment. Tap an option to execute "
      "the selected action.\n"
      "Players can join the game at any time. To leave a game, "
      "use /leave. If a player takes more than 90 seconds to play, "
      "you can use /skip to skip that player. Use /notify_me to "
      "receive a private message when a new game is started.\n\n"
      "Language and other settings: /settings\n"
      "Other commands (only game creator):\n"
      "/close - Close lobby\n"
      "/open - Open lobby\n"
      "/kill - Terminate the game\n"
      "/kick - Select a player to kick "
      "by replying to him or her\n"
      "/enable_translations - Translate relevant texts into all "
      "languages spoken in a game\n"
      "/disable_translations - Use English for those texts\n\n"
      "Experimental: Play in multiple groups at the same time. "
      "Press the Current game: ... button and select the "
      "group you want to play a card in.\n"
      "If you enjoy this bot, "
      ""
      "rate me, join the "
      "update channel"
      " and buy an UNO card game.")

    send_async(bot, update.message.chat_id, text=help_text,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def modes(bot, update):
    """Handler for the /help command"""
    modes_explanation = _("This UNO bot has four game modes: Classic, Sanic, Wild and Text.\n\n"
      " 🎻 The Classic mode uses the conventional UNO deck and there is no auto skip.\n"
      " 🚀 The Sanic mode uses the conventional UNO deck and the bot automatically skips a player if he/she takes too long to play its turn\n"
      " 🐉 The Wild mode uses a deck with more special cards, less number variety and no auto skip.\n"
      " ✍ The Text mode uses the conventional UNO deck but instead of stickers it uses the text.\n\n"
      "To change the game mode, the GAME CREATOR has to type the bot nickname and a space, "
      "just like when playing a card, and all gamemode options should appear.")
    send_async(bot, update.message.chat_id, text=modes_explanation,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def source(bot, update):
    """Handler for the /help command"""
    source_text = _("This bot is Free Software and licensed under the AGPL. "
      "The code is available here: \n"
      "https://github.com/jh0ker/mau_mau_bot")
    attributions = _("Attributions:\n"
      'Draw icon by '
      'Faithtoken\n'
      'Pass icon by '
      'Delapouite\n'
      "Originals available on http://game-icons.net\n"
      "Icons edited by ɳick")

    send_async(bot, update.message.chat_id, text=source_text + '\n' +
                                                 attributions,
               parse_mode=ParseMode.HTML, disable_web_page_preview=True)

@user_locale
def news(bot, update):
    """Handler for the /news command"""
    send_async(bot, update.message.chat_id,
               text=_("All news here: https://telegram.me/unobotupdates"),
               disable_web_page_preview=True)

@user_locale
def stats(bot, update):
    user = update.message.from_user
    us = UserSetting.get(id=user.id)
    if not us or not us.stats:
        send_async(bot, update.message.chat_id,
                   text=_("You did not enable statistics. Use /settings in "
                          "a private chat with the bot to enable them."))
    else:
        stats_text = list()

        n = us.games_played
        stats_text.append(
            _("{number} game played",
              "{number} games played",
              n).format(number=n)
        )

        n = us.first_places
        m = round((us.first_places / us.games_played) * 100) if games_played else 0
        stats_text.append(
            _("{number} first place ({percent}%)",
              "{number} first places ({percent}%)",
              n).format(number=n, percent=m)
        )

        n = us.cards_played
        stats_text.append(
            _("{number} card played",
              "{number} cards played",
              n).format(number=n)
        )

        send_async(bot, update.message.chat_id,
                   text='\n'.join(stats_text))

def register():
    dispatcher.add_handler(CommandHandler('help', help_handler))
    dispatcher.add_handler(CommandHandler('source', source))
    dispatcher.add_handler(CommandHandler('news', news))
    dispatcher.add_handler(CommandHandler('stats', stats))
    dispatcher.add_handler(CommandHandler('modes', modes))
<|endoftext|>


<commit_before>from TSL2561 import TSL2561
from MCP9808 import MCP9808
import time

def main():
    tsl = TSL2561(debug=0)
    mcp = MCP9808(debug=0)
    #tsl.set_gain(16)
    while True:
        full = tsl.read_full()
        ir = tsl.read_IR()
        lux = tsl.read_lux()
        print("%d,%d = %d lux" % (full, ir, lux))
        temp = mcp.read_temp()
        print("%0.2f degC" % temp)
        time.sleep(1)

if __name__ == '__main__':
    main()
<commit_msg>Add light PWM control to test code.<commit_after>from TSL2561 import TSL2561
from MCP9808 import MCP9808
import time
import wiringpi2 as wiringpi

def main():
    wiringpi.wiringPiSetupGpio()
    wiringpi.pinMode(18,2)  # enable PWM mode on pin 18
    tsl = TSL2561(debug=0)
    mcp = MCP9808(debug=0)
    #tsl.set_gain(16)
    while True:
        full = tsl.read_full()
        ir = tsl.read_IR()
        lux = tsl.read_lux()
        print("%d,%d = %d lux" % (full, ir, lux))
        temp = mcp.read_temp()
        print("%0.2f degC" % temp)
        wiringpi.pwmWrite(18, min(int(lux), 1024))
        time.sleep(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import sys
import warnings
import distutils
from distutils.dir_util import copy_tree, remove_tree
from setuptools.command.sdist import sdist
from setuptools import Extension

# Customization of the sdist
class BuildSourceDistribution(sdist):
    """
    Override some behavior on sdist

    Copy the build/lib to native/jars to remove ant/jdk dependency
    """
    def run(self):
        # We need to build a jar cache for the source distribution
        self.run_command("build_java")
        dest = os.path.join('native','jars')
        src = os.path.join('build','lib')
        if not os.path.exists(src):
            distutils.log.error("Jar source file is missing from build")
            raise distutils.errors.DistutilsPlatformError("Error copying jar file")
        copy_tree(src, dest)

        # Collect the sources
        super(sdist, self).run()

        # Clean up the jar cache after sdist
        remove_tree(dest)

<commit_msg>Need to work with old style classes as well.<commit_after># -*- coding: utf-8 -*-
import os
import sys
import warnings
import distutils
from distutils.dir_util import copy_tree, remove_tree
from setuptools.command.sdist import sdist
from setuptools import Extension

# Customization of the sdist
class BuildSourceDistribution(sdist):
    """
    Override some behavior on sdist

    Copy the build/lib to native/jars to remove ant/jdk dependency
    """
    def run(self):
        # We need to build a jar cache for the source distribution
        self.run_command("build_java")
        dest = os.path.join('native','jars')
        src = os.path.join('build','lib')
        if not os.path.exists(src):
            distutils.log.error("Jar source file is missing from build")
            raise distutils.errors.DistutilsPlatformError("Error copying jar file")
        copy_tree(src, dest)

        # Collect the sources
        sdist.run(self)

        # Clean up the jar cache after sdist
        remove_tree(dest)

<|endoftext|>